Commit a7d8cd591c47536b0a1e359bf3b1806fc057ffe9 | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-11-04 14:24:59-04:00
committer	GitHub <noreply@github.com> 2023-11-04 14:24:59-04:00
log	a7d8cd591c47536b0a1e359bf3b1806fc057ffe9
tree	069ca321e822ea95d7265ece0b79173832a0745a
parent	f6de3ec963e3a7d96cd4f6c72b0f076f0437c45d
parent	095c4294aa8b275da0627adefad046923fcaae46
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17788 from jacobly0/x86_64

x86_64: pass more tests

49 files changed, 1650 insertions(+), 962 deletions(-)

lib
- std
  - Thread
    Condition.zig
    Mutex.zig
    RwLock.zig
    Semaphore.zig
  - atomic
    Atomic.zig
    queue.zig
  - base64.zig
  - bit_set.zig
  - compress
    zstandard.zig
  - crypto
    25519
    ed25519.zig
    Certificate.zig
    Certificate
    Bundle.zig
    aes.zig
    aes_gcm.zig
    argon2.zig
    bcrypt.zig
    cmac.zig
    ecdsa.zig
    ghash_polyval.zig
    pcurves
    p256.zig
    phc_encoding.zig
    sha2.zig
  - fmt
    parse_float.zig
  - hash
    xxhash.zig
  - http
    Client.zig
    Server.zig
    protocol.zig
  - math.zig
  - math
    big
    int.zig
    int_test.zig
  - mem.zig
  - multi_array_list.zig
  - net
    test.zig
  - once.zig
  - os
    test.zig
  - rand
    test.zig
  - zig
    tokenizer.zig
src
- Compilation.zig
- arch
  - x86_64
    CodeGen.zig
    Encoding.zig
    Mir.zig
    bits.zig
    encodings.zig
- codegen.zig
- link
  - Coff.zig
- main.zig
- print_zir.zig
- register_manager.zig
test
- behavior
  - vector.zig

lib/std/Thread/Condition.zig-10

...	...	@@ -324,8 +324,6 @@ test "Condition - wait and signal" {
324	324	return error.SkipZigTest;
325	325	}
326	326
327		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
328
329	327	const num_threads = 4;
330	328
331	329	const MultiWait = struct {
...	...	@@ -371,8 +369,6 @@ test "Condition - signal" {
371	369	return error.SkipZigTest;
372	370	}
373	371
374		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
375
376	372	const num_threads = 4;
377	373
378	374	const SignalTest = struct {
...	...	@@ -440,8 +436,6 @@ test "Condition - multi signal" {
440	436	return error.SkipZigTest;
441	437	}
442	438
443		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
444
445	439	const num_threads = 4;
446	440	const num_iterations = 4;
447	441
...	...	@@ -504,8 +498,6 @@ test "Condition - broadcasting" {
504	498	return error.SkipZigTest;
505	499	}
506	500
507		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
508
509	501	const num_threads = 10;
510	502
511	503	const BroadcastTest = struct {
...	...	@@ -573,8 +565,6 @@ test "Condition - broadcasting - wake all threads" {
573	565	return error.SkipZigTest;
574	566	}
575	567
576		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
577
578	568	var num_runs: usize = 1;
579	569	const num_threads = 10;
580	570

lib/std/Thread/Mutex.zig-2

...	...	@@ -289,8 +289,6 @@ test "Mutex - many contended" {
289	289	return error.SkipZigTest;
290	290	}
291	291
292		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
293
294	292	const num_threads = 4;
295	293	const num_increments = 1000;
296	294

lib/std/Thread/RwLock.zig-2

...	...	@@ -297,8 +297,6 @@ test "RwLock - concurrent access" {
297	297	if (builtin.single_threaded)
298	298	return;
299	299
300		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
301
302	300	const num_writers: usize = 2;
303	301	const num_readers: usize = 4;
304	302	const num_writes: usize = 10000;

lib/std/Thread/Semaphore.zig-2

...	...	@@ -39,8 +39,6 @@ test "Thread.Semaphore" {
39	39	return error.SkipZigTest;
40	40	}
41	41
42		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
43
44	42	const TestContext = struct {
45	43	sem: *Semaphore,
46	44	n: *i32,

lib/std/atomic/Atomic.zig-2

...	...	@@ -467,8 +467,6 @@ test "Atomic.fetchSub" {
467	467	}
468	468
469	469	test "Atomic.fetchMin" {
470		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
471
472	470	inline for (atomicIntTypes()) \|Int\| {
473	471	inline for (atomic_rmw_orderings) \|ordering\| {
474	472	var x = Atomic(Int).init(5);

lib/std/atomic/queue.zig-2

...	...	@@ -175,8 +175,6 @@ const puts_per_thread = 500;
175	175	const put_thread_count = 3;
176	176
177	177	test "std.atomic.Queue" {
178		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
179
180	178	var plenty_of_memory = try std.heap.page_allocator.alloc(u8, 300 * 1024);
181	179	defer std.heap.page_allocator.free(plenty_of_memory);
182	180

lib/std/base64.zig-4

...	...	@@ -355,8 +355,6 @@ pub const Base64DecoderWithIgnore = struct {
355	355	};
356	356
357	357	test "base64" {
358		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
359
360	358	@setEvalBranchQuota(8000);
361	359	try testBase64();
362	360	try comptime testAllApis(standard, "comptime", "Y29tcHRpbWU=");
...	...	@@ -377,8 +375,6 @@ test "base64 padding dest overflow" {
377	375	}
378	376
379	377	test "base64 url_safe_no_pad" {
380		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
381
382	378	@setEvalBranchQuota(8000);
383	379	try testBase64UrlSafeNoPad();
384	380	try comptime testAllApis(url_safe_no_pad, "comptime", "Y29tcHRpbWU");

lib/std/bit_set.zig-3

...	...	@@ -1638,7 +1638,6 @@ fn testStaticBitSet(comptime Set: type) !void {
1638	1638
1639	1639	test "IntegerBitSet" {
1640	1640	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
1641		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1642	1641
1643	1642	try testStaticBitSet(IntegerBitSet(0));
1644	1643	try testStaticBitSet(IntegerBitSet(1));
...	...	@@ -1651,8 +1650,6 @@ test "IntegerBitSet" {
1651	1650	}
1652	1651
1653	1652	test "ArrayBitSet" {
1654		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1655
1656	1653	inline for (.{ 0, 1, 2, 31, 32, 33, 63, 64, 65, 254, 500, 3000 }) \|size\| {
1657	1654	try testStaticBitSet(ArrayBitSet(u8, size));
1658	1655	try testStaticBitSet(ArrayBitSet(u16, size));

lib/std/compress/zstandard.zig-2

...	...	@@ -264,8 +264,6 @@ fn testReader(data: []const u8, comptime expected: []const u8) !void {
264	264	}
265	265
266	266	test "zstandard decompression" {
267		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
268
269	267	const uncompressed = @embedFile("testdata/rfc8478.txt");
270	268	const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
271	269	const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");

lib/std/crypto/25519/ed25519.zig+2-13

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("std");
2		const builtin = @import("builtin");
3	2	const crypto = std.crypto;
4	3	const debug = std.debug;
5	4	const fmt = std.fmt;
...	...	@@ -276,8 +275,8 @@ pub const Ed25519 = struct {
276	275	pub fn fromSecretKey(secret_key: SecretKey) (NonCanonicalError \|\| EncodingError \|\| IdentityElementError)!KeyPair {
277	276	// It is critical for EdDSA to use the correct public key.
278	277	// In order to enforce this, a SecretKey implicitly includes a copy of the public key.
279		// In Debug mode, we can still afford checking that the public key is correct for extra safety.
280		if (builtin.mode == .Debug) {
	278	// With runtime safety, we can still afford checking that the public key is correct.
	279	if (std.debug.runtime_safety) {
281	280	const pk_p = try Curve.fromBytes(secret_key.publicKeyBytes());
282	281	const recomputed_kp = try create(secret_key.seed());
283	282	debug.assert(mem.eql(u8, &recomputed_kp.public_key.toBytes(), &pk_p.toBytes()));
...	...	@@ -493,8 +492,6 @@ test "ed25519 key pair creation" {
493	492	}
494	493
495	494	test "ed25519 signature" {
496		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
497
498	495	var seed: [32]u8 = undefined;
499	496	_ = try fmt.hexToBytes(seed[0..], "8052030376d47112be7f73ed7a019293dd12ad910b654455798b4667d73de166");
500	497	const key_pair = try Ed25519.KeyPair.create(seed);
...	...	@@ -507,8 +504,6 @@ test "ed25519 signature" {
507	504	}
508	505
509	506	test "ed25519 batch verification" {
510		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
511
512	507	var i: usize = 0;
513	508	while (i < 100) : (i += 1) {
514	509	const key_pair = try Ed25519.KeyPair.create(null);
...	...	@@ -538,8 +533,6 @@ test "ed25519 batch verification" {
538	533	}
539	534
540	535	test "ed25519 test vectors" {
541		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
542
543	536	const Vec = struct {
544	537	msg_hex: *const [64:0]u8,
545	538	public_key_hex: *const [64:0]u8,
...	...	@@ -642,8 +635,6 @@ test "ed25519 test vectors" {
642	635	}
643	636
644	637	test "ed25519 with blind keys" {
645		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
646
647	638	const BlindKeyPair = Ed25519.key_blinding.BlindKeyPair;
648	639
649	640	// Create a standard Ed25519 key pair
...	...	@@ -667,8 +658,6 @@ test "ed25519 with blind keys" {
667	658	}
668	659
669	660	test "ed25519 signatures with streaming" {
670		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
671
672	661	const kp = try Ed25519.KeyPair.create(null);
673	662
674	663	var signer = try kp.signer(null);

lib/std/crypto/Certificate.zig+13-13

...	...	@@ -614,18 +614,18 @@ const Date = struct {
614	614	};
615	615
616	616	pub fn parseTimeDigits(text: *const [2]u8, min: u8, max: u8) !u8 {
617		const nn: @Vector(2, u16) = .{ text[0], text[1] };
618		const zero: @Vector(2, u16) = .{ '0', '0' };
619		const mm: @Vector(2, u16) = .{ 10, 1 };
620		const result = @reduce(.Add, (nn -% zero) *% mm);
	617	const result = if (use_vectors) result: {
	618	const nn: @Vector(2, u16) = .{ text[0], text[1] };
	619	const zero: @Vector(2, u16) = .{ '0', '0' };
	620	const mm: @Vector(2, u16) = .{ 10, 1 };
	621	break :result @reduce(.Add, (nn -% zero) *% mm);
	622	} else std.fmt.parseInt(u8, text, 10) catch return error.CertificateTimeInvalid;
621	623	if (result < min) return error.CertificateTimeInvalid;
622	624	if (result > max) return error.CertificateTimeInvalid;
623	625	return @truncate(result);
624	626	}
625	627
626	628	test parseTimeDigits {
627		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
628
629	629	const expectEqual = std.testing.expectEqual;
630	630	try expectEqual(@as(u8, 0), try parseTimeDigits("00", 0, 99));
631	631	try expectEqual(@as(u8, 99), try parseTimeDigits("99", 0, 99));
...	...	@@ -638,17 +638,17 @@ test parseTimeDigits {
638	638	}
639	639
640	640	pub fn parseYear4(text: *const [4]u8) !u16 {
641		const nnnn: @Vector(4, u32) = .{ text[0], text[1], text[2], text[3] };
642		const zero: @Vector(4, u32) = .{ '0', '0', '0', '0' };
643		const mmmm: @Vector(4, u32) = .{ 1000, 100, 10, 1 };
644		const result = @reduce(.Add, (nnnn -% zero) *% mmmm);
	641	const result = if (use_vectors) result: {
	642	const nnnn: @Vector(4, u32) = .{ text[0], text[1], text[2], text[3] };
	643	const zero: @Vector(4, u32) = .{ '0', '0', '0', '0' };
	644	const mmmm: @Vector(4, u32) = .{ 1000, 100, 10, 1 };
	645	break :result @reduce(.Add, (nnnn -% zero) *% mmmm);
	646	} else std.fmt.parseInt(u16, text, 10) catch return error.CertificateTimeInvalid;
645	647	if (result > 9999) return error.CertificateTimeInvalid;
646	648	return @truncate(result);
647	649	}
648	650
649	651	test parseYear4 {
650		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
651
652	652	const expectEqual = std.testing.expectEqual;
653	653	try expectEqual(@as(u16, 0), try parseYear4("0000"));
654	654	try expectEqual(@as(u16, 9999), try parseYear4("9999"));
...	...	@@ -1124,4 +1124,4 @@ pub const rsa = struct {
1124	1124	}
1125	1125	};
1126	1126
1127		const builtin = @import("builtin");
	1127	const use_vectors = @import("builtin").zig_backend != .stage2_x86_64;

lib/std/crypto/Certificate/Bundle.zig-2

...	...	@@ -318,8 +318,6 @@ const MapContext = struct {
318	318	test "scan for OS-provided certificates" {
319	319	if (builtin.os.tag == .wasi) return error.SkipZigTest;
320	320
321		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
322
323	321	var bundle: Bundle = .{};
324	322	defer bundle.deinit(std.testing.allocator);
325	323

lib/std/crypto/aes.zig-2

...	...	@@ -28,8 +28,6 @@ pub const Aes128 = impl.Aes128;
28	28	pub const Aes256 = impl.Aes256;
29	29
30	30	test "ctr" {
31		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
32
33	31	// NIST SP 800-38A pp 55-58
34	32	const ctr = @import("modes.zig").ctr;
35	33

lib/std/crypto/aes_gcm.zig+2-11

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("std");
2		const builtin = @import("builtin");
3	2	const assert = std.debug.assert;
4	3	const crypto = std.crypto;
5	4	const debug = std.debug;
...	...	@@ -42,7 +41,7 @@ fn AesGcm(comptime Aes: anytype) type {
42	41	mac.pad();
43	42
44	43	mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);
45		modes.ctr(@TypeOf(aes), aes, c, m, j, std.builtin.Endian.big);
	44	modes.ctr(@TypeOf(aes), aes, c, m, j, .big);
46	45	mac.update(c[0..m.len][0..]);
47	46	mac.pad();
48	47
...	...	@@ -104,7 +103,7 @@ fn AesGcm(comptime Aes: anytype) type {
104	103	}
105	104
106	105	mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);
107		modes.ctr(@TypeOf(aes), aes, m, c, j, std.builtin.Endian.big);
	106	modes.ctr(@TypeOf(aes), aes, m, c, j, .big);
108	107	}
109	108	};
110	109	}
...	...	@@ -113,8 +112,6 @@ const htest = @import("test.zig");
113	112	const testing = std.testing;
114	113
115	114	test "Aes256Gcm - Empty message and no associated data" {
116		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
117
118	115	const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
119	116	const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
120	117	const ad = "";
...	...	@@ -127,8 +124,6 @@ test "Aes256Gcm - Empty message and no associated data" {
127	124	}
128	125
129	126	test "Aes256Gcm - Associated data only" {
130		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
131
132	127	const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
133	128	const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
134	129	const m = "";
...	...	@@ -141,8 +136,6 @@ test "Aes256Gcm - Associated data only" {
141	136	}
142	137
143	138	test "Aes256Gcm - Message only" {
144		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
145
146	139	const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
147	140	const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
148	141	const m = "Test with message only";
...	...	@@ -160,8 +153,6 @@ test "Aes256Gcm - Message only" {
160	153	}
161	154
162	155	test "Aes256Gcm - Message and associated data" {
163		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
164
165	156	const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
166	157	const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
167	158	const m = "Test with message";

lib/std/crypto/argon2.zig-4

...	...	@@ -896,8 +896,6 @@ test "kdf" {
896	896	}
897	897
898	898	test "phc format hasher" {
899		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
900
901	899	const allocator = std.testing.allocator;
902	900	const password = "testpass";
903	901
...	...	@@ -913,8 +911,6 @@ test "phc format hasher" {
913	911	}
914	912
915	913	test "password hash and password verify" {
916		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
917
918	914	const allocator = std.testing.allocator;
919	915	const password = "testpass";
920	916

lib/std/crypto/bcrypt.zig-7

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("std");
2		const builtin = @import("builtin");
3	2	const base64 = std.base64;
4	3	const crypto = std.crypto;
5	4	const debug = std.debug;
...	...	@@ -754,8 +753,6 @@ pub fn strVerify(
754	753	}
755	754
756	755	test "bcrypt codec" {
757		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
758
759	756	var salt: [salt_length]u8 = undefined;
760	757	crypto.random.bytes(&salt);
761	758	var salt_str: [salt_str_length]u8 = undefined;
...	...	@@ -766,8 +763,6 @@ test "bcrypt codec" {
766	763	}
767	764
768	765	test "bcrypt crypt format" {
769		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
770
771	766	var hash_options = HashOptions{
772	767	.params = .{ .rounds_log = 5 },
773	768	.encoding = .crypt,
...	...	@@ -808,8 +803,6 @@ test "bcrypt crypt format" {
808	803	}
809	804
810	805	test "bcrypt phc format" {
811		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
812
813	806	var hash_options = HashOptions{
814	807	.params = .{ .rounds_log = 5 },
815	808	.encoding = .phc,

lib/std/crypto/cmac.zig-9

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("std");
2		const builtin = @import("builtin");
3	2	const crypto = std.crypto;
4	3	const mem = std.mem;
5	4
...	...	@@ -94,8 +93,6 @@ pub fn Cmac(comptime BlockCipher: type) type {
94	93	const testing = std.testing;
95	94
96	95	test "CmacAes128 - Example 1: len = 0" {
97		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
98
99	96	const key = [_]u8{
100	97	0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
101	98	};
...	...	@@ -109,8 +106,6 @@ test "CmacAes128 - Example 1: len = 0" {
109	106	}
110	107
111	108	test "CmacAes128 - Example 2: len = 16" {
112		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
113
114	109	const key = [_]u8{
115	110	0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
116	111	};
...	...	@@ -126,8 +121,6 @@ test "CmacAes128 - Example 2: len = 16" {
126	121	}
127	122
128	123	test "CmacAes128 - Example 3: len = 40" {
129		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
130
131	124	const key = [_]u8{
132	125	0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
133	126	};
...	...	@@ -145,8 +138,6 @@ test "CmacAes128 - Example 3: len = 40" {
145	138	}
146	139
147	140	test "CmacAes128 - Example 4: len = 64" {
148		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
149
150	141	const key = [_]u8{
151	142	0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
152	143	};

lib/std/crypto/ecdsa.zig-5

...	...	@@ -373,7 +373,6 @@ pub fn Ecdsa(comptime Curve: type, comptime Hash: type) type {
373	373
374	374	test "ECDSA - Basic operations over EcdsaP384Sha384" {
375	375	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
376		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
377	376
378	377	const Scheme = EcdsaP384Sha384;
379	378	const kp = try Scheme.KeyPair.create(null);
...	...	@@ -407,7 +406,6 @@ test "ECDSA - Basic operations over Secp256k1" {
407	406
408	407	test "ECDSA - Basic operations over EcdsaP384Sha256" {
409	408	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
410		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
411	409
412	410	const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);
413	411	const kp = try Scheme.KeyPair.create(null);
...	...	@@ -424,7 +422,6 @@ test "ECDSA - Basic operations over EcdsaP384Sha256" {
424	422
425	423	test "ECDSA - Verifying a existing signature with EcdsaP384Sha256" {
426	424	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
427		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
428	425
429	426	const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);
430	427	// zig fmt: off
...	...	@@ -469,7 +466,6 @@ const TestVector = struct {
469	466
470	467	test "ECDSA - Test vectors from Project Wycheproof" {
471	468	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
472		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
473	469
474	470	const vectors = [_]TestVector{
475	471	.{ .key = "042927b10512bae3eddcfe467828128bad2903269919f7086069c8c4df6c732838c7787964eaac00e5921fb1498a60f4606766b3d9685001558d1a974e7341513e", .msg = "313233343030", .sig = "304402202ba3a8be6b94d5ec80a6d9d1190a436effe50d85a1eee859b8cc6af9bd5c2e1802204cd60b855d442f5b3c7b11eb6c4e0ae7525fe710fab9aa7c77a67f79e6fadd76", .result = .valid },
...	...	@@ -884,7 +880,6 @@ fn tvTry(vector: TestVector) !void {
884	880
885	881	test "ECDSA - Sec1 encoding/decoding" {
886	882	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
887		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
888	883
889	884	const Scheme = EcdsaP384Sha384;
890	885	const kp = try Scheme.KeyPair.create(null);

lib/std/crypto/ghash_polyval.zig-4

...	...	@@ -422,8 +422,6 @@ fn Hash(comptime endian: std.builtin.Endian, comptime shift_key: bool) type {
422	422	const htest = @import("test.zig");
423	423
424	424	test "ghash" {
425		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
426
427	425	const key = [_]u8{0x42} ** 16;
428	426	const m = [_]u8{0x69} ** 256;
429	427
...	...	@@ -441,8 +439,6 @@ test "ghash" {
441	439	}
442	440
443	441	test "ghash2" {
444		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
445
446	442	var key: [16]u8 = undefined;
447	443	var i: usize = 0;
448	444	while (i < key.len) : (i += 1) {

lib/std/crypto/pcurves/p256.zig-2

...	...	@@ -478,7 +478,5 @@ pub const AffineCoordinates = struct {
478	478	};
479	479
480	480	test {
481		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
482
483	481	_ = @import("tests/p256.zig");
484	482	}

lib/std/crypto/phc_encoding.zig-3

...	...	@@ -1,7 +1,6 @@
1	1	// https://github.com/P-H-C/phc-string-format
2	2
3	3	const std = @import("std");
4		const builtin = @import("builtin");
5	4	const fmt = std.fmt;
6	5	const io = std.io;
7	6	const mem = std.mem;
...	...	@@ -264,8 +263,6 @@ fn kvSplit(str: []const u8) !struct { key: []const u8, value: []const u8 } {
264	263	}
265	264
266	265	test "phc format - encoding/decoding" {
267		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
268
269	266	const Input = struct {
270	267	str: []const u8,
271	268	HashResult: type,

lib/std/crypto/sha2.zig+1-1

...	...	@@ -238,7 +238,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {
238	238	return;
239	239	},
240	240	// C backend doesn't currently support passing vectors to inline asm.
241		.x86_64 => if (builtin.zig_backend != .stage2_c and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {
	241	.x86_64 => if (builtin.zig_backend != .stage2_c and builtin.zig_backend != .stage2_x86_64 and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {
242	242	var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };
243	243	var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };
244	244	const s_v = @as(*[16]v4u32, @ptrCast(&s));

lib/std/fmt/parse_float.zig-2

...	...	@@ -83,8 +83,6 @@ test "fmt.parseFloat #11169" {
83	83	}
84	84
85	85	test "fmt.parseFloat hex.special" {
86		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
87
88	86	try testing.expect(math.isNan(try parseFloat(f32, "nAn")));
89	87	try testing.expect(math.isPositiveInf(try parseFloat(f32, "iNf")));
90	88	try testing.expect(math.isPositiveInf(try parseFloat(f32, "+Inf")));

lib/std/hash/xxhash.zig+2-1

...	...	@@ -2,6 +2,7 @@ const std = @import("std");
2	2	const builtin = @import("builtin");
3	3	const mem = std.mem;
4	4	const expectEqual = std.testing.expectEqual;
	5	const native_endian = builtin.cpu.arch.endian();
5	6
6	7	const rotl = std.math.rotl;
7	8
...	...	@@ -472,7 +473,7 @@ pub const XxHash3 = struct {
472	473	}
473	474
474	475	inline fn swap(x: anytype) @TypeOf(x) {
475		return if (builtin.cpu.arch.endian() == .big) @byteSwap(x) else x;
	476	return if (native_endian == .big) @byteSwap(x) else x;
476	477	}
477	478
478	479	inline fn disableAutoVectorization(x: anytype) void {

lib/std/http/Client.zig+16-10

...	...	@@ -9,6 +9,7 @@ const net = std.net;
9	9	const Uri = std.Uri;
10	10	const Allocator = mem.Allocator;
11	11	const assert = std.debug.assert;
	12	const use_vectors = builtin.zig_backend != .stage2_x86_64;
12	13
13	14	const Client = @This();
14	15	const proto = @import("protocol.zig");
...	...	@@ -408,7 +409,7 @@ pub const Response = struct {
408	409	else => return error.HttpHeadersInvalid,
409	410	};
410	411	if (first_line[8] != ' ') return error.HttpHeadersInvalid;
411		const status = @as(http.Status, @enumFromInt(parseInt3(first_line[9..12].*)));
	412	const status: http.Status = @enumFromInt(parseInt3(first_line[9..12]));
412	413	const reason = mem.trimLeft(u8, first_line[12..], " ");
413	414
414	415	res.version = version;
...	...	@@ -481,20 +482,24 @@ pub const Response = struct {
481	482	}
482	483
483	484	inline fn int64(array: *const [8]u8) u64 {
484		return @as(u64, @bitCast(array.*));
	485	return @bitCast(array.*);
485	486	}
486	487
487		fn parseInt3(nnn: @Vector(3, u8)) u10 {
488		const zero: @Vector(3, u8) = .{ '0', '0', '0' };
489		const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
490		return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
	488	fn parseInt3(text: *const [3]u8) u10 {
	489	if (use_vectors) {
	490	const nnn: @Vector(3, u8) = text.*;
	491	const zero: @Vector(3, u8) = .{ '0', '0', '0' };
	492	const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
	493	return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
	494	}
	495	return std.fmt.parseInt(u10, text, 10) catch unreachable;
491	496	}
492	497
493	498	test parseInt3 {
494	499	const expectEqual = testing.expectEqual;
495		try expectEqual(@as(u10, 0), parseInt3("000".*));
496		try expectEqual(@as(u10, 418), parseInt3("418".*));
497		try expectEqual(@as(u10, 999), parseInt3("999".*));
	500	try expectEqual(@as(u10, 0), parseInt3("000"));
	501	try expectEqual(@as(u10, 418), parseInt3("418"));
	502	try expectEqual(@as(u10, 999), parseInt3("999"));
498	503	}
499	504
500	505	version: http.Version,
...	...	@@ -1588,7 +1593,8 @@ test {
1588	1593
1589	1594	if (builtin.os.tag == .wasi) return error.SkipZigTest;
1590	1595
1591		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	1596	if (builtin.zig_backend == .stage2_x86_64 and
	1597	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .avx)) return error.SkipZigTest;
1592	1598
1593	1599	std.testing.refAllDecls(@This());
1594	1600	}

lib/std/http/Server.zig-2

...	...	@@ -736,8 +736,6 @@ test "HTTP server handles a chunked transfer coding request" {
736	736	return error.SkipZigTest;
737	737	}
738	738
739		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
740
741	739	const native_endian = comptime builtin.cpu.arch.endian();
742	740	if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
743	741	// https://github.com/ziglang/zig/issues/13782

lib/std/http/protocol.zig+23-12

...	...	@@ -1,8 +1,10 @@
1	1	const std = @import("../std.zig");
	2	const builtin = @import("builtin");
2	3	const testing = std.testing;
3	4	const mem = std.mem;
4	5
5	6	const assert = std.debug.assert;
	7	const use_vectors = builtin.zig_backend != .stage2_x86_64;
6	8
7	9	pub const State = enum {
8	10	/// Begin header parsing states.
...	...	@@ -83,7 +85,7 @@ pub const HeadersParser = struct {
83	85	/// first byte of content is located at `bytes[result]`.
84	86	pub fn findHeadersEnd(r: *HeadersParser, bytes: []const u8) u32 {
85	87	const vector_len: comptime_int = @max(std.simd.suggestVectorSize(u8) orelse 1, 8);
86		const len = @as(u32, @intCast(bytes.len));
	88	const len: u32 = @intCast(bytes.len);
87	89	var index: u32 = 0;
88	90
89	91	while (true) {
...	...	@@ -175,18 +177,27 @@ pub const HeadersParser = struct {
175	177	continue;
176	178	},
177	179	else => {
178		const Vector = @Vector(vector_len, u8);
179		// const BoolVector = @Vector(vector_len, bool);
180		const BitVector = @Vector(vector_len, u1);
181		const SizeVector = @Vector(vector_len, u8);
182
183	180	const chunk = bytes[index..][0..vector_len];
184		const v: Vector = chunk.*;
185		const matches_r = @as(BitVector, @bitCast(v == @as(Vector, @splat('\r'))));
186		const matches_n = @as(BitVector, @bitCast(v == @as(Vector, @splat('\n'))));
187		const matches_or: SizeVector = matches_r \| matches_n;
188
189		const matches = @reduce(.Add, matches_or);
	181	const matches = if (use_vectors) matches: {
	182	const Vector = @Vector(vector_len, u8);
	183	// const BoolVector = @Vector(vector_len, bool);
	184	const BitVector = @Vector(vector_len, u1);
	185	const SizeVector = @Vector(vector_len, u8);
	186
	187	const v: Vector = chunk.*;
	188	const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));
	189	const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
	190	const matches_or: SizeVector = matches_r \| matches_n;
	191
	192	break :matches @reduce(.Add, matches_or);
	193	} else matches: {
	194	var matches: u8 = 0;
	195	for (chunk) \|byte\| switch (byte) {
	196	'\r', '\n' => matches += 1,
	197	else => {},
	198	};
	199	break :matches matches;
	200	};
190	201	switch (matches) {
191	202	0 => {},
192	203	1 => switch (chunk[vector_len - 1]) {

lib/std/math.zig+2-21

...	...	@@ -492,8 +492,6 @@ pub fn shl(comptime T: type, a: T, shift_amt: anytype) T {
492	492	}
493	493
494	494	test "shl" {
495		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
496
497	495	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
498	496	// https://github.com/ziglang/zig/issues/12012
499	497	return error.SkipZigTest;
...	...	@@ -539,8 +537,6 @@ pub fn shr(comptime T: type, a: T, shift_amt: anytype) T {
539	537	}
540	538
541	539	test "shr" {
542		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
543
544	540	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
545	541	// https://github.com/ziglang/zig/issues/12012
546	542	return error.SkipZigTest;
...	...	@@ -587,8 +583,6 @@ pub fn rotr(comptime T: type, x: T, r: anytype) T {
587	583	}
588	584
589	585	test "rotr" {
590		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
591
592	586	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
593	587	// https://github.com/ziglang/zig/issues/12012
594	588	return error.SkipZigTest;
...	...	@@ -634,8 +628,6 @@ pub fn rotl(comptime T: type, x: T, r: anytype) T {
634	628	}
635	629
636	630	test "rotl" {
637		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
638
639	631	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
640	632	// https://github.com/ziglang/zig/issues/12012
641	633	return error.SkipZigTest;
...	...	@@ -764,8 +756,6 @@ pub fn divTrunc(comptime T: type, numerator: T, denominator: T) !T {
764	756	}
765	757
766	758	test "divTrunc" {
767		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
768
769	759	try testDivTrunc();
770	760	try comptime testDivTrunc();
771	761	}
...	...	@@ -790,8 +780,6 @@ pub fn divFloor(comptime T: type, numerator: T, denominator: T) !T {
790	780	}
791	781
792	782	test "divFloor" {
793		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
794
795	783	try testDivFloor();
796	784	try comptime testDivFloor();
797	785	}
...	...	@@ -829,8 +817,6 @@ pub fn divCeil(comptime T: type, numerator: T, denominator: T) !T {
829	817	}
830	818
831	819	test "divCeil" {
832		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
833
834	820	try testDivCeil();
835	821	try comptime testDivCeil();
836	822	}
...	...	@@ -875,8 +861,6 @@ pub fn divExact(comptime T: type, numerator: T, denominator: T) !T {
875	861	}
876	862
877	863	test "divExact" {
878		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
879
880	864	try testDivExact();
881	865	try comptime testDivExact();
882	866	}
...	...	@@ -903,8 +887,6 @@ pub fn mod(comptime T: type, numerator: T, denominator: T) !T {
903	887	}
904	888
905	889	test "mod" {
906		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
907
908	890	try testMod();
909	891	try comptime testMod();
910	892	}
...	...	@@ -931,8 +913,6 @@ pub fn rem(comptime T: type, numerator: T, denominator: T) !T {
931	913	}
932	914
933	915	test "rem" {
934		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
935
936	916	try testRem();
937	917	try comptime testRem();
938	918	}
...	...	@@ -1285,7 +1265,8 @@ pub fn lerp(a: anytype, b: anytype, t: anytype) @TypeOf(a, b, t) {
1285	1265	}
1286	1266
1287	1267	test "lerp" {
1288		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	1268	if (builtin.zig_backend == .stage2_x86_64 and
	1269	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .fma)) return error.SkipZigTest;
1289	1270
1290	1271	try testing.expectEqual(@as(f64, 75), lerp(50, 100, 0.5));
1291	1272	try testing.expectEqual(@as(f32, 43.75), lerp(50, 25, 0.25));

lib/std/math/big/int.zig+3-3

...	...	@@ -1318,7 +1318,7 @@ pub const Mutable = struct {
1318	1318	///
1319	1319	/// `limbs_buffer` is used for temporary storage.
1320	1320	/// The amount required is given by `calcPowLimbsBufferLen`.
1321		pub fn pow(r: *Mutable, a: Const, b: u32, limbs_buffer: []Limb) !void {
	1321	pub fn pow(r: *Mutable, a: Const, b: u32, limbs_buffer: []Limb) void {
1322	1322	assert(r.limbs.ptr != a.limbs.ptr); // illegal aliasing
1323	1323
1324	1324	// Handle all the trivial cases first
...	...	@@ -3213,7 +3213,7 @@ pub const Managed = struct {
3213	3213	var m = try Managed.initCapacity(rma.allocator, needed_limbs);
3214	3214	errdefer m.deinit();
3215	3215	var m_mut = m.toMutable();
3216		try m_mut.pow(a.toConst(), b, limbs_buffer);
	3216	m_mut.pow(a.toConst(), b, limbs_buffer);
3217	3217	m.setMetadata(m_mut.positive, m_mut.len);
3218	3218
3219	3219	rma.deinit();
...	...	@@ -3221,7 +3221,7 @@ pub const Managed = struct {
3221	3221	} else {
3222	3222	try rma.ensureCapacity(needed_limbs);
3223	3223	var rma_mut = rma.toMutable();
3224		try rma_mut.pow(a.toConst(), b, limbs_buffer);
	3224	rma_mut.pow(a.toConst(), b, limbs_buffer);
3225	3225	rma.setMetadata(rma_mut.positive, rma_mut.len);
3226	3226	}
3227	3227	}

lib/std/math/big/int_test.zig-6

...	...	@@ -2568,8 +2568,6 @@ test "big.int const to managed" {
2568	2568	}
2569	2569
2570	2570	test "big.int pow" {
2571		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2572
2573	2571	{
2574	2572	var a = try Managed.initSet(testing.allocator, -3);
2575	2573	defer a.deinit();
...	...	@@ -2763,8 +2761,6 @@ fn popCountTest(val: *const Managed, bit_count: usize, expected: usize) !void {
2763	2761	}
2764	2762
2765	2763	test "big int conversion read/write twos complement" {
2766		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2767
2768	2764	var a = try Managed.initSet(testing.allocator, (1 << 493) - 1);
2769	2765	defer a.deinit();
2770	2766	var b = try Managed.initSet(testing.allocator, (1 << 493) - 1);
...	...	@@ -2863,8 +2859,6 @@ test "big int write twos complement +/- zero" {
2863	2859	}
2864	2860
2865	2861	test "big int conversion write twos complement with padding" {
2866		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2867
2868	2862	var a = try Managed.initSet(testing.allocator, 0x01_ffffffff_ffffffff_ffffffff);
2869	2863	defer a.deinit();
2870	2864

lib/std/mem.zig-4

...	...	@@ -315,8 +315,6 @@ pub fn zeroes(comptime T: type) T {
315	315	}
316	316
317	317	test "zeroes" {
318		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
319
320	318	const C_struct = extern struct {
321	319	x: u32,
322	320	y: u32 align(128),
...	...	@@ -4342,8 +4340,6 @@ pub fn alignInSlice(slice: anytype, comptime new_alignment: usize) ?AlignedSlice
4342	4340	}
4343	4341
4344	4342	test "read/write(Var)PackedInt" {
4345		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
4346
4347	4343	switch (builtin.cpu.arch) {
4348	4344	// This test generates too much code to execute on WASI.
4349	4345	// LLVM backend fails with "too many locals: locals exceed maximum"

lib/std/multi_array_list.zig+1-1

...	...	@@ -106,7 +106,7 @@ pub fn MultiArrayList(comptime T: type) type {
106	106	}
107	107
108	108	pub fn toMultiArrayList(self: Slice) Self {
109		if (self.ptrs.len == 0) {
	109	if (self.ptrs.len == 0 or self.capacity == 0) {
110	110	return .{};
111	111	}
112	112	const unaligned_ptr = self.ptrs[sizes.fields[0]];

lib/std/net/test.zig+1-1

...	...	@@ -60,7 +60,7 @@ test "parse and render IPv6 addresses" {
60	60	}
61	61
62	62	test "invalid but parseable IPv6 scope ids" {
63		if (builtin.os.tag != .linux or comptime !builtin.os.tag.isDarwin()) {
	63	if (builtin.os.tag != .linux and comptime !builtin.os.tag.isDarwin()) {
64	64	// Currently, resolveIp6 with alphanumerical scope IDs only works on Linux.
65	65	// TODO Make this test pass on other operating systems.
66	66	return error.SkipZigTest;

lib/std/once.zig-2

...	...	@@ -46,8 +46,6 @@ fn incr() void {
46	46	}
47	47
48	48	test "Once executes its function just once" {
49		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
50
51	49	if (builtin.single_threaded) {
52	50	global_once.call();
53	51	global_once.call();

lib/std/os/test.zig-4

...	...	@@ -375,8 +375,6 @@ fn testThreadIdFn(thread_id: *Thread.Id) void {
375	375	test "std.Thread.getCurrentId" {
376	376	if (builtin.single_threaded) return error.SkipZigTest;
377	377
378		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
379
380	378	var thread_current_id: Thread.Id = undefined;
381	379	const thread = try Thread.spawn(.{}, testThreadIdFn, .{&thread_current_id});
382	380	thread.join();
...	...	@@ -420,8 +418,6 @@ test "cpu count" {
420	418	test "thread local storage" {
421	419	if (builtin.single_threaded) return error.SkipZigTest;
422	420
423		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
424
425	421	const thread1 = try Thread.spawn(.{}, testTls, .{});
426	422	const thread2 = try Thread.spawn(.{}, testTls, .{});
427	423	try testTls();

lib/std/rand/test.zig-7

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("../std.zig");
2		const builtin = @import("builtin");
3	2	const math = std.math;
4	3	const DefaultPrng = std.rand.DefaultPrng;
5	4	const Random = std.rand.Random;
...	...	@@ -200,8 +199,6 @@ fn testRandomIntLessThan() !void {
200	199	}
201	200
202	201	test "Random intAtMost" {
203		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
204
205	202	@setEvalBranchQuota(10000);
206	203	try testRandomIntAtMost();
207	204	try comptime testRandomIntAtMost();
...	...	@@ -242,8 +239,6 @@ fn testRandomIntAtMost() !void {
242	239	}
243	240
244	241	test "Random Biased" {
245		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
246
247	242	var prng = DefaultPrng.init(0);
248	243	const random = prng.random();
249	244	// Not thoroughly checking the logic here.
...	...	@@ -452,8 +447,6 @@ test "CSPRNG" {
452	447	}
453	448
454	449	test "Random weightedIndex" {
455		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
456
457	450	// Make sure weightedIndex works for various integers and floats
458	451	inline for (.{ u64, i4, f32, f64 }) \|T\| {
459	452	var prng = DefaultPrng.init(0);

lib/std/zig/tokenizer.zig-9

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("../std.zig");
2		const builtin = @import("builtin");
3	2
4	3	pub const Token = struct {
5	4	tag: Tag,
...	...	@@ -1450,8 +1449,6 @@ test "chars" {
1450	1449	}
1451	1450
1452	1451	test "invalid token characters" {
1453		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1454
1455	1452	try testTokenize("#", &.{.invalid});
1456	1453	try testTokenize("`", &.{.invalid});
1457	1454	try testTokenize("'c", &.{.invalid});
...	...	@@ -1571,8 +1568,6 @@ test "pipe and then invalid" {
1571	1568	}
1572	1569
1573	1570	test "line comment and doc comment" {
1574		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1575
1576	1571	try testTokenize("//", &.{});
1577	1572	try testTokenize("// a / b", &.{});
1578	1573	try testTokenize("// /", &.{});
...	...	@@ -1647,8 +1642,6 @@ test "range literals" {
1647	1642	}
1648	1643
1649	1644	test "number literals decimal" {
1650		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1651
1652	1645	try testTokenize("0", &.{.number_literal});
1653	1646	try testTokenize("1", &.{.number_literal});
1654	1647	try testTokenize("2", &.{.number_literal});
...	...	@@ -1897,8 +1890,6 @@ test "invalid token with unfinished escape right before eof" {
1897	1890	}
1898	1891
1899	1892	test "saturating operators" {
1900		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1901
1902	1893	try testTokenize("<<", &.{.angle_bracket_angle_bracket_left});
1903	1894	try testTokenize("<<\|", &.{.angle_bracket_angle_bracket_left_pipe});
1904	1895	try testTokenize("<<\|=", &.{.angle_bracket_angle_bracket_left_pipe_equal});

src/Compilation.zig+3-1

...	...	@@ -1121,7 +1121,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1121	1121	const include_compiler_rt = options.want_compiler_rt orelse needs_c_symbols;
1122	1122
1123	1123	const must_single_thread = target_util.isSingleThreaded(options.target);
1124		const single_threaded = options.single_threaded orelse must_single_thread;
	1124	const single_threaded = options.single_threaded orelse must_single_thread or
	1125	// x86_64 codegen doesn't support TLV for most object formats
	1126	(!use_llvm and options.target.cpu.arch == .x86_64 and options.target.ofmt != .macho);
1125	1127	if (must_single_thread and !single_threaded) {
1126	1128	return error.TargetRequiresSingleThreaded;
1127	1129	}

src/arch/x86_64/CodeGen.zig+1440-681

...	...	@@ -388,7 +388,7 @@ pub const MCValue = union(enum) {
388	388	};
389	389	}
390	390
391		fn mem(mcv: MCValue, size: Memory.Size) Memory {
	391	fn mem(mcv: MCValue, function: *Self, size: Memory.Size) !Memory {
392	392	return switch (mcv) {
393	393	.none,
394	394	.unreach,
...	...	@@ -409,7 +409,6 @@ pub const MCValue = union(enum) {
409	409	.lea_frame,
410	410	.reserved_frame,
411	411	.air_ref,
412		.load_symbol,
413	412	.lea_symbol,
414	413	=> unreachable,
415	414	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\| .{
...	...	@@ -433,6 +432,19 @@ pub const MCValue = union(enum) {
433	432	.disp = frame_addr.off,
434	433	} },
435	434	},
	435	.load_symbol => \|sym_off\| {
	436	assert(sym_off.off == 0);
	437	return .{
	438	.base = .{ .reloc = .{
	439	.atom_index = try function.owner.getSymbolIndex(function),
	440	.sym_index = sym_off.sym,
	441	} },
	442	.mod = .{ .rm = .{
	443	.size = size,
	444	.disp = sym_off.off,
	445	} },
	446	};
	447	},
436	448	};
437	449	}
438	450
...	...	@@ -722,12 +734,14 @@ const InstTracking = struct {
722	734
723	735	const FrameAlloc = struct {
724	736	abi_size: u31,
	737	spill_pad: u3,
725	738	abi_align: Alignment,
726	739	ref_count: u16,
727	740
728		fn init(alloc_abi: struct { size: u64, alignment: Alignment }) FrameAlloc {
	741	fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc {
729	742	return .{
730	743	.abi_size = @intCast(alloc_abi.size),
	744	.spill_pad = alloc_abi.pad,
731	745	.abi_align = alloc_abi.alignment,
732	746	.ref_count = 0,
733	747	};
...	...	@@ -738,6 +752,20 @@ const FrameAlloc = struct {
738	752	.alignment = ty.abiAlignment(mod),
739	753	});
740	754	}
	755	fn initSpill(ty: Type, mod: *Module) FrameAlloc {
	756	const abi_size = ty.abiSize(mod);
	757	const spill_size = if (abi_size < 8)
	758	math.ceilPowerOfTwoAssert(u64, abi_size)
	759	else
	760	std.mem.alignForward(u64, abi_size, 8);
	761	return init(.{
	762	.size = spill_size,
	763	.pad = @intCast(spill_size - abi_size),
	764	.alignment = ty.abiAlignment(mod).maxStrict(
	765	Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
	766	),
	767	});
	768	}
741	769	};
742	770
743	771	const StackAllocation = struct {
...	...	@@ -1668,8 +1696,7 @@ fn gen(self: *Self) InnerError!void {
1668	1696	// The address where to store the return value for the caller is in a
1669	1697	// register which the callee is free to clobber. Therefore, we purposely
1670	1698	// spill it to stack immediately.
1671		const frame_index =
1672		try self.allocFrameIndex(FrameAlloc.initType(Type.usize, mod));
	1699	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(Type.usize, mod));
1673	1700	try self.genSetMem(
1674	1701	.{ .frame = frame_index },
1675	1702	0,
...	...	@@ -2434,7 +2461,7 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
2434	2461	}
2435	2462	}
2436	2463
2437		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ty, mod));
	2464	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, mod));
2438	2465	return .{ .load_frame = .{ .index = frame_index } };
2439	2466	}
2440	2467
...	...	@@ -2445,7 +2472,10 @@ fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {
2445	2472	80 => abi.RegisterClass.x87,
2446	2473	else => abi.RegisterClass.sse,
2447	2474	},
2448		.Vector => abi.RegisterClass.sse,
	2475	.Vector => switch (ty.childType(mod).toIntern()) {
	2476	.bool_type => abi.RegisterClass.gp,
	2477	else => abi.RegisterClass.sse,
	2478	},
2449	2479	else => abi.RegisterClass.gp,
2450	2480	};
2451	2481	}
...	...	@@ -2503,6 +2533,19 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2503	2533	) \|inst, *tracking\| tracking.resurrect(inst, state.scope_generation);
2504	2534	for (deaths) \|death\| try self.processDeath(death);
2505	2535
	2536	const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;
	2537	var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
	2538	if (opts.update_tracking) ({}) else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
	2539
	2540	var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(
	2541	stack.get(),
	2542	@typeInfo(ExpectedContents).Array.len,
	2543	);
	2544	defer if (!opts.update_tracking) {
	2545	for (reg_locks.items) \|lock\| self.register_manager.unlockReg(lock);
	2546	reg_locks.deinit();
	2547	};
	2548
2506	2549	for (0..state.registers.len) \|index\| {
2507	2550	const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
2508	2551	null
...	...	@@ -2519,11 +2562,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2519	2562	try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);
2520	2563	}
2521	2564	if (target_maybe_inst) \|target_inst\| {
2522		try self.inst_tracking.getPtr(target_inst).?.materialize(
2523		self,
2524		target_inst,
2525		state.reg_tracking[index],
2526		);
	2565	const target_tracking = self.inst_tracking.getPtr(target_inst).?;
	2566	try target_tracking.materialize(self, target_inst, state.reg_tracking[index]);
2527	2567	}
2528	2568	}
2529	2569	if (opts.update_tracking) {
...	...	@@ -2541,7 +2581,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2541	2581	state.reg_tracking[index],
2542	2582	);
2543	2583	}
2544		}
	2584	} else if (target_maybe_inst) \|_\|
	2585	try reg_locks.append(self.register_manager.lockRegIndexAssumeUnused(@intCast(index)));
2545	2586	}
2546	2587	if (opts.emit_instructions) if (self.eflags_inst) \|inst\|
2547	2588	try self.inst_tracking.getPtr(inst).?.spill(self, inst);
...	...	@@ -2578,8 +2619,17 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
2578	2619	}
2579	2620	}
2580	2621
2581		pub fn spillRegisters(self: *Self, registers: []const Register) !void {
2582		for (registers) \|reg\| try self.register_manager.getReg(reg, null);
	2622	pub fn spillCallerPreservedRegs(self: *Self, cc: std.builtin.CallingConvention) !void {
	2623	switch (cc) {
	2624	inline .SysV, .Win64 => \|known_cc\| try self.spillRegisters(
	2625	comptime abi.getCallerPreservedRegs(known_cc),
	2626	),
	2627	else => unreachable,
	2628	}
	2629	}
	2630
	2631	pub fn spillRegisters(self: *Self, comptime registers: []const Register) !void {
	2632	inline for (registers) \|reg\| try self.register_manager.getKnownReg(reg, null);
2583	2633	}
2584	2634
2585	2635	/// Copies a value to a register without tracking the register. The register is not considered
...	...	@@ -2699,7 +2749,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2699	2749	.{ .v_ss, .cvtsd2 },
2700	2750	dst_reg,
2701	2751	dst_reg,
2702		src_mcv.mem(.qword),
	2752	try src_mcv.mem(self, .qword),
2703	2753	) else try self.asmRegisterRegisterRegister(
2704	2754	.{ .v_ss, .cvtsd2 },
2705	2755	dst_reg,
...	...	@@ -2711,7 +2761,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2711	2761	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2712	2762	.{ ._ss, .cvtsd2 },
2713	2763	dst_reg,
2714		src_mcv.mem(.qword),
	2764	try src_mcv.mem(self, .qword),
2715	2765	) else try self.asmRegisterRegister(
2716	2766	.{ ._ss, .cvtsd2 },
2717	2767	dst_reg,
...	...	@@ -2798,7 +2848,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2798	2848	.{ .v_sd, .cvtss2 },
2799	2849	dst_reg,
2800	2850	dst_reg,
2801		src_mcv.mem(.dword),
	2851	try src_mcv.mem(self, .dword),
2802	2852	) else try self.asmRegisterRegisterRegister(
2803	2853	.{ .v_sd, .cvtss2 },
2804	2854	dst_reg,
...	...	@@ -2810,7 +2860,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2810	2860	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2811	2861	.{ ._sd, .cvtss2 },
2812	2862	dst_reg,
2813		src_mcv.mem(.dword),
	2863	try src_mcv.mem(self, .dword),
2814	2864	) else try self.asmRegisterRegister(
2815	2865	.{ ._sd, .cvtss2 },
2816	2866	dst_reg,
...	...	@@ -2851,8 +2901,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2851	2901	};
2852	2902
2853	2903	const dst_mcv = if (dst_int_info.bits <= src_storage_bits and
2854		std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
2855		std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and
	2904	math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
	2905	math.divCeil(u32, src_storage_bits, 64) catch unreachable and
2856	2906	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
2857	2907	const dst_mcv = try self.allocRegOrMem(inst, true);
2858	2908	try self.genCopy(min_ty, dst_mcv, src_mcv);
...	...	@@ -2869,22 +2919,28 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2869	2919	break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) };
2870	2920	}
2871	2921
2872		const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
2873		const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;
	2922	const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
	2923	const dst_limbs_len = math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;
2874	2924
2875		const high_mcv = dst_mcv.address().offset((src_limbs_len - 1) * 8).deref();
2876		const high_reg = try self.copyToTmpRegister(switch (src_int_info.signedness) {
2877		.signed => Type.isize,
2878		.unsigned => Type.usize,
2879		}, high_mcv);
	2925	const high_mcv: MCValue = if (dst_mcv.isMemory())
	2926	dst_mcv.address().offset((src_limbs_len - 1) * 8).deref()
	2927	else
	2928	.{ .register = dst_mcv.register_pair[1] };
	2929	const high_reg = if (high_mcv.isRegister())
	2930	high_mcv.getReg().?
	2931	else
	2932	try self.copyToTmpRegister(switch (src_int_info.signedness) {
	2933	.signed => Type.isize,
	2934	.unsigned => Type.usize,
	2935	}, high_mcv);
2880	2936	const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);
2881	2937	defer self.register_manager.unlockReg(high_lock);
2882	2938
2883	2939	const high_bits = src_int_info.bits % 64;
2884	2940	if (high_bits > 0) {
2885		const high_ty = try mod.intType(extend, high_bits);
2886		try self.truncateRegister(high_ty, high_reg);
2887		try self.genCopy(Type.usize, high_mcv, .{ .register = high_reg });
	2941	try self.truncateRegister(src_ty, high_reg);
	2942	const high_ty = if (dst_int_info.bits >= 64) Type.usize else dst_ty;
	2943	try self.genCopy(high_ty, high_mcv, .{ .register = high_reg });
2888	2944	}
2889	2945
2890	2946	if (dst_limbs_len > src_limbs_len) try self.genInlineMemset(
...	...	@@ -2995,14 +3051,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
2995	3051	.{ .vp_, .@"and" },
2996	3052	dst_reg,
2997	3053	dst_reg,
2998		splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),
	3054	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
2999	3055	);
3000	3056	try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);
3001	3057	} else {
3002	3058	try self.asmRegisterMemory(
3003	3059	.{ .p_, .@"and" },
3004	3060	dst_reg,
3005		splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),
	3061	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
3006	3062	);
3007	3063	try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);
3008	3064	}
...	...	@@ -3048,7 +3104,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
3048	3104	const len = try self.resolveInst(bin_op.rhs);
3049	3105	const len_ty = self.typeOf(bin_op.rhs);
3050	3106
3051		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, mod));
	3107	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod));
3052	3108	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
3053	3109	try self.genSetMem(
3054	3110	.{ .frame = frame_index },
...	...	@@ -3068,8 +3124,36 @@ fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
3068	3124	}
3069	3125
3070	3126	fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
	3127	const mod = self.bin_file.options.module.?;
3071	3128	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
3072	3129	const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
	3130
	3131	const dst_ty = self.typeOfIndex(inst);
	3132	if (dst_ty.isAbiInt(mod)) {
	3133	const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
	3134	const bit_size: u32 = @intCast(dst_ty.bitSize(mod));
	3135	if (abi_size * 8 > bit_size) {
	3136	const dst_lock = switch (dst_mcv) {
	3137	.register => \|dst_reg\| self.register_manager.lockRegAssumeUnused(dst_reg),
	3138	else => null,
	3139	};
	3140	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	3141
	3142	if (dst_mcv.isRegister()) {
	3143	try self.truncateRegister(dst_ty, dst_mcv.getReg().?);
	3144	} else {
	3145	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	3146	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	3147	defer self.register_manager.unlockReg(tmp_lock);
	3148
	3149	const hi_ty = try mod.intType(.unsigned, @intCast((dst_ty.bitSize(mod) - 1) % 64 + 1));
	3150	const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
	3151	try self.genSetReg(tmp_reg, hi_ty, hi_mcv);
	3152	try self.truncateRegister(dst_ty, tmp_reg);
	3153	try self.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg });
	3154	}
	3155	}
	3156	}
3073	3157	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
3074	3158	}
3075	3159
...	...	@@ -3176,7 +3260,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3176	3260	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3177	3261	.{ ._, .mov },
3178	3262	tmp_reg,
3179		mat_lhs_mcv.address().offset(8).deref().mem(.qword),
	3263	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3180	3264	) else try self.asmRegisterRegister(
3181	3265	.{ ._, .mov },
3182	3266	tmp_reg,
...	...	@@ -3200,7 +3284,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3200	3284	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3201	3285	.{ ._, .xor },
3202	3286	tmp_reg,
3203		mat_rhs_mcv.address().offset(8).deref().mem(.qword),
	3287	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3204	3288	) else try self.asmRegisterRegister(
3205	3289	.{ ._, .xor },
3206	3290	tmp_reg,
...	...	@@ -3300,12 +3384,12 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3300	3384	try self.asmRegisterMemory(
3301	3385	.{ ._, .add },
3302	3386	tmp_regs[0],
3303		mat_rhs_mcv.mem(.qword),
	3387	try mat_rhs_mcv.mem(self, .qword),
3304	3388	);
3305	3389	try self.asmRegisterMemory(
3306	3390	.{ ._, .adc },
3307	3391	tmp_regs[1],
3308		mat_rhs_mcv.address().offset(8).deref().mem(.qword),
	3392	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3309	3393	);
3310	3394	} else for (
3311	3395	[_]Mir.Inst.Tag{ .add, .adc },
...	...	@@ -3534,7 +3618,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3534	3618	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3535	3619	.{ ._, .mov },
3536	3620	tmp_reg,
3537		mat_lhs_mcv.address().offset(8).deref().mem(.qword),
	3621	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3538	3622	) else try self.asmRegisterRegister(
3539	3623	.{ ._, .mov },
3540	3624	tmp_reg,
...	...	@@ -3558,7 +3642,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3558	3642	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3559	3643	.{ ._, .xor },
3560	3644	tmp_reg,
3561		mat_rhs_mcv.address().offset(8).deref().mem(.qword),
	3645	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3562	3646	) else try self.asmRegisterRegister(
3563	3647	.{ ._, .xor },
3564	3648	tmp_reg,
...	...	@@ -3567,7 +3651,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3567	3651
3568	3652	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));
3569	3653	try self.asmRegister(.{ ._, .not }, tmp_reg);
3570		try self.asmMemoryImmediate(.{ ._, .cmp }, overflow.mem(.dword), Immediate.s(0));
	3654	try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), Immediate.s(0));
3571	3655	try self.freeValue(overflow);
3572	3656	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
3573	3657	try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63));
...	...	@@ -3665,7 +3749,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3665	3749	}
3666	3750
3667	3751	const frame_index =
3668		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	3752	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3669	3753	try self.genSetMem(
3670	3754	.{ .frame = frame_index },
3671	3755	@intCast(tuple_ty.structFieldOffset(1, mod)),
...	...	@@ -3682,7 +3766,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3682	3766	}
3683	3767
3684	3768	const frame_index =
3685		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	3769	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3686	3770	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
3687	3771	break :result .{ .load_frame = .{ .index = frame_index } };
3688	3772	},
...	...	@@ -3738,7 +3822,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3738	3822	}
3739	3823
3740	3824	const frame_index =
3741		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	3825	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3742	3826	try self.genSetMem(
3743	3827	.{ .frame = frame_index },
3744	3828	@intCast(tuple_ty.structFieldOffset(1, mod)),
...	...	@@ -3755,7 +3839,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3755	3839	}
3756	3840
3757	3841	const frame_index =
3758		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	3842	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3759	3843	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
3760	3844	break :result .{ .load_frame = .{ .index = frame_index } };
3761	3845	},
...	...	@@ -3874,7 +3958,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3874	3958	);
3875	3959	try self.asmMemoryImmediate(
3876	3960	.{ ._, .cmp },
3877		overflow.mem(self.memSize(Type.c_int)),
	3961	try overflow.mem(self, self.memSize(Type.c_int)),
3878	3962	Immediate.s(0),
3879	3963	);
3880	3964	try self.genSetMem(
...	...	@@ -3926,14 +4010,19 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3926	4010	};
3927	4011	defer if (mat_rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3928	4012
3929		if (mat_lhs_mcv.isMemory())
3930		try self.asmRegisterMemory(.{ ._, .mov }, .rax, mat_lhs_mcv.mem(.qword))
3931		else
3932		try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);
	4013	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
	4014	.{ ._, .mov },
	4015	.rax,
	4016	try mat_lhs_mcv.mem(self, .qword),
	4017	) else try self.asmRegisterRegister(
	4018	.{ ._, .mov },
	4019	.rax,
	4020	mat_lhs_mcv.register_pair[0],
	4021	);
3933	4022	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3934	4023	.{ ._, .mov },
3935	4024	tmp_regs[0],
3936		mat_rhs_mcv.address().offset(8).deref().mem(.qword),
	4025	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3937	4026	) else try self.asmRegisterRegister(
3938	4027	.{ ._, .mov },
3939	4028	tmp_regs[0],
...	...	@@ -3944,7 +4033,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3944	4033	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);
3945	4034	try self.asmSetccRegister(.o, tmp_regs[2].to8());
3946	4035	if (mat_rhs_mcv.isMemory())
3947		try self.asmMemory(.{ ._, .mul }, mat_rhs_mcv.mem(.qword))
	4036	try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
3948	4037	else
3949	4038	try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
3950	4039	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
...	...	@@ -3953,7 +4042,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3953	4042	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3954	4043	.{ ._, .mov },
3955	4044	tmp_regs[0],
3956		mat_lhs_mcv.address().offset(8).deref().mem(.qword),
	4045	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3957	4046	) else try self.asmRegisterRegister(
3958	4047	.{ ._, .mov },
3959	4048	tmp_regs[0],
...	...	@@ -3967,14 +4056,15 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3967	4056	tmp_regs[3].to8(),
3968	4057	);
3969	4058	try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
3970		if (mat_rhs_mcv.isMemory())
3971		try self.asmRegisterMemory(.{ .i_, .mul }, tmp_regs[0], mat_rhs_mcv.mem(.qword))
3972		else
3973		try self.asmRegisterRegister(
3974		.{ .i_, .mul },
3975		tmp_regs[0],
3976		mat_rhs_mcv.register_pair[0],
3977		);
	4059	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
	4060	.{ .i_, .mul },
	4061	tmp_regs[0],
	4062	try mat_rhs_mcv.mem(self, .qword),
	4063	) else try self.asmRegisterRegister(
	4064	.{ .i_, .mul },
	4065	tmp_regs[0],
	4066	mat_rhs_mcv.register_pair[0],
	4067	);
3978	4068	try self.asmSetccRegister(.o, tmp_regs[2].to8());
3979	4069	try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
3980	4070	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
...	...	@@ -4020,8 +4110,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4020	4110	self.eflags_inst = inst;
4021	4111	break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
4022	4112	} else {
4023		const frame_index =
4024		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	4113	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
4025	4114	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
4026	4115	break :result .{ .load_frame = .{ .index = frame_index } };
4027	4116	},
...	...	@@ -4032,8 +4121,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4032	4121	src_ty.fmt(mod), dst_ty.fmt(mod),
4033	4122	});
4034	4123
4035		const frame_index =
4036		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	4124	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
4037	4125	if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
4038	4126	try self.genSetMem(
4039	4127	.{ .frame = frame_index },
...	...	@@ -4106,7 +4194,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
4106	4194	.register => \|reg\| try self.asmRegister(tag, registerAlias(reg, abi_size)),
4107	4195	.memory, .indirect, .load_frame => try self.asmMemory(
4108	4196	tag,
4109		mat_rhs.mem(Memory.Size.fromSize(abi_size)),
	4197	try mat_rhs.mem(self, Memory.Size.fromSize(abi_size)),
4110	4198	),
4111	4199	else => unreachable,
4112	4200	}
...	...	@@ -4160,8 +4248,8 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
4160	4248	);
4161	4249	try self.asmCmovccRegisterRegister(
4162	4250	.z,
4163		registerAlias(divisor, abi_size),
4164		registerAlias(.rdx, abi_size),
	4251	registerAlias(divisor, @max(abi_size, 2)),
	4252	registerAlias(.rdx, @max(abi_size, 2)),
4165	4253	);
4166	4254	try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });
4167	4255	return MCValue{ .register = divisor };
...	...	@@ -4171,47 +4259,268 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
4171	4259	const mod = self.bin_file.options.module.?;
4172	4260	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
4173	4261
4174		try self.spillRegisters(&.{.rcx});
4175
4176		const tag = self.air.instructions.items(.tag)[inst];
4177		try self.register_manager.getReg(.rcx, null);
4178		const lhs = try self.resolveInst(bin_op.lhs);
4179		const rhs = try self.resolveInst(bin_op.rhs);
	4262	const air_tags = self.air.instructions.items(.tag);
	4263	const tag = air_tags[inst];
4180	4264	const lhs_ty = self.typeOf(bin_op.lhs);
4181	4265	const rhs_ty = self.typeOf(bin_op.rhs);
	4266	const result: MCValue = result: {
	4267	switch (lhs_ty.zigTypeTag(mod)) {
	4268	.Int => {
	4269	try self.spillRegisters(&.{.rcx});
	4270	try self.register_manager.getReg(.rcx, null);
	4271	const lhs_mcv = try self.resolveInst(bin_op.lhs);
	4272	const rhs_mcv = try self.resolveInst(bin_op.rhs);
4182	4273
4183		const dst_mcv = try self.genShiftBinOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
4184		switch (tag) {
4185		.shr, .shr_exact, .shl_exact => {},
4186		.shl => switch (dst_mcv) {
4187		.register => \|dst_reg\| try self.truncateRegister(lhs_ty, dst_reg),
4188		.register_pair => \|dst_regs\| try self.truncateRegister(lhs_ty, dst_regs[1]),
4189		.load_frame => \|frame_addr\| {
4190		const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
4191		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
4192		defer self.register_manager.unlockReg(tmp_lock);
	4274	const dst_mcv = try self.genShiftBinOp(tag, inst, lhs_mcv, rhs_mcv, lhs_ty, rhs_ty);
	4275	switch (tag) {
	4276	.shr, .shr_exact, .shl_exact => {},
	4277	.shl => switch (dst_mcv) {
	4278	.register => \|dst_reg\| try self.truncateRegister(lhs_ty, dst_reg),
	4279	.register_pair => \|dst_regs\| try self.truncateRegister(lhs_ty, dst_regs[1]),
	4280	.load_frame => \|frame_addr\| {
	4281	const tmp_reg =
	4282	try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	4283	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	4284	defer self.register_manager.unlockReg(tmp_lock);
4193	4285
4194		const lhs_bits: u31 = @intCast(lhs_ty.bitSize(mod));
4195		const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;
4196		const off = frame_addr.off + lhs_bits / 64 * 8;
4197		try self.genSetReg(
4198		tmp_reg,
4199		tmp_ty,
4200		.{ .load_frame = .{ .index = frame_addr.index, .off = off } },
4201		);
4202		try self.truncateRegister(lhs_ty, tmp_reg);
4203		try self.genSetMem(
4204		.{ .frame = frame_addr.index },
4205		off,
4206		tmp_ty,
4207		.{ .register = tmp_reg },
4208		);
	4286	const lhs_bits: u31 = @intCast(lhs_ty.bitSize(mod));
	4287	const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;
	4288	const off = frame_addr.off + (lhs_bits - 1) / 64 * 8;
	4289	try self.genSetReg(
	4290	tmp_reg,
	4291	tmp_ty,
	4292	.{ .load_frame = .{ .index = frame_addr.index, .off = off } },
	4293	);
	4294	try self.truncateRegister(lhs_ty, tmp_reg);
	4295	try self.genSetMem(
	4296	.{ .frame = frame_addr.index },
	4297	off,
	4298	tmp_ty,
	4299	.{ .register = tmp_reg },
	4300	);
	4301	},
	4302	else => {},
	4303	},
	4304	else => unreachable,
	4305	}
	4306	break :result dst_mcv;
	4307	},
	4308	.Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
	4309	.Int => if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.childType(mod).intInfo(mod).bits) {
	4310	else => null,
	4311	16 => switch (lhs_ty.vectorLen(mod)) {
	4312	else => null,
	4313	1...8 => switch (tag) {
	4314	else => unreachable,
	4315	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4316	.signed => if (self.hasFeature(.avx))
	4317	.{ .vp_w, .sra }
	4318	else
	4319	.{ .p_w, .sra },
	4320	.unsigned => if (self.hasFeature(.avx))
	4321	.{ .vp_w, .srl }
	4322	else
	4323	.{ .p_w, .srl },
	4324	},
	4325	.shl, .shl_exact => if (self.hasFeature(.avx))
	4326	.{ .vp_w, .sll }
	4327	else
	4328	.{ .p_w, .sll },
	4329	},
	4330	9...16 => switch (tag) {
	4331	else => unreachable,
	4332	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4333	.signed => if (self.hasFeature(.avx2)) .{ .vp_w, .sra } else null,
	4334	.unsigned => if (self.hasFeature(.avx2)) .{ .vp_w, .srl } else null,
	4335	},
	4336	.shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_w, .sll } else null,
	4337	},
	4338	},
	4339	32 => switch (lhs_ty.vectorLen(mod)) {
	4340	else => null,
	4341	1...4 => switch (tag) {
	4342	else => unreachable,
	4343	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4344	.signed => if (self.hasFeature(.avx))
	4345	.{ .vp_d, .sra }
	4346	else
	4347	.{ .p_d, .sra },
	4348	.unsigned => if (self.hasFeature(.avx))
	4349	.{ .vp_d, .srl }
	4350	else
	4351	.{ .p_d, .srl },
	4352	},
	4353	.shl, .shl_exact => if (self.hasFeature(.avx))
	4354	.{ .vp_d, .sll }
	4355	else
	4356	.{ .p_d, .sll },
	4357	},
	4358	5...8 => switch (tag) {
	4359	else => unreachable,
	4360	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4361	.signed => if (self.hasFeature(.avx2)) .{ .vp_d, .sra } else null,
	4362	.unsigned => if (self.hasFeature(.avx2)) .{ .vp_d, .srl } else null,
	4363	},
	4364	.shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_d, .sll } else null,
	4365	},
	4366	},
	4367	64 => switch (lhs_ty.vectorLen(mod)) {
	4368	else => null,
	4369	1...2 => switch (tag) {
	4370	else => unreachable,
	4371	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4372	.signed => if (self.hasFeature(.avx))
	4373	.{ .vp_q, .sra }
	4374	else
	4375	.{ .p_q, .sra },
	4376	.unsigned => if (self.hasFeature(.avx))
	4377	.{ .vp_q, .srl }
	4378	else
	4379	.{ .p_q, .srl },
	4380	},
	4381	.shl, .shl_exact => if (self.hasFeature(.avx))
	4382	.{ .vp_q, .sll }
	4383	else
	4384	.{ .p_q, .sll },
	4385	},
	4386	3...4 => switch (tag) {
	4387	else => unreachable,
	4388	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4389	.signed => if (self.hasFeature(.avx2)) .{ .vp_q, .sra } else null,
	4390	.unsigned => if (self.hasFeature(.avx2)) .{ .vp_q, .srl } else null,
	4391	},
	4392	.shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_q, .sll } else null,
	4393	},
	4394	},
	4395	})) \|mir_tag\| if (try self.air.value(bin_op.rhs, mod)) \|rhs_val\| {
	4396	switch (mod.intern_pool.indexToKey(rhs_val.toIntern())) {
	4397	.aggregate => \|rhs_aggregate\| switch (rhs_aggregate.storage) {
	4398	.repeated_elem => \|rhs_elem\| {
	4399	const abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
	4400
	4401	const lhs_mcv = try self.resolveInst(bin_op.lhs);
	4402	const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and
	4403	self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
	4404	.{lhs_mcv.getReg().?} ** 2
	4405	else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{
	4406	try self.register_manager.allocReg(inst, abi.RegisterClass.sse),
	4407	lhs_mcv.getReg().?,
	4408	} else .{(try self.copyToRegisterWithInstTracking(
	4409	inst,
	4410	lhs_ty,
	4411	lhs_mcv,
	4412	)).register} ** 2;
	4413	const reg_locks =
	4414	self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg });
	4415	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\|
	4416	self.register_manager.unlockReg(lock);
	4417
	4418	const shift_imm =
	4419	Immediate.u(@intCast(rhs_elem.toValue().toUnsignedInt(mod)));
	4420	if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate(
	4421	mir_tag,
	4422	registerAlias(dst_reg, abi_size),
	4423	registerAlias(lhs_reg, abi_size),
	4424	shift_imm,
	4425	) else {
	4426	assert(dst_reg.id() == lhs_reg.id());
	4427	try self.asmRegisterImmediate(
	4428	mir_tag,
	4429	registerAlias(dst_reg, abi_size),
	4430	shift_imm,
	4431	);
	4432	}
	4433	break :result .{ .register = dst_reg };
	4434	},
	4435	else => {},
	4436	},
	4437	else => {},
	4438	}
	4439	} else if (Air.refToIndex(bin_op.rhs)) \|rhs_inst\| switch (air_tags[rhs_inst]) {
	4440	.splat => {
	4441	const abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
	4442
	4443	const lhs_mcv = try self.resolveInst(bin_op.lhs);
	4444	const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and
	4445	self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
	4446	.{lhs_mcv.getReg().?} ** 2
	4447	else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{
	4448	try self.register_manager.allocReg(inst, abi.RegisterClass.sse),
	4449	lhs_mcv.getReg().?,
	4450	} else .{(try self.copyToRegisterWithInstTracking(
	4451	inst,
	4452	lhs_ty,
	4453	lhs_mcv,
	4454	)).register} ** 2;
	4455	const reg_locks = self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg });
	4456	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\|
	4457	self.register_manager.unlockReg(lock);
	4458
	4459	const shift_reg =
	4460	try self.copyToTmpRegister(rhs_ty, .{ .air_ref = bin_op.rhs });
	4461	const shift_lock = self.register_manager.lockRegAssumeUnused(shift_reg);
	4462	defer self.register_manager.unlockReg(shift_lock);
	4463
	4464	const mask_ty = try mod.vectorType(.{ .len = 16, .child = .u8_type });
	4465	const mask_mcv = try self.genTypedValue(.{
	4466	.ty = mask_ty,
	4467	.val = (try mod.intern(.{ .aggregate = .{
	4468	.ty = mask_ty.toIntern(),
	4469	.storage = .{ .elems = &([1]InternPool.Index{
	4470	(try rhs_ty.childType(mod).maxIntScalar(mod, Type.u8)).toIntern(),
	4471	} ++ [1]InternPool.Index{
	4472	(try mod.intValue(Type.u8, 0)).toIntern(),
	4473	} ** 15) },
	4474	} })).toValue(),
	4475	});
	4476	const mask_addr_reg =
	4477	try self.copyToTmpRegister(Type.usize, mask_mcv.address());
	4478	const mask_addr_lock = self.register_manager.lockRegAssumeUnused(mask_addr_reg);
	4479	defer self.register_manager.unlockReg(mask_addr_lock);
	4480
	4481	if (self.hasFeature(.avx)) {
	4482	try self.asmRegisterRegisterMemory(
	4483	.{ .vp_, .@"and" },
	4484	shift_reg.to128(),
	4485	shift_reg.to128(),
	4486	.{
	4487	.base = .{ .reg = mask_addr_reg },
	4488	.mod = .{ .rm = .{ .size = .xword } },
	4489	},
	4490	);
	4491	try self.asmRegisterRegisterRegister(
	4492	mir_tag,
	4493	registerAlias(dst_reg, abi_size),
	4494	registerAlias(lhs_reg, abi_size),
	4495	shift_reg.to128(),
	4496	);
	4497	} else {
	4498	try self.asmRegisterMemory(
	4499	.{ .p_, .@"and" },
	4500	shift_reg.to128(),
	4501	.{
	4502	.base = .{ .reg = mask_addr_reg },
	4503	.mod = .{ .rm = .{ .size = .xword } },
	4504	},
	4505	);
	4506	assert(dst_reg.id() == lhs_reg.id());
	4507	try self.asmRegisterRegister(
	4508	mir_tag,
	4509	registerAlias(dst_reg, abi_size),
	4510	shift_reg.to128(),
	4511	);
	4512	}
	4513	break :result .{ .register = dst_reg };
	4514	},
	4515	else => {},
	4516	},
	4517	else => {},
4209	4518	},
4210	4519	else => {},
4211		},
4212		else => unreachable,
4213		}
4214		return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
	4520	}
	4521	return self.fail("TODO implement airShlShrBinOp for {}", .{lhs_ty.fmt(mod)});
	4522	};
	4523	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4215	4524	}
4216	4525
4217	4526	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -4230,12 +4539,18 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
4230	4539
4231	4540	const opt_mcv = try self.resolveInst(ty_op.operand);
4232	4541	if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {
4233		switch (opt_mcv) {
4234		.register => \|reg\| try self.truncateRegister(pl_ty, reg),
4235		.register_overflow => \|ro\| try self.truncateRegister(pl_ty, ro.reg),
	4542	const pl_mcv: MCValue = switch (opt_mcv) {
	4543	.register_overflow => \|ro\| pl: {
	4544	self.eflags_inst = null; // actually stop tracking the overflow part
	4545	break :pl .{ .register = ro.reg };
	4546	},
	4547	else => opt_mcv,
	4548	};
	4549	switch (pl_mcv) {
	4550	.register => \|pl_reg\| try self.truncateRegister(pl_ty, pl_reg),
4236	4551	else => {},
4237	4552	}
4238		break :result opt_mcv;
	4553	break :result pl_mcv;
4239	4554	}
4240	4555
4241	4556	const pl_mcv = try self.allocRegOrMem(inst, true);
...	...	@@ -4472,8 +4787,9 @@ fn genUnwrapErrUnionPayloadMir(
4472	4787	const eu_lock = self.register_manager.lockReg(reg);
4473	4788	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
4474	4789
4475		const result_mcv: MCValue = if (maybe_inst) \|inst\|
4476		try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)
	4790	const payload_in_gp = self.regClassForType(payload_ty).supersetOf(abi.RegisterClass.gp);
	4791	const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)
	4792	try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)
4477	4793	else
4478	4794	.{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
4479	4795	if (payload_off > 0) try self.genShiftBinOpMir(
...	...	@@ -4482,7 +4798,12 @@ fn genUnwrapErrUnionPayloadMir(
4482	4798	result_mcv,
4483	4799	.{ .immediate = @as(u6, @intCast(payload_off * 8)) },
4484	4800	) else try self.truncateRegister(payload_ty, result_mcv.register);
4485		break :result result_mcv;
	4801	break :result if (payload_in_gp)
	4802	result_mcv
	4803	else if (maybe_inst) \|inst\|
	4804	try self.copyToRegisterWithInstTracking(inst, payload_ty, result_mcv)
	4805	else
	4806	.{ .register = try self.copyToTmpRegister(payload_ty, result_mcv) };
4486	4807	},
4487	4808	else => return self.fail("TODO implement genUnwrapErrUnionPayloadMir for {}", .{err_union}),
4488	4809	}
...	...	@@ -4593,7 +4914,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
4593	4914	const result: MCValue = result: {
4594	4915	if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .{ .immediate = 0 };
4595	4916
4596		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, mod));
	4917	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, mod));
4597	4918	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
4598	4919	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
4599	4920	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
...	...	@@ -4615,7 +4936,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
4615	4936	const result: MCValue = result: {
4616	4937	if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result try self.resolveInst(ty_op.operand);
4617	4938
4618		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, mod));
	4939	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, mod));
4619	4940	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
4620	4941	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
4621	4942	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);
...	...	@@ -4770,14 +5091,19 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
4770	5091	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
4771	5092	const mod = self.bin_file.options.module.?;
4772	5093	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
4773		const slice_ty = self.typeOf(bin_op.lhs);
4774	5094
4775		const slice_ptr_field_type = slice_ty.slicePtrFieldType(mod);
4776		const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
4777		const dst_mcv = try self.allocRegOrMem(inst, false);
4778		try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
	5095	const result: MCValue = result: {
	5096	const elem_ty = self.typeOfIndex(inst);
	5097	if (!elem_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .none;
4779	5098
4780		return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
	5099	const slice_ty = self.typeOf(bin_op.lhs);
	5100	const slice_ptr_field_type = slice_ty.slicePtrFieldType(mod);
	5101	const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
	5102	const dst_mcv = try self.allocRegOrMem(inst, false);
	5103	try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
	5104	break :result dst_mcv;
	5105	};
	5106	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4781	5107	}
4782	5108
4783	5109	fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -4810,11 +5136,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
4810	5136	};
4811	5137	defer if (index_lock) \|lock\| self.register_manager.unlockReg(lock);
4812	5138
4813		const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
4814		const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
4815		defer self.register_manager.unlockReg(offset_reg_lock);
4816
4817	5139	const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	5140	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	5141	defer self.register_manager.unlockReg(addr_lock);
	5142
4818	5143	switch (array) {
4819	5144	.register => {
4820	5145	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod));
...	...	@@ -4843,6 +5168,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
4843	5168	else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
4844	5169	}
4845	5170
	5171	const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
	5172	const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
	5173	defer self.register_manager.unlockReg(offset_lock);
	5174
4846	5175	// TODO we could allocate register here, but need to expect addr register and potentially
4847	5176	// offset register.
4848	5177	try self.spillEflagsIfOccupied();
...	...	@@ -5093,7 +5422,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
5093	5422	.{ ._, .sub },
5094	5423	dst_ty,
5095	5424	dst_mcv,
5096		.{ .immediate = 8 + self.regExtraBits(src_ty) },
	5425	.{ .immediate = 32 - src_bits },
5097	5426	);
5098	5427	} else if (src_bits <= 64) {
5099	5428	try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);
...	...	@@ -5361,7 +5690,9 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
5361	5690	mat_src_mcv
5362	5691	else
5363	5692	.{ .register = mat_src_mcv.register_pair[0] }, false);
5364		try self.genPopCount(tmp_regs[1], Type.usize, if (mat_src_mcv.isMemory())
	5693	const src_info = src_ty.intInfo(mod);
	5694	const hi_ty = try mod.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1);
	5695	try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isMemory())
5365	5696	mat_src_mcv.address().offset(8).deref()
5366	5697	else
5367	5698	.{ .register = mat_src_mcv.register_pair[1] }, false);
...	...	@@ -5383,9 +5714,13 @@ fn genPopCount(
5383	5714	const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
5384	5715	if (self.hasFeature(.popcnt)) return self.genBinOpMir(
5385	5716	.{ ._, .popcnt },
5386		if (src_abi_size > 1) src_ty else Type.u16,
	5717	if (src_abi_size > 1) src_ty else Type.u32,
5387	5718	.{ .register = dst_reg },
5388		src_mcv,
	5719	if (src_abi_size > 1) src_mcv else src: {
	5720	if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv);
	5721	try self.truncateRegister(try src_ty.toUnsigned(mod), dst_reg);
	5722	break :src .{ .register = dst_reg };
	5723	},
5389	5724	);
5390	5725
5391	5726	const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
...	...	@@ -5517,9 +5852,9 @@ fn genByteSwap(
5517	5852	try self.asmRegisterMemory(
5518	5853	.{ ._, .movbe },
5519	5854	dst_regs[0],
5520		src_mcv.address().offset(8).deref().mem(.qword),
	5855	try src_mcv.address().offset(8).deref().mem(self, .qword),
5521	5856	);
5522		try self.asmRegisterMemory(.{ ._, .movbe }, dst_regs[1], src_mcv.mem(.qword));
	5857	try self.asmRegisterMemory(.{ ._, .movbe }, dst_regs[1], try src_mcv.mem(self, .qword));
5523	5858	} else for (dst_regs, src_mcv.register_pair) \|dst_reg, src_reg\| {
5524	5859	try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to64(), src_reg.to64());
5525	5860	try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
...	...	@@ -5762,7 +6097,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
5762	6097	else => unreachable,
5763	6098	} });
5764	6099	const sign_mem: Memory = if (sign_mcv.isMemory())
5765		sign_mcv.mem(Memory.Size.fromSize(abi_size))
	6100	try sign_mcv.mem(self, Memory.Size.fromSize(abi_size))
5766	6101	else
5767	6102	.{
5768	6103	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
...	...	@@ -5945,7 +6280,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5945	6280	mir_tag,
5946	6281	dst_alias,
5947	6282	dst_alias,
5948		src_mcv.mem(Memory.Size.fromSize(abi_size)),
	6283	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
5949	6284	Immediate.u(@as(u5, @bitCast(mode))),
5950	6285	) else try self.asmRegisterRegisterRegisterImmediate(
5951	6286	mir_tag,
...	...	@@ -5960,7 +6295,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5960	6295	else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
5961	6296	mir_tag,
5962	6297	dst_alias,
5963		src_mcv.mem(Memory.Size.fromSize(abi_size)),
	6298	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
5964	6299	Immediate.u(@as(u5, @bitCast(mode))),
5965	6300	) else try self.asmRegisterRegisterImmediate(
5966	6301	mir_tag,
...	...	@@ -6000,7 +6335,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6000	6335	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
6001	6336	.l,
6002	6337	registerAlias(dst_mcv.register, cmov_abi_size),
6003		src_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),
	6338	try src_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
6004	6339	),
6005	6340	else => {
6006	6341	const val_reg = try self.copyToTmpRegister(ty, src_mcv);
...	...	@@ -6100,7 +6435,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6100	6435	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6101	6436	mir_tag,
6102	6437	dst_alias,
6103		src_mcv.mem(self.memSize(ty)),
	6438	try src_mcv.mem(self, self.memSize(ty)),
6104	6439	) else try self.asmRegisterRegister(
6105	6440	mir_tag,
6106	6441	dst_alias,
...	...	@@ -6206,7 +6541,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6206	6541	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6207	6542	.{ .v_ps, .cvtph2 },
6208	6543	wide_reg,
6209		src_mcv.mem(Memory.Size.fromSize(
	6544	try src_mcv.mem(self, Memory.Size.fromSize(
6210	6545	@intCast(@divExact(wide_reg.bitSize(), 16)),
6211	6546	)),
6212	6547	) else try self.asmRegisterRegister(
...	...	@@ -6254,7 +6589,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6254	6589	mir_tag,
6255	6590	dst_reg,
6256	6591	dst_reg,
6257		src_mcv.mem(Memory.Size.fromSize(abi_size)),
	6592	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
6258	6593	) else try self.asmRegisterRegisterRegister(
6259	6594	mir_tag,
6260	6595	dst_reg,
...	...	@@ -6267,7 +6602,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6267	6602	else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
6268	6603	mir_tag,
6269	6604	dst_reg,
6270		src_mcv.mem(Memory.Size.fromSize(abi_size)),
	6605	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
6271	6606	) else try self.asmRegisterRegister(
6272	6607	mir_tag,
6273	6608	dst_reg,
...	...	@@ -6332,7 +6667,7 @@ fn reuseOperandAdvanced(
6332	6667	return false;
6333	6668
6334	6669	switch (mcv) {
6335		.register, .register_pair => for (mcv.getRegs()) \|reg\| {
	6670	.register, .register_pair, .register_overflow => for (mcv.getRegs()) \|reg\| {
6336	6671	// If it's in the registers table, need to associate the register(s) with the
6337	6672	// new instruction.
6338	6673	if (maybe_tracked_inst) \|tracked_inst\| {
...	...	@@ -6346,6 +6681,10 @@ fn reuseOperandAdvanced(
6346	6681	.load_frame => \|frame_addr\| if (frame_addr.index.isNamed()) return false,
6347	6682	else => return false,
6348	6683	}
	6684	switch (mcv) {
	6685	.eflags, .register_overflow => self.eflags_inst = maybe_tracked_inst,
	6686	else => {},
	6687	}
6349	6688
6350	6689	// Prevent the operand deaths processing code from deallocating it.
6351	6690	self.liveness.clearOperandDeath(inst, op_index);
...	...	@@ -6363,11 +6702,36 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6363	6702	if (!val_ty.hasRuntimeBitsIgnoreComptime(mod)) return;
6364	6703	const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
6365	6704
	6705	if (ptr_info.packed_offset.bit_offset % 8 == 0) {
	6706	try self.load(
	6707	dst_mcv,
	6708	ptr_ty,
	6709	ptr_mcv.offset(@intCast(@divExact(ptr_info.packed_offset.bit_offset, 8))),
	6710	);
	6711	const val_bit_size: u32 = @intCast(val_ty.bitSize(mod));
	6712	if (val_abi_size * 8 > val_bit_size) {
	6713	if (dst_mcv.isRegister()) {
	6714	try self.truncateRegister(val_ty, dst_mcv.getReg().?);
	6715	} else {
	6716	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	6717	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	6718	defer self.register_manager.unlockReg(tmp_lock);
	6719
	6720	const hi_mcv = dst_mcv.address().offset(@intCast(val_bit_size / 64 * 8)).deref();
	6721	try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
	6722	try self.truncateRegister(val_ty, tmp_reg);
	6723	try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
	6724	}
	6725	}
	6726	return;
	6727	}
	6728
6366	6729	if (val_abi_size > 8) return self.fail("TODO implement packed load of {}", .{val_ty.fmt(mod)});
6367	6730
6368	6731	const limb_abi_size: u32 = @min(val_abi_size, 8);
6369	6732	const limb_abi_bits = limb_abi_size * 8;
6370		const val_byte_off: i32 = @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size);
	6733	const val_byte_off: i32 =
	6734	@intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size);
6371	6735	const val_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits;
6372	6736	const val_extra_bits = self.regExtraBits(val_ty);
6373	6737
...	...	@@ -6393,6 +6757,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6393	6757	.disp = val_byte_off,
6394	6758	} },
6395	6759	});
	6760	try self.spillEflagsIfOccupied();
6396	6761	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
6397	6762	} else {
6398	6763	const tmp_reg =
...	...	@@ -6415,6 +6780,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6415	6780	.disp = val_byte_off + 1,
6416	6781	} },
6417	6782	});
	6783	try self.spillEflagsIfOccupied();
6418	6784	try self.asmRegisterRegisterImmediate(
6419	6785	.{ ._rd, .sh },
6420	6786	dst_alias,
...	...	@@ -6498,6 +6864,27 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
6498	6864	} else {
6499	6865	try self.load(dst_mcv, ptr_ty, ptr_mcv);
6500	6866	}
	6867
	6868	if (elem_ty.isAbiInt(mod) and elem_size * 8 > elem_ty.bitSize(mod)) {
	6869	const high_mcv: MCValue = switch (dst_mcv) {
	6870	.register => \|dst_reg\| .{ .register = dst_reg },
	6871	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
	6872	else => dst_mcv.address().offset(@intCast((elem_size - 1) / 8 * 8)).deref(),
	6873	};
	6874	const high_reg = if (high_mcv.isRegister())
	6875	high_mcv.getReg().?
	6876	else
	6877	try self.copyToTmpRegister(Type.usize, high_mcv);
	6878	const high_lock = self.register_manager.lockReg(high_reg);
	6879	defer if (high_lock) \|lock\| self.register_manager.unlockReg(lock);
	6880
	6881	try self.truncateRegister(elem_ty, high_reg);
	6882	if (!high_mcv.isRegister()) try self.genCopy(
	6883	if (elem_size <= 8) elem_ty else Type.usize,
	6884	high_mcv,
	6885	.{ .register = high_reg },
	6886	);
	6887	}
6501	6888	break :result dst_mcv;
6502	6889	};
6503	6890	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -6530,7 +6917,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
6530	6917	.base = .{ .reg = ptr_reg },
6531	6918	.mod = .{ .rm = .{
6532	6919	.size = Memory.Size.fromSize(limb_abi_size),
6533		.disp = src_byte_off + limb_i * limb_abi_bits,
	6920	.disp = src_byte_off + limb_i * limb_abi_size,
6534	6921	} },
6535	6922	};
6536	6923
...	...	@@ -6575,6 +6962,22 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
6575	6962	limb_mem,
6576	6963	registerAlias(tmp_reg, limb_abi_size),
6577	6964	);
	6965	} else if (src_bit_size <= 128 and src_bit_off == 0) {
	6966	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	6967	const tmp_mcv = MCValue{ .register = tmp_reg };
	6968	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	6969	defer self.register_manager.unlockReg(tmp_lock);
	6970
	6971	try self.genSetReg(tmp_reg, limb_ty, switch (limb_i) {
	6972	0 => src_mcv,
	6973	else => src_mcv.address().offset(limb_i * limb_abi_size).deref(),
	6974	});
	6975	try self.genBinOpMir(.{ ._, .@"and" }, limb_ty, tmp_mcv, .{ .immediate = part_mask });
	6976	try self.asmMemoryRegister(
	6977	.{ ._, .@"or" },
	6978	limb_mem,
	6979	registerAlias(tmp_reg, limb_abi_size),
	6980	);
6578	6981	} else return self.fail("TODO: implement packed store of {}", .{src_ty.fmt(mod)});
6579	6982	}
6580	6983	}
...	...	@@ -6627,6 +7030,11 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
6627	7030	} else {
6628	7031	// TODO if the value is undef, don't lower this instruction
6629	7032	}
	7033
	7034	try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
	7035	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
	7036	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
	7037
6630	7038	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
6631	7039	const ptr_mcv = try self.resolveInst(bin_op.lhs);
6632	7040	const ptr_ty = self.typeOf(bin_op.lhs);
...	...	@@ -6719,12 +7127,15 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6719	7127	const dst_lock = self.register_manager.lockReg(dst_reg);
6720	7128	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
6721	7129
6722		if (field_off > 0) try self.genShiftBinOpMir(
6723		.{ ._r, .sh },
6724		Type.usize,
6725		dst_mcv,
6726		.{ .immediate = field_off },
6727		);
	7130	if (field_off > 0) {
	7131	try self.spillEflagsIfOccupied();
	7132	try self.genShiftBinOpMir(
	7133	.{ ._r, .sh },
	7134	Type.usize,
	7135	dst_mcv,
	7136	.{ .immediate = field_off },
	7137	);
	7138	}
6728	7139	if (abi.RegisterClass.gp.isSet(RegisterManager.indexOfRegIntoTracked(dst_reg).?) and
6729	7140	container_ty.abiSize(mod) * 8 > field_ty.bitSize(mod))
6730	7141	try self.truncateRegister(field_ty, dst_reg);
...	...	@@ -6759,12 +7170,15 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6759	7170	defer for (dst_locks) \|dst_lock\| if (dst_lock) \|lock\|
6760	7171	self.register_manager.unlockReg(lock);
6761	7172
6762		if (field_off > 0) try self.genShiftBinOpMir(
6763		.{ ._r, .sh },
6764		Type.u128,
6765		dst_mcv,
6766		.{ .immediate = field_off },
6767		);
	7173	if (field_off > 0) {
	7174	try self.spillEflagsIfOccupied();
	7175	try self.genShiftBinOpMir(
	7176	.{ ._r, .sh },
	7177	Type.u128,
	7178	dst_mcv,
	7179	.{ .immediate = field_off },
	7180	);
	7181	}
6768	7182
6769	7183	if (field_bit_size <= 64) {
6770	7184	if (self.regExtraBits(field_ty) > 0)
...	...	@@ -6792,15 +7206,18 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6792	7206	const dst_lock = self.register_manager.lockReg(dst_reg);
6793	7207	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
6794	7208
6795		if (field_off % 64 > 0) try self.genShiftBinOpMir(
6796		.{ ._r, .sh },
6797		Type.usize,
6798		dst_mcv,
6799		.{ .immediate = field_off % 64 },
6800		);
6801		if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg);
6802
6803		break :result if (field_rc.supersetOf(abi.RegisterClass.gp))
	7209	if (field_off % 64 > 0) {
	7210	try self.spillEflagsIfOccupied();
	7211	try self.genShiftBinOpMir(
	7212	.{ ._r, .sh },
	7213	Type.usize,
	7214	dst_mcv,
	7215	.{ .immediate = field_off % 64 },
	7216	);
	7217	}
	7218	if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg);
	7219
	7220	break :result if (field_rc.supersetOf(abi.RegisterClass.gp))
6804	7221	dst_mcv
6805	7222	else
6806	7223	try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
...	...	@@ -6808,17 +7225,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6808	7225	.register_overflow => \|ro\| {
6809	7226	switch (index) {
6810	7227	// Get wrapped value for overflow operation.
6811		0 => break :result if (self.liveness.operandDies(inst, 0))
6812		.{ .register = ro.reg }
6813		else
6814		try self.copyToRegisterWithInstTracking(
6815		inst,
6816		Type.usize,
6817		.{ .register = ro.reg },
6818		),
	7228	0 => if (self.reuseOperand(inst, extra.struct_operand, 0, src_mcv)) {
	7229	self.eflags_inst = null; // actually stop tracking the overflow part
	7230	break :result .{ .register = ro.reg };
	7231	} else break :result try self.copyToRegisterWithInstTracking(
	7232	inst,
	7233	Type.usize,
	7234	.{ .register = ro.reg },
	7235	),
6819	7236	// Get overflow bit.
6820		1 => if (self.liveness.operandDies(inst, 0)) {
6821		self.eflags_inst = inst;
	7237	1 => if (self.reuseOperandAdvanced(inst, extra.struct_operand, 0, src_mcv, null)) {
	7238	self.eflags_inst = inst; // actually keep tracking the overflow part
6822	7239	break :result .{ .eflags = ro.eflags };
6823	7240	} else {
6824	7241	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
...	...	@@ -6833,11 +7250,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6833	7250	if (field_off % 8 == 0) {
6834	7251	const off_mcv =
6835	7252	src_mcv.address().offset(@intCast(@divExact(field_off, 8))).deref();
	7253	const field_bit_size = field_ty.bitSize(mod);
6836	7254
6837	7255	if (field_abi_size <= 8) {
6838	7256	const int_ty = try mod.intType(
6839	7257	if (field_ty.isAbiInt(mod)) field_ty.intInfo(mod).signedness else .unsigned,
6840		@intCast(field_ty.bitSize(mod)),
	7258	@intCast(field_bit_size),
6841	7259	);
6842	7260
6843	7261	const dst_reg = try self.register_manager.allocReg(
...	...	@@ -6856,10 +7274,24 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6856	7274	try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
6857	7275	}
6858	7276
6859		if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result off_mcv;
6860
6861		const dst_mcv = try self.allocRegOrMem(inst, true);
6862		try self.genCopy(field_ty, dst_mcv, off_mcv);
	7277	const dst_mcv = if (self.reuseOperand(inst, operand, 0, src_mcv))
	7278	off_mcv
	7279	else dst: {
	7280	const dst_mcv = try self.allocRegOrMem(inst, true);
	7281	try self.genCopy(field_ty, dst_mcv, off_mcv);
	7282	break :dst dst_mcv;
	7283	};
	7284	if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) {
	7285	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	7286	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	7287	defer self.register_manager.unlockReg(tmp_lock);
	7288
	7289	const hi_mcv =
	7290	dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();
	7291	try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
	7292	try self.truncateRegister(field_ty, tmp_reg);
	7293	try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
	7294	}
6863	7295	break :result dst_mcv;
6864	7296	}
6865	7297
...	...	@@ -6888,6 +7320,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6888	7320	.disp = frame_addr.off + field_byte_off,
6889	7321	} },
6890	7322	});
	7323	try self.spillEflagsIfOccupied();
6891	7324	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
6892	7325	} else {
6893	7326	const tmp_reg = registerAlias(
...	...	@@ -6916,6 +7349,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6916	7349	.disp = frame_addr.off + field_byte_off + limb_abi_size,
6917	7350	} },
6918	7351	});
	7352	try self.spillEflagsIfOccupied();
6919	7353	try self.asmRegisterRegisterImmediate(
6920	7354	.{ ._rd, .sh },
6921	7355	dst_alias,
...	...	@@ -6960,13 +7394,18 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
6960	7394	fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
6961	7395	const mod = self.bin_file.options.module.?;
6962	7396	const src_ty = self.typeOf(src_air);
6963		const src_mcv = try self.resolveInst(src_air);
6964	7397	if (src_ty.zigTypeTag(mod) == .Vector)
6965	7398	return self.fail("TODO implement genUnOp for {}", .{src_ty.fmt(mod)});
6966	7399
	7400	var src_mcv = try self.resolveInst(src_air);
6967	7401	switch (src_mcv) {
6968	7402	.eflags => \|cc\| switch (tag) {
6969		.not => return .{ .eflags = cc.negate() },
	7403	.not => {
	7404	if (maybe_inst) \|inst\| if (self.reuseOperand(inst, src_air, 0, src_mcv))
	7405	return .{ .eflags = cc.negate() };
	7406	try self.spillEflagsIfOccupied();
	7407	src_mcv = try self.resolveInst(src_air);
	7408	},
6970	7409	else => {},
6971	7410	},
6972	7411	else => {},
...	...	@@ -7013,7 +7452,25 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
7013	7452	} else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
7014	7453	}
7015	7454	},
7016		.neg => try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv),
	7455	.neg => {
	7456	try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv);
	7457	const abi_size: u16 = @intCast(src_ty.abiSize(mod));
	7458	const bit_size = src_ty.intInfo(mod).bits;
	7459	if (abi_size * 8 > bit_size) {
	7460	if (dst_mcv.isRegister()) {
	7461	try self.truncateRegister(src_ty, dst_mcv.getReg().?);
	7462	} else {
	7463	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	7464	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	7465	defer self.register_manager.unlockReg(tmp_lock);
	7466
	7467	const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
	7468	try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
	7469	try self.truncateRegister(src_ty, tmp_reg);
	7470	try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
	7471	}
	7472	}
	7473	},
7017	7474	else => unreachable,
7018	7475	}
7019	7476	return dst_mcv;
...	...	@@ -7054,7 +7511,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
7054	7511	},
7055	7512	.indirect, .load_frame => try self.asmMemory(
7056	7513	mir_tag,
7057		dst_mcv.mem(Memory.Size.fromSize(abi_size)),
	7514	try dst_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7058	7515	),
7059	7516	}
7060	7517	}
...	...	@@ -7552,27 +8009,27 @@ fn genMulDivBinOp(
7552	8009	defer self.register_manager.unlockReg(tmp_lock);
7553	8010
7554	8011	if (mat_lhs_mcv.isMemory())
7555		try self.asmRegisterMemory(.{ ._, .mov }, .rax, mat_lhs_mcv.mem(.qword))
	8012	try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .qword))
7556	8013	else
7557	8014	try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);
7558	8015	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
7559	8016	.{ ._, .mov },
7560	8017	tmp_reg,
7561		mat_rhs_mcv.address().offset(8).deref().mem(.qword),
	8018	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
7562	8019	) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);
7563	8020	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);
7564	8021	if (mat_rhs_mcv.isMemory())
7565		try self.asmMemory(.{ ._, .mul }, mat_rhs_mcv.mem(.qword))
	8022	try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
7566	8023	else
7567	8024	try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
7568	8025	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
7569	8026	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
7570	8027	.{ ._, .mov },
7571	8028	tmp_reg,
7572		mat_lhs_mcv.address().offset(8).deref().mem(.qword),
	8029	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
7573	8030	) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);
7574	8031	if (mat_rhs_mcv.isMemory())
7575		try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.mem(.qword))
	8032	try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .qword))
7576	8033	else
7577	8034	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);
7578	8035	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
...	...	@@ -7833,7 +8290,7 @@ fn genBinOp(
7833	8290	.{ .vp_w, .insr },
7834	8291	dst_reg,
7835	8292	dst_reg,
7836		rhs_mcv.mem(.word),
	8293	try rhs_mcv.mem(self, .word),
7837	8294	Immediate.u(1),
7838	8295	) else try self.asmRegisterRegisterRegister(
7839	8296	.{ .vp_, .unpcklwd },
...	...	@@ -7858,7 +8315,7 @@ fn genBinOp(
7858	8315	mir_tag,
7859	8316	dst_reg,
7860	8317	dst_reg,
7861		src_mcv.mem(Memory.Size.fromBitSize(float_bits)),
	8318	try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
7862	8319	) else try self.asmRegisterRegisterRegister(
7863	8320	mir_tag,
7864	8321	dst_reg,
...	...	@@ -7877,7 +8334,7 @@ fn genBinOp(
7877	8334	if (src_mcv.isMemory()) try self.asmRegisterMemory(
7878	8335	mir_tag,
7879	8336	dst_reg,
7880		src_mcv.mem(Memory.Size.fromBitSize(float_bits)),
	8337	try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
7881	8338	) else try self.asmRegisterRegister(
7882	8339	mir_tag,
7883	8340	dst_reg,
...	...	@@ -7919,12 +8376,18 @@ fn genBinOp(
7919	8376	};
7920	8377	}
7921	8378
7922		if ((lhs_ty.scalarType(mod).isRuntimeFloat() and
	8379	const sse_op = switch (lhs_ty.zigTypeTag(mod)) {
	8380	else => false,
	8381	.Float => true,
	8382	.Vector => switch (lhs_ty.childType(mod).toIntern()) {
	8383	.bool_type => false,
	8384	else => true,
	8385	},
	8386	};
	8387	if (sse_op and ((lhs_ty.scalarType(mod).isRuntimeFloat() and
7923	8388	lhs_ty.scalarType(mod).floatBits(self.target.*) == 80) or
7924		lhs_ty.abiSize(mod) > @as(u6, if (self.hasFeature(.avx)) 32 else 16))
7925		return self.fail("TODO implement genBinOp for {s} {}", .{
7926		@tagName(air_tag), lhs_ty.fmt(mod),
7927		});
	8389	lhs_ty.abiSize(mod) > @as(u6, if (self.hasFeature(.avx)) 32 else 16)))
	8390	return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(air_tag), lhs_ty.fmt(mod) });
7928	8391
7929	8392	const maybe_mask_reg = switch (air_tag) {
7930	8393	else => null,
...	...	@@ -7941,10 +8404,16 @@ fn genBinOp(
7941	8404	if (maybe_mask_reg) \|mask_reg\| self.register_manager.lockRegAssumeUnused(mask_reg) else null;
7942	8405	defer if (mask_lock) \|lock\| self.register_manager.unlockReg(lock);
7943	8406
7944		const ordered_air = if (lhs_ty.isVector(mod) and lhs_ty.childType(mod).isAbiInt(mod) and
7945		switch (air_tag) {
7946		.cmp_lt, .cmp_gte => true,
7947		else => false,
	8407	const ordered_air = if (lhs_ty.isVector(mod) and switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
	8408	.Int => switch (air_tag) {
	8409	.cmp_lt, .cmp_gte => true,
	8410	else => false,
	8411	},
	8412	.Float => switch (air_tag) {
	8413	.cmp_gte, .cmp_gt => true,
	8414	else => false,
	8415	},
	8416	else => unreachable,
7948	8417	}) .{ .lhs = rhs_air, .rhs = lhs_air } else .{ .lhs = lhs_air, .rhs = rhs_air };
7949	8418
7950	8419	const lhs_mcv = try self.resolveInst(ordered_air.lhs);
...	...	@@ -7971,14 +8440,12 @@ fn genBinOp(
7971	8440	.xor,
7972	8441	.min,
7973	8442	.max,
	8443	.cmp_eq,
	8444	.cmp_neq,
7974	8445	=> true,
7975	8446
7976	8447	else => false,
7977	8448	};
7978		const vec_op = switch (lhs_ty.zigTypeTag(mod)) {
7979		else => false,
7980		.Float, .Vector => true,
7981		};
7982	8449
7983	8450	const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) {
7984	8451	.register => \|lhs_reg\| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null },
...	...	@@ -8000,23 +8467,23 @@ fn genBinOp(
8000	8467	var flipped = false;
8001	8468	var copied_to_dst = true;
8002	8469	const dst_mcv: MCValue = dst: {
	8470	const tracked_inst = switch (air_tag) {
	8471	else => maybe_inst,
	8472	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => null,
	8473	};
8003	8474	if (maybe_inst) \|inst\| {
8004		const tracked_inst = switch (air_tag) {
8005		else => inst,
8006		.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => null,
8007		};
8008		if ((!vec_op or lhs_mcv.isRegister()) and
	8475	if ((!sse_op or lhs_mcv.isRegister()) and
8009	8476	self.reuseOperandAdvanced(inst, ordered_air.lhs, 0, lhs_mcv, tracked_inst))
8010	8477	break :dst lhs_mcv;
8011		if (is_commutative and (!vec_op or rhs_mcv.isRegister()) and
	8478	if (is_commutative and (!sse_op or rhs_mcv.isRegister()) and
8012	8479	self.reuseOperandAdvanced(inst, ordered_air.rhs, 1, rhs_mcv, tracked_inst))
8013	8480	{
8014	8481	flipped = true;
8015	8482	break :dst rhs_mcv;
8016	8483	}
8017	8484	}
8018		const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
8019		if (vec_op and lhs_mcv.isRegister() and self.hasFeature(.avx))
	8485	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, tracked_inst, true);
	8486	if (sse_op and lhs_mcv.isRegister() and self.hasFeature(.avx))
8020	8487	copied_to_dst = false
8021	8488	else
8022	8489	try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
...	...	@@ -8046,7 +8513,7 @@ fn genBinOp(
8046	8513	};
8047	8514	defer for (src_locks) \|src_lock\| if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
8048	8515
8049		if (!vec_op) {
	8516	if (!sse_op) {
8050	8517	switch (air_tag) {
8051	8518	.add,
8052	8519	.add_wrap,
...	...	@@ -8130,17 +8597,25 @@ fn genBinOp(
8130	8597
8131	8598	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
8132	8599	if (src_mcv.isMemory()) {
8133		try self.asmRegisterMemory(.{ ._, .cmp }, dst_regs[0], src_mcv.mem(.qword));
	8600	try self.asmRegisterMemory(
	8601	.{ ._, .cmp },
	8602	dst_regs[0],
	8603	try src_mcv.mem(self, .qword),
	8604	);
8134	8605	try self.asmRegisterMemory(
8135	8606	.{ ._, .sbb },
8136	8607	tmp_reg,
8137		src_mcv.address().offset(8).deref().mem(.qword),
	8608	try src_mcv.address().offset(8).deref().mem(self, .qword),
	8609	);
	8610	try self.asmCmovccRegisterMemory(
	8611	cc,
	8612	dst_regs[0],
	8613	try src_mcv.mem(self, .qword),
8138	8614	);
8139		try self.asmCmovccRegisterMemory(cc, dst_regs[0], src_mcv.mem(.qword));
8140	8615	try self.asmCmovccRegisterMemory(
8141	8616	cc,
8142	8617	dst_regs[1],
8143		src_mcv.address().offset(8).deref().mem(.qword),
	8618	try src_mcv.address().offset(8).deref().mem(self, .qword),
8144	8619	);
8145	8620	} else {
8146	8621	try self.asmRegisterRegister(
...	...	@@ -8292,7 +8767,7 @@ fn genBinOp(
8292	8767	.{ .vp_w, .insr },
8293	8768	dst_reg,
8294	8769	dst_reg,
8295		src_mcv.mem(.word),
	8770	try src_mcv.mem(self, .word),
8296	8771	Immediate.u(1),
8297	8772	) else try self.asmRegisterRegisterRegister(
8298	8773	.{ .vp_, .unpcklwd },
...	...	@@ -8738,7 +9213,7 @@ fn genBinOp(
8738	9213	.{ .vp_w, .insr },
8739	9214	dst_reg,
8740	9215	dst_reg,
8741		src_mcv.mem(.word),
	9216	try src_mcv.mem(self, .word),
8742	9217	Immediate.u(1),
8743	9218	) else try self.asmRegisterRegisterRegister(
8744	9219	.{ .vp_, .unpcklwd },
...	...	@@ -8784,7 +9259,7 @@ fn genBinOp(
8784	9259	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
8785	9260	.{ .vp_d, .insr },
8786	9261	dst_reg,
8787		src_mcv.mem(.dword),
	9262	try src_mcv.mem(self, .dword),
8788	9263	Immediate.u(1),
8789	9264	) else try self.asmRegisterRegisterRegister(
8790	9265	.{ .v_ps, .unpckl },
...	...	@@ -8836,7 +9311,7 @@ fn genBinOp(
8836	9311	if (src_mcv.isMemory()) try self.asmRegisterMemory(
8837	9312	.{ .v_ps, .cvtph2 },
8838	9313	tmp_reg,
8839		src_mcv.mem(.qword),
	9314	try src_mcv.mem(self, .qword),
8840	9315	) else try self.asmRegisterRegister(
8841	9316	.{ .v_ps, .cvtph2 },
8842	9317	tmp_reg,
...	...	@@ -8879,7 +9354,7 @@ fn genBinOp(
8879	9354	if (src_mcv.isMemory()) try self.asmRegisterMemory(
8880	9355	.{ .v_ps, .cvtph2 },
8881	9356	tmp_reg,
8882		src_mcv.mem(.xword),
	9357	try src_mcv.mem(self, .xword),
8883	9358	) else try self.asmRegisterRegister(
8884	9359	.{ .v_ps, .cvtph2 },
8885	9360	tmp_reg,
...	...	@@ -8925,6 +9400,13 @@ fn genBinOp(
8925	9400	=> if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
8926	9401	.max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
8927	9402	.min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
	9403	.cmp_lt,
	9404	.cmp_lte,
	9405	.cmp_eq,
	9406	.cmp_gte,
	9407	.cmp_gt,
	9408	.cmp_neq,
	9409	=> if (self.hasFeature(.avx)) .{ .v_ss, .cmp } else .{ ._ss, .cmp },
8928	9410	else => unreachable,
8929	9411	},
8930	9412	2...4 => switch (air_tag) {
...	...	@@ -8938,6 +9420,13 @@ fn genBinOp(
8938	9420	=> if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div },
8939	9421	.max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max },
8940	9422	.min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min },
	9423	.cmp_lt,
	9424	.cmp_lte,
	9425	.cmp_eq,
	9426	.cmp_gte,
	9427	.cmp_gt,
	9428	.cmp_neq,
	9429	=> if (self.hasFeature(.avx)) .{ .v_ps, .cmp } else .{ ._ps, .cmp },
8941	9430	else => unreachable,
8942	9431	},
8943	9432	5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
...	...	@@ -8947,6 +9436,7 @@ fn genBinOp(
8947	9436	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
8948	9437	.max => .{ .v_ps, .max },
8949	9438	.min => .{ .v_ps, .min },
	9439	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_ps, .cmp },
8950	9440	else => unreachable,
8951	9441	} else null,
8952	9442	else => null,
...	...	@@ -8963,6 +9453,13 @@ fn genBinOp(
8963	9453	=> if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
8964	9454	.max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
8965	9455	.min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
	9456	.cmp_lt,
	9457	.cmp_lte,
	9458	.cmp_eq,
	9459	.cmp_gte,
	9460	.cmp_gt,
	9461	.cmp_neq,
	9462	=> if (self.hasFeature(.avx)) .{ .v_sd, .cmp } else .{ ._sd, .cmp },
8966	9463	else => unreachable,
8967	9464	},
8968	9465	2 => switch (air_tag) {
...	...	@@ -8976,6 +9473,13 @@ fn genBinOp(
8976	9473	=> if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div },
8977	9474	.max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max },
8978	9475	.min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min },
	9476	.cmp_lt,
	9477	.cmp_lte,
	9478	.cmp_eq,
	9479	.cmp_gte,
	9480	.cmp_gt,
	9481	.cmp_neq,
	9482	=> if (self.hasFeature(.avx)) .{ .v_pd, .cmp } else .{ ._pd, .cmp },
8979	9483	else => unreachable,
8980	9484	},
8981	9485	3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
...	...	@@ -8984,6 +9488,7 @@ fn genBinOp(
8984	9488	.mul => .{ .v_pd, .mul },
8985	9489	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div },
8986	9490	.max => .{ .v_pd, .max },
	9491	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_pd, .cmp },
8987	9492	.min => .{ .v_pd, .min },
8988	9493	else => unreachable,
8989	9494	} else null,
...	...	@@ -9004,43 +9509,96 @@ fn genBinOp(
9004	9509	const lhs_copy_lock = if (lhs_copy_reg) \|reg\| self.register_manager.lockReg(reg) else null;
9005	9510	defer if (lhs_copy_lock) \|lock\| self.register_manager.unlockReg(lock);
9006	9511
9007		if (self.hasFeature(.avx)) {
9008		const lhs_reg =
9009		if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
9010		if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
9011		mir_tag,
9012		dst_reg,
9013		lhs_reg,
9014		src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {
9015		else => Memory.Size.fromSize(abi_size),
9016		.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9017		}),
9018		) else try self.asmRegisterRegisterRegister(
9019		mir_tag,
9020		dst_reg,
9021		lhs_reg,
9022		registerAlias(if (src_mcv.isRegister())
9023		src_mcv.getReg().?
9024		else
9025		try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9026		);
9027		} else {
9028		assert(copied_to_dst);
9029		if (src_mcv.isMemory()) try self.asmRegisterMemory(
9030		mir_tag,
9031		dst_reg,
9032		src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {
9033		else => Memory.Size.fromSize(abi_size),
9034		.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9035		}),
9036		) else try self.asmRegisterRegister(
9037		mir_tag,
9038		dst_reg,
9039		registerAlias(if (src_mcv.isRegister())
9040		src_mcv.getReg().?
9041		else
9042		try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9043		);
	9512	switch (mir_tag[1]) {
	9513	else => if (self.hasFeature(.avx)) {
	9514	const lhs_reg =
	9515	if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
	9516	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
	9517	mir_tag,
	9518	dst_reg,
	9519	lhs_reg,
	9520	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
	9521	else => Memory.Size.fromSize(abi_size),
	9522	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
	9523	}),
	9524	) else try self.asmRegisterRegisterRegister(
	9525	mir_tag,
	9526	dst_reg,
	9527	lhs_reg,
	9528	registerAlias(if (src_mcv.isRegister())
	9529	src_mcv.getReg().?
	9530	else
	9531	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
	9532	);
	9533	} else {
	9534	assert(copied_to_dst);
	9535	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	9536	mir_tag,
	9537	dst_reg,
	9538	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
	9539	else => Memory.Size.fromSize(abi_size),
	9540	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
	9541	}),
	9542	) else try self.asmRegisterRegister(
	9543	mir_tag,
	9544	dst_reg,
	9545	registerAlias(if (src_mcv.isRegister())
	9546	src_mcv.getReg().?
	9547	else
	9548	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
	9549	);
	9550	},
	9551	.cmp => {
	9552	const imm = Immediate.u(switch (air_tag) {
	9553	.cmp_eq => 0,
	9554	.cmp_lt, .cmp_gt => 1,
	9555	.cmp_lte, .cmp_gte => 2,
	9556	.cmp_neq => 4,
	9557	else => unreachable,
	9558	});
	9559	if (self.hasFeature(.avx)) {
	9560	const lhs_reg =
	9561	if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
	9562	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
	9563	mir_tag,
	9564	dst_reg,
	9565	lhs_reg,
	9566	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
	9567	else => Memory.Size.fromSize(abi_size),
	9568	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
	9569	}),
	9570	imm,
	9571	) else try self.asmRegisterRegisterRegisterImmediate(
	9572	mir_tag,
	9573	dst_reg,
	9574	lhs_reg,
	9575	registerAlias(if (src_mcv.isRegister())
	9576	src_mcv.getReg().?
	9577	else
	9578	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
	9579	imm,
	9580	);
	9581	} else {
	9582	assert(copied_to_dst);
	9583	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
	9584	mir_tag,
	9585	dst_reg,
	9586	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
	9587	else => Memory.Size.fromSize(abi_size),
	9588	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
	9589	}),
	9590	imm,
	9591	) else try self.asmRegisterRegisterImmediate(
	9592	mir_tag,
	9593	dst_reg,
	9594	registerAlias(if (src_mcv.isRegister())
	9595	src_mcv.getReg().?
	9596	else
	9597	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
	9598	imm,
	9599	);
	9600	}
	9601	},
9044	9602	}
9045	9603
9046	9604	switch (air_tag) {
...	...	@@ -9281,48 +9839,46 @@ fn genBinOp(
9281	9839	);
9282	9840	}
9283	9841	},
9284		.cmp_lt,
9285		.cmp_lte,
9286		.cmp_eq,
9287		.cmp_gte,
9288		.cmp_gt,
9289		.cmp_neq,
9290		=> {
9291		switch (air_tag) {
9292		.cmp_lt,
9293		.cmp_eq,
9294		.cmp_gt,
9295		=> {},
9296		.cmp_lte,
9297		.cmp_gte,
9298		.cmp_neq,
9299		=> {
9300		const unsigned_ty = try lhs_ty.toUnsigned(mod);
9301		const not_mcv = try self.genTypedValue(.{
9302		.ty = lhs_ty,
9303		.val = try unsigned_ty.maxInt(mod, unsigned_ty),
9304		});
9305		const not_mem: Memory = if (not_mcv.isMemory())
9306		not_mcv.mem(Memory.Size.fromSize(abi_size))
9307		else
9308		.{ .base = .{
9309		.reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
9310		}, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };
9311		switch (mir_tag[0]) {
9312		.vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
9313		.{ .vp_, .xor },
9314		dst_reg,
9315		dst_reg,
9316		not_mem,
9317		),
9318		.p_b, .p_d, .p_q, .p_w => try self.asmRegisterMemory(
9319		.{ .p_, .xor },
9320		dst_reg,
9321		not_mem,
9322		),
9323		else => unreachable,
9324		}
	9842	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => {
	9843	switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
	9844	.Int => switch (air_tag) {
	9845	.cmp_lt,
	9846	.cmp_eq,
	9847	.cmp_gt,
	9848	=> {},
	9849	.cmp_lte,
	9850	.cmp_gte,
	9851	.cmp_neq,
	9852	=> {
	9853	const unsigned_ty = try lhs_ty.toUnsigned(mod);
	9854	const not_mcv = try self.genTypedValue(.{
	9855	.ty = lhs_ty,
	9856	.val = try unsigned_ty.maxInt(mod, unsigned_ty),
	9857	});
	9858	const not_mem: Memory = if (not_mcv.isMemory())
	9859	try not_mcv.mem(self, Memory.Size.fromSize(abi_size))
	9860	else
	9861	.{ .base = .{
	9862	.reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
	9863	}, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };
	9864	switch (mir_tag[0]) {
	9865	.vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
	9866	.{ .vp_, .xor },
	9867	dst_reg,
	9868	dst_reg,
	9869	not_mem,
	9870	),
	9871	.p_b, .p_d, .p_q, .p_w => try self.asmRegisterMemory(
	9872	.{ .p_, .xor },
	9873	dst_reg,
	9874	not_mem,
	9875	),
	9876	else => unreachable,
	9877	}
	9878	},
	9879	else => unreachable,
9325	9880	},
	9881	.Float => {},
9326	9882	else => unreachable,
9327	9883	}
9328	9884
...	...	@@ -9331,8 +9887,12 @@ fn genBinOp(
9331	9887	defer self.register_manager.unlockReg(gp_lock);
9332	9888
9333	9889	try self.asmRegisterRegister(switch (mir_tag[0]) {
9334		.vp_b, .vp_d, .vp_q, .vp_w => .{ .vp_b, .movmsk },
	9890	._pd, ._sd => .{ ._pd, .movmsk },
	9891	._ps, ._ss => .{ ._ps, .movmsk },
9335	9892	.p_b, .p_d, .p_q, .p_w => .{ .p_b, .movmsk },
	9893	.v_pd, .v_sd => .{ .v_pd, .movmsk },
	9894	.v_ps, .v_ss => .{ .v_ps, .movmsk },
	9895	.vp_b, .vp_d, .vp_q, .vp_w => .{ .vp_b, .movmsk },
9336	9896	else => unreachable,
9337	9897	}, gp_reg.to32(), dst_reg);
9338	9898	return .{ .register = gp_reg };
...	...	@@ -9459,13 +10019,13 @@ fn genBinOpMir(
9459	10019	.load_frame,
9460	10020	.lea_frame,
9461	10021	=> {
9462		blk: {
9463		return self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) {
	10022	direct: {
	10023	try self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) {
9464	10024	.memory => \|addr\| .{
9465	10025	.base = .{ .reg = .ds },
9466	10026	.mod = .{ .rm = .{
9467	10027	.size = Memory.Size.fromSize(limb_abi_size),
9468		.disp = math.cast(i32, addr + off) orelse break :blk,
	10028	.disp = math.cast(i32, addr + off) orelse break :direct,
9469	10029	} },
9470	10030	},
9471	10031	.indirect => \|reg_off\| .{
...	...	@@ -9482,8 +10042,9 @@ fn genBinOpMir(
9482	10042	.disp = frame_addr.off + off,
9483	10043	} },
9484	10044	},
9485		else => break :blk,
	10045	else => break :direct,
9486	10046	});
	10047	continue;
9487	10048	}
9488	10049
9489	10050	switch (src_mcv) {
...	...	@@ -10142,30 +10703,30 @@ fn genCall(self: *Self, info: union(enum) {
10142	10703	}
10143	10704
10144	10705	try self.spillEflagsIfOccupied();
10145		try self.spillRegisters(abi.getCallerPreservedRegs(resolved_cc));
	10706	try self.spillCallerPreservedRegs(resolved_cc);
10146	10707
10147	10708	// set stack arguments first because this can clobber registers
10148	10709	// also clobber spill arguments as we go
10149	10710	switch (call_info.return_value.long) {
10150	10711	.none, .unreach => {},
10151		.indirect => \|reg_off\| try self.spillRegisters(&.{reg_off.reg}),
	10712	.indirect => \|reg_off\| try self.register_manager.getReg(reg_off.reg, null),
10152	10713	else => unreachable,
10153	10714	}
10154	10715	for (call_info.args, arg_types, args, frame_indices) \|dst_arg, arg_ty, src_arg, *frame_index\|
10155	10716	switch (dst_arg) {
10156	10717	.none => {},
10157	10718	.register => \|reg\| {
10158		try self.spillRegisters(&.{reg});
	10719	try self.register_manager.getReg(reg, null);
10159	10720	try reg_locks.append(self.register_manager.lockReg(reg));
10160	10721	},
10161	10722	.register_pair => \|regs\| {
10162		try self.spillRegisters(&regs);
	10723	for (regs) \|reg\| try self.register_manager.getReg(reg, null);
10163	10724	try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs));
10164	10725	},
10165	10726	.indirect => \|reg_off\| {
10166	10727	frame_index.* = try self.allocFrameIndex(FrameAlloc.initType(arg_ty, mod));
10167	10728	try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg);
10168		try self.spillRegisters(&.{reg_off.reg});
	10729	try self.register_manager.getReg(reg_off.reg, null);
10169	10730	try reg_locks.append(self.register_manager.lockReg(reg_off.reg));
10170	10731	},
10171	10732	.load_frame => {
...	...	@@ -10180,7 +10741,7 @@ fn genCall(self: *Self, info: union(enum) {
10180	10741	.none, .unreach => {},
10181	10742	.indirect => \|reg_off\| {
10182	10743	const ret_ty = fn_info.return_type.toType();
10183		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ret_ty, mod));
	10744	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ret_ty, mod));
10184	10745	try self.genSetReg(reg_off.reg, Type.usize, .{
10185	10746	.lea_frame = .{ .index = frame_index, .off = -reg_off.off },
10186	10747	});
...	...	@@ -10306,19 +10867,20 @@ fn genCall(self: *Self, info: union(enum) {
10306	10867	fn airRet(self: *Self, inst: Air.Inst.Index) !void {
10307	10868	const mod = self.bin_file.options.module.?;
10308	10869	const un_op = self.air.instructions.items(.data)[inst].un_op;
10309		const operand = try self.resolveInst(un_op);
10310	10870
10311	10871	const ret_ty = self.fn_type.fnReturnType(mod);
10312	10872	switch (self.ret_mcv.short) {
10313	10873	.none => {},
10314		.register, .register_pair => try self.genCopy(ret_ty, self.ret_mcv.short, operand),
	10874	.register,
	10875	.register_pair,
	10876	=> try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }),
10315	10877	.indirect => \|reg_off\| {
10316	10878	try self.register_manager.getReg(reg_off.reg, null);
10317	10879	const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
10318	10880	defer self.register_manager.unlockReg(lock);
10319	10881
10320	10882	try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long);
10321		try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, operand);
	10883	try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, .{ .air_ref = un_op });
10322	10884	},
10323	10885	else => unreachable,
10324	10886	}
...	...	@@ -10593,7 +11155,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
10593	11155	const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
10594	11156	defer for (locks) \|lock\| self.register_manager.unlockReg(lock);
10595	11157
10596		const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable;
	11158	const limbs_len = math.divCeil(u16, abi_size, 8) catch unreachable;
10597	11159	var limb_i: u16 = 0;
10598	11160	while (limb_i < limbs_len) : (limb_i += 1) {
10599	11161	const off = limb_i * 8;
...	...	@@ -10688,7 +11250,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
10688	11250	.{ .vp_w, .insr },
10689	11251	tmp1_reg,
10690	11252	dst_reg.to128(),
10691		src_mcv.mem(.word),
	11253	try src_mcv.mem(self, .word),
10692	11254	Immediate.u(1),
10693	11255	) else try self.asmRegisterRegisterRegister(
10694	11256	.{ .vp_, .unpcklwd },
...	...	@@ -10892,8 +11454,8 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
10892	11454	},
10893	11455	.register => \|reg\| {
10894	11456	try self.spillEflagsIfOccupied();
10895		try self.asmRegisterImmediate(.{ ._, .@"test" }, reg, Immediate.u(1));
10896		return self.asmJccReloc(.e, undefined);
	11457	try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), Immediate.u(1));
	11458	return self.asmJccReloc(.z, undefined);
10897	11459	},
10898	11460	.immediate,
10899	11461	.load_frame,
...	...	@@ -11366,8 +11928,18 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
11366	11928	try self.spillEflagsIfOccupied();
11367	11929	for (items, relocs, 0..) \|item, *reloc, i\| {
11368	11930	const item_mcv = try self.resolveInst(item);
11369		try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);
11370		reloc.* = try self.asmJccReloc(if (i < relocs.len - 1) .e else .ne, undefined);
	11931	const cc: Condition = switch (condition) {
	11932	.eflags => \|cc\| switch (item_mcv.immediate) {
	11933	0 => cc.negate(),
	11934	1 => cc,
	11935	else => unreachable,
	11936	},
	11937	else => cc: {
	11938	try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);
	11939	break :cc .e;
	11940	},
	11941	};
	11942	reloc.* = try self.asmJccReloc(if (i < relocs.len - 1) cc else cc.negate(), undefined);
11371	11943	}
11372	11944
11373	11945	for (liveness.deaths[case_i]) \|operand\| try self.processDeath(operand);
...	...	@@ -11433,12 +12005,12 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
11433	12005	if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
11434	12006	if (first_br) break :result src_mcv;
11435	12007
11436		if (block_tracking.getReg()) \|block_reg\|
	12008	for (block_tracking.getRegs()) \|block_reg\|
11437	12009	try self.register_manager.getReg(block_reg, br.block_inst);
11438	12010	// .long = .none to avoid merging operand and block result stack frames.
11439	12011	var current_tracking = InstTracking{ .long = .none, .short = src_mcv };
11440	12012	try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*);
11441		if (src_mcv.getReg()) \|src_reg\| self.register_manager.freeReg(src_reg);
	12013	for (src_mcv.getRegs()) \|src_reg\| self.register_manager.freeReg(src_reg);
11442	12014	break :result block_tracking.short;
11443	12015	}
11444	12016
...	...	@@ -11492,8 +12064,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11492	12064	var args = std.ArrayList(MCValue).init(self.gpa);
11493	12065	try args.ensureTotalCapacity(outputs.len + inputs.len);
11494	12066	defer {
11495		for (args.items) \|arg\| if (arg.getReg()) \|reg\|
11496		self.register_manager.unlockReg(.{ .register = reg });
	12067	for (args.items) \|arg\| if (arg.getReg()) \|reg\| self.register_manager.unlockReg(.{
	12068	.tracked_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue,
	12069	});
11497	12070	args.deinit();
11498	12071	}
11499	12072	var arg_map = std.StringHashMap(u8).init(self.gpa);
...	...	@@ -12177,16 +12750,87 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
12177	12750	.general_purpose, .segment => return .{ .move = .{ ._, .mov } },
12178	12751	.x87 => return .x87_load_store,
12179	12752	.mmx => {},
12180		.sse => {
12181		switch (ty.zigTypeTag(mod)) {
12182		else => {
12183		const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
12184		assert(std.mem.indexOfNone(abi.Class, classes, &.{
12185		.integer, .sse, .float, .float_combine,
12186		}) == null);
12187		const abi_size = ty.abiSize(mod);
12188		if (abi_size < 4 or
12189		std.mem.indexOfScalar(abi.Class, classes, .integer) != null) switch (abi_size) {
	12753	.sse => switch (ty.zigTypeTag(mod)) {
	12754	else => {
	12755	const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
	12756	assert(std.mem.indexOfNone(abi.Class, classes, &.{
	12757	.integer, .sse, .float, .float_combine,
	12758	}) == null);
	12759	const abi_size = ty.abiSize(mod);
	12760	if (abi_size < 4 or
	12761	std.mem.indexOfScalar(abi.Class, classes, .integer) != null) switch (abi_size) {
	12762	1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
	12763	.insert = .{ .vp_b, .insr },
	12764	.extract = .{ .vp_b, .extr },
	12765	} } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
	12766	.insert = .{ .p_b, .insr },
	12767	.extract = .{ .p_b, .extr },
	12768	} },
	12769	2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	12770	.insert = .{ .vp_w, .insr },
	12771	.extract = .{ .vp_w, .extr },
	12772	} } else .{ .insert_extract = .{
	12773	.insert = .{ .p_w, .insr },
	12774	.extract = .{ .p_w, .extr },
	12775	} },
	12776	3...4 => return .{ .move = if (self.hasFeature(.avx))
	12777	.{ .v_d, .mov }
	12778	else
	12779	.{ ._d, .mov } },
	12780	5...8 => return .{ .move = if (self.hasFeature(.avx))
	12781	.{ .v_q, .mov }
	12782	else
	12783	.{ ._q, .mov } },
	12784	9...16 => return .{ .move = if (self.hasFeature(.avx))
	12785	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12786	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12787	17...32 => if (self.hasFeature(.avx))
	12788	return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
	12789	else => {},
	12790	} else switch (abi_size) {
	12791	4 => return .{ .move = if (self.hasFeature(.avx))
	12792	.{ .v_ss, .mov }
	12793	else
	12794	.{ ._ss, .mov } },
	12795	5...8 => return .{ .move = if (self.hasFeature(.avx))
	12796	.{ .v_sd, .mov }
	12797	else
	12798	.{ ._sd, .mov } },
	12799	9...16 => return .{ .move = if (self.hasFeature(.avx))
	12800	if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
	12801	else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
	12802	17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned)
	12803	.{ .v_pd, .mova }
	12804	else
	12805	.{ .v_pd, .movu } },
	12806	else => {},
	12807	}
	12808	},
	12809	.Float => switch (ty.floatBits(self.target.*)) {
	12810	16 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	12811	.insert = .{ .vp_w, .insr },
	12812	.extract = .{ .vp_w, .extr },
	12813	} } else .{ .insert_extract = .{
	12814	.insert = .{ .p_w, .insr },
	12815	.extract = .{ .p_w, .extr },
	12816	} },
	12817	32 => return .{ .move = if (self.hasFeature(.avx))
	12818	.{ .v_ss, .mov }
	12819	else
	12820	.{ ._ss, .mov } },
	12821	64 => return .{ .move = if (self.hasFeature(.avx))
	12822	.{ .v_sd, .mov }
	12823	else
	12824	.{ ._sd, .mov } },
	12825	128 => return .{ .move = if (self.hasFeature(.avx))
	12826	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12827	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12828	else => {},
	12829	},
	12830	.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
	12831	.Bool => {},
	12832	.Int => switch (ty.childType(mod).intInfo(mod).bits) {
	12833	8 => switch (ty.vectorLen(mod)) {
12190	12834	1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
12191	12835	.insert = .{ .vp_b, .insr },
12192	12836	.extract = .{ .vp_b, .extr },
...	...	@@ -12213,242 +12857,169 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
12213	12857	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12214	12858	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12215	12859	17...32 => if (self.hasFeature(.avx))
12216		return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
	12860	return .{ .move = if (aligned)
	12861	.{ .v_, .movdqa }
	12862	else
	12863	.{ .v_, .movdqu } },
12217	12864	else => {},
12218		} else switch (abi_size) {
12219		4 => return .{ .move = if (self.hasFeature(.avx))
12220		.{ .v_ss, .mov }
	12865	},
	12866	16 => switch (ty.vectorLen(mod)) {
	12867	1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	12868	.insert = .{ .vp_w, .insr },
	12869	.extract = .{ .vp_w, .extr },
	12870	} } else .{ .insert_extract = .{
	12871	.insert = .{ .p_w, .insr },
	12872	.extract = .{ .p_w, .extr },
	12873	} },
	12874	2 => return .{ .move = if (self.hasFeature(.avx))
	12875	.{ .v_d, .mov }
12221	12876	else
12222		.{ ._ss, .mov } },
	12877	.{ ._d, .mov } },
	12878	3...4 => return .{ .move = if (self.hasFeature(.avx))
	12879	.{ .v_q, .mov }
	12880	else
	12881	.{ ._q, .mov } },
12223	12882	5...8 => return .{ .move = if (self.hasFeature(.avx))
12224		.{ .v_sd, .mov }
	12883	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12884	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12885	9...16 => if (self.hasFeature(.avx))
	12886	return .{ .move = if (aligned)
	12887	.{ .v_, .movdqa }
	12888	else
	12889	.{ .v_, .movdqu } },
	12890	else => {},
	12891	},
	12892	32 => switch (ty.vectorLen(mod)) {
	12893	1 => return .{ .move = if (self.hasFeature(.avx))
	12894	.{ .v_d, .mov }
12225	12895	else
12226		.{ ._sd, .mov } },
12227		9...16 => return .{ .move = if (self.hasFeature(.avx))
12228		if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
12229		else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
12230		17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned)
12231		.{ .v_pd, .mova }
	12896	.{ ._d, .mov } },
	12897	2 => return .{ .move = if (self.hasFeature(.avx))
	12898	.{ .v_q, .mov }
12232	12899	else
12233		.{ .v_pd, .movu } },
12234		else => {},
12235		}
12236		},
12237		.Float => switch (ty.floatBits(self.target.*)) {
12238		16 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12239		.insert = .{ .vp_w, .insr },
12240		.extract = .{ .vp_w, .extr },
12241		} } else .{ .insert_extract = .{
12242		.insert = .{ .p_w, .insr },
12243		.extract = .{ .p_w, .extr },
12244		} },
12245		32 => return .{ .move = if (self.hasFeature(.avx))
12246		.{ .v_ss, .mov }
12247		else
12248		.{ ._ss, .mov } },
12249		64 => return .{ .move = if (self.hasFeature(.avx))
12250		.{ .v_sd, .mov }
12251		else
12252		.{ ._sd, .mov } },
12253		128 => return .{ .move = if (self.hasFeature(.avx))
12254		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12255		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12256		else => {},
12257		},
12258		.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
12259		.Bool => return .{ .move = .{ ._, .mov } },
12260		.Int => switch (ty.childType(mod).intInfo(mod).bits) {
12261		8 => switch (ty.vectorLen(mod)) {
12262		1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
12263		.insert = .{ .vp_b, .insr },
12264		.extract = .{ .vp_b, .extr },
12265		} } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
12266		.insert = .{ .p_b, .insr },
12267		.extract = .{ .p_b, .extr },
12268		} },
12269		2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12270		.insert = .{ .vp_w, .insr },
12271		.extract = .{ .vp_w, .extr },
12272		} } else .{ .insert_extract = .{
12273		.insert = .{ .p_w, .insr },
12274		.extract = .{ .p_w, .extr },
12275		} },
12276		3...4 => return .{ .move = if (self.hasFeature(.avx))
12277		.{ .v_d, .mov }
12278		else
12279		.{ ._d, .mov } },
12280		5...8 => return .{ .move = if (self.hasFeature(.avx))
12281		.{ .v_q, .mov }
12282		else
12283		.{ ._q, .mov } },
12284		9...16 => return .{ .move = if (self.hasFeature(.avx))
12285		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12286		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12287		17...32 => if (self.hasFeature(.avx))
12288		return .{ .move = if (aligned)
12289		.{ .v_, .movdqa }
12290		else
12291		.{ .v_, .movdqu } },
12292		else => {},
12293		},
12294		16 => switch (ty.vectorLen(mod)) {
12295		1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12296		.insert = .{ .vp_w, .insr },
12297		.extract = .{ .vp_w, .extr },
12298		} } else .{ .insert_extract = .{
12299		.insert = .{ .p_w, .insr },
12300		.extract = .{ .p_w, .extr },
12301		} },
12302		2 => return .{ .move = if (self.hasFeature(.avx))
12303		.{ .v_d, .mov }
12304		else
12305		.{ ._d, .mov } },
12306		3...4 => return .{ .move = if (self.hasFeature(.avx))
12307		.{ .v_q, .mov }
12308		else
12309		.{ ._q, .mov } },
12310		5...8 => return .{ .move = if (self.hasFeature(.avx))
12311		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12312		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12313		9...16 => if (self.hasFeature(.avx))
12314		return .{ .move = if (aligned)
12315		.{ .v_, .movdqa }
12316		else
12317		.{ .v_, .movdqu } },
12318		else => {},
12319		},
12320		32 => switch (ty.vectorLen(mod)) {
12321		1 => return .{ .move = if (self.hasFeature(.avx))
12322		.{ .v_d, .mov }
	12900	.{ ._q, .mov } },
	12901	3...4 => return .{ .move = if (self.hasFeature(.avx))
	12902	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12903	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12904	5...8 => if (self.hasFeature(.avx))
	12905	return .{ .move = if (aligned)
	12906	.{ .v_, .movdqa }
12323	12907	else
12324		.{ ._d, .mov } },
12325		2 => return .{ .move = if (self.hasFeature(.avx))
12326		.{ .v_q, .mov }
	12908	.{ .v_, .movdqu } },
	12909	else => {},
	12910	},
	12911	64 => switch (ty.vectorLen(mod)) {
	12912	1 => return .{ .move = if (self.hasFeature(.avx))
	12913	.{ .v_q, .mov }
	12914	else
	12915	.{ ._q, .mov } },
	12916	2 => return .{ .move = if (self.hasFeature(.avx))
	12917	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12918	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12919	3...4 => if (self.hasFeature(.avx))
	12920	return .{ .move = if (aligned)
	12921	.{ .v_, .movdqa }
12327	12922	else
12328		.{ ._q, .mov } },
12329		3...4 => return .{ .move = if (self.hasFeature(.avx))
12330		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12331		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12332		5...8 => if (self.hasFeature(.avx))
12333		return .{ .move = if (aligned)
12334		.{ .v_, .movdqa }
12335		else
12336		.{ .v_, .movdqu } },
12337		else => {},
12338		},
12339		64 => switch (ty.vectorLen(mod)) {
12340		1 => return .{ .move = if (self.hasFeature(.avx))
12341		.{ .v_q, .mov }
	12923	.{ .v_, .movdqu } },
	12924	else => {},
	12925	},
	12926	128 => switch (ty.vectorLen(mod)) {
	12927	1 => return .{ .move = if (self.hasFeature(.avx))
	12928	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12929	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12930	2 => if (self.hasFeature(.avx))
	12931	return .{ .move = if (aligned)
	12932	.{ .v_, .movdqa }
12342	12933	else
12343		.{ ._q, .mov } },
12344		2 => return .{ .move = if (self.hasFeature(.avx))
12345		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12346		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12347		3...4 => if (self.hasFeature(.avx))
12348		return .{ .move = if (aligned)
12349		.{ .v_, .movdqa }
12350		else
12351		.{ .v_, .movdqu } },
12352		else => {},
12353		},
12354		128 => switch (ty.vectorLen(mod)) {
12355		1 => return .{ .move = if (self.hasFeature(.avx))
12356		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12357		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12358		2 => if (self.hasFeature(.avx))
12359		return .{ .move = if (aligned)
12360		.{ .v_, .movdqa }
12361		else
12362		.{ .v_, .movdqu } },
12363		else => {},
12364		},
12365		256 => switch (ty.vectorLen(mod)) {
12366		1 => if (self.hasFeature(.avx))
12367		return .{ .move = if (aligned)
12368		.{ .v_, .movdqa }
12369		else
12370		.{ .v_, .movdqu } },
12371		else => {},
12372		},
	12934	.{ .v_, .movdqu } },
12373	12935	else => {},
12374	12936	},
12375		.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
12376		16 => switch (ty.vectorLen(mod)) {
12377		1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12378		.insert = .{ .vp_w, .insr },
12379		.extract = .{ .vp_w, .extr },
12380		} } else .{ .insert_extract = .{
12381		.insert = .{ .p_w, .insr },
12382		.extract = .{ .p_w, .extr },
12383		} },
12384		2 => return .{ .move = if (self.hasFeature(.avx))
12385		.{ .v_d, .mov }
	12937	256 => switch (ty.vectorLen(mod)) {
	12938	1 => if (self.hasFeature(.avx))
	12939	return .{ .move = if (aligned)
	12940	.{ .v_, .movdqa }
12386	12941	else
12387		.{ ._d, .mov } },
12388		3...4 => return .{ .move = if (self.hasFeature(.avx))
12389		.{ .v_q, .mov }
	12942	.{ .v_, .movdqu } },
	12943	else => {},
	12944	},
	12945	else => {},
	12946	},
	12947	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
	12948	16 => switch (ty.vectorLen(mod)) {
	12949	1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	12950	.insert = .{ .vp_w, .insr },
	12951	.extract = .{ .vp_w, .extr },
	12952	} } else .{ .insert_extract = .{
	12953	.insert = .{ .p_w, .insr },
	12954	.extract = .{ .p_w, .extr },
	12955	} },
	12956	2 => return .{ .move = if (self.hasFeature(.avx))
	12957	.{ .v_d, .mov }
	12958	else
	12959	.{ ._d, .mov } },
	12960	3...4 => return .{ .move = if (self.hasFeature(.avx))
	12961	.{ .v_q, .mov }
	12962	else
	12963	.{ ._q, .mov } },
	12964	5...8 => return .{ .move = if (self.hasFeature(.avx))
	12965	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12966	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12967	9...16 => if (self.hasFeature(.avx))
	12968	return .{ .move = if (aligned)
	12969	.{ .v_, .movdqa }
12390	12970	else
12391		.{ ._q, .mov } },
12392		5...8 => return .{ .move = if (self.hasFeature(.avx))
12393		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12394		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12395		9...16 => if (self.hasFeature(.avx))
12396		return .{ .move = if (aligned)
12397		.{ .v_, .movdqa }
12398		else
12399		.{ .v_, .movdqu } },
12400		else => {},
12401		},
12402		32 => switch (ty.vectorLen(mod)) {
12403		1 => return .{ .move = if (self.hasFeature(.avx))
12404		.{ .v_ss, .mov }
	12971	.{ .v_, .movdqu } },
	12972	else => {},
	12973	},
	12974	32 => switch (ty.vectorLen(mod)) {
	12975	1 => return .{ .move = if (self.hasFeature(.avx))
	12976	.{ .v_ss, .mov }
	12977	else
	12978	.{ ._ss, .mov } },
	12979	2 => return .{ .move = if (self.hasFeature(.avx))
	12980	.{ .v_sd, .mov }
	12981	else
	12982	.{ ._sd, .mov } },
	12983	3...4 => return .{ .move = if (self.hasFeature(.avx))
	12984	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
	12985	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
	12986	5...8 => if (self.hasFeature(.avx))
	12987	return .{ .move = if (aligned)
	12988	.{ .v_ps, .mova }
12405	12989	else
12406		.{ ._ss, .mov } },
12407		2 => return .{ .move = if (self.hasFeature(.avx))
12408		.{ .v_sd, .mov }
	12990	.{ .v_ps, .movu } },
	12991	else => {},
	12992	},
	12993	64 => switch (ty.vectorLen(mod)) {
	12994	1 => return .{ .move = if (self.hasFeature(.avx))
	12995	.{ .v_sd, .mov }
	12996	else
	12997	.{ ._sd, .mov } },
	12998	2 => return .{ .move = if (self.hasFeature(.avx))
	12999	if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
	13000	else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
	13001	3...4 => if (self.hasFeature(.avx))
	13002	return .{ .move = if (aligned)
	13003	.{ .v_pd, .mova }
12409	13004	else
12410		.{ ._sd, .mov } },
12411		3...4 => return .{ .move = if (self.hasFeature(.avx))
12412		if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
12413		else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
12414		5...8 => if (self.hasFeature(.avx))
12415		return .{ .move = if (aligned)
12416		.{ .v_ps, .mova }
12417		else
12418		.{ .v_ps, .movu } },
12419		else => {},
12420		},
12421		64 => switch (ty.vectorLen(mod)) {
12422		1 => return .{ .move = if (self.hasFeature(.avx))
12423		.{ .v_sd, .mov }
	13005	.{ .v_pd, .movu } },
	13006	else => {},
	13007	},
	13008	128 => switch (ty.vectorLen(mod)) {
	13009	1 => return .{ .move = if (self.hasFeature(.avx))
	13010	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	13011	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	13012	2 => if (self.hasFeature(.avx))
	13013	return .{ .move = if (aligned)
	13014	.{ .v_, .movdqa }
12424	13015	else
12425		.{ ._sd, .mov } },
12426		2 => return .{ .move = if (self.hasFeature(.avx))
12427		if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
12428		else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
12429		3...4 => if (self.hasFeature(.avx))
12430		return .{ .move = if (aligned)
12431		.{ .v_pd, .mova }
12432		else
12433		.{ .v_pd, .movu } },
12434		else => {},
12435		},
12436		128 => switch (ty.vectorLen(mod)) {
12437		1 => return .{ .move = if (self.hasFeature(.avx))
12438		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12439		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12440		2 => if (self.hasFeature(.avx))
12441		return .{ .move = if (aligned)
12442		.{ .v_, .movdqa }
12443		else
12444		.{ .v_, .movdqu } },
12445		else => {},
12446		},
	13016	.{ .v_, .movdqu } },
12447	13017	else => {},
12448	13018	},
12449	13019	else => {},
12450	13020	},
12451		}
	13021	else => {},
	13022	},
12452	13023	},
12453	13024	}
12454	13025	return self.fail("TODO moveStrategy for {}", .{ty.fmt(mod)});
...	...	@@ -12514,32 +13085,18 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
12514	13085	};
12515	13086	defer if (src_info) \|info\| self.register_manager.unlockReg(info.addr_lock);
12516	13087
12517		const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
12518		for (dst_regs, classes, 0..) \|dst_reg, class, dst_reg_i\| {
12519		const class_ty = switch (class) {
12520		.integer => Type.usize,
12521		.sse, .float, .float_combine => Type.f64,
12522		else => unreachable,
12523		};
12524		const off: i32 = @intCast(dst_reg_i * 8);
12525		switch (src_mcv) {
12526		.register_pair => \|src_regs\| try self.genSetReg(
12527		dst_reg,
12528		class_ty,
12529		.{ .register = src_regs[dst_reg_i] },
12530		),
12531		.memory, .indirect, .load_frame => try self.genSetReg(
12532		dst_reg,
12533		class_ty,
12534		src_mcv.address().offset(off).deref(),
12535		),
12536		.load_symbol, .load_direct, .load_got, .load_tlv => try self.genSetReg(
12537		dst_reg,
12538		class_ty,
12539		.{ .indirect = .{ .reg = src_info.?.addr_reg, .off = off } },
12540		),
	13088	var part_disp: i32 = 0;
	13089	for (dst_regs, try self.splitType(ty), 0..) \|dst_reg, dst_ty, part_i\| {
	13090	try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) {
	13091	.register_pair => \|src_regs\| .{ .register = src_regs[part_i] },
	13092	.memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
	13093	.load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{
	13094	.reg = src_info.?.addr_reg,
	13095	.off = part_disp,
	13096	} },
12541	13097	else => unreachable,
12542		}
	13098	});
	13099	part_disp += @intCast(dst_ty.abiSize(mod));
12543	13100	}
12544	13101	},
12545	13102	.indirect => \|reg_off\| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),
...	...	@@ -12584,6 +13141,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12584	13141	if (imm == 0) {
12585	13142	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
12586	13143	// register is the fastest way to zero a register.
	13144	try self.spillEflagsIfOccupied();
12587	13145	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
12588	13146	} else if (abi_size > 4 and math.cast(u32, imm) != null) {
12589	13147	// 32-bit moves zero-extend to 64-bit.
...	...	@@ -12933,44 +13491,65 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12933	13491	.eflags => \|cc\| try self.asmSetccMemory(cc, .{ .base = base, .mod = .{
12934	13492	.rm = .{ .size = .byte, .disp = disp },
12935	13493	} }),
12936		.register => \|src_reg\| try (try self.moveStrategy(ty, src_reg.class(), switch (base) {
12937		.none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12938		.reg => \|reg\| switch (reg) {
12939		.es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12940		else => false,
12941		},
12942		.frame => \|frame_index\| self.getFrameAddrAlignment(
12943		.{ .index = frame_index, .off = disp },
12944		).compare(.gte, ty.abiAlignment(mod)),
12945		.reloc => false,
12946		})).write(
12947		self,
12948		.{ .base = base, .mod = .{ .rm = .{
12949		.size = self.memSize(ty),
12950		.disp = disp,
12951		} } },
12952		registerAlias(src_reg, abi_size),
12953		),
12954		.register_pair => \|src_regs\| for (src_regs, 0..) \|src_reg, src_reg_i\| {
12955		const part_size: u16 = @min(abi_size - src_reg_i * 8, 8);
12956		try (try self.moveStrategy(
12957		try mod.intType(.unsigned, part_size * 8),
12958		src_reg.class(),
12959		switch (base) {
12960		.none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12961		.reg => \|reg\| switch (reg) {
12962		.es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12963		else => false,
12964		},
12965		.frame => \|frame_index\| self.getFrameAddrAlignment(
12966		.{ .index = frame_index, .off = disp },
12967		).compare(.gte, ty.abiAlignment(mod)),
12968		.reloc => false,
	13494	.register => \|src_reg\| {
	13495	const mem_size = switch (base) {
	13496	.frame => \|base_fi\| mem_size: {
	13497	assert(disp >= 0);
	13498	const frame_abi_size = self.frame_allocs.items(.abi_size)[@intFromEnum(base_fi)];
	13499	const frame_spill_pad = self.frame_allocs.items(.spill_pad)[@intFromEnum(base_fi)];
	13500	assert(frame_abi_size - frame_spill_pad - disp >= abi_size);
	13501	break :mem_size if (frame_abi_size - frame_spill_pad - disp == abi_size)
	13502	frame_abi_size
	13503	else
	13504	abi_size;
12969	13505	},
12970		)).write(self, .{ .base = base, .mod = .{ .rm = .{
12971		.size = Memory.Size.fromSize(part_size),
12972		.disp = disp + @as(i32, @intCast(src_reg_i * 8)),
12973		} } }, registerAlias(src_reg, part_size));
	13506	else => abi_size,
	13507	};
	13508	const src_alias = registerAlias(src_reg, abi_size);
	13509	const src_size: u32 = @intCast(switch (src_alias.class()) {
	13510	.general_purpose, .segment, .x87 => @divExact(src_alias.bitSize(), 8),
	13511	.mmx, .sse => abi_size,
	13512	});
	13513	if (src_size > mem_size) {
	13514	const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
	13515	.size = src_size,
	13516	.alignment = Alignment.fromNonzeroByteUnits(src_size),
	13517	}));
	13518	const frame_mcv: MCValue = .{ .load_frame = .{ .index = frame_index } };
	13519	try (try self.moveStrategy(ty, src_alias.class(), true)).write(
	13520	self,
	13521	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{
	13522	.size = Memory.Size.fromSize(src_size),
	13523	} } },
	13524	src_alias,
	13525	);
	13526	try self.genSetMem(base, disp, ty, frame_mcv);
	13527	try self.freeValue(frame_mcv);
	13528	} else try (try self.moveStrategy(ty, src_alias.class(), switch (base) {
	13529	.none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
	13530	.reg => \|reg\| switch (reg) {
	13531	.es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
	13532	else => false,
	13533	},
	13534	.frame => \|frame_index\| self.getFrameAddrAlignment(
	13535	.{ .index = frame_index, .off = disp },
	13536	).compare(.gte, ty.abiAlignment(mod)),
	13537	.reloc => false,
	13538	})).write(
	13539	self,
	13540	.{ .base = base, .mod = .{ .rm = .{
	13541	.size = self.memSize(ty),
	13542	.disp = disp,
	13543	} } },
	13544	src_alias,
	13545	);
	13546	},
	13547	.register_pair => \|src_regs\| {
	13548	var part_disp: i32 = disp;
	13549	for (try self.splitType(ty), src_regs) \|src_ty, src_reg\| {
	13550	try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg });
	13551	part_disp += @intCast(src_ty.abiSize(mod));
	13552	}
12974	13553	},
12975	13554	.register_overflow => \|ro\| switch (ty.zigTypeTag(mod)) {
12976	13555	.Struct => {
...	...	@@ -13226,50 +13805,43 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
13226	13805	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
13227	13806	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
13228	13807
13229		const dst_mcv = if (dst_rc.supersetOf(src_rc) and
13230		self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
13231		src_mcv
13232		else dst: {
	13808	const dst_mcv = if (dst_rc.supersetOf(src_rc) and dst_ty.abiSize(mod) <= src_ty.abiSize(mod) and
	13809	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
13233	13810	const dst_mcv = try self.allocRegOrMem(inst, true);
13234		try self.genCopy(
13235		if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
13236		dst_mcv,
13237		src_mcv,
13238		);
	13811	try self.genCopy(switch (math.order(dst_ty.abiSize(mod), src_ty.abiSize(mod))) {
	13812	.lt => dst_ty,
	13813	.eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
	13814	.gt => src_ty,
	13815	}, dst_mcv, src_mcv);
13239	13816	break :dst dst_mcv;
13240	13817	};
13241	13818
13242	13819	if (dst_ty.isRuntimeFloat()) break :result dst_mcv;
13243	13820
13244		const dst_signedness =
13245		if (dst_ty.isAbiInt(mod)) dst_ty.intInfo(mod).signedness else .unsigned;
13246		if (!src_ty.isRuntimeFloat() or src_ty.floatBits(self.target.*) != 80) {
13247		const src_signedness =
13248		if (src_ty.isAbiInt(mod)) src_ty.intInfo(mod).signedness else .unsigned;
13249		if (dst_signedness == src_signedness) break :result dst_mcv;
13250		}
	13821	if (dst_ty.isAbiInt(mod) and src_ty.isAbiInt(mod) and
	13822	dst_ty.intInfo(mod).signedness == src_ty.intInfo(mod).signedness) break :result dst_mcv;
13251	13823
13252		const abi_size: u16 = @intCast(dst_ty.abiSize(mod));
13253		const bit_size: u16 = @intCast(dst_ty.bitSize(mod));
13254		if (abi_size * 8 <= bit_size) break :result dst_mcv;
	13824	const abi_size = dst_ty.abiSize(mod);
	13825	const bit_size = dst_ty.bitSize(mod);
	13826	if (abi_size * 8 <= bit_size or dst_ty.isVector(mod)) break :result dst_mcv;
13255	13827
13256		const dst_limbs_len = math.divCeil(i32, bit_size, 64) catch unreachable;
13257		const high_reg = if (dst_mcv.isRegister())
13258		dst_mcv.getReg().?
	13828	const dst_limbs_len = math.divCeil(i32, @intCast(bit_size), 64) catch unreachable;
	13829	const high_mcv: MCValue = switch (dst_mcv) {
	13830	.register => \|dst_reg\| .{ .register = dst_reg },
	13831	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
	13832	else => dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
	13833	};
	13834	const high_reg = if (high_mcv.isRegister())
	13835	high_mcv.getReg().?
13259	13836	else
13260		try self.copyToTmpRegister(
13261		Type.usize,
13262		dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
13263		);
	13837	try self.copyToTmpRegister(Type.usize, high_mcv);
13264	13838	const high_lock = self.register_manager.lockReg(high_reg);
13265	13839	defer if (high_lock) \|lock\| self.register_manager.unlockReg(lock);
13266	13840
13267		const high_ty = try mod.intType(dst_signedness, bit_size % 64);
13268
13269		try self.truncateRegister(high_ty, high_reg);
13270		if (!dst_mcv.isRegister()) try self.genCopy(
13271		Type.usize,
13272		dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
	13841	try self.truncateRegister(dst_ty, high_reg);
	13842	if (!high_mcv.isRegister()) try self.genCopy(
	13843	if (abi_size <= 8) dst_ty else Type.usize,
	13844	high_mcv,
13273	13845	.{ .register = high_reg },
13274	13846	);
13275	13847	break :result dst_mcv;
...	...	@@ -13287,7 +13859,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
13287	13859	const array_ty = ptr_ty.childType(mod);
13288	13860	const array_len = array_ty.arrayLen(mod);
13289	13861
13290		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, mod));
	13862	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod));
13291	13863	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
13292	13864	try self.genSetMem(
13293	13865	.{ .frame = frame_index },
...	...	@@ -13497,7 +14069,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
13497	14069	const ptr_mcv = try self.resolveInst(extra.ptr);
13498	14070	const mem_size = Memory.Size.fromSize(val_abi_size);
13499	14071	const ptr_mem: Memory = switch (ptr_mcv) {
13500		.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),
	14072	.immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
13501	14073	else => .{
13502	14074	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13503	14075	.mod = .{ .rm = .{ .size = mem_size } },
...	...	@@ -13563,7 +14135,7 @@ fn atomicOp(
13563	14135	const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
13564	14136	const mem_size = Memory.Size.fromSize(val_abi_size);
13565	14137	const ptr_mem: Memory = switch (ptr_mcv) {
13566		.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),
	14138	.immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
13567	14139	else => .{
13568	14140	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13569	14141	.mod = .{ .rm = .{ .size = mem_size } },
...	...	@@ -13671,27 +14243,41 @@ fn atomicOp(
13671	14243	},
13672	14244	};
13673	14245
13674		try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
13675	14246	const cmov_abi_size = @max(val_abi_size, 2);
13676	14247	switch (val_mcv) {
13677		.register => \|val_reg\| try self.asmCmovccRegisterRegister(
13678		cc,
13679		registerAlias(tmp_reg, cmov_abi_size),
13680		registerAlias(val_reg, cmov_abi_size),
13681		),
13682		.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
13683		cc,
13684		registerAlias(tmp_reg, cmov_abi_size),
13685		val_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),
13686		),
13687		else => {
13688		const val_reg = try self.copyToTmpRegister(val_ty, val_mcv);
	14248	.register => \|val_reg\| {
	14249	try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
13689	14250	try self.asmCmovccRegisterRegister(
13690	14251	cc,
13691	14252	registerAlias(tmp_reg, cmov_abi_size),
13692	14253	registerAlias(val_reg, cmov_abi_size),
13693	14254	);
13694	14255	},
	14256	.memory, .indirect, .load_frame => {
	14257	try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
	14258	try self.asmCmovccRegisterMemory(
	14259	cc,
	14260	registerAlias(tmp_reg, cmov_abi_size),
	14261	try val_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
	14262	);
	14263	},
	14264	else => {
	14265	const mat_reg = try self.copyToTmpRegister(val_ty, val_mcv);
	14266	const mat_lock = self.register_manager.lockRegAssumeUnused(mat_reg);
	14267	defer self.register_manager.unlockReg(mat_lock);
	14268
	14269	try self.genBinOpMir(
	14270	.{ ._, .cmp },
	14271	val_ty,
	14272	tmp_mcv,
	14273	.{ .register = mat_reg },
	14274	);
	14275	try self.asmCmovccRegisterRegister(
	14276	cc,
	14277	registerAlias(tmp_reg, cmov_abi_size),
	14278	registerAlias(mat_reg, cmov_abi_size),
	14279	);
	14280	},
13695	14281	}
13696	14282	},
13697	14283	};
...	...	@@ -13728,8 +14314,8 @@ fn atomicOp(
13728	14314	.reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),
13729	14315	} },
13730	14316	};
13731		const val_lo_mem = val_mem_mcv.mem(.qword);
13732		const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);
	14317	const val_lo_mem = try val_mem_mcv.mem(self, .qword);
	14318	const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .qword);
13733	14319	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
13734	14320	try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
13735	14321	try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
...	...	@@ -13859,6 +14445,10 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
13859	14445
13860	14446	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
13861	14447
	14448	try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
	14449	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
	14450	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
	14451
13862	14452	const dst_ptr = try self.resolveInst(bin_op.lhs);
13863	14453	const dst_ptr_ty = self.typeOf(bin_op.lhs);
13864	14454	const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
...	...	@@ -13976,6 +14566,10 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
13976	14566	const mod = self.bin_file.options.module.?;
13977	14567	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
13978	14568
	14569	try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
	14570	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
	14571	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
	14572
13979	14573	const dst_ptr = try self.resolveInst(bin_op.lhs);
13980	14574	const dst_ptr_ty = self.typeOf(bin_op.lhs);
13981	14575	const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
...	...	@@ -14000,7 +14594,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
14000	14594	try self.asmRegisterMemoryImmediate(
14001	14595	.{ .i_, .mul },
14002	14596	len_reg,
14003		dst_ptr.address().offset(8).deref().mem(.qword),
	14597	try dst_ptr.address().offset(8).deref().mem(self, .qword),
14004	14598	Immediate.s(@intCast(dst_ptr_ty.childType(mod).abiSize(mod))),
14005	14599	);
14006	14600	break :len .{ .register = len_reg };
...	...	@@ -14046,7 +14640,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
14046	14640	}
14047	14641
14048	14642	try self.spillEflagsIfOccupied();
14049		try self.spillRegisters(abi.getCallerPreservedRegs(resolved_cc));
	14643	try self.spillCallerPreservedRegs(resolved_cc);
14050	14644
14051	14645	const param_regs = abi.getCAbiIntParamRegs(resolved_cc);
14052	14646
...	...	@@ -14171,28 +14765,162 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14171	14765	const mod = self.bin_file.options.module.?;
14172	14766	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
14173	14767	const vector_ty = self.typeOfIndex(inst);
	14768	const vector_len = vector_ty.vectorLen(mod);
14174	14769	const dst_rc = self.regClassForType(vector_ty);
14175		const scalar_ty = vector_ty.scalarType(mod);
	14770	const scalar_ty = self.typeOf(ty_op.operand);
14176	14771
14177		const src_mcv = try self.resolveInst(ty_op.operand);
14178	14772	const result: MCValue = result: {
14179	14773	switch (scalar_ty.zigTypeTag(mod)) {
14180	14774	else => {},
	14775	.Bool => {
	14776	const regs =
	14777	try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp);
	14778	const reg_locks = self.register_manager.lockRegsAssumeUnused(2, regs);
	14779	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
	14780
	14781	try self.genSetReg(regs[1], vector_ty, .{ .immediate = 0 });
	14782	try self.genSetReg(
	14783	regs[1],
	14784	vector_ty,
	14785	.{ .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - vector_len) },
	14786	);
	14787	const src_mcv = try self.resolveInst(ty_op.operand);
	14788	const abi_size = @max(math.divCeil(u32, vector_len, 8) catch unreachable, 4);
	14789	try self.asmCmovccRegisterRegister(
	14790	switch (src_mcv) {
	14791	.eflags => \|cc\| cc,
	14792	.register => \|src_reg\| cc: {
	14793	try self.asmRegisterImmediate(
	14794	.{ ._, .@"test" },
	14795	src_reg.to8(),
	14796	Immediate.u(1),
	14797	);
	14798	break :cc .nz;
	14799	},
	14800	else => cc: {
	14801	try self.asmMemoryImmediate(
	14802	.{ ._, .@"test" },
	14803	try src_mcv.mem(self, .byte),
	14804	Immediate.u(1),
	14805	);
	14806	break :cc .nz;
	14807	},
	14808	},
	14809	registerAlias(regs[0], abi_size),
	14810	registerAlias(regs[1], abi_size),
	14811	);
	14812	break :result .{ .register = regs[0] };
	14813	},
	14814	.Int => if (self.hasFeature(.avx2)) avx2: {
	14815	const mir_tag = @as(?Mir.Inst.FixedTag, switch (scalar_ty.intInfo(mod).bits) {
	14816	else => null,
	14817	1...8 => switch (vector_len) {
	14818	else => null,
	14819	1...32 => .{ .vp_b, .broadcast },
	14820	},
	14821	9...16 => switch (vector_len) {
	14822	else => null,
	14823	1...16 => .{ .vp_w, .broadcast },
	14824	},
	14825	17...32 => switch (vector_len) {
	14826	else => null,
	14827	1...8 => .{ .vp_d, .broadcast },
	14828	},
	14829	33...64 => switch (vector_len) {
	14830	else => null,
	14831	1...4 => .{ .vp_q, .broadcast },
	14832	},
	14833	65...128 => switch (vector_len) {
	14834	else => null,
	14835	1...2 => .{ .vp_i128, .broadcast },
	14836	},
	14837	}) orelse break :avx2;
	14838
	14839	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
	14840	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
	14841	defer self.register_manager.unlockReg(dst_lock);
	14842
	14843	const src_mcv = try self.resolveInst(ty_op.operand);
	14844	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	14845	mir_tag,
	14846	registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
	14847	try src_mcv.mem(self, self.memSize(scalar_ty)),
	14848	) else {
	14849	if (mir_tag[0] == .vp_i128) break :avx2;
	14850	try self.genSetReg(dst_reg, scalar_ty, src_mcv);
	14851	try self.asmRegisterRegister(
	14852	mir_tag,
	14853	registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
	14854	registerAlias(dst_reg, @intCast(scalar_ty.abiSize(mod))),
	14855	);
	14856	}
	14857	break :result .{ .register = dst_reg };
	14858	} else {
	14859	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
	14860	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
	14861	defer self.register_manager.unlockReg(dst_lock);
	14862
	14863	try self.genSetReg(dst_reg, scalar_ty, .{ .air_ref = ty_op.operand });
	14864	if (vector_len == 1) break :result .{ .register = dst_reg };
	14865
	14866	const dst_alias = registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod)));
	14867	const scalar_bits = scalar_ty.intInfo(mod).bits;
	14868	if (switch (scalar_bits) {
	14869	1...8 => true,
	14870	9...128 => false,
	14871	else => unreachable,
	14872	}) if (self.hasFeature(.avx)) try self.asmRegisterRegisterRegister(
	14873	.{ .vp_, .unpcklbw },
	14874	dst_alias,
	14875	dst_alias,
	14876	dst_alias,
	14877	) else try self.asmRegisterRegister(
	14878	.{ .p_, .unpcklbw },
	14879	dst_alias,
	14880	dst_alias,
	14881	);
	14882	if (switch (scalar_bits) {
	14883	1...8 => vector_len > 2,
	14884	9...16 => true,
	14885	17...128 => false,
	14886	else => unreachable,
	14887	}) try self.asmRegisterRegisterImmediate(
	14888	.{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
	14889	dst_alias,
	14890	dst_alias,
	14891	Immediate.u(0),
	14892	);
	14893	if (switch (scalar_bits) {
	14894	1...8 => vector_len > 4,
	14895	9...16 => vector_len > 2,
	14896	17...64 => true,
	14897	65...128 => false,
	14898	else => unreachable,
	14899	}) try self.asmRegisterRegisterImmediate(
	14900	.{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf },
	14901	dst_alias,
	14902	dst_alias,
	14903	Immediate.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),
	14904	);
	14905	break :result .{ .register = dst_reg };
	14906	},
14181	14907	.Float => switch (scalar_ty.floatBits(self.target.*)) {
14182		32 => switch (vector_ty.vectorLen(mod)) {
	14908	32 => switch (vector_len) {
14183	14909	1 => {
	14910	const src_mcv = try self.resolveInst(ty_op.operand);
14184	14911	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14185	14912	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14186	14913	try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14187	14914	break :result .{ .register = dst_reg };
14188	14915	},
14189	14916	2...4 => {
	14917	const src_mcv = try self.resolveInst(ty_op.operand);
14190	14918	if (self.hasFeature(.avx)) {
14191	14919	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14192	14920	if (src_mcv.isMemory()) try self.asmRegisterMemory(
14193	14921	.{ .v_ss, .broadcast },
14194	14922	dst_reg.to128(),
14195		src_mcv.mem(.dword),
	14923	try src_mcv.mem(self, .dword),
14196	14924	) else {
14197	14925	const src_reg = if (src_mcv.isRegister())
14198	14926	src_mcv.getReg().?
...	...	@@ -14224,11 +14952,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14224	14952	}
14225	14953	},
14226	14954	5...8 => if (self.hasFeature(.avx)) {
	14955	const src_mcv = try self.resolveInst(ty_op.operand);
14227	14956	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14228	14957	if (src_mcv.isMemory()) try self.asmRegisterMemory(
14229	14958	.{ .v_ss, .broadcast },
14230	14959	dst_reg.to256(),
14231		src_mcv.mem(.dword),
	14960	try src_mcv.mem(self, .dword),
14232	14961	) else {
14233	14962	const src_reg = if (src_mcv.isRegister())
14234	14963	src_mcv.getReg().?
...	...	@@ -14259,20 +14988,22 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14259	14988	},
14260	14989	else => {},
14261	14990	},
14262		64 => switch (vector_ty.vectorLen(mod)) {
	14991	64 => switch (vector_len) {
14263	14992	1 => {
	14993	const src_mcv = try self.resolveInst(ty_op.operand);
14264	14994	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14265	14995	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14266	14996	try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14267	14997	break :result .{ .register = dst_reg };
14268	14998	},
14269	14999	2 => {
	15000	const src_mcv = try self.resolveInst(ty_op.operand);
14270	15001	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14271	15002	if (self.hasFeature(.sse3)) {
14272	15003	if (src_mcv.isMemory()) try self.asmRegisterMemory(
14273	15004	if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
14274	15005	dst_reg.to128(),
14275		src_mcv.mem(.qword),
	15006	try src_mcv.mem(self, .qword),
14276	15007	) else try self.asmRegisterRegister(
14277	15008	if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
14278	15009	dst_reg.to128(),
...	...	@@ -14292,11 +15023,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14292	15023	);
14293	15024	},
14294	15025	3...4 => if (self.hasFeature(.avx)) {
	15026	const src_mcv = try self.resolveInst(ty_op.operand);
14295	15027	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14296	15028	if (src_mcv.isMemory()) try self.asmRegisterMemory(
14297	15029	.{ .v_sd, .broadcast },
14298	15030	dst_reg.to256(),
14299		src_mcv.mem(.qword),
	15031	try src_mcv.mem(self, .qword),
14300	15032	) else {
14301	15033	const src_reg = if (src_mcv.isRegister())
14302	15034	src_mcv.getReg().?
...	...	@@ -14325,19 +15057,21 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14325	15057	},
14326	15058	else => {},
14327	15059	},
14328		128 => switch (vector_ty.vectorLen(mod)) {
	15060	128 => switch (vector_len) {
14329	15061	1 => {
	15062	const src_mcv = try self.resolveInst(ty_op.operand);
14330	15063	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14331	15064	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14332	15065	try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14333	15066	break :result .{ .register = dst_reg };
14334	15067	},
14335	15068	2 => if (self.hasFeature(.avx)) {
	15069	const src_mcv = try self.resolveInst(ty_op.operand);
14336	15070	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14337	15071	if (src_mcv.isMemory()) try self.asmRegisterMemory(
14338	15072	.{ .v_f128, .broadcast },
14339	15073	dst_reg.to256(),
14340		src_mcv.mem(.xword),
	15074	try src_mcv.mem(self, .xword),
14341	15075	) else {
14342	15076	const src_reg = if (src_mcv.isRegister())
14343	15077	src_mcv.getReg().?
...	...	@@ -14389,7 +15123,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
14389	15123	try self.spillEflagsIfOccupied();
14390	15124
14391	15125	const operand_mcv = try self.resolveInst(reduce.operand);
14392		const mask_len = (std.math.cast(u6, operand_ty.vectorLen(mod)) orelse
	15126	const mask_len = (math.cast(u6, operand_ty.vectorLen(mod)) orelse
14393	15127	return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(mod)}));
14394	15128	const mask = (@as(u64, 1) << mask_len) - 1;
14395	15129	const abi_size: u32 = @intCast(operand_ty.abiSize(mod));
...	...	@@ -14397,7 +15131,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
14397	15131	.Or => {
14398	15132	if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
14399	15133	.{ ._, .@"test" },
14400		operand_mcv.mem(Memory.Size.fromSize(abi_size)),
	15134	try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)),
14401	15135	Immediate.u(mask),
14402	15136	) else {
14403	15137	const operand_reg = registerAlias(if (operand_mcv.isRegister())
...	...	@@ -14445,8 +15179,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
14445	15179	const result: MCValue = result: {
14446	15180	switch (result_ty.zigTypeTag(mod)) {
14447	15181	.Struct => {
14448		const frame_index =
14449		try self.allocFrameIndex(FrameAlloc.initType(result_ty, mod));
	15182	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, mod));
14450	15183	if (result_ty.containerLayout(mod) == .Packed) {
14451	15184	const struct_type = mod.typeToStruct(result_ty).?;
14452	15185	try self.genInlineMemset(
...	...	@@ -14542,8 +15275,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
14542	15275	break :result .{ .load_frame = .{ .index = frame_index } };
14543	15276	},
14544	15277	.Array => {
14545		const frame_index =
14546		try self.allocFrameIndex(FrameAlloc.initType(result_ty, mod));
	15278	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, mod));
14547	15279	const elem_ty = result_ty.childType(mod);
14548	15280	const elem_size: u32 = @intCast(elem_ty.abiSize(mod));
14549	15281
...	...	@@ -14789,7 +15521,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
14789	15521	mir_tag,
14790	15522	mop1_reg,
14791	15523	mop2_reg,
14792		mops[2].mem(Memory.Size.fromSize(abi_size)),
	15524	try mops[2].mem(self, Memory.Size.fromSize(abi_size)),
14793	15525	);
14794	15526	break :result mops[0];
14795	15527	};
...	...	@@ -14807,7 +15539,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
14807	15539	)) {
14808	15540	.SysV => result: {
14809	15541	const info = self.va_info.sysv;
14810		const dst_fi = try self.allocFrameIndex(FrameAlloc.initType(va_list_ty, mod));
	15542	const dst_fi = try self.allocFrameIndex(FrameAlloc.initSpill(va_list_ty, mod));
14811	15543	var field_off: u31 = 0;
14812	15544	// gp_offset: c_uint,
14813	15545	try self.genSetMem(
...	...	@@ -15015,7 +15747,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
15015	15747	.{ .v_ss, .cvtsd2 },
15016	15748	dst_reg,
15017	15749	dst_reg,
15018		promote_mcv.mem(.qword),
	15750	try promote_mcv.mem(self, .qword),
15019	15751	) else try self.asmRegisterRegisterRegister(
15020	15752	.{ .v_ss, .cvtsd2 },
15021	15753	dst_reg,
...	...	@@ -15027,7 +15759,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
15027	15759	) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(
15028	15760	.{ ._ss, .cvtsd2 },
15029	15761	dst_reg,
15030		promote_mcv.mem(.qword),
	15762	try promote_mcv.mem(self, .qword),
15031	15763	) else try self.asmRegisterRegister(
15032	15764	.{ ._ss, .cvtsd2 },
15033	15765	dst_reg,
...	...	@@ -15473,6 +16205,33 @@ fn memSize(self: *Self, ty: Type) Memory.Size {
15473	16205	};
15474	16206	}
15475	16207
	16208	fn splitType(self: *Self, ty: Type) ![2]Type {
	16209	const mod = self.bin_file.options.module.?;
	16210	const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
	16211	var parts: [2]Type = undefined;
	16212	if (classes.len == 2) for (&parts, classes, 0..) \|*part, class, part_i\| {
	16213	part.* = switch (class) {
	16214	.integer => switch (part_i) {
	16215	0 => Type.u64,
	16216	1 => part: {
	16217	const elem_size = ty.abiAlignment(mod).minStrict(.@"8").toByteUnitsOptional().?;
	16218	const elem_ty = try mod.intType(.unsigned, @intCast(elem_size * 8));
	16219	break :part switch (@divExact(ty.abiSize(mod) - 8, elem_size)) {
	16220	1 => elem_ty,
	16221	else => \|len\| try mod.arrayType(.{ .len = len, .child = elem_ty.toIntern() }),
	16222	};
	16223	},
	16224	else => unreachable,
	16225	},
	16226	.float => Type.f32,
	16227	.float_combine => try mod.vectorType(.{ .len = 2, .child = .f32_type }),
	16228	.sse => Type.f64,
	16229	else => break,
	16230	};
	16231	} else if (parts[0].abiSize(mod) + parts[1].abiSize(mod) == ty.abiSize(mod)) return parts;
	16232	return self.fail("TODO implement splitType for {}", .{ty.fmt(mod)});
	16233	}
	16234
15476	16235	/// Truncates the value in the register in place.
15477	16236	/// Clobbers any remaining bits.
15478	16237	fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {

src/arch/x86_64/Encoding.zig+13-17

...	...	@@ -62,8 +62,7 @@ pub fn findByMnemonic(
62	62	.rex, .rex_short => if (!rex_required) continue,
63	63	else => {},
64	64	}
65		for (input_ops, data.ops) \|input_op, data_op\|
66		if (!input_op.isSubset(data_op)) continue :next;
	65	for (input_ops, data.ops) \|input_op, data_op\| if (!input_op.isSubset(data_op)) continue :next;
67	66
68	67	const enc = Encoding{ .mnemonic = mnemonic, .data = data };
69	68	if (shortest_enc) \|previous_shortest_enc\| {
...	...	@@ -410,6 +409,8 @@ pub const Mnemonic = enum {
410	409	vfmadd132ps, vfmadd213ps, vfmadd231ps,
411	410	vfmadd132sd, vfmadd213sd, vfmadd231sd,
412	411	vfmadd132ss, vfmadd213ss, vfmadd231ss,
	412	// AVX2
	413	vpbroadcastb, vpbroadcastd, vpbroadcasti128, vpbroadcastq, vpbroadcastw,
413	414	// zig fmt: on
414	415	};
415	416
...	...	@@ -444,7 +445,7 @@ pub const Op = enum {
444	445	moffs,
445	446	sreg,
446	447	st, mm, mm_m64,
447		xmm0, xmm, xmm_m32, xmm_m64, xmm_m128,
	448	xmm0, xmm, xmm_m8, xmm_m16, xmm_m32, xmm_m64, xmm_m128,
448	449	ymm, ymm_m256,
449	450	// zig fmt: on
450	451
...	...	@@ -534,7 +535,7 @@ pub const Op = enum {
534	535	.eax, .r32, .rm32, .r32_m16 => unreachable,
535	536	.rax, .r64, .rm64, .r64_m16 => unreachable,
536	537	.st, .mm, .mm_m64 => unreachable,
537		.xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
	538	.xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
538	539	.ymm, .ymm_m256 => unreachable,
539	540	.m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
540	541	.unity => 1,
...	...	@@ -556,7 +557,7 @@ pub const Op = enum {
556	557	.eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,
557	558	.rax, .r64, .rm64, .r64_m16, .mm, .mm_m64 => 64,
558	559	.st => 80,
559		.xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
	560	.xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
560	561	.ymm, .ymm_m256 => 256,
561	562	};
562	563	}
...	...	@@ -568,8 +569,8 @@ pub const Op = enum {
568	569	.rel8, .rel16, .rel32 => unreachable,
569	570	.al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64 => unreachable,
570	571	.st, .mm, .xmm0, .xmm, .ymm => unreachable,
571		.m8, .rm8, .r32_m8 => 8,
572		.m16, .rm16, .r32_m16, .r64_m16 => 16,
	572	.m8, .rm8, .r32_m8, .xmm_m8 => 8,
	573	.m16, .rm16, .r32_m16, .r64_m16, .xmm_m16 => 16,
573	574	.m32, .rm32, .xmm_m32 => 32,
574	575	.m64, .rm64, .mm_m64, .xmm_m64 => 64,
575	576	.m80 => 80,
...	...	@@ -600,7 +601,7 @@ pub const Op = enum {
600	601	.rm8, .rm16, .rm32, .rm64,
601	602	.r32_m8, .r32_m16, .r64_m16,
602	603	.st, .mm, .mm_m64,
603		.xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128,
	604	.xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128,
604	605	.ymm, .ymm_m256,
605	606	=> true,
606	607	else => false,
...	...	@@ -629,7 +630,7 @@ pub const Op = enum {
629	630	.m8, .m16, .m32, .m64, .m80, .m128, .m256,
630	631	.m,
631	632	.mm_m64,
632		.xmm_m32, .xmm_m64, .xmm_m128,
	633	.xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128,
633	634	.ymm_m256,
634	635	=> true,
635	636	else => false,
...	...	@@ -654,7 +655,7 @@ pub const Op = enum {
654	655	.sreg => .segment,
655	656	.st => .x87,
656	657	.mm, .mm_m64 => .mmx,
657		.xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .sse,
	658	.xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => .sse,
658	659	.ymm, .ymm_m256 => .sse,
659	660	};
660	661	}
...	...	@@ -826,18 +827,13 @@ const mnemonic_to_encodings_map = init: {
826	827	for (&data_storage, entries, 0..) \|*data, entry, data_index\| {
827	828	data.* = .{
828	829	.op_en = entry[1],
829		.ops = undefined,
	830	.ops = (entry[2] ++ .{.none} ** (data.ops.len - entry[2].len)).*,
830	831	.opc_len = entry[3].len,
831		.opc = undefined,
	832	.opc = (entry[3] ++ .{undefined} ** (data.opc.len - entry[3].len)).*,
832	833	.modrm_ext = entry[4],
833	834	.mode = entry[5],
834	835	.feature = entry[6],
835	836	};
836		// TODO: use `@memcpy` for these. When I did that, I got a false positive
837		// compile error for this copy happening at compile time.
838		std.mem.copyForwards(Op, &data.ops, entry[2]);
839		std.mem.copyForwards(u8, &data.opc, entry[3]);
840
841	837	while (mnemonic_int < @intFromEnum(entry[0])) : (mnemonic_int += 1) {
842	838	mnemonic_map[mnemonic_int] = data_storage[mnemonic_start..data_index];
843	839	mnemonic_start = data_index;

src/arch/x86_64/Mir.zig+2

...	...	@@ -255,6 +255,8 @@ pub const Inst = struct {
255	255	vp_q,
256	256	/// VEX-Encoded Packed ___ Double Quadword
257	257	vp_dq,
	258	/// VEX-Encoded Packed ___ Integer Data
	259	vp_i128,
258	260	/// VEX-Encoded ___ Scalar Single-Precision Values
259	261	v_ss,
260	262	/// VEX-Encoded ___ Packed Single-Precision Values

src/arch/x86_64/bits.zig+2-2

...	...	@@ -222,7 +222,7 @@ pub const Register = enum(u7) {
222	222	@intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax),
223	223	@intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax),
224	224	@intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al),
225		@intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah) - 4,
	225	@intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah),
226	226
227	227	@intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm15) => @intFromEnum(Register.ymm0) - 16,
228	228	@intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm15) => @intFromEnum(Register.xmm0) - 16,
...	...	@@ -237,7 +237,7 @@ pub const Register = enum(u7) {
237	237	return @intCast(@intFromEnum(reg) - base);
238	238	}
239	239
240		pub fn bitSize(reg: Register) u64 {
	240	pub fn bitSize(reg: Register) u10 {
241	241	return switch (@intFromEnum(reg)) {
242	242	// zig fmt: off
243	243	@intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => 64,

src/arch/x86_64/encodings.zig+10

...	...	@@ -1742,6 +1742,16 @@ pub const table = [_]Entry{
1742	1742
1743	1743	.{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },
1744	1744
	1745	.{ .vpbroadcastb, .rm, &.{ .xmm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 },
	1746	.{ .vpbroadcastb, .rm, &.{ .ymm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 },
	1747	.{ .vpbroadcastw, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 },
	1748	.{ .vpbroadcastw, .rm, &.{ .ymm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_256_w0, .avx2 },
	1749	.{ .vpbroadcastd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_128_w0, .avx2 },
	1750	.{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 },
	1751	.{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 },
	1752	.{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 },
	1753	.{ .vpbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
	1754
1745	1755	.{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },
1746	1756	.{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },
1747	1757	.{ .vpcmpeqd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x76 }, 0, .vex_256_wig, .avx2 },

src/codegen.zig+4-1

...	...	@@ -376,7 +376,10 @@ pub fn generateSymbol(
376	376	.val = switch (aggregate.storage) {
377	377	.bytes => unreachable,
378	378	.elems => \|elems\| elems[@as(usize, @intCast(index))],
379		.repeated_elem => \|elem\| elem,
	379	.repeated_elem => \|elem\| if (index < array_type.len)
	380	elem
	381	else
	382	array_type.sentinel,
380	383	}.toValue(),
381	384	}, code, debug_output, reloc_info)) {
382	385	.ok => {},

src/link/Coff.zig+1

...	...	@@ -388,6 +388,7 @@ fn populateMissingMetadata(self: *Coff) !void {
388	388	self.rdata_section_index = try self.allocateSection(".rdata", file_size, .{
389	389	.CNT_INITIALIZED_DATA = 1,
390	390	.MEM_READ = 1,
	391	.MEM_WRITE = 1,
391	392	});
392	393	}
393	394

src/main.zig+1

...	...	@@ -6492,6 +6492,7 @@ pub fn cmdDumpZir(
6492	6492	.mod = undefined,
6493	6493	.root_decl = .none,
6494	6494	};
	6495	defer file.zir.deinit(gpa);
6495	6496
6496	6497	{
6497	6498	const instruction_bytes = file.zir.instructions.len *

src/print_zir.zig+6-9

...	...	@@ -2456,20 +2456,15 @@ const Writer = struct {
2456	2456	const inst_data = self.code.instructions.items(.data)[@intFromEnum(inst)].save_err_ret_index;
2457	2457
2458	2458	try self.writeInstRef(stream, inst_data.operand);
	2459
2459	2460	try stream.writeAll(")");
2460	2461	}
2461	2462
2462	2463	fn writeRestoreErrRetIndex(self: *Writer, stream: anytype, inst: Zir.Inst.Index) !void {
2463	2464	const inst_data = self.code.instructions.items(.data)[@intFromEnum(inst)].restore_err_ret_index;
2464	2465
2465		if (inst_data.block != .none) {
2466		try self.writeInstRef(stream, inst_data.block);
2467		}
2468
2469		if (inst_data.operand != .none) {
2470		if (inst_data.block != .none) try stream.writeAll(", ");
2471		try self.writeInstRef(stream, inst_data.operand);
2472		}
	2466	try self.writeInstRef(stream, inst_data.block);
	2467	try self.writeInstRef(stream, inst_data.operand);
2473	2468
2474	2469	try stream.writeAll(")");
2475	2470	}
...	...	@@ -2615,7 +2610,9 @@ const Writer = struct {
2615	2610	}
2616	2611
2617	2612	fn writeInstRef(self: *Writer, stream: anytype, ref: Zir.Inst.Ref) !void {
2618		if (ref.toIndex()) \|i\| {
	2613	if (ref == .none) {
	2614	return stream.writeAll(".none");
	2615	} else if (ref.toIndex()) \|i\| {
2619	2616	return self.writeInstIndex(stream, i);
2620	2617	} else {
2621	2618	const val: InternPool.Index = @enumFromInt(@intFromEnum(ref));

src/register_manager.zig+102-60

...	...	@@ -55,6 +55,7 @@ pub fn RegisterManager(
55	55	const Self = @This();
56	56
57	57	pub const TrackedRegisters = [tracked_registers.len]Air.Inst.Index;
	58	pub const TrackedIndex = std.math.IntFittingRange(0, tracked_registers.len - 1);
58	59	pub const RegisterBitSet = StaticBitSet(tracked_registers.len);
59	60
60	61	fn getFunction(self: Self) Function {
...	...	@@ -66,45 +67,64 @@ pub fn RegisterManager(
66	67	return !register_class.isSet(index);
67	68	}
68	69
	70	fn markRegIndexAllocated(self: *Self, tracked_index: TrackedIndex) void {
	71	self.allocated_registers.set(tracked_index);
	72	}
69	73	fn markRegAllocated(self: *Self, reg: Register) void {
70		const index = indexOfRegIntoTracked(reg) orelse return;
71		self.allocated_registers.set(index);
	74	self.markRegIndexAllocated(indexOfRegIntoTracked(reg) orelse return);
72	75	}
73	76
	77	fn markRegIndexUsed(self: *Self, tracked_index: TrackedIndex) void {
	78	self.free_registers.unset(tracked_index);
	79	}
74	80	fn markRegUsed(self: *Self, reg: Register) void {
75		const index = indexOfRegIntoTracked(reg) orelse return;
76		self.free_registers.unset(index);
	81	self.markRegIndexUsed(indexOfRegIntoTracked(reg) orelse return);
77	82	}
78	83
	84	fn markRegIndexFree(self: *Self, tracked_index: TrackedIndex) void {
	85	self.free_registers.set(tracked_index);
	86	}
79	87	fn markRegFree(self: *Self, reg: Register) void {
80		const index = indexOfRegIntoTracked(reg) orelse return;
81		self.free_registers.set(index);
	88	self.markRegIndexFree(indexOfRegIntoTracked(reg) orelse return);
82	89	}
83	90
84	91	pub fn indexOfReg(
85		comptime registers: []const Register,
	92	comptime set: []const Register,
86	93	reg: Register,
87		) ?std.math.IntFittingRange(0, registers.len - 1) {
88		inline for (tracked_registers, 0..) \|cpreg, i\| {
89		if (reg.id() == cpreg.id()) return i;
	94	) ?std.math.IntFittingRange(0, set.len - 1) {
	95	const Id = @TypeOf(reg.id());
	96	comptime var min_id: Id = std.math.maxInt(Id);
	97	comptime var max_id: Id = std.math.minInt(Id);
	98	inline for (set) \|elem\| {
	99	const elem_id = comptime elem.id();
	100	min_id = @min(elem_id, min_id);
	101	max_id = @max(elem_id, max_id);
90	102	}
91		return null;
	103
	104	const OptionalIndex = std.math.IntFittingRange(0, set.len);
	105	comptime var map = [1]OptionalIndex{set.len} ** (max_id + 1 - min_id);
	106	inline for (set, 0..) \|elem, elem_index\| map[comptime elem.id() - min_id] = elem_index;
	107
	108	const id_index = reg.id() -% min_id;
	109	if (id_index >= map.len) return null;
	110	const set_index = map[id_index];
	111	return if (set_index < set.len) @intCast(set_index) else null;
92	112	}
93	113
94		pub fn indexOfRegIntoTracked(
95		reg: Register,
96		) ?std.math.IntFittingRange(0, tracked_registers.len) {
	114	pub fn indexOfRegIntoTracked(reg: Register) ?TrackedIndex {
97	115	return indexOfReg(tracked_registers, reg);
98	116	}
99	117
100		pub fn regAtTrackedIndex(index: std.math.IntFittingRange(0, tracked_registers.len)) Register {
101		return tracked_registers[index];
	118	pub fn regAtTrackedIndex(tracked_index: TrackedIndex) Register {
	119	return tracked_registers[tracked_index];
102	120	}
103	121
104	122	/// Returns true when this register is not tracked
	123	pub fn isRegIndexFree(self: Self, tracked_index: TrackedIndex) bool {
	124	return self.free_registers.isSet(tracked_index);
	125	}
105	126	pub fn isRegFree(self: Self, reg: Register) bool {
106		const index = indexOfRegIntoTracked(reg) orelse return true;
107		return self.free_registers.isSet(index);
	127	return self.isRegIndexFree(indexOfRegIntoTracked(reg) orelse return true);
108	128	}
109	129
110	130	/// Returns whether this register was allocated in the course
...	...	@@ -119,14 +139,14 @@ pub fn RegisterManager(
119	139	/// Returns whether this register is locked
120	140	///
121	141	/// Returns false when this register is not tracked
	142	fn isRegIndexLocked(self: Self, tracked_index: TrackedIndex) bool {
	143	return self.locked_registers.isSet(tracked_index);
	144	}
122	145	pub fn isRegLocked(self: Self, reg: Register) bool {
123		const index = indexOfRegIntoTracked(reg) orelse return false;
124		return self.locked_registers.isSet(index);
	146	return self.isRegIndexLocked(indexOfRegIntoTracked(reg) orelse return false);
125	147	}
126	148
127		pub const RegisterLock = struct {
128		register: Register,
129		};
	149	pub const RegisterLock = struct { tracked_index: TrackedIndex };
130	150
131	151	/// Prevents the register from being allocated until they are
132	152	/// unlocked again.
...	...	@@ -134,25 +154,29 @@ pub fn RegisterManager(
134	154	/// locked, or `null` otherwise.
135	155	/// Only the owner of the `RegisterLock` can unlock the
136	156	/// register later.
137		pub fn lockReg(self: *Self, reg: Register) ?RegisterLock {
138		log.debug("locking {}", .{reg});
139		if (self.isRegLocked(reg)) {
	157	pub fn lockRegIndex(self: *Self, tracked_index: TrackedIndex) ?RegisterLock {
	158	log.debug("locking {}", .{regAtTrackedIndex(tracked_index)});
	159	if (self.isRegIndexLocked(tracked_index)) {
140	160	log.debug(" register already locked", .{});
141	161	return null;
142	162	}
143		const index = indexOfRegIntoTracked(reg) orelse return null;
144		self.locked_registers.set(index);
145		return RegisterLock{ .register = reg };
	163	self.locked_registers.set(tracked_index);
	164	return RegisterLock{ .tracked_index = tracked_index };
	165	}
	166	pub fn lockReg(self: *Self, reg: Register) ?RegisterLock {
	167	return self.lockRegIndex(indexOfRegIntoTracked(reg) orelse return null);
146	168	}
147	169
148	170	/// Like `lockReg` but asserts the register was unused always
149	171	/// returning a valid lock.
	172	pub fn lockRegIndexAssumeUnused(self: *Self, tracked_index: TrackedIndex) RegisterLock {
	173	log.debug("locking asserting free {}", .{regAtTrackedIndex(tracked_index)});
	174	assert(!self.isRegIndexLocked(tracked_index));
	175	self.locked_registers.set(tracked_index);
	176	return RegisterLock{ .tracked_index = tracked_index };
	177	}
150	178	pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock {
151		log.debug("locking asserting free {}", .{reg});
152		assert(!self.isRegLocked(reg));
153		const index = indexOfRegIntoTracked(reg) orelse unreachable;
154		self.locked_registers.set(index);
155		return RegisterLock{ .register = reg };
	179	return self.lockRegIndexAssumeUnused(indexOfRegIntoTracked(reg) orelse unreachable);
156	180	}
157	181
158	182	/// Like `lockReg` but locks multiple registers.
...	...	@@ -181,9 +205,8 @@ pub fn RegisterManager(
181	205	/// Requires `RegisterLock` to unlock a register.
182	206	/// Call `lockReg` to obtain the lock first.
183	207	pub fn unlockReg(self: *Self, lock: RegisterLock) void {
184		log.debug("unlocking {}", .{lock.register});
185		const index = indexOfRegIntoTracked(lock.register) orelse return;
186		self.locked_registers.unset(index);
	208	log.debug("unlocking {}", .{regAtTrackedIndex(lock.tracked_index)});
	209	self.locked_registers.unset(lock.tracked_index);
187	210	}
188	211
189	212	/// Returns true when at least one register is locked
...	...	@@ -319,44 +342,63 @@ pub fn RegisterManager(
319	342	/// Spills the register if it is currently allocated. If a
320	343	/// corresponding instruction is passed, will also track this
321	344	/// register.
322		pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {
323		const index = indexOfRegIntoTracked(reg) orelse return;
324		log.debug("getReg {} for inst {?}", .{ reg, inst });
325
326		if (!self.isRegFree(reg)) {
327		self.markRegAllocated(reg);
	345	fn getRegIndex(
	346	self: *Self,
	347	tracked_index: TrackedIndex,
	348	inst: ?Air.Inst.Index,
	349	) AllocateRegistersError!void {
	350	log.debug("getReg {} for inst {?}", .{ regAtTrackedIndex(tracked_index), inst });
	351	if (!self.isRegIndexFree(tracked_index)) {
	352	self.markRegIndexAllocated(tracked_index);
328	353
329	354	// Move the instruction that was previously there to a
330	355	// stack allocation.
331		const spilled_inst = self.registers[index];
332		if (inst) \|tracked_inst\| self.registers[index] = tracked_inst;
333		try self.getFunction().spillInstruction(reg, spilled_inst);
334		if (inst == null) self.freeReg(reg);
335		} else self.getRegAssumeFree(reg, inst);
	356	const spilled_inst = self.registers[tracked_index];
	357	if (inst) \|tracked_inst\| self.registers[tracked_index] = tracked_inst;
	358	try self.getFunction().spillInstruction(regAtTrackedIndex(tracked_index), spilled_inst);
	359	if (inst == null) self.freeRegIndex(tracked_index);
	360	} else self.getRegIndexAssumeFree(tracked_index, inst);
	361	}
	362	pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {
	363	return self.getRegIndex(indexOfRegIntoTracked(reg) orelse return, inst);
	364	}
	365	pub fn getKnownReg(
	366	self: *Self,
	367	comptime reg: Register,
	368	inst: ?Air.Inst.Index,
	369	) AllocateRegistersError!void {
	370	return self.getRegIndex((comptime indexOfRegIntoTracked(reg)) orelse return, inst);
336	371	}
337	372
338	373	/// Allocates the specified register with the specified
339	374	/// instruction. Asserts that the register is free and no
340	375	/// spilling is necessary.
341		pub fn getRegAssumeFree(self: *Self, reg: Register, inst: ?Air.Inst.Index) void {
342		const index = indexOfRegIntoTracked(reg) orelse return;
343		log.debug("getRegAssumeFree {} for inst {?}", .{ reg, inst });
344		self.markRegAllocated(reg);
	376	fn getRegIndexAssumeFree(
	377	self: *Self,
	378	tracked_index: TrackedIndex,
	379	inst: ?Air.Inst.Index,
	380	) void {
	381	log.debug("getRegAssumeFree {} for inst {?}", .{ regAtTrackedIndex(tracked_index), inst });
	382	self.markRegIndexAllocated(tracked_index);
345	383
346		assert(self.isRegFree(reg));
	384	assert(self.isRegIndexFree(tracked_index));
347	385	if (inst) \|tracked_inst\| {
348		self.registers[index] = tracked_inst;
349		self.markRegUsed(reg);
	386	self.registers[tracked_index] = tracked_inst;
	387	self.markRegIndexUsed(tracked_index);
350	388	}
351	389	}
	390	pub fn getRegAssumeFree(self: *Self, reg: Register, inst: ?Air.Inst.Index) void {
	391	self.getRegIndexAssumeFree(indexOfRegIntoTracked(reg) orelse return, inst);
	392	}
352	393
353	394	/// Marks the specified register as free
	395	fn freeRegIndex(self: *Self, tracked_index: TrackedIndex) void {
	396	log.debug("freeing register {}", .{regAtTrackedIndex(tracked_index)});
	397	self.registers[tracked_index] = undefined;
	398	self.markRegIndexFree(tracked_index);
	399	}
354	400	pub fn freeReg(self: *Self, reg: Register) void {
355		const index = indexOfRegIntoTracked(reg) orelse return;
356		log.debug("freeing register {}", .{reg});
357
358		self.registers[index] = undefined;
359		self.markRegFree(reg);
	401	self.freeRegIndex(indexOfRegIntoTracked(reg) orelse return);
360	402	}
361	403	};
362	404	}

test/behavior/vector.zig-1

...	...	@@ -1260,7 +1260,6 @@ test "zero multiplicand" {
1260	1260
1261	1261	test "@intCast to u0" {
1262	1262	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1263		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1264	1263	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1265	1264	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1266	1265	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO