Commit 509be7cf1f10c5d329d2b0524f2af6bfcabd52de | zig

author	Jacob Young <jacobly@ziglang.org> 2023-11-03 23:18:21-04:00
committer	Jacob Young <jacobly@ziglang.org> 2023-11-03 23:18:21-04:00
log	509be7cf1f10c5d329d2b0524f2af6bfcabd52de
tree	655825e5800a1c79e8ccc0abe5011dd47655a96a
parent	f6de3ec963e3a7d96cd4f6c72b0f076f0437c45d

x86_64: fix std test failures

45 files changed, 1413 insertions(+), 842 deletions(-)

lib
- std
  - Thread
    Condition.zig
    Mutex.zig
    RwLock.zig
    Semaphore.zig
  - atomic
    Atomic.zig
    queue.zig
  - base64.zig
  - bit_set.zig
  - compress
    zstandard.zig
  - crypto
    25519
    ed25519.zig
    Certificate.zig
    Certificate
    Bundle.zig
    aes.zig
    aes_gcm.zig
    argon2.zig
    bcrypt.zig
    cmac.zig
    ecdsa.zig
    ghash_polyval.zig
    pcurves
    p256.zig
    phc_encoding.zig
    sha2.zig
  - fmt
    parse_float.zig
  - hash
    xxhash.zig
  - http
    Client.zig
    Server.zig
    protocol.zig
  - math.zig
  - math
    big
    int.zig
    int_test.zig
  - mem.zig
  - net
    test.zig
  - once.zig
  - os
    test.zig
  - rand
    test.zig
  - zig
    tokenizer.zig
src
- Compilation.zig
- arch
  - x86_64
    CodeGen.zig
    Encoding.zig
    Mir.zig
    bits.zig
    encodings.zig
- codegen.zig
- link
  - Coff.zig
test
- behavior
  - vector.zig

lib/std/Thread/Condition.zig-10

...	...	@@ -324,8 +324,6 @@ test "Condition - wait and signal" {
324	324	return error.SkipZigTest;
325	325	}
326	326
327		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
328
329	327	const num_threads = 4;
330	328
331	329	const MultiWait = struct {
...	...	@@ -371,8 +369,6 @@ test "Condition - signal" {
371	369	return error.SkipZigTest;
372	370	}
373	371
374		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
375
376	372	const num_threads = 4;
377	373
378	374	const SignalTest = struct {
...	...	@@ -440,8 +436,6 @@ test "Condition - multi signal" {
440	436	return error.SkipZigTest;
441	437	}
442	438
443		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
444
445	439	const num_threads = 4;
446	440	const num_iterations = 4;
447	441
...	...	@@ -504,8 +498,6 @@ test "Condition - broadcasting" {
504	498	return error.SkipZigTest;
505	499	}
506	500
507		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
508
509	501	const num_threads = 10;
510	502
511	503	const BroadcastTest = struct {
...	...	@@ -573,8 +565,6 @@ test "Condition - broadcasting - wake all threads" {
573	565	return error.SkipZigTest;
574	566	}
575	567
576		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
577
578	568	var num_runs: usize = 1;
579	569	const num_threads = 10;
580	570

lib/std/Thread/Mutex.zig-2

...	...	@@ -289,8 +289,6 @@ test "Mutex - many contended" {
289	289	return error.SkipZigTest;
290	290	}
291	291
292		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
293
294	292	const num_threads = 4;
295	293	const num_increments = 1000;
296	294

lib/std/Thread/RwLock.zig-2

...	...	@@ -297,8 +297,6 @@ test "RwLock - concurrent access" {
297	297	if (builtin.single_threaded)
298	298	return;
299	299
300		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
301
302	300	const num_writers: usize = 2;
303	301	const num_readers: usize = 4;
304	302	const num_writes: usize = 10000;

lib/std/Thread/Semaphore.zig-2

...	...	@@ -39,8 +39,6 @@ test "Thread.Semaphore" {
39	39	return error.SkipZigTest;
40	40	}
41	41
42		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
43
44	42	const TestContext = struct {
45	43	sem: *Semaphore,
46	44	n: *i32,

lib/std/atomic/Atomic.zig-2

...	...	@@ -467,8 +467,6 @@ test "Atomic.fetchSub" {
467	467	}
468	468
469	469	test "Atomic.fetchMin" {
470		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
471
472	470	inline for (atomicIntTypes()) \|Int\| {
473	471	inline for (atomic_rmw_orderings) \|ordering\| {
474	472	var x = Atomic(Int).init(5);

lib/std/atomic/queue.zig-2

...	...	@@ -175,8 +175,6 @@ const puts_per_thread = 500;
175	175	const put_thread_count = 3;
176	176
177	177	test "std.atomic.Queue" {
178		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
179
180	178	var plenty_of_memory = try std.heap.page_allocator.alloc(u8, 300 * 1024);
181	179	defer std.heap.page_allocator.free(plenty_of_memory);
182	180

lib/std/base64.zig-4

...	...	@@ -355,8 +355,6 @@ pub const Base64DecoderWithIgnore = struct {
355	355	};
356	356
357	357	test "base64" {
358		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
359
360	358	@setEvalBranchQuota(8000);
361	359	try testBase64();
362	360	try comptime testAllApis(standard, "comptime", "Y29tcHRpbWU=");
...	...	@@ -377,8 +375,6 @@ test "base64 padding dest overflow" {
377	375	}
378	376
379	377	test "base64 url_safe_no_pad" {
380		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
381
382	378	@setEvalBranchQuota(8000);
383	379	try testBase64UrlSafeNoPad();
384	380	try comptime testAllApis(url_safe_no_pad, "comptime", "Y29tcHRpbWU");

lib/std/bit_set.zig-3

...	...	@@ -1638,7 +1638,6 @@ fn testStaticBitSet(comptime Set: type) !void {
1638	1638
1639	1639	test "IntegerBitSet" {
1640	1640	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
1641		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1642	1641
1643	1642	try testStaticBitSet(IntegerBitSet(0));
1644	1643	try testStaticBitSet(IntegerBitSet(1));
...	...	@@ -1651,8 +1650,6 @@ test "IntegerBitSet" {
1651	1650	}
1652	1651
1653	1652	test "ArrayBitSet" {
1654		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1655
1656	1653	inline for (.{ 0, 1, 2, 31, 32, 33, 63, 64, 65, 254, 500, 3000 }) \|size\| {
1657	1654	try testStaticBitSet(ArrayBitSet(u8, size));
1658	1655	try testStaticBitSet(ArrayBitSet(u16, size));

lib/std/compress/zstandard.zig-2

...	...	@@ -264,8 +264,6 @@ fn testReader(data: []const u8, comptime expected: []const u8) !void {
264	264	}
265	265
266	266	test "zstandard decompression" {
267		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
268
269	267	const uncompressed = @embedFile("testdata/rfc8478.txt");
270	268	const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
271	269	const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");

lib/std/crypto/25519/ed25519.zig+2-13

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("std");
2		const builtin = @import("builtin");
3	2	const crypto = std.crypto;
4	3	const debug = std.debug;
5	4	const fmt = std.fmt;
...	...	@@ -276,8 +275,8 @@ pub const Ed25519 = struct {
276	275	pub fn fromSecretKey(secret_key: SecretKey) (NonCanonicalError \|\| EncodingError \|\| IdentityElementError)!KeyPair {
277	276	// It is critical for EdDSA to use the correct public key.
278	277	// In order to enforce this, a SecretKey implicitly includes a copy of the public key.
279		// In Debug mode, we can still afford checking that the public key is correct for extra safety.
280		if (builtin.mode == .Debug) {
	278	// With runtime safety, we can still afford checking that the public key is correct.
	279	if (std.debug.runtime_safety) {
281	280	const pk_p = try Curve.fromBytes(secret_key.publicKeyBytes());
282	281	const recomputed_kp = try create(secret_key.seed());
283	282	debug.assert(mem.eql(u8, &recomputed_kp.public_key.toBytes(), &pk_p.toBytes()));
...	...	@@ -493,8 +492,6 @@ test "ed25519 key pair creation" {
493	492	}
494	493
495	494	test "ed25519 signature" {
496		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
497
498	495	var seed: [32]u8 = undefined;
499	496	_ = try fmt.hexToBytes(seed[0..], "8052030376d47112be7f73ed7a019293dd12ad910b654455798b4667d73de166");
500	497	const key_pair = try Ed25519.KeyPair.create(seed);
...	...	@@ -507,8 +504,6 @@ test "ed25519 signature" {
507	504	}
508	505
509	506	test "ed25519 batch verification" {
510		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
511
512	507	var i: usize = 0;
513	508	while (i < 100) : (i += 1) {
514	509	const key_pair = try Ed25519.KeyPair.create(null);
...	...	@@ -538,8 +533,6 @@ test "ed25519 batch verification" {
538	533	}
539	534
540	535	test "ed25519 test vectors" {
541		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
542
543	536	const Vec = struct {
544	537	msg_hex: *const [64:0]u8,
545	538	public_key_hex: *const [64:0]u8,
...	...	@@ -642,8 +635,6 @@ test "ed25519 test vectors" {
642	635	}
643	636
644	637	test "ed25519 with blind keys" {
645		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
646
647	638	const BlindKeyPair = Ed25519.key_blinding.BlindKeyPair;
648	639
649	640	// Create a standard Ed25519 key pair
...	...	@@ -667,8 +658,6 @@ test "ed25519 with blind keys" {
667	658	}
668	659
669	660	test "ed25519 signatures with streaming" {
670		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
671
672	661	const kp = try Ed25519.KeyPair.create(null);
673	662
674	663	var signer = try kp.signer(null);

lib/std/crypto/Certificate.zig+13-13

...	...	@@ -614,18 +614,18 @@ const Date = struct {
614	614	};
615	615
616	616	pub fn parseTimeDigits(text: *const [2]u8, min: u8, max: u8) !u8 {
617		const nn: @Vector(2, u16) = .{ text[0], text[1] };
618		const zero: @Vector(2, u16) = .{ '0', '0' };
619		const mm: @Vector(2, u16) = .{ 10, 1 };
620		const result = @reduce(.Add, (nn -% zero) *% mm);
	617	const result = if (use_vectors) result: {
	618	const nn: @Vector(2, u16) = .{ text[0], text[1] };
	619	const zero: @Vector(2, u16) = .{ '0', '0' };
	620	const mm: @Vector(2, u16) = .{ 10, 1 };
	621	break :result @reduce(.Add, (nn -% zero) *% mm);
	622	} else std.fmt.parseInt(u8, text, 10) catch return error.CertificateTimeInvalid;
621	623	if (result < min) return error.CertificateTimeInvalid;
622	624	if (result > max) return error.CertificateTimeInvalid;
623	625	return @truncate(result);
624	626	}
625	627
626	628	test parseTimeDigits {
627		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
628
629	629	const expectEqual = std.testing.expectEqual;
630	630	try expectEqual(@as(u8, 0), try parseTimeDigits("00", 0, 99));
631	631	try expectEqual(@as(u8, 99), try parseTimeDigits("99", 0, 99));
...	...	@@ -638,17 +638,17 @@ test parseTimeDigits {
638	638	}
639	639
640	640	pub fn parseYear4(text: *const [4]u8) !u16 {
641		const nnnn: @Vector(4, u32) = .{ text[0], text[1], text[2], text[3] };
642		const zero: @Vector(4, u32) = .{ '0', '0', '0', '0' };
643		const mmmm: @Vector(4, u32) = .{ 1000, 100, 10, 1 };
644		const result = @reduce(.Add, (nnnn -% zero) *% mmmm);
	641	const result = if (use_vectors) result: {
	642	const nnnn: @Vector(4, u32) = .{ text[0], text[1], text[2], text[3] };
	643	const zero: @Vector(4, u32) = .{ '0', '0', '0', '0' };
	644	const mmmm: @Vector(4, u32) = .{ 1000, 100, 10, 1 };
	645	break :result @reduce(.Add, (nnnn -% zero) *% mmmm);
	646	} else std.fmt.parseInt(u16, text, 10) catch return error.CertificateTimeInvalid;
645	647	if (result > 9999) return error.CertificateTimeInvalid;
646	648	return @truncate(result);
647	649	}
648	650
649	651	test parseYear4 {
650		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
651
652	652	const expectEqual = std.testing.expectEqual;
653	653	try expectEqual(@as(u16, 0), try parseYear4("0000"));
654	654	try expectEqual(@as(u16, 9999), try parseYear4("9999"));
...	...	@@ -1124,4 +1124,4 @@ pub const rsa = struct {
1124	1124	}
1125	1125	};
1126	1126
1127		const builtin = @import("builtin");
	1127	const use_vectors = @import("builtin").zig_backend != .stage2_x86_64;

lib/std/crypto/Certificate/Bundle.zig-2

...	...	@@ -318,8 +318,6 @@ const MapContext = struct {
318	318	test "scan for OS-provided certificates" {
319	319	if (builtin.os.tag == .wasi) return error.SkipZigTest;
320	320
321		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
322
323	321	var bundle: Bundle = .{};
324	322	defer bundle.deinit(std.testing.allocator);
325	323

lib/std/crypto/aes.zig-2

...	...	@@ -28,8 +28,6 @@ pub const Aes128 = impl.Aes128;
28	28	pub const Aes256 = impl.Aes256;
29	29
30	30	test "ctr" {
31		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
32
33	31	// NIST SP 800-38A pp 55-58
34	32	const ctr = @import("modes.zig").ctr;
35	33

lib/std/crypto/aes_gcm.zig+2-11

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("std");
2		const builtin = @import("builtin");
3	2	const assert = std.debug.assert;
4	3	const crypto = std.crypto;
5	4	const debug = std.debug;
...	...	@@ -42,7 +41,7 @@ fn AesGcm(comptime Aes: anytype) type {
42	41	mac.pad();
43	42
44	43	mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);
45		modes.ctr(@TypeOf(aes), aes, c, m, j, std.builtin.Endian.big);
	44	modes.ctr(@TypeOf(aes), aes, c, m, j, .big);
46	45	mac.update(c[0..m.len][0..]);
47	46	mac.pad();
48	47
...	...	@@ -104,7 +103,7 @@ fn AesGcm(comptime Aes: anytype) type {
104	103	}
105	104
106	105	mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);
107		modes.ctr(@TypeOf(aes), aes, m, c, j, std.builtin.Endian.big);
	106	modes.ctr(@TypeOf(aes), aes, m, c, j, .big);
108	107	}
109	108	};
110	109	}
...	...	@@ -113,8 +112,6 @@ const htest = @import("test.zig");
113	112	const testing = std.testing;
114	113
115	114	test "Aes256Gcm - Empty message and no associated data" {
116		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
117
118	115	const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
119	116	const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
120	117	const ad = "";
...	...	@@ -127,8 +124,6 @@ test "Aes256Gcm - Empty message and no associated data" {
127	124	}
128	125
129	126	test "Aes256Gcm - Associated data only" {
130		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
131
132	127	const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
133	128	const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
134	129	const m = "";
...	...	@@ -141,8 +136,6 @@ test "Aes256Gcm - Associated data only" {
141	136	}
142	137
143	138	test "Aes256Gcm - Message only" {
144		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
145
146	139	const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
147	140	const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
148	141	const m = "Test with message only";
...	...	@@ -160,8 +153,6 @@ test "Aes256Gcm - Message only" {
160	153	}
161	154
162	155	test "Aes256Gcm - Message and associated data" {
163		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
164
165	156	const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
166	157	const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
167	158	const m = "Test with message";

lib/std/crypto/argon2.zig-4

...	...	@@ -896,8 +896,6 @@ test "kdf" {
896	896	}
897	897
898	898	test "phc format hasher" {
899		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
900
901	899	const allocator = std.testing.allocator;
902	900	const password = "testpass";
903	901
...	...	@@ -913,8 +911,6 @@ test "phc format hasher" {
913	911	}
914	912
915	913	test "password hash and password verify" {
916		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
917
918	914	const allocator = std.testing.allocator;
919	915	const password = "testpass";
920	916

lib/std/crypto/bcrypt.zig-7

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("std");
2		const builtin = @import("builtin");
3	2	const base64 = std.base64;
4	3	const crypto = std.crypto;
5	4	const debug = std.debug;
...	...	@@ -754,8 +753,6 @@ pub fn strVerify(
754	753	}
755	754
756	755	test "bcrypt codec" {
757		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
758
759	756	var salt: [salt_length]u8 = undefined;
760	757	crypto.random.bytes(&salt);
761	758	var salt_str: [salt_str_length]u8 = undefined;
...	...	@@ -766,8 +763,6 @@ test "bcrypt codec" {
766	763	}
767	764
768	765	test "bcrypt crypt format" {
769		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
770
771	766	var hash_options = HashOptions{
772	767	.params = .{ .rounds_log = 5 },
773	768	.encoding = .crypt,
...	...	@@ -808,8 +803,6 @@ test "bcrypt crypt format" {
808	803	}
809	804
810	805	test "bcrypt phc format" {
811		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
812
813	806	var hash_options = HashOptions{
814	807	.params = .{ .rounds_log = 5 },
815	808	.encoding = .phc,

lib/std/crypto/cmac.zig-9

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("std");
2		const builtin = @import("builtin");
3	2	const crypto = std.crypto;
4	3	const mem = std.mem;
5	4
...	...	@@ -94,8 +93,6 @@ pub fn Cmac(comptime BlockCipher: type) type {
94	93	const testing = std.testing;
95	94
96	95	test "CmacAes128 - Example 1: len = 0" {
97		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
98
99	96	const key = [_]u8{
100	97	0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
101	98	};
...	...	@@ -109,8 +106,6 @@ test "CmacAes128 - Example 1: len = 0" {
109	106	}
110	107
111	108	test "CmacAes128 - Example 2: len = 16" {
112		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
113
114	109	const key = [_]u8{
115	110	0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
116	111	};
...	...	@@ -126,8 +121,6 @@ test "CmacAes128 - Example 2: len = 16" {
126	121	}
127	122
128	123	test "CmacAes128 - Example 3: len = 40" {
129		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
130
131	124	const key = [_]u8{
132	125	0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
133	126	};
...	...	@@ -145,8 +138,6 @@ test "CmacAes128 - Example 3: len = 40" {
145	138	}
146	139
147	140	test "CmacAes128 - Example 4: len = 64" {
148		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
149
150	141	const key = [_]u8{
151	142	0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
152	143	};

lib/std/crypto/ecdsa.zig-5

...	...	@@ -373,7 +373,6 @@ pub fn Ecdsa(comptime Curve: type, comptime Hash: type) type {
373	373
374	374	test "ECDSA - Basic operations over EcdsaP384Sha384" {
375	375	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
376		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
377	376
378	377	const Scheme = EcdsaP384Sha384;
379	378	const kp = try Scheme.KeyPair.create(null);
...	...	@@ -407,7 +406,6 @@ test "ECDSA - Basic operations over Secp256k1" {
407	406
408	407	test "ECDSA - Basic operations over EcdsaP384Sha256" {
409	408	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
410		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
411	409
412	410	const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);
413	411	const kp = try Scheme.KeyPair.create(null);
...	...	@@ -424,7 +422,6 @@ test "ECDSA - Basic operations over EcdsaP384Sha256" {
424	422
425	423	test "ECDSA - Verifying a existing signature with EcdsaP384Sha256" {
426	424	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
427		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
428	425
429	426	const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);
430	427	// zig fmt: off
...	...	@@ -469,7 +466,6 @@ const TestVector = struct {
469	466
470	467	test "ECDSA - Test vectors from Project Wycheproof" {
471	468	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
472		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
473	469
474	470	const vectors = [_]TestVector{
475	471	.{ .key = "042927b10512bae3eddcfe467828128bad2903269919f7086069c8c4df6c732838c7787964eaac00e5921fb1498a60f4606766b3d9685001558d1a974e7341513e", .msg = "313233343030", .sig = "304402202ba3a8be6b94d5ec80a6d9d1190a436effe50d85a1eee859b8cc6af9bd5c2e1802204cd60b855d442f5b3c7b11eb6c4e0ae7525fe710fab9aa7c77a67f79e6fadd76", .result = .valid },
...	...	@@ -884,7 +880,6 @@ fn tvTry(vector: TestVector) !void {
884	880
885	881	test "ECDSA - Sec1 encoding/decoding" {
886	882	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
887		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
888	883
889	884	const Scheme = EcdsaP384Sha384;
890	885	const kp = try Scheme.KeyPair.create(null);

lib/std/crypto/ghash_polyval.zig-4

...	...	@@ -422,8 +422,6 @@ fn Hash(comptime endian: std.builtin.Endian, comptime shift_key: bool) type {
422	422	const htest = @import("test.zig");
423	423
424	424	test "ghash" {
425		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
426
427	425	const key = [_]u8{0x42} ** 16;
428	426	const m = [_]u8{0x69} ** 256;
429	427
...	...	@@ -441,8 +439,6 @@ test "ghash" {
441	439	}
442	440
443	441	test "ghash2" {
444		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
445
446	442	var key: [16]u8 = undefined;
447	443	var i: usize = 0;
448	444	while (i < key.len) : (i += 1) {

lib/std/crypto/pcurves/p256.zig-2

...	...	@@ -478,7 +478,5 @@ pub const AffineCoordinates = struct {
478	478	};
479	479
480	480	test {
481		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
482
483	481	_ = @import("tests/p256.zig");
484	482	}

lib/std/crypto/phc_encoding.zig-3

...	...	@@ -1,7 +1,6 @@
1	1	// https://github.com/P-H-C/phc-string-format
2	2
3	3	const std = @import("std");
4		const builtin = @import("builtin");
5	4	const fmt = std.fmt;
6	5	const io = std.io;
7	6	const mem = std.mem;
...	...	@@ -264,8 +263,6 @@ fn kvSplit(str: []const u8) !struct { key: []const u8, value: []const u8 } {
264	263	}
265	264
266	265	test "phc format - encoding/decoding" {
267		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
268
269	266	const Input = struct {
270	267	str: []const u8,
271	268	HashResult: type,

lib/std/crypto/sha2.zig+1-1

...	...	@@ -238,7 +238,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {
238	238	return;
239	239	},
240	240	// C backend doesn't currently support passing vectors to inline asm.
241		.x86_64 => if (builtin.zig_backend != .stage2_c and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {
	241	.x86_64 => if (builtin.zig_backend != .stage2_c and builtin.zig_backend != .stage2_x86_64 and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {
242	242	var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };
243	243	var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };
244	244	const s_v = @as(*[16]v4u32, @ptrCast(&s));

lib/std/fmt/parse_float.zig-2

...	...	@@ -83,8 +83,6 @@ test "fmt.parseFloat #11169" {
83	83	}
84	84
85	85	test "fmt.parseFloat hex.special" {
86		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
87
88	86	try testing.expect(math.isNan(try parseFloat(f32, "nAn")));
89	87	try testing.expect(math.isPositiveInf(try parseFloat(f32, "iNf")));
90	88	try testing.expect(math.isPositiveInf(try parseFloat(f32, "+Inf")));

lib/std/hash/xxhash.zig+2-1

...	...	@@ -2,6 +2,7 @@ const std = @import("std");
2	2	const builtin = @import("builtin");
3	3	const mem = std.mem;
4	4	const expectEqual = std.testing.expectEqual;
	5	const native_endian = builtin.cpu.arch.endian();
5	6
6	7	const rotl = std.math.rotl;
7	8
...	...	@@ -472,7 +473,7 @@ pub const XxHash3 = struct {
472	473	}
473	474
474	475	inline fn swap(x: anytype) @TypeOf(x) {
475		return if (builtin.cpu.arch.endian() == .big) @byteSwap(x) else x;
	476	return if (native_endian == .big) @byteSwap(x) else x;
476	477	}
477	478
478	479	inline fn disableAutoVectorization(x: anytype) void {

lib/std/http/Client.zig+16-10

...	...	@@ -9,6 +9,7 @@ const net = std.net;
9	9	const Uri = std.Uri;
10	10	const Allocator = mem.Allocator;
11	11	const assert = std.debug.assert;
	12	const use_vectors = builtin.zig_backend != .stage2_x86_64;
12	13
13	14	const Client = @This();
14	15	const proto = @import("protocol.zig");
...	...	@@ -408,7 +409,7 @@ pub const Response = struct {
408	409	else => return error.HttpHeadersInvalid,
409	410	};
410	411	if (first_line[8] != ' ') return error.HttpHeadersInvalid;
411		const status = @as(http.Status, @enumFromInt(parseInt3(first_line[9..12].*)));
	412	const status: http.Status = @enumFromInt(parseInt3(first_line[9..12]));
412	413	const reason = mem.trimLeft(u8, first_line[12..], " ");
413	414
414	415	res.version = version;
...	...	@@ -481,20 +482,24 @@ pub const Response = struct {
481	482	}
482	483
483	484	inline fn int64(array: *const [8]u8) u64 {
484		return @as(u64, @bitCast(array.*));
	485	return @bitCast(array.*);
485	486	}
486	487
487		fn parseInt3(nnn: @Vector(3, u8)) u10 {
488		const zero: @Vector(3, u8) = .{ '0', '0', '0' };
489		const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
490		return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
	488	fn parseInt3(text: *const [3]u8) u10 {
	489	if (use_vectors) {
	490	const nnn: @Vector(3, u8) = text.*;
	491	const zero: @Vector(3, u8) = .{ '0', '0', '0' };
	492	const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
	493	return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
	494	}
	495	return std.fmt.parseInt(u10, text, 10) catch unreachable;
491	496	}
492	497
493	498	test parseInt3 {
494	499	const expectEqual = testing.expectEqual;
495		try expectEqual(@as(u10, 0), parseInt3("000".*));
496		try expectEqual(@as(u10, 418), parseInt3("418".*));
497		try expectEqual(@as(u10, 999), parseInt3("999".*));
	500	try expectEqual(@as(u10, 0), parseInt3("000"));
	501	try expectEqual(@as(u10, 418), parseInt3("418"));
	502	try expectEqual(@as(u10, 999), parseInt3("999"));
498	503	}
499	504
500	505	version: http.Version,
...	...	@@ -1588,7 +1593,8 @@ test {
1588	1593
1589	1594	if (builtin.os.tag == .wasi) return error.SkipZigTest;
1590	1595
1591		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	1596	if (builtin.zig_backend == .stage2_x86_64 and
	1597	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .avx)) return error.SkipZigTest;
1592	1598
1593	1599	std.testing.refAllDecls(@This());
1594	1600	}

lib/std/http/Server.zig-2

...	...	@@ -736,8 +736,6 @@ test "HTTP server handles a chunked transfer coding request" {
736	736	return error.SkipZigTest;
737	737	}
738	738
739		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
740
741	739	const native_endian = comptime builtin.cpu.arch.endian();
742	740	if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
743	741	// https://github.com/ziglang/zig/issues/13782

lib/std/http/protocol.zig+23-12

...	...	@@ -1,8 +1,10 @@
1	1	const std = @import("../std.zig");
	2	const builtin = @import("builtin");
2	3	const testing = std.testing;
3	4	const mem = std.mem;
4	5
5	6	const assert = std.debug.assert;
	7	const use_vectors = builtin.zig_backend != .stage2_x86_64;
6	8
7	9	pub const State = enum {
8	10	/// Begin header parsing states.
...	...	@@ -83,7 +85,7 @@ pub const HeadersParser = struct {
83	85	/// first byte of content is located at `bytes[result]`.
84	86	pub fn findHeadersEnd(r: *HeadersParser, bytes: []const u8) u32 {
85	87	const vector_len: comptime_int = @max(std.simd.suggestVectorSize(u8) orelse 1, 8);
86		const len = @as(u32, @intCast(bytes.len));
	88	const len: u32 = @intCast(bytes.len);
87	89	var index: u32 = 0;
88	90
89	91	while (true) {
...	...	@@ -175,18 +177,27 @@ pub const HeadersParser = struct {
175	177	continue;
176	178	},
177	179	else => {
178		const Vector = @Vector(vector_len, u8);
179		// const BoolVector = @Vector(vector_len, bool);
180		const BitVector = @Vector(vector_len, u1);
181		const SizeVector = @Vector(vector_len, u8);
182
183	180	const chunk = bytes[index..][0..vector_len];
184		const v: Vector = chunk.*;
185		const matches_r = @as(BitVector, @bitCast(v == @as(Vector, @splat('\r'))));
186		const matches_n = @as(BitVector, @bitCast(v == @as(Vector, @splat('\n'))));
187		const matches_or: SizeVector = matches_r \| matches_n;
188
189		const matches = @reduce(.Add, matches_or);
	181	const matches = if (use_vectors) matches: {
	182	const Vector = @Vector(vector_len, u8);
	183	// const BoolVector = @Vector(vector_len, bool);
	184	const BitVector = @Vector(vector_len, u1);
	185	const SizeVector = @Vector(vector_len, u8);
	186
	187	const v: Vector = chunk.*;
	188	const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));
	189	const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
	190	const matches_or: SizeVector = matches_r \| matches_n;
	191
	192	break :matches @reduce(.Add, matches_or);
	193	} else matches: {
	194	var matches: u8 = 0;
	195	for (chunk) \|byte\| switch (byte) {
	196	'\r', '\n' => matches += 1,
	197	else => {},
	198	};
	199	break :matches matches;
	200	};
190	201	switch (matches) {
191	202	0 => {},
192	203	1 => switch (chunk[vector_len - 1]) {

lib/std/math.zig+2-21

...	...	@@ -492,8 +492,6 @@ pub fn shl(comptime T: type, a: T, shift_amt: anytype) T {
492	492	}
493	493
494	494	test "shl" {
495		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
496
497	495	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
498	496	// https://github.com/ziglang/zig/issues/12012
499	497	return error.SkipZigTest;
...	...	@@ -539,8 +537,6 @@ pub fn shr(comptime T: type, a: T, shift_amt: anytype) T {
539	537	}
540	538
541	539	test "shr" {
542		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
543
544	540	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
545	541	// https://github.com/ziglang/zig/issues/12012
546	542	return error.SkipZigTest;
...	...	@@ -587,8 +583,6 @@ pub fn rotr(comptime T: type, x: T, r: anytype) T {
587	583	}
588	584
589	585	test "rotr" {
590		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
591
592	586	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
593	587	// https://github.com/ziglang/zig/issues/12012
594	588	return error.SkipZigTest;
...	...	@@ -634,8 +628,6 @@ pub fn rotl(comptime T: type, x: T, r: anytype) T {
634	628	}
635	629
636	630	test "rotl" {
637		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
638
639	631	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
640	632	// https://github.com/ziglang/zig/issues/12012
641	633	return error.SkipZigTest;
...	...	@@ -764,8 +756,6 @@ pub fn divTrunc(comptime T: type, numerator: T, denominator: T) !T {
764	756	}
765	757
766	758	test "divTrunc" {
767		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
768
769	759	try testDivTrunc();
770	760	try comptime testDivTrunc();
771	761	}
...	...	@@ -790,8 +780,6 @@ pub fn divFloor(comptime T: type, numerator: T, denominator: T) !T {
790	780	}
791	781
792	782	test "divFloor" {
793		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
794
795	783	try testDivFloor();
796	784	try comptime testDivFloor();
797	785	}
...	...	@@ -829,8 +817,6 @@ pub fn divCeil(comptime T: type, numerator: T, denominator: T) !T {
829	817	}
830	818
831	819	test "divCeil" {
832		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
833
834	820	try testDivCeil();
835	821	try comptime testDivCeil();
836	822	}
...	...	@@ -875,8 +861,6 @@ pub fn divExact(comptime T: type, numerator: T, denominator: T) !T {
875	861	}
876	862
877	863	test "divExact" {
878		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
879
880	864	try testDivExact();
881	865	try comptime testDivExact();
882	866	}
...	...	@@ -903,8 +887,6 @@ pub fn mod(comptime T: type, numerator: T, denominator: T) !T {
903	887	}
904	888
905	889	test "mod" {
906		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
907
908	890	try testMod();
909	891	try comptime testMod();
910	892	}
...	...	@@ -931,8 +913,6 @@ pub fn rem(comptime T: type, numerator: T, denominator: T) !T {
931	913	}
932	914
933	915	test "rem" {
934		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
935
936	916	try testRem();
937	917	try comptime testRem();
938	918	}
...	...	@@ -1285,7 +1265,8 @@ pub fn lerp(a: anytype, b: anytype, t: anytype) @TypeOf(a, b, t) {
1285	1265	}
1286	1266
1287	1267	test "lerp" {
1288		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	1268	if (builtin.zig_backend == .stage2_x86_64 and
	1269	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .fma)) return error.SkipZigTest;
1289	1270
1290	1271	try testing.expectEqual(@as(f64, 75), lerp(50, 100, 0.5));
1291	1272	try testing.expectEqual(@as(f32, 43.75), lerp(50, 25, 0.25));

lib/std/math/big/int.zig+3-3

...	...	@@ -1318,7 +1318,7 @@ pub const Mutable = struct {
1318	1318	///
1319	1319	/// `limbs_buffer` is used for temporary storage.
1320	1320	/// The amount required is given by `calcPowLimbsBufferLen`.
1321		pub fn pow(r: *Mutable, a: Const, b: u32, limbs_buffer: []Limb) !void {
	1321	pub fn pow(r: *Mutable, a: Const, b: u32, limbs_buffer: []Limb) void {
1322	1322	assert(r.limbs.ptr != a.limbs.ptr); // illegal aliasing
1323	1323
1324	1324	// Handle all the trivial cases first
...	...	@@ -3213,7 +3213,7 @@ pub const Managed = struct {
3213	3213	var m = try Managed.initCapacity(rma.allocator, needed_limbs);
3214	3214	errdefer m.deinit();
3215	3215	var m_mut = m.toMutable();
3216		try m_mut.pow(a.toConst(), b, limbs_buffer);
	3216	m_mut.pow(a.toConst(), b, limbs_buffer);
3217	3217	m.setMetadata(m_mut.positive, m_mut.len);
3218	3218
3219	3219	rma.deinit();
...	...	@@ -3221,7 +3221,7 @@ pub const Managed = struct {
3221	3221	} else {
3222	3222	try rma.ensureCapacity(needed_limbs);
3223	3223	var rma_mut = rma.toMutable();
3224		try rma_mut.pow(a.toConst(), b, limbs_buffer);
	3224	rma_mut.pow(a.toConst(), b, limbs_buffer);
3225	3225	rma.setMetadata(rma_mut.positive, rma_mut.len);
3226	3226	}
3227	3227	}

lib/std/math/big/int_test.zig-6

...	...	@@ -2568,8 +2568,6 @@ test "big.int const to managed" {
2568	2568	}
2569	2569
2570	2570	test "big.int pow" {
2571		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2572
2573	2571	{
2574	2572	var a = try Managed.initSet(testing.allocator, -3);
2575	2573	defer a.deinit();
...	...	@@ -2763,8 +2761,6 @@ fn popCountTest(val: *const Managed, bit_count: usize, expected: usize) !void {
2763	2761	}
2764	2762
2765	2763	test "big int conversion read/write twos complement" {
2766		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2767
2768	2764	var a = try Managed.initSet(testing.allocator, (1 << 493) - 1);
2769	2765	defer a.deinit();
2770	2766	var b = try Managed.initSet(testing.allocator, (1 << 493) - 1);
...	...	@@ -2863,8 +2859,6 @@ test "big int write twos complement +/- zero" {
2863	2859	}
2864	2860
2865	2861	test "big int conversion write twos complement with padding" {
2866		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2867
2868	2862	var a = try Managed.initSet(testing.allocator, 0x01_ffffffff_ffffffff_ffffffff);
2869	2863	defer a.deinit();
2870	2864

lib/std/mem.zig-4

...	...	@@ -315,8 +315,6 @@ pub fn zeroes(comptime T: type) T {
315	315	}
316	316
317	317	test "zeroes" {
318		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
319
320	318	const C_struct = extern struct {
321	319	x: u32,
322	320	y: u32 align(128),
...	...	@@ -4342,8 +4340,6 @@ pub fn alignInSlice(slice: anytype, comptime new_alignment: usize) ?AlignedSlice
4342	4340	}
4343	4341
4344	4342	test "read/write(Var)PackedInt" {
4345		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
4346
4347	4343	switch (builtin.cpu.arch) {
4348	4344	// This test generates too much code to execute on WASI.
4349	4345	// LLVM backend fails with "too many locals: locals exceed maximum"

lib/std/net/test.zig+1-1

...	...	@@ -60,7 +60,7 @@ test "parse and render IPv6 addresses" {
60	60	}
61	61
62	62	test "invalid but parseable IPv6 scope ids" {
63		if (builtin.os.tag != .linux or comptime !builtin.os.tag.isDarwin()) {
	63	if (builtin.os.tag != .linux and comptime !builtin.os.tag.isDarwin()) {
64	64	// Currently, resolveIp6 with alphanumerical scope IDs only works on Linux.
65	65	// TODO Make this test pass on other operating systems.
66	66	return error.SkipZigTest;

lib/std/once.zig-2

...	...	@@ -46,8 +46,6 @@ fn incr() void {
46	46	}
47	47
48	48	test "Once executes its function just once" {
49		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
50
51	49	if (builtin.single_threaded) {
52	50	global_once.call();
53	51	global_once.call();

lib/std/os/test.zig-4

...	...	@@ -375,8 +375,6 @@ fn testThreadIdFn(thread_id: *Thread.Id) void {
375	375	test "std.Thread.getCurrentId" {
376	376	if (builtin.single_threaded) return error.SkipZigTest;
377	377
378		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
379
380	378	var thread_current_id: Thread.Id = undefined;
381	379	const thread = try Thread.spawn(.{}, testThreadIdFn, .{&thread_current_id});
382	380	thread.join();
...	...	@@ -420,8 +418,6 @@ test "cpu count" {
420	418	test "thread local storage" {
421	419	if (builtin.single_threaded) return error.SkipZigTest;
422	420
423		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
424
425	421	const thread1 = try Thread.spawn(.{}, testTls, .{});
426	422	const thread2 = try Thread.spawn(.{}, testTls, .{});
427	423	try testTls();

lib/std/rand/test.zig-7

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("../std.zig");
2		const builtin = @import("builtin");
3	2	const math = std.math;
4	3	const DefaultPrng = std.rand.DefaultPrng;
5	4	const Random = std.rand.Random;
...	...	@@ -200,8 +199,6 @@ fn testRandomIntLessThan() !void {
200	199	}
201	200
202	201	test "Random intAtMost" {
203		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
204
205	202	@setEvalBranchQuota(10000);
206	203	try testRandomIntAtMost();
207	204	try comptime testRandomIntAtMost();
...	...	@@ -242,8 +239,6 @@ fn testRandomIntAtMost() !void {
242	239	}
243	240
244	241	test "Random Biased" {
245		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
246
247	242	var prng = DefaultPrng.init(0);
248	243	const random = prng.random();
249	244	// Not thoroughly checking the logic here.
...	...	@@ -452,8 +447,6 @@ test "CSPRNG" {
452	447	}
453	448
454	449	test "Random weightedIndex" {
455		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
456
457	450	// Make sure weightedIndex works for various integers and floats
458	451	inline for (.{ u64, i4, f32, f64 }) \|T\| {
459	452	var prng = DefaultPrng.init(0);

lib/std/zig/tokenizer.zig-9

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("../std.zig");
2		const builtin = @import("builtin");
3	2
4	3	pub const Token = struct {
5	4	tag: Tag,
...	...	@@ -1450,8 +1449,6 @@ test "chars" {
1450	1449	}
1451	1450
1452	1451	test "invalid token characters" {
1453		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1454
1455	1452	try testTokenize("#", &.{.invalid});
1456	1453	try testTokenize("`", &.{.invalid});
1457	1454	try testTokenize("'c", &.{.invalid});
...	...	@@ -1571,8 +1568,6 @@ test "pipe and then invalid" {
1571	1568	}
1572	1569
1573	1570	test "line comment and doc comment" {
1574		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1575
1576	1571	try testTokenize("//", &.{});
1577	1572	try testTokenize("// a / b", &.{});
1578	1573	try testTokenize("// /", &.{});
...	...	@@ -1647,8 +1642,6 @@ test "range literals" {
1647	1642	}
1648	1643
1649	1644	test "number literals decimal" {
1650		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1651
1652	1645	try testTokenize("0", &.{.number_literal});
1653	1646	try testTokenize("1", &.{.number_literal});
1654	1647	try testTokenize("2", &.{.number_literal});
...	...	@@ -1897,8 +1890,6 @@ test "invalid token with unfinished escape right before eof" {
1897	1890	}
1898	1891
1899	1892	test "saturating operators" {
1900		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1901
1902	1893	try testTokenize("<<", &.{.angle_bracket_angle_bracket_left});
1903	1894	try testTokenize("<<\|", &.{.angle_bracket_angle_bracket_left_pipe});
1904	1895	try testTokenize("<<\|=", &.{.angle_bracket_angle_bracket_left_pipe_equal});

src/Compilation.zig+3-1

...	...	@@ -1121,7 +1121,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1121	1121	const include_compiler_rt = options.want_compiler_rt orelse needs_c_symbols;
1122	1122
1123	1123	const must_single_thread = target_util.isSingleThreaded(options.target);
1124		const single_threaded = options.single_threaded orelse must_single_thread;
	1124	const single_threaded = options.single_threaded orelse must_single_thread or
	1125	// x86_64 codegen doesn't support TLV for most object formats
	1126	(!use_llvm and options.target.cpu.arch == .x86_64 and options.target.ofmt != .macho);
1125	1127	if (must_single_thread and !single_threaded) {
1126	1128	return error.TargetRequiresSingleThreaded;
1127	1129	}

src/arch/x86_64/CodeGen.zig+1317-641

...	...	@@ -388,7 +388,7 @@ pub const MCValue = union(enum) {
388	388	};
389	389	}
390	390
391		fn mem(mcv: MCValue, size: Memory.Size) Memory {
	391	fn mem(mcv: MCValue, function: *Self, size: Memory.Size) !Memory {
392	392	return switch (mcv) {
393	393	.none,
394	394	.unreach,
...	...	@@ -409,7 +409,6 @@ pub const MCValue = union(enum) {
409	409	.lea_frame,
410	410	.reserved_frame,
411	411	.air_ref,
412		.load_symbol,
413	412	.lea_symbol,
414	413	=> unreachable,
415	414	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\| .{
...	...	@@ -433,6 +432,19 @@ pub const MCValue = union(enum) {
433	432	.disp = frame_addr.off,
434	433	} },
435	434	},
	435	.load_symbol => \|sym_off\| {
	436	assert(sym_off.off == 0);
	437	return .{
	438	.base = .{ .reloc = .{
	439	.atom_index = try function.owner.getSymbolIndex(function),
	440	.sym_index = sym_off.sym,
	441	} },
	442	.mod = .{ .rm = .{
	443	.size = size,
	444	.disp = sym_off.off,
	445	} },
	446	};
	447	},
436	448	};
437	449	}
438	450
...	...	@@ -722,12 +734,14 @@ const InstTracking = struct {
722	734
723	735	const FrameAlloc = struct {
724	736	abi_size: u31,
	737	spill_pad: u3,
725	738	abi_align: Alignment,
726	739	ref_count: u16,
727	740
728		fn init(alloc_abi: struct { size: u64, alignment: Alignment }) FrameAlloc {
	741	fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc {
729	742	return .{
730	743	.abi_size = @intCast(alloc_abi.size),
	744	.spill_pad = alloc_abi.pad,
731	745	.abi_align = alloc_abi.alignment,
732	746	.ref_count = 0,
733	747	};
...	...	@@ -738,6 +752,20 @@ const FrameAlloc = struct {
738	752	.alignment = ty.abiAlignment(mod),
739	753	});
740	754	}
	755	fn initSpill(ty: Type, mod: *Module) FrameAlloc {
	756	const abi_size = ty.abiSize(mod);
	757	const spill_size = if (abi_size < 8)
	758	math.ceilPowerOfTwoAssert(u64, abi_size)
	759	else
	760	std.mem.alignForward(u64, abi_size, 8);
	761	return init(.{
	762	.size = spill_size,
	763	.pad = @intCast(spill_size - abi_size),
	764	.alignment = ty.abiAlignment(mod).maxStrict(
	765	Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
	766	),
	767	});
	768	}
741	769	};
742	770
743	771	const StackAllocation = struct {
...	...	@@ -1668,8 +1696,7 @@ fn gen(self: *Self) InnerError!void {
1668	1696	// The address where to store the return value for the caller is in a
1669	1697	// register which the callee is free to clobber. Therefore, we purposely
1670	1698	// spill it to stack immediately.
1671		const frame_index =
1672		try self.allocFrameIndex(FrameAlloc.initType(Type.usize, mod));
	1699	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(Type.usize, mod));
1673	1700	try self.genSetMem(
1674	1701	.{ .frame = frame_index },
1675	1702	0,
...	...	@@ -2434,7 +2461,7 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
2434	2461	}
2435	2462	}
2436	2463
2437		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ty, mod));
	2464	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, mod));
2438	2465	return .{ .load_frame = .{ .index = frame_index } };
2439	2466	}
2440	2467
...	...	@@ -2445,7 +2472,10 @@ fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {
2445	2472	80 => abi.RegisterClass.x87,
2446	2473	else => abi.RegisterClass.sse,
2447	2474	},
2448		.Vector => abi.RegisterClass.sse,
	2475	.Vector => switch (ty.childType(mod).toIntern()) {
	2476	.bool_type => abi.RegisterClass.gp,
	2477	else => abi.RegisterClass.sse,
	2478	},
2449	2479	else => abi.RegisterClass.gp,
2450	2480	};
2451	2481	}
...	...	@@ -2699,7 +2729,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2699	2729	.{ .v_ss, .cvtsd2 },
2700	2730	dst_reg,
2701	2731	dst_reg,
2702		src_mcv.mem(.qword),
	2732	try src_mcv.mem(self, .qword),
2703	2733	) else try self.asmRegisterRegisterRegister(
2704	2734	.{ .v_ss, .cvtsd2 },
2705	2735	dst_reg,
...	...	@@ -2711,7 +2741,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2711	2741	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2712	2742	.{ ._ss, .cvtsd2 },
2713	2743	dst_reg,
2714		src_mcv.mem(.qword),
	2744	try src_mcv.mem(self, .qword),
2715	2745	) else try self.asmRegisterRegister(
2716	2746	.{ ._ss, .cvtsd2 },
2717	2747	dst_reg,
...	...	@@ -2798,7 +2828,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2798	2828	.{ .v_sd, .cvtss2 },
2799	2829	dst_reg,
2800	2830	dst_reg,
2801		src_mcv.mem(.dword),
	2831	try src_mcv.mem(self, .dword),
2802	2832	) else try self.asmRegisterRegisterRegister(
2803	2833	.{ .v_sd, .cvtss2 },
2804	2834	dst_reg,
...	...	@@ -2810,7 +2840,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2810	2840	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2811	2841	.{ ._sd, .cvtss2 },
2812	2842	dst_reg,
2813		src_mcv.mem(.dword),
	2843	try src_mcv.mem(self, .dword),
2814	2844	) else try self.asmRegisterRegister(
2815	2845	.{ ._sd, .cvtss2 },
2816	2846	dst_reg,
...	...	@@ -2851,8 +2881,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2851	2881	};
2852	2882
2853	2883	const dst_mcv = if (dst_int_info.bits <= src_storage_bits and
2854		std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
2855		std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and
	2884	math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
	2885	math.divCeil(u32, src_storage_bits, 64) catch unreachable and
2856	2886	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
2857	2887	const dst_mcv = try self.allocRegOrMem(inst, true);
2858	2888	try self.genCopy(min_ty, dst_mcv, src_mcv);
...	...	@@ -2869,22 +2899,28 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2869	2899	break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) };
2870	2900	}
2871	2901
2872		const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
2873		const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;
	2902	const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
	2903	const dst_limbs_len = math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;
2874	2904
2875		const high_mcv = dst_mcv.address().offset((src_limbs_len - 1) * 8).deref();
2876		const high_reg = try self.copyToTmpRegister(switch (src_int_info.signedness) {
2877		.signed => Type.isize,
2878		.unsigned => Type.usize,
2879		}, high_mcv);
	2905	const high_mcv: MCValue = if (dst_mcv.isMemory())
	2906	dst_mcv.address().offset((src_limbs_len - 1) * 8).deref()
	2907	else
	2908	.{ .register = dst_mcv.register_pair[1] };
	2909	const high_reg = if (high_mcv.isRegister())
	2910	high_mcv.getReg().?
	2911	else
	2912	try self.copyToTmpRegister(switch (src_int_info.signedness) {
	2913	.signed => Type.isize,
	2914	.unsigned => Type.usize,
	2915	}, high_mcv);
2880	2916	const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);
2881	2917	defer self.register_manager.unlockReg(high_lock);
2882	2918
2883	2919	const high_bits = src_int_info.bits % 64;
2884	2920	if (high_bits > 0) {
2885		const high_ty = try mod.intType(extend, high_bits);
2886		try self.truncateRegister(high_ty, high_reg);
2887		try self.genCopy(Type.usize, high_mcv, .{ .register = high_reg });
	2921	try self.truncateRegister(src_ty, high_reg);
	2922	const high_ty = if (dst_int_info.bits >= 64) Type.usize else dst_ty;
	2923	try self.genCopy(high_ty, high_mcv, .{ .register = high_reg });
2888	2924	}
2889	2925
2890	2926	if (dst_limbs_len > src_limbs_len) try self.genInlineMemset(
...	...	@@ -2995,14 +3031,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
2995	3031	.{ .vp_, .@"and" },
2996	3032	dst_reg,
2997	3033	dst_reg,
2998		splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),
	3034	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
2999	3035	);
3000	3036	try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);
3001	3037	} else {
3002	3038	try self.asmRegisterMemory(
3003	3039	.{ .p_, .@"and" },
3004	3040	dst_reg,
3005		splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),
	3041	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
3006	3042	);
3007	3043	try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);
3008	3044	}
...	...	@@ -3048,7 +3084,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
3048	3084	const len = try self.resolveInst(bin_op.rhs);
3049	3085	const len_ty = self.typeOf(bin_op.rhs);
3050	3086
3051		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, mod));
	3087	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod));
3052	3088	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
3053	3089	try self.genSetMem(
3054	3090	.{ .frame = frame_index },
...	...	@@ -3068,8 +3104,36 @@ fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
3068	3104	}
3069	3105
3070	3106	fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
	3107	const mod = self.bin_file.options.module.?;
3071	3108	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
3072	3109	const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
	3110
	3111	const dst_ty = self.typeOfIndex(inst);
	3112	if (dst_ty.isAbiInt(mod)) {
	3113	const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
	3114	const bit_size: u32 = @intCast(dst_ty.bitSize(mod));
	3115	if (abi_size * 8 > bit_size) {
	3116	const dst_lock = switch (dst_mcv) {
	3117	.register => \|dst_reg\| self.register_manager.lockRegAssumeUnused(dst_reg),
	3118	else => null,
	3119	};
	3120	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	3121
	3122	if (dst_mcv.isRegister()) {
	3123	try self.truncateRegister(dst_ty, dst_mcv.getReg().?);
	3124	} else {
	3125	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	3126	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	3127	defer self.register_manager.unlockReg(tmp_lock);
	3128
	3129	const hi_ty = try mod.intType(.unsigned, @intCast((dst_ty.bitSize(mod) - 1) % 64 + 1));
	3130	const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
	3131	try self.genSetReg(tmp_reg, hi_ty, hi_mcv);
	3132	try self.truncateRegister(dst_ty, tmp_reg);
	3133	try self.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg });
	3134	}
	3135	}
	3136	}
3073	3137	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
3074	3138	}
3075	3139
...	...	@@ -3176,7 +3240,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3176	3240	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3177	3241	.{ ._, .mov },
3178	3242	tmp_reg,
3179		mat_lhs_mcv.address().offset(8).deref().mem(.qword),
	3243	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3180	3244	) else try self.asmRegisterRegister(
3181	3245	.{ ._, .mov },
3182	3246	tmp_reg,
...	...	@@ -3200,7 +3264,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3200	3264	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3201	3265	.{ ._, .xor },
3202	3266	tmp_reg,
3203		mat_rhs_mcv.address().offset(8).deref().mem(.qword),
	3267	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3204	3268	) else try self.asmRegisterRegister(
3205	3269	.{ ._, .xor },
3206	3270	tmp_reg,
...	...	@@ -3300,12 +3364,12 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3300	3364	try self.asmRegisterMemory(
3301	3365	.{ ._, .add },
3302	3366	tmp_regs[0],
3303		mat_rhs_mcv.mem(.qword),
	3367	try mat_rhs_mcv.mem(self, .qword),
3304	3368	);
3305	3369	try self.asmRegisterMemory(
3306	3370	.{ ._, .adc },
3307	3371	tmp_regs[1],
3308		mat_rhs_mcv.address().offset(8).deref().mem(.qword),
	3372	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3309	3373	);
3310	3374	} else for (
3311	3375	[_]Mir.Inst.Tag{ .add, .adc },
...	...	@@ -3534,7 +3598,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3534	3598	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3535	3599	.{ ._, .mov },
3536	3600	tmp_reg,
3537		mat_lhs_mcv.address().offset(8).deref().mem(.qword),
	3601	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3538	3602	) else try self.asmRegisterRegister(
3539	3603	.{ ._, .mov },
3540	3604	tmp_reg,
...	...	@@ -3558,7 +3622,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3558	3622	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3559	3623	.{ ._, .xor },
3560	3624	tmp_reg,
3561		mat_rhs_mcv.address().offset(8).deref().mem(.qword),
	3625	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3562	3626	) else try self.asmRegisterRegister(
3563	3627	.{ ._, .xor },
3564	3628	tmp_reg,
...	...	@@ -3567,7 +3631,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3567	3631
3568	3632	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));
3569	3633	try self.asmRegister(.{ ._, .not }, tmp_reg);
3570		try self.asmMemoryImmediate(.{ ._, .cmp }, overflow.mem(.dword), Immediate.s(0));
	3634	try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), Immediate.s(0));
3571	3635	try self.freeValue(overflow);
3572	3636	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
3573	3637	try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63));
...	...	@@ -3665,7 +3729,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3665	3729	}
3666	3730
3667	3731	const frame_index =
3668		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	3732	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3669	3733	try self.genSetMem(
3670	3734	.{ .frame = frame_index },
3671	3735	@intCast(tuple_ty.structFieldOffset(1, mod)),
...	...	@@ -3682,7 +3746,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3682	3746	}
3683	3747
3684	3748	const frame_index =
3685		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	3749	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3686	3750	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
3687	3751	break :result .{ .load_frame = .{ .index = frame_index } };
3688	3752	},
...	...	@@ -3738,7 +3802,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3738	3802	}
3739	3803
3740	3804	const frame_index =
3741		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	3805	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3742	3806	try self.genSetMem(
3743	3807	.{ .frame = frame_index },
3744	3808	@intCast(tuple_ty.structFieldOffset(1, mod)),
...	...	@@ -3755,7 +3819,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3755	3819	}
3756	3820
3757	3821	const frame_index =
3758		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	3822	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3759	3823	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
3760	3824	break :result .{ .load_frame = .{ .index = frame_index } };
3761	3825	},
...	...	@@ -3874,7 +3938,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3874	3938	);
3875	3939	try self.asmMemoryImmediate(
3876	3940	.{ ._, .cmp },
3877		overflow.mem(self.memSize(Type.c_int)),
	3941	try overflow.mem(self, self.memSize(Type.c_int)),
3878	3942	Immediate.s(0),
3879	3943	);
3880	3944	try self.genSetMem(
...	...	@@ -3926,14 +3990,19 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3926	3990	};
3927	3991	defer if (mat_rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3928	3992
3929		if (mat_lhs_mcv.isMemory())
3930		try self.asmRegisterMemory(.{ ._, .mov }, .rax, mat_lhs_mcv.mem(.qword))
3931		else
3932		try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);
	3993	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
	3994	.{ ._, .mov },
	3995	.rax,
	3996	try mat_lhs_mcv.mem(self, .qword),
	3997	) else try self.asmRegisterRegister(
	3998	.{ ._, .mov },
	3999	.rax,
	4000	mat_lhs_mcv.register_pair[0],
	4001	);
3933	4002	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3934	4003	.{ ._, .mov },
3935	4004	tmp_regs[0],
3936		mat_rhs_mcv.address().offset(8).deref().mem(.qword),
	4005	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3937	4006	) else try self.asmRegisterRegister(
3938	4007	.{ ._, .mov },
3939	4008	tmp_regs[0],
...	...	@@ -3944,7 +4013,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3944	4013	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);
3945	4014	try self.asmSetccRegister(.o, tmp_regs[2].to8());
3946	4015	if (mat_rhs_mcv.isMemory())
3947		try self.asmMemory(.{ ._, .mul }, mat_rhs_mcv.mem(.qword))
	4016	try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
3948	4017	else
3949	4018	try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
3950	4019	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
...	...	@@ -3953,7 +4022,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3953	4022	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3954	4023	.{ ._, .mov },
3955	4024	tmp_regs[0],
3956		mat_lhs_mcv.address().offset(8).deref().mem(.qword),
	4025	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3957	4026	) else try self.asmRegisterRegister(
3958	4027	.{ ._, .mov },
3959	4028	tmp_regs[0],
...	...	@@ -3967,14 +4036,15 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3967	4036	tmp_regs[3].to8(),
3968	4037	);
3969	4038	try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
3970		if (mat_rhs_mcv.isMemory())
3971		try self.asmRegisterMemory(.{ .i_, .mul }, tmp_regs[0], mat_rhs_mcv.mem(.qword))
3972		else
3973		try self.asmRegisterRegister(
3974		.{ .i_, .mul },
3975		tmp_regs[0],
3976		mat_rhs_mcv.register_pair[0],
3977		);
	4039	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
	4040	.{ .i_, .mul },
	4041	tmp_regs[0],
	4042	try mat_rhs_mcv.mem(self, .qword),
	4043	) else try self.asmRegisterRegister(
	4044	.{ .i_, .mul },
	4045	tmp_regs[0],
	4046	mat_rhs_mcv.register_pair[0],
	4047	);
3978	4048	try self.asmSetccRegister(.o, tmp_regs[2].to8());
3979	4049	try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
3980	4050	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
...	...	@@ -4020,8 +4090,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4020	4090	self.eflags_inst = inst;
4021	4091	break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
4022	4092	} else {
4023		const frame_index =
4024		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	4093	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
4025	4094	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
4026	4095	break :result .{ .load_frame = .{ .index = frame_index } };
4027	4096	},
...	...	@@ -4032,8 +4101,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4032	4101	src_ty.fmt(mod), dst_ty.fmt(mod),
4033	4102	});
4034	4103
4035		const frame_index =
4036		try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
	4104	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
4037	4105	if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
4038	4106	try self.genSetMem(
4039	4107	.{ .frame = frame_index },
...	...	@@ -4106,7 +4174,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
4106	4174	.register => \|reg\| try self.asmRegister(tag, registerAlias(reg, abi_size)),
4107	4175	.memory, .indirect, .load_frame => try self.asmMemory(
4108	4176	tag,
4109		mat_rhs.mem(Memory.Size.fromSize(abi_size)),
	4177	try mat_rhs.mem(self, Memory.Size.fromSize(abi_size)),
4110	4178	),
4111	4179	else => unreachable,
4112	4180	}
...	...	@@ -4160,8 +4228,8 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
4160	4228	);
4161	4229	try self.asmCmovccRegisterRegister(
4162	4230	.z,
4163		registerAlias(divisor, abi_size),
4164		registerAlias(.rdx, abi_size),
	4231	registerAlias(divisor, @max(abi_size, 2)),
	4232	registerAlias(.rdx, @max(abi_size, 2)),
4165	4233	);
4166	4234	try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });
4167	4235	return MCValue{ .register = divisor };
...	...	@@ -4171,47 +4239,268 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
4171	4239	const mod = self.bin_file.options.module.?;
4172	4240	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
4173	4241
4174		try self.spillRegisters(&.{.rcx});
4175
4176		const tag = self.air.instructions.items(.tag)[inst];
4177		try self.register_manager.getReg(.rcx, null);
4178		const lhs = try self.resolveInst(bin_op.lhs);
4179		const rhs = try self.resolveInst(bin_op.rhs);
	4242	const air_tags = self.air.instructions.items(.tag);
	4243	const tag = air_tags[inst];
4180	4244	const lhs_ty = self.typeOf(bin_op.lhs);
4181	4245	const rhs_ty = self.typeOf(bin_op.rhs);
	4246	const result: MCValue = result: {
	4247	switch (lhs_ty.zigTypeTag(mod)) {
	4248	.Int => {
	4249	try self.spillRegisters(&.{.rcx});
	4250	try self.register_manager.getReg(.rcx, null);
	4251	const lhs_mcv = try self.resolveInst(bin_op.lhs);
	4252	const rhs_mcv = try self.resolveInst(bin_op.rhs);
4182	4253
4183		const dst_mcv = try self.genShiftBinOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
4184		switch (tag) {
4185		.shr, .shr_exact, .shl_exact => {},
4186		.shl => switch (dst_mcv) {
4187		.register => \|dst_reg\| try self.truncateRegister(lhs_ty, dst_reg),
4188		.register_pair => \|dst_regs\| try self.truncateRegister(lhs_ty, dst_regs[1]),
4189		.load_frame => \|frame_addr\| {
4190		const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
4191		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
4192		defer self.register_manager.unlockReg(tmp_lock);
	4254	const dst_mcv = try self.genShiftBinOp(tag, inst, lhs_mcv, rhs_mcv, lhs_ty, rhs_ty);
	4255	switch (tag) {
	4256	.shr, .shr_exact, .shl_exact => {},
	4257	.shl => switch (dst_mcv) {
	4258	.register => \|dst_reg\| try self.truncateRegister(lhs_ty, dst_reg),
	4259	.register_pair => \|dst_regs\| try self.truncateRegister(lhs_ty, dst_regs[1]),
	4260	.load_frame => \|frame_addr\| {
	4261	const tmp_reg =
	4262	try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	4263	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	4264	defer self.register_manager.unlockReg(tmp_lock);
4193	4265
4194		const lhs_bits: u31 = @intCast(lhs_ty.bitSize(mod));
4195		const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;
4196		const off = frame_addr.off + lhs_bits / 64 * 8;
4197		try self.genSetReg(
4198		tmp_reg,
4199		tmp_ty,
4200		.{ .load_frame = .{ .index = frame_addr.index, .off = off } },
4201		);
4202		try self.truncateRegister(lhs_ty, tmp_reg);
4203		try self.genSetMem(
4204		.{ .frame = frame_addr.index },
4205		off,
4206		tmp_ty,
4207		.{ .register = tmp_reg },
4208		);
	4266	const lhs_bits: u31 = @intCast(lhs_ty.bitSize(mod));
	4267	const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;
	4268	const off = frame_addr.off + (lhs_bits - 1) / 64 * 8;
	4269	try self.genSetReg(
	4270	tmp_reg,
	4271	tmp_ty,
	4272	.{ .load_frame = .{ .index = frame_addr.index, .off = off } },
	4273	);
	4274	try self.truncateRegister(lhs_ty, tmp_reg);
	4275	try self.genSetMem(
	4276	.{ .frame = frame_addr.index },
	4277	off,
	4278	tmp_ty,
	4279	.{ .register = tmp_reg },
	4280	);
	4281	},
	4282	else => {},
	4283	},
	4284	else => unreachable,
	4285	}
	4286	break :result dst_mcv;
	4287	},
	4288	.Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
	4289	.Int => if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.childType(mod).intInfo(mod).bits) {
	4290	else => null,
	4291	16 => switch (lhs_ty.vectorLen(mod)) {
	4292	else => null,
	4293	1...8 => switch (tag) {
	4294	else => unreachable,
	4295	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4296	.signed => if (self.hasFeature(.avx))
	4297	.{ .vp_w, .sra }
	4298	else
	4299	.{ .p_w, .sra },
	4300	.unsigned => if (self.hasFeature(.avx))
	4301	.{ .vp_w, .srl }
	4302	else
	4303	.{ .p_w, .srl },
	4304	},
	4305	.shl, .shl_exact => if (self.hasFeature(.avx))
	4306	.{ .vp_w, .sll }
	4307	else
	4308	.{ .p_w, .sll },
	4309	},
	4310	9...16 => switch (tag) {
	4311	else => unreachable,
	4312	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4313	.signed => if (self.hasFeature(.avx2)) .{ .vp_w, .sra } else null,
	4314	.unsigned => if (self.hasFeature(.avx2)) .{ .vp_w, .srl } else null,
	4315	},
	4316	.shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_w, .sll } else null,
	4317	},
	4318	},
	4319	32 => switch (lhs_ty.vectorLen(mod)) {
	4320	else => null,
	4321	1...4 => switch (tag) {
	4322	else => unreachable,
	4323	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4324	.signed => if (self.hasFeature(.avx))
	4325	.{ .vp_d, .sra }
	4326	else
	4327	.{ .p_d, .sra },
	4328	.unsigned => if (self.hasFeature(.avx))
	4329	.{ .vp_d, .srl }
	4330	else
	4331	.{ .p_d, .srl },
	4332	},
	4333	.shl, .shl_exact => if (self.hasFeature(.avx))
	4334	.{ .vp_d, .sll }
	4335	else
	4336	.{ .p_d, .sll },
	4337	},
	4338	5...8 => switch (tag) {
	4339	else => unreachable,
	4340	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4341	.signed => if (self.hasFeature(.avx2)) .{ .vp_d, .sra } else null,
	4342	.unsigned => if (self.hasFeature(.avx2)) .{ .vp_d, .srl } else null,
	4343	},
	4344	.shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_d, .sll } else null,
	4345	},
	4346	},
	4347	64 => switch (lhs_ty.vectorLen(mod)) {
	4348	else => null,
	4349	1...2 => switch (tag) {
	4350	else => unreachable,
	4351	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4352	.signed => if (self.hasFeature(.avx))
	4353	.{ .vp_q, .sra }
	4354	else
	4355	.{ .p_q, .sra },
	4356	.unsigned => if (self.hasFeature(.avx))
	4357	.{ .vp_q, .srl }
	4358	else
	4359	.{ .p_q, .srl },
	4360	},
	4361	.shl, .shl_exact => if (self.hasFeature(.avx))
	4362	.{ .vp_q, .sll }
	4363	else
	4364	.{ .p_q, .sll },
	4365	},
	4366	3...4 => switch (tag) {
	4367	else => unreachable,
	4368	.shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
	4369	.signed => if (self.hasFeature(.avx2)) .{ .vp_q, .sra } else null,
	4370	.unsigned => if (self.hasFeature(.avx2)) .{ .vp_q, .srl } else null,
	4371	},
	4372	.shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_q, .sll } else null,
	4373	},
	4374	},
	4375	})) \|mir_tag\| if (try self.air.value(bin_op.rhs, mod)) \|rhs_val\| {
	4376	switch (mod.intern_pool.indexToKey(rhs_val.toIntern())) {
	4377	.aggregate => \|rhs_aggregate\| switch (rhs_aggregate.storage) {
	4378	.repeated_elem => \|rhs_elem\| {
	4379	const abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
	4380
	4381	const lhs_mcv = try self.resolveInst(bin_op.lhs);
	4382	const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and
	4383	self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
	4384	.{lhs_mcv.getReg().?} ** 2
	4385	else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{
	4386	try self.register_manager.allocReg(inst, abi.RegisterClass.sse),
	4387	lhs_mcv.getReg().?,
	4388	} else .{(try self.copyToRegisterWithInstTracking(
	4389	inst,
	4390	lhs_ty,
	4391	lhs_mcv,
	4392	)).register} ** 2;
	4393	const reg_locks =
	4394	self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg });
	4395	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\|
	4396	self.register_manager.unlockReg(lock);
	4397
	4398	const shift_imm =
	4399	Immediate.u(@intCast(rhs_elem.toValue().toUnsignedInt(mod)));
	4400	if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate(
	4401	mir_tag,
	4402	registerAlias(dst_reg, abi_size),
	4403	registerAlias(lhs_reg, abi_size),
	4404	shift_imm,
	4405	) else {
	4406	assert(dst_reg.id() == lhs_reg.id());
	4407	try self.asmRegisterImmediate(
	4408	mir_tag,
	4409	registerAlias(dst_reg, abi_size),
	4410	shift_imm,
	4411	);
	4412	}
	4413	break :result .{ .register = dst_reg };
	4414	},
	4415	else => {},
	4416	},
	4417	else => {},
	4418	}
	4419	} else if (Air.refToIndex(bin_op.rhs)) \|rhs_inst\| switch (air_tags[rhs_inst]) {
	4420	.splat => {
	4421	const abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
	4422
	4423	const lhs_mcv = try self.resolveInst(bin_op.lhs);
	4424	const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and
	4425	self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
	4426	.{lhs_mcv.getReg().?} ** 2
	4427	else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{
	4428	try self.register_manager.allocReg(inst, abi.RegisterClass.sse),
	4429	lhs_mcv.getReg().?,
	4430	} else .{(try self.copyToRegisterWithInstTracking(
	4431	inst,
	4432	lhs_ty,
	4433	lhs_mcv,
	4434	)).register} ** 2;
	4435	const reg_locks = self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg });
	4436	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\|
	4437	self.register_manager.unlockReg(lock);
	4438
	4439	const shift_reg =
	4440	try self.copyToTmpRegister(rhs_ty, .{ .air_ref = bin_op.rhs });
	4441	const shift_lock = self.register_manager.lockRegAssumeUnused(shift_reg);
	4442	defer self.register_manager.unlockReg(shift_lock);
	4443
	4444	const mask_ty = try mod.vectorType(.{ .len = 16, .child = .u8_type });
	4445	const mask_mcv = try self.genTypedValue(.{
	4446	.ty = mask_ty,
	4447	.val = (try mod.intern(.{ .aggregate = .{
	4448	.ty = mask_ty.toIntern(),
	4449	.storage = .{ .elems = &([1]InternPool.Index{
	4450	(try rhs_ty.childType(mod).maxIntScalar(mod, Type.u8)).toIntern(),
	4451	} ++ [1]InternPool.Index{
	4452	(try mod.intValue(Type.u8, 0)).toIntern(),
	4453	} ** 15) },
	4454	} })).toValue(),
	4455	});
	4456	const mask_addr_reg =
	4457	try self.copyToTmpRegister(Type.usize, mask_mcv.address());
	4458	const mask_addr_lock = self.register_manager.lockRegAssumeUnused(mask_addr_reg);
	4459	defer self.register_manager.unlockReg(mask_addr_lock);
	4460
	4461	if (self.hasFeature(.avx)) {
	4462	try self.asmRegisterRegisterMemory(
	4463	.{ .vp_, .@"and" },
	4464	shift_reg.to128(),
	4465	shift_reg.to128(),
	4466	.{
	4467	.base = .{ .reg = mask_addr_reg },
	4468	.mod = .{ .rm = .{ .size = .xword } },
	4469	},
	4470	);
	4471	try self.asmRegisterRegisterRegister(
	4472	mir_tag,
	4473	registerAlias(dst_reg, abi_size),
	4474	registerAlias(lhs_reg, abi_size),
	4475	shift_reg.to128(),
	4476	);
	4477	} else {
	4478	try self.asmRegisterMemory(
	4479	.{ .p_, .@"and" },
	4480	shift_reg.to128(),
	4481	.{
	4482	.base = .{ .reg = mask_addr_reg },
	4483	.mod = .{ .rm = .{ .size = .xword } },
	4484	},
	4485	);
	4486	assert(dst_reg.id() == lhs_reg.id());
	4487	try self.asmRegisterRegister(
	4488	mir_tag,
	4489	registerAlias(dst_reg, abi_size),
	4490	shift_reg.to128(),
	4491	);
	4492	}
	4493	break :result .{ .register = dst_reg };
	4494	},
	4495	else => {},
	4496	},
	4497	else => {},
4209	4498	},
4210	4499	else => {},
4211		},
4212		else => unreachable,
4213		}
4214		return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
	4500	}
	4501	return self.fail("TODO implement airShlShrBinOp for {}", .{lhs_ty.fmt(mod)});
	4502	};
	4503	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4215	4504	}
4216	4505
4217	4506	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -4230,12 +4519,18 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
4230	4519
4231	4520	const opt_mcv = try self.resolveInst(ty_op.operand);
4232	4521	if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {
4233		switch (opt_mcv) {
4234		.register => \|reg\| try self.truncateRegister(pl_ty, reg),
4235		.register_overflow => \|ro\| try self.truncateRegister(pl_ty, ro.reg),
	4522	const pl_mcv: MCValue = switch (opt_mcv) {
	4523	.register_overflow => \|ro\| pl: {
	4524	self.eflags_inst = null; // actually stop tracking the overflow part
	4525	break :pl .{ .register = ro.reg };
	4526	},
	4527	else => opt_mcv,
	4528	};
	4529	switch (pl_mcv) {
	4530	.register => \|pl_reg\| try self.truncateRegister(pl_ty, pl_reg),
4236	4531	else => {},
4237	4532	}
4238		break :result opt_mcv;
	4533	break :result pl_mcv;
4239	4534	}
4240	4535
4241	4536	const pl_mcv = try self.allocRegOrMem(inst, true);
...	...	@@ -4472,8 +4767,9 @@ fn genUnwrapErrUnionPayloadMir(
4472	4767	const eu_lock = self.register_manager.lockReg(reg);
4473	4768	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
4474	4769
4475		const result_mcv: MCValue = if (maybe_inst) \|inst\|
4476		try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)
	4770	const payload_in_gp = self.regClassForType(payload_ty).supersetOf(abi.RegisterClass.gp);
	4771	const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)
	4772	try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)
4477	4773	else
4478	4774	.{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
4479	4775	if (payload_off > 0) try self.genShiftBinOpMir(
...	...	@@ -4482,7 +4778,12 @@ fn genUnwrapErrUnionPayloadMir(
4482	4778	result_mcv,
4483	4779	.{ .immediate = @as(u6, @intCast(payload_off * 8)) },
4484	4780	) else try self.truncateRegister(payload_ty, result_mcv.register);
4485		break :result result_mcv;
	4781	break :result if (payload_in_gp)
	4782	result_mcv
	4783	else if (maybe_inst) \|inst\|
	4784	try self.copyToRegisterWithInstTracking(inst, payload_ty, result_mcv)
	4785	else
	4786	.{ .register = try self.copyToTmpRegister(payload_ty, result_mcv) };
4486	4787	},
4487	4788	else => return self.fail("TODO implement genUnwrapErrUnionPayloadMir for {}", .{err_union}),
4488	4789	}
...	...	@@ -4593,7 +4894,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
4593	4894	const result: MCValue = result: {
4594	4895	if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .{ .immediate = 0 };
4595	4896
4596		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, mod));
	4897	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, mod));
4597	4898	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
4598	4899	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
4599	4900	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
...	...	@@ -4615,7 +4916,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
4615	4916	const result: MCValue = result: {
4616	4917	if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result try self.resolveInst(ty_op.operand);
4617	4918
4618		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, mod));
	4919	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, mod));
4619	4920	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
4620	4921	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
4621	4922	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);
...	...	@@ -4770,14 +5071,19 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
4770	5071	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
4771	5072	const mod = self.bin_file.options.module.?;
4772	5073	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
4773		const slice_ty = self.typeOf(bin_op.lhs);
4774	5074
4775		const slice_ptr_field_type = slice_ty.slicePtrFieldType(mod);
4776		const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
4777		const dst_mcv = try self.allocRegOrMem(inst, false);
4778		try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
	5075	const result: MCValue = result: {
	5076	const elem_ty = self.typeOfIndex(inst);
	5077	if (!elem_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .none;
4779	5078
4780		return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
	5079	const slice_ty = self.typeOf(bin_op.lhs);
	5080	const slice_ptr_field_type = slice_ty.slicePtrFieldType(mod);
	5081	const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
	5082	const dst_mcv = try self.allocRegOrMem(inst, false);
	5083	try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
	5084	break :result dst_mcv;
	5085	};
	5086	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4781	5087	}
4782	5088
4783	5089	fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -4810,11 +5116,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
4810	5116	};
4811	5117	defer if (index_lock) \|lock\| self.register_manager.unlockReg(lock);
4812	5118
4813		const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
4814		const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
4815		defer self.register_manager.unlockReg(offset_reg_lock);
4816
4817	5119	const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	5120	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	5121	defer self.register_manager.unlockReg(addr_lock);
	5122
4818	5123	switch (array) {
4819	5124	.register => {
4820	5125	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod));
...	...	@@ -4843,6 +5148,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
4843	5148	else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
4844	5149	}
4845	5150
	5151	const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
	5152	const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
	5153	defer self.register_manager.unlockReg(offset_lock);
	5154
4846	5155	// TODO we could allocate register here, but need to expect addr register and potentially
4847	5156	// offset register.
4848	5157	try self.spillEflagsIfOccupied();
...	...	@@ -5093,7 +5402,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
5093	5402	.{ ._, .sub },
5094	5403	dst_ty,
5095	5404	dst_mcv,
5096		.{ .immediate = 8 + self.regExtraBits(src_ty) },
	5405	.{ .immediate = 32 - src_bits },
5097	5406	);
5098	5407	} else if (src_bits <= 64) {
5099	5408	try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);
...	...	@@ -5361,7 +5670,9 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
5361	5670	mat_src_mcv
5362	5671	else
5363	5672	.{ .register = mat_src_mcv.register_pair[0] }, false);
5364		try self.genPopCount(tmp_regs[1], Type.usize, if (mat_src_mcv.isMemory())
	5673	const src_info = src_ty.intInfo(mod);
	5674	const hi_ty = try mod.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1);
	5675	try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isMemory())
5365	5676	mat_src_mcv.address().offset(8).deref()
5366	5677	else
5367	5678	.{ .register = mat_src_mcv.register_pair[1] }, false);
...	...	@@ -5383,9 +5694,13 @@ fn genPopCount(
5383	5694	const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
5384	5695	if (self.hasFeature(.popcnt)) return self.genBinOpMir(
5385	5696	.{ ._, .popcnt },
5386		if (src_abi_size > 1) src_ty else Type.u16,
	5697	if (src_abi_size > 1) src_ty else Type.u32,
5387	5698	.{ .register = dst_reg },
5388		src_mcv,
	5699	if (src_abi_size > 1) src_mcv else src: {
	5700	if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv);
	5701	try self.truncateRegister(try src_ty.toUnsigned(mod), dst_reg);
	5702	break :src .{ .register = dst_reg };
	5703	},
5389	5704	);
5390	5705
5391	5706	const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
...	...	@@ -5517,9 +5832,9 @@ fn genByteSwap(
5517	5832	try self.asmRegisterMemory(
5518	5833	.{ ._, .movbe },
5519	5834	dst_regs[0],
5520		src_mcv.address().offset(8).deref().mem(.qword),
	5835	try src_mcv.address().offset(8).deref().mem(self, .qword),
5521	5836	);
5522		try self.asmRegisterMemory(.{ ._, .movbe }, dst_regs[1], src_mcv.mem(.qword));
	5837	try self.asmRegisterMemory(.{ ._, .movbe }, dst_regs[1], try src_mcv.mem(self, .qword));
5523	5838	} else for (dst_regs, src_mcv.register_pair) \|dst_reg, src_reg\| {
5524	5839	try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to64(), src_reg.to64());
5525	5840	try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
...	...	@@ -5762,7 +6077,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
5762	6077	else => unreachable,
5763	6078	} });
5764	6079	const sign_mem: Memory = if (sign_mcv.isMemory())
5765		sign_mcv.mem(Memory.Size.fromSize(abi_size))
	6080	try sign_mcv.mem(self, Memory.Size.fromSize(abi_size))
5766	6081	else
5767	6082	.{
5768	6083	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
...	...	@@ -5945,7 +6260,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5945	6260	mir_tag,
5946	6261	dst_alias,
5947	6262	dst_alias,
5948		src_mcv.mem(Memory.Size.fromSize(abi_size)),
	6263	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
5949	6264	Immediate.u(@as(u5, @bitCast(mode))),
5950	6265	) else try self.asmRegisterRegisterRegisterImmediate(
5951	6266	mir_tag,
...	...	@@ -5960,7 +6275,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5960	6275	else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
5961	6276	mir_tag,
5962	6277	dst_alias,
5963		src_mcv.mem(Memory.Size.fromSize(abi_size)),
	6278	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
5964	6279	Immediate.u(@as(u5, @bitCast(mode))),
5965	6280	) else try self.asmRegisterRegisterImmediate(
5966	6281	mir_tag,
...	...	@@ -6000,7 +6315,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6000	6315	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
6001	6316	.l,
6002	6317	registerAlias(dst_mcv.register, cmov_abi_size),
6003		src_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),
	6318	try src_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
6004	6319	),
6005	6320	else => {
6006	6321	const val_reg = try self.copyToTmpRegister(ty, src_mcv);
...	...	@@ -6100,7 +6415,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6100	6415	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6101	6416	mir_tag,
6102	6417	dst_alias,
6103		src_mcv.mem(self.memSize(ty)),
	6418	try src_mcv.mem(self, self.memSize(ty)),
6104	6419	) else try self.asmRegisterRegister(
6105	6420	mir_tag,
6106	6421	dst_alias,
...	...	@@ -6206,7 +6521,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6206	6521	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6207	6522	.{ .v_ps, .cvtph2 },
6208	6523	wide_reg,
6209		src_mcv.mem(Memory.Size.fromSize(
	6524	try src_mcv.mem(self, Memory.Size.fromSize(
6210	6525	@intCast(@divExact(wide_reg.bitSize(), 16)),
6211	6526	)),
6212	6527	) else try self.asmRegisterRegister(
...	...	@@ -6254,7 +6569,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6254	6569	mir_tag,
6255	6570	dst_reg,
6256	6571	dst_reg,
6257		src_mcv.mem(Memory.Size.fromSize(abi_size)),
	6572	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
6258	6573	) else try self.asmRegisterRegisterRegister(
6259	6574	mir_tag,
6260	6575	dst_reg,
...	...	@@ -6267,7 +6582,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6267	6582	else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
6268	6583	mir_tag,
6269	6584	dst_reg,
6270		src_mcv.mem(Memory.Size.fromSize(abi_size)),
	6585	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
6271	6586	) else try self.asmRegisterRegister(
6272	6587	mir_tag,
6273	6588	dst_reg,
...	...	@@ -6332,7 +6647,7 @@ fn reuseOperandAdvanced(
6332	6647	return false;
6333	6648
6334	6649	switch (mcv) {
6335		.register, .register_pair => for (mcv.getRegs()) \|reg\| {
	6650	.register, .register_pair, .register_overflow => for (mcv.getRegs()) \|reg\| {
6336	6651	// If it's in the registers table, need to associate the register(s) with the
6337	6652	// new instruction.
6338	6653	if (maybe_tracked_inst) \|tracked_inst\| {
...	...	@@ -6346,6 +6661,10 @@ fn reuseOperandAdvanced(
6346	6661	.load_frame => \|frame_addr\| if (frame_addr.index.isNamed()) return false,
6347	6662	else => return false,
6348	6663	}
	6664	switch (mcv) {
	6665	.eflags, .register_overflow => self.eflags_inst = maybe_tracked_inst,
	6666	else => {},
	6667	}
6349	6668
6350	6669	// Prevent the operand deaths processing code from deallocating it.
6351	6670	self.liveness.clearOperandDeath(inst, op_index);
...	...	@@ -6363,11 +6682,36 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6363	6682	if (!val_ty.hasRuntimeBitsIgnoreComptime(mod)) return;
6364	6683	const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
6365	6684
	6685	if (ptr_info.packed_offset.bit_offset % 8 == 0) {
	6686	try self.load(
	6687	dst_mcv,
	6688	ptr_ty,
	6689	ptr_mcv.offset(@intCast(@divExact(ptr_info.packed_offset.bit_offset, 8))),
	6690	);
	6691	const val_bit_size: u32 = @intCast(val_ty.bitSize(mod));
	6692	if (val_abi_size * 8 > val_bit_size) {
	6693	if (dst_mcv.isRegister()) {
	6694	try self.truncateRegister(val_ty, dst_mcv.getReg().?);
	6695	} else {
	6696	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	6697	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	6698	defer self.register_manager.unlockReg(tmp_lock);
	6699
	6700	const hi_mcv = dst_mcv.address().offset(@intCast(val_bit_size / 64 * 8)).deref();
	6701	try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
	6702	try self.truncateRegister(val_ty, tmp_reg);
	6703	try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
	6704	}
	6705	}
	6706	return;
	6707	}
	6708
6366	6709	if (val_abi_size > 8) return self.fail("TODO implement packed load of {}", .{val_ty.fmt(mod)});
6367	6710
6368	6711	const limb_abi_size: u32 = @min(val_abi_size, 8);
6369	6712	const limb_abi_bits = limb_abi_size * 8;
6370		const val_byte_off: i32 = @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size);
	6713	const val_byte_off: i32 =
	6714	@intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size);
6371	6715	const val_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits;
6372	6716	const val_extra_bits = self.regExtraBits(val_ty);
6373	6717
...	...	@@ -6530,7 +6874,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
6530	6874	.base = .{ .reg = ptr_reg },
6531	6875	.mod = .{ .rm = .{
6532	6876	.size = Memory.Size.fromSize(limb_abi_size),
6533		.disp = src_byte_off + limb_i * limb_abi_bits,
	6877	.disp = src_byte_off + limb_i * limb_abi_size,
6534	6878	} },
6535	6879	};
6536	6880
...	...	@@ -6575,6 +6919,22 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
6575	6919	limb_mem,
6576	6920	registerAlias(tmp_reg, limb_abi_size),
6577	6921	);
	6922	} else if (src_bit_size <= 128 and src_bit_off == 0) {
	6923	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	6924	const tmp_mcv = MCValue{ .register = tmp_reg };
	6925	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	6926	defer self.register_manager.unlockReg(tmp_lock);
	6927
	6928	try self.genSetReg(tmp_reg, limb_ty, switch (limb_i) {
	6929	0 => src_mcv,
	6930	else => src_mcv.address().offset(limb_i * limb_abi_size).deref(),
	6931	});
	6932	try self.genBinOpMir(.{ ._, .@"and" }, limb_ty, tmp_mcv, .{ .immediate = part_mask });
	6933	try self.asmMemoryRegister(
	6934	.{ ._, .@"or" },
	6935	limb_mem,
	6936	registerAlias(tmp_reg, limb_abi_size),
	6937	);
6578	6938	} else return self.fail("TODO: implement packed store of {}", .{src_ty.fmt(mod)});
6579	6939	}
6580	6940	}
...	...	@@ -6808,17 +7168,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6808	7168	.register_overflow => \|ro\| {
6809	7169	switch (index) {
6810	7170	// Get wrapped value for overflow operation.
6811		0 => break :result if (self.liveness.operandDies(inst, 0))
6812		.{ .register = ro.reg }
6813		else
6814		try self.copyToRegisterWithInstTracking(
6815		inst,
6816		Type.usize,
6817		.{ .register = ro.reg },
6818		),
	7171	0 => if (self.reuseOperand(inst, extra.struct_operand, 0, src_mcv)) {
	7172	self.eflags_inst = null; // actually stop tracking the overflow part
	7173	break :result .{ .register = ro.reg };
	7174	} else break :result try self.copyToRegisterWithInstTracking(
	7175	inst,
	7176	Type.usize,
	7177	.{ .register = ro.reg },
	7178	),
6819	7179	// Get overflow bit.
6820		1 => if (self.liveness.operandDies(inst, 0)) {
6821		self.eflags_inst = inst;
	7180	1 => if (self.reuseOperandAdvanced(inst, extra.struct_operand, 0, src_mcv, null)) {
	7181	self.eflags_inst = inst; // actually keep tracking the overflow part
6822	7182	break :result .{ .eflags = ro.eflags };
6823	7183	} else {
6824	7184	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
...	...	@@ -6833,11 +7193,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6833	7193	if (field_off % 8 == 0) {
6834	7194	const off_mcv =
6835	7195	src_mcv.address().offset(@intCast(@divExact(field_off, 8))).deref();
	7196	const field_bit_size = field_ty.bitSize(mod);
6836	7197
6837	7198	if (field_abi_size <= 8) {
6838	7199	const int_ty = try mod.intType(
6839	7200	if (field_ty.isAbiInt(mod)) field_ty.intInfo(mod).signedness else .unsigned,
6840		@intCast(field_ty.bitSize(mod)),
	7201	@intCast(field_bit_size),
6841	7202	);
6842	7203
6843	7204	const dst_reg = try self.register_manager.allocReg(
...	...	@@ -6856,10 +7217,24 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6856	7217	try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
6857	7218	}
6858	7219
6859		if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result off_mcv;
6860
6861		const dst_mcv = try self.allocRegOrMem(inst, true);
6862		try self.genCopy(field_ty, dst_mcv, off_mcv);
	7220	const dst_mcv = if (self.reuseOperand(inst, operand, 0, src_mcv))
	7221	off_mcv
	7222	else dst: {
	7223	const dst_mcv = try self.allocRegOrMem(inst, true);
	7224	try self.genCopy(field_ty, dst_mcv, off_mcv);
	7225	break :dst dst_mcv;
	7226	};
	7227	if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) {
	7228	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	7229	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	7230	defer self.register_manager.unlockReg(tmp_lock);
	7231
	7232	const hi_mcv =
	7233	dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();
	7234	try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
	7235	try self.truncateRegister(field_ty, tmp_reg);
	7236	try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
	7237	}
6863	7238	break :result dst_mcv;
6864	7239	}
6865	7240
...	...	@@ -7013,7 +7388,25 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
7013	7388	} else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
7014	7389	}
7015	7390	},
7016		.neg => try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv),
	7391	.neg => {
	7392	try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv);
	7393	const abi_size: u16 = @intCast(src_ty.abiSize(mod));
	7394	const bit_size = src_ty.intInfo(mod).bits;
	7395	if (abi_size * 8 > bit_size) {
	7396	if (dst_mcv.isRegister()) {
	7397	try self.truncateRegister(src_ty, dst_mcv.getReg().?);
	7398	} else {
	7399	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	7400	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	7401	defer self.register_manager.unlockReg(tmp_lock);
	7402
	7403	const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
	7404	try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
	7405	try self.truncateRegister(src_ty, tmp_reg);
	7406	try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
	7407	}
	7408	}
	7409	},
7017	7410	else => unreachable,
7018	7411	}
7019	7412	return dst_mcv;
...	...	@@ -7054,7 +7447,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
7054	7447	},
7055	7448	.indirect, .load_frame => try self.asmMemory(
7056	7449	mir_tag,
7057		dst_mcv.mem(Memory.Size.fromSize(abi_size)),
	7450	try dst_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7058	7451	),
7059	7452	}
7060	7453	}
...	...	@@ -7552,27 +7945,27 @@ fn genMulDivBinOp(
7552	7945	defer self.register_manager.unlockReg(tmp_lock);
7553	7946
7554	7947	if (mat_lhs_mcv.isMemory())
7555		try self.asmRegisterMemory(.{ ._, .mov }, .rax, mat_lhs_mcv.mem(.qword))
	7948	try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .qword))
7556	7949	else
7557	7950	try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);
7558	7951	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
7559	7952	.{ ._, .mov },
7560	7953	tmp_reg,
7561		mat_rhs_mcv.address().offset(8).deref().mem(.qword),
	7954	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
7562	7955	) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);
7563	7956	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);
7564	7957	if (mat_rhs_mcv.isMemory())
7565		try self.asmMemory(.{ ._, .mul }, mat_rhs_mcv.mem(.qword))
	7958	try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
7566	7959	else
7567	7960	try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
7568	7961	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
7569	7962	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
7570	7963	.{ ._, .mov },
7571	7964	tmp_reg,
7572		mat_lhs_mcv.address().offset(8).deref().mem(.qword),
	7965	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
7573	7966	) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);
7574	7967	if (mat_rhs_mcv.isMemory())
7575		try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.mem(.qword))
	7968	try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .qword))
7576	7969	else
7577	7970	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);
7578	7971	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
...	...	@@ -7833,7 +8226,7 @@ fn genBinOp(
7833	8226	.{ .vp_w, .insr },
7834	8227	dst_reg,
7835	8228	dst_reg,
7836		rhs_mcv.mem(.word),
	8229	try rhs_mcv.mem(self, .word),
7837	8230	Immediate.u(1),
7838	8231	) else try self.asmRegisterRegisterRegister(
7839	8232	.{ .vp_, .unpcklwd },
...	...	@@ -7858,7 +8251,7 @@ fn genBinOp(
7858	8251	mir_tag,
7859	8252	dst_reg,
7860	8253	dst_reg,
7861		src_mcv.mem(Memory.Size.fromBitSize(float_bits)),
	8254	try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
7862	8255	) else try self.asmRegisterRegisterRegister(
7863	8256	mir_tag,
7864	8257	dst_reg,
...	...	@@ -7877,7 +8270,7 @@ fn genBinOp(
7877	8270	if (src_mcv.isMemory()) try self.asmRegisterMemory(
7878	8271	mir_tag,
7879	8272	dst_reg,
7880		src_mcv.mem(Memory.Size.fromBitSize(float_bits)),
	8273	try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
7881	8274	) else try self.asmRegisterRegister(
7882	8275	mir_tag,
7883	8276	dst_reg,
...	...	@@ -7919,12 +8312,18 @@ fn genBinOp(
7919	8312	};
7920	8313	}
7921	8314
7922		if ((lhs_ty.scalarType(mod).isRuntimeFloat() and
	8315	const sse_op = switch (lhs_ty.zigTypeTag(mod)) {
	8316	else => false,
	8317	.Float => true,
	8318	.Vector => switch (lhs_ty.childType(mod).toIntern()) {
	8319	.bool_type => false,
	8320	else => true,
	8321	},
	8322	};
	8323	if (sse_op and ((lhs_ty.scalarType(mod).isRuntimeFloat() and
7923	8324	lhs_ty.scalarType(mod).floatBits(self.target.*) == 80) or
7924		lhs_ty.abiSize(mod) > @as(u6, if (self.hasFeature(.avx)) 32 else 16))
7925		return self.fail("TODO implement genBinOp for {s} {}", .{
7926		@tagName(air_tag), lhs_ty.fmt(mod),
7927		});
	8325	lhs_ty.abiSize(mod) > @as(u6, if (self.hasFeature(.avx)) 32 else 16)))
	8326	return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(air_tag), lhs_ty.fmt(mod) });
7928	8327
7929	8328	const maybe_mask_reg = switch (air_tag) {
7930	8329	else => null,
...	...	@@ -7941,10 +8340,16 @@ fn genBinOp(
7941	8340	if (maybe_mask_reg) \|mask_reg\| self.register_manager.lockRegAssumeUnused(mask_reg) else null;
7942	8341	defer if (mask_lock) \|lock\| self.register_manager.unlockReg(lock);
7943	8342
7944		const ordered_air = if (lhs_ty.isVector(mod) and lhs_ty.childType(mod).isAbiInt(mod) and
7945		switch (air_tag) {
7946		.cmp_lt, .cmp_gte => true,
7947		else => false,
	8343	const ordered_air = if (lhs_ty.isVector(mod) and switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
	8344	.Int => switch (air_tag) {
	8345	.cmp_lt, .cmp_gte => true,
	8346	else => false,
	8347	},
	8348	.Float => switch (air_tag) {
	8349	.cmp_gte, .cmp_gt => true,
	8350	else => false,
	8351	},
	8352	else => unreachable,
7948	8353	}) .{ .lhs = rhs_air, .rhs = lhs_air } else .{ .lhs = lhs_air, .rhs = rhs_air };
7949	8354
7950	8355	const lhs_mcv = try self.resolveInst(ordered_air.lhs);
...	...	@@ -7971,14 +8376,12 @@ fn genBinOp(
7971	8376	.xor,
7972	8377	.min,
7973	8378	.max,
	8379	.cmp_eq,
	8380	.cmp_neq,
7974	8381	=> true,
7975	8382
7976	8383	else => false,
7977	8384	};
7978		const vec_op = switch (lhs_ty.zigTypeTag(mod)) {
7979		else => false,
7980		.Float, .Vector => true,
7981		};
7982	8385
7983	8386	const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) {
7984	8387	.register => \|lhs_reg\| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null },
...	...	@@ -8000,23 +8403,23 @@ fn genBinOp(
8000	8403	var flipped = false;
8001	8404	var copied_to_dst = true;
8002	8405	const dst_mcv: MCValue = dst: {
	8406	const tracked_inst = switch (air_tag) {
	8407	else => maybe_inst,
	8408	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => null,
	8409	};
8003	8410	if (maybe_inst) \|inst\| {
8004		const tracked_inst = switch (air_tag) {
8005		else => inst,
8006		.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => null,
8007		};
8008		if ((!vec_op or lhs_mcv.isRegister()) and
	8411	if ((!sse_op or lhs_mcv.isRegister()) and
8009	8412	self.reuseOperandAdvanced(inst, ordered_air.lhs, 0, lhs_mcv, tracked_inst))
8010	8413	break :dst lhs_mcv;
8011		if (is_commutative and (!vec_op or rhs_mcv.isRegister()) and
	8414	if (is_commutative and (!sse_op or rhs_mcv.isRegister()) and
8012	8415	self.reuseOperandAdvanced(inst, ordered_air.rhs, 1, rhs_mcv, tracked_inst))
8013	8416	{
8014	8417	flipped = true;
8015	8418	break :dst rhs_mcv;
8016	8419	}
8017	8420	}
8018		const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
8019		if (vec_op and lhs_mcv.isRegister() and self.hasFeature(.avx))
	8421	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, tracked_inst, true);
	8422	if (sse_op and lhs_mcv.isRegister() and self.hasFeature(.avx))
8020	8423	copied_to_dst = false
8021	8424	else
8022	8425	try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
...	...	@@ -8046,7 +8449,7 @@ fn genBinOp(
8046	8449	};
8047	8450	defer for (src_locks) \|src_lock\| if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
8048	8451
8049		if (!vec_op) {
	8452	if (!sse_op) {
8050	8453	switch (air_tag) {
8051	8454	.add,
8052	8455	.add_wrap,
...	...	@@ -8130,17 +8533,25 @@ fn genBinOp(
8130	8533
8131	8534	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
8132	8535	if (src_mcv.isMemory()) {
8133		try self.asmRegisterMemory(.{ ._, .cmp }, dst_regs[0], src_mcv.mem(.qword));
	8536	try self.asmRegisterMemory(
	8537	.{ ._, .cmp },
	8538	dst_regs[0],
	8539	try src_mcv.mem(self, .qword),
	8540	);
8134	8541	try self.asmRegisterMemory(
8135	8542	.{ ._, .sbb },
8136	8543	tmp_reg,
8137		src_mcv.address().offset(8).deref().mem(.qword),
	8544	try src_mcv.address().offset(8).deref().mem(self, .qword),
	8545	);
	8546	try self.asmCmovccRegisterMemory(
	8547	cc,
	8548	dst_regs[0],
	8549	try src_mcv.mem(self, .qword),
8138	8550	);
8139		try self.asmCmovccRegisterMemory(cc, dst_regs[0], src_mcv.mem(.qword));
8140	8551	try self.asmCmovccRegisterMemory(
8141	8552	cc,
8142	8553	dst_regs[1],
8143		src_mcv.address().offset(8).deref().mem(.qword),
	8554	try src_mcv.address().offset(8).deref().mem(self, .qword),
8144	8555	);
8145	8556	} else {
8146	8557	try self.asmRegisterRegister(
...	...	@@ -8292,7 +8703,7 @@ fn genBinOp(
8292	8703	.{ .vp_w, .insr },
8293	8704	dst_reg,
8294	8705	dst_reg,
8295		src_mcv.mem(.word),
	8706	try src_mcv.mem(self, .word),
8296	8707	Immediate.u(1),
8297	8708	) else try self.asmRegisterRegisterRegister(
8298	8709	.{ .vp_, .unpcklwd },
...	...	@@ -8738,7 +9149,7 @@ fn genBinOp(
8738	9149	.{ .vp_w, .insr },
8739	9150	dst_reg,
8740	9151	dst_reg,
8741		src_mcv.mem(.word),
	9152	try src_mcv.mem(self, .word),
8742	9153	Immediate.u(1),
8743	9154	) else try self.asmRegisterRegisterRegister(
8744	9155	.{ .vp_, .unpcklwd },
...	...	@@ -8784,7 +9195,7 @@ fn genBinOp(
8784	9195	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
8785	9196	.{ .vp_d, .insr },
8786	9197	dst_reg,
8787		src_mcv.mem(.dword),
	9198	try src_mcv.mem(self, .dword),
8788	9199	Immediate.u(1),
8789	9200	) else try self.asmRegisterRegisterRegister(
8790	9201	.{ .v_ps, .unpckl },
...	...	@@ -8836,7 +9247,7 @@ fn genBinOp(
8836	9247	if (src_mcv.isMemory()) try self.asmRegisterMemory(
8837	9248	.{ .v_ps, .cvtph2 },
8838	9249	tmp_reg,
8839		src_mcv.mem(.qword),
	9250	try src_mcv.mem(self, .qword),
8840	9251	) else try self.asmRegisterRegister(
8841	9252	.{ .v_ps, .cvtph2 },
8842	9253	tmp_reg,
...	...	@@ -8879,7 +9290,7 @@ fn genBinOp(
8879	9290	if (src_mcv.isMemory()) try self.asmRegisterMemory(
8880	9291	.{ .v_ps, .cvtph2 },
8881	9292	tmp_reg,
8882		src_mcv.mem(.xword),
	9293	try src_mcv.mem(self, .xword),
8883	9294	) else try self.asmRegisterRegister(
8884	9295	.{ .v_ps, .cvtph2 },
8885	9296	tmp_reg,
...	...	@@ -8925,6 +9336,13 @@ fn genBinOp(
8925	9336	=> if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
8926	9337	.max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
8927	9338	.min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
	9339	.cmp_lt,
	9340	.cmp_lte,
	9341	.cmp_eq,
	9342	.cmp_gte,
	9343	.cmp_gt,
	9344	.cmp_neq,
	9345	=> if (self.hasFeature(.avx)) .{ .v_ss, .cmp } else .{ ._ss, .cmp },
8928	9346	else => unreachable,
8929	9347	},
8930	9348	2...4 => switch (air_tag) {
...	...	@@ -8938,7 +9356,14 @@ fn genBinOp(
8938	9356	=> if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div },
8939	9357	.max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max },
8940	9358	.min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min },
8941		else => unreachable,
	9359	.cmp_lt,
	9360	.cmp_lte,
	9361	.cmp_eq,
	9362	.cmp_gte,
	9363	.cmp_gt,
	9364	.cmp_neq,
	9365	=> if (self.hasFeature(.avx)) .{ .v_ps, .cmp } else .{ ._ps, .cmp },
	9366	else => unreachable,
8942	9367	},
8943	9368	5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
8944	9369	.add => .{ .v_ps, .add },
...	...	@@ -8947,6 +9372,7 @@ fn genBinOp(
8947	9372	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
8948	9373	.max => .{ .v_ps, .max },
8949	9374	.min => .{ .v_ps, .min },
	9375	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_ps, .cmp },
8950	9376	else => unreachable,
8951	9377	} else null,
8952	9378	else => null,
...	...	@@ -8963,6 +9389,13 @@ fn genBinOp(
8963	9389	=> if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
8964	9390	.max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
8965	9391	.min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
	9392	.cmp_lt,
	9393	.cmp_lte,
	9394	.cmp_eq,
	9395	.cmp_gte,
	9396	.cmp_gt,
	9397	.cmp_neq,
	9398	=> if (self.hasFeature(.avx)) .{ .v_sd, .cmp } else .{ ._sd, .cmp },
8966	9399	else => unreachable,
8967	9400	},
8968	9401	2 => switch (air_tag) {
...	...	@@ -8976,6 +9409,13 @@ fn genBinOp(
8976	9409	=> if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div },
8977	9410	.max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max },
8978	9411	.min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min },
	9412	.cmp_lt,
	9413	.cmp_lte,
	9414	.cmp_eq,
	9415	.cmp_gte,
	9416	.cmp_gt,
	9417	.cmp_neq,
	9418	=> if (self.hasFeature(.avx)) .{ .v_pd, .cmp } else .{ ._pd, .cmp },
8979	9419	else => unreachable,
8980	9420	},
8981	9421	3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
...	...	@@ -8984,6 +9424,7 @@ fn genBinOp(
8984	9424	.mul => .{ .v_pd, .mul },
8985	9425	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div },
8986	9426	.max => .{ .v_pd, .max },
	9427	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_pd, .cmp },
8987	9428	.min => .{ .v_pd, .min },
8988	9429	else => unreachable,
8989	9430	} else null,
...	...	@@ -9004,43 +9445,96 @@ fn genBinOp(
9004	9445	const lhs_copy_lock = if (lhs_copy_reg) \|reg\| self.register_manager.lockReg(reg) else null;
9005	9446	defer if (lhs_copy_lock) \|lock\| self.register_manager.unlockReg(lock);
9006	9447
9007		if (self.hasFeature(.avx)) {
9008		const lhs_reg =
9009		if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
9010		if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
9011		mir_tag,
9012		dst_reg,
9013		lhs_reg,
9014		src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {
9015		else => Memory.Size.fromSize(abi_size),
9016		.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9017		}),
9018		) else try self.asmRegisterRegisterRegister(
9019		mir_tag,
9020		dst_reg,
9021		lhs_reg,
9022		registerAlias(if (src_mcv.isRegister())
9023		src_mcv.getReg().?
9024		else
9025		try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9026		);
9027		} else {
9028		assert(copied_to_dst);
9029		if (src_mcv.isMemory()) try self.asmRegisterMemory(
9030		mir_tag,
9031		dst_reg,
9032		src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {
9033		else => Memory.Size.fromSize(abi_size),
9034		.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9035		}),
9036		) else try self.asmRegisterRegister(
9037		mir_tag,
9038		dst_reg,
9039		registerAlias(if (src_mcv.isRegister())
9040		src_mcv.getReg().?
9041		else
9042		try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9043		);
	9448	switch (mir_tag[1]) {
	9449	else => if (self.hasFeature(.avx)) {
	9450	const lhs_reg =
	9451	if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
	9452	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
	9453	mir_tag,
	9454	dst_reg,
	9455	lhs_reg,
	9456	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
	9457	else => Memory.Size.fromSize(abi_size),
	9458	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
	9459	}),
	9460	) else try self.asmRegisterRegisterRegister(
	9461	mir_tag,
	9462	dst_reg,
	9463	lhs_reg,
	9464	registerAlias(if (src_mcv.isRegister())
	9465	src_mcv.getReg().?
	9466	else
	9467	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
	9468	);
	9469	} else {
	9470	assert(copied_to_dst);
	9471	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	9472	mir_tag,
	9473	dst_reg,
	9474	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
	9475	else => Memory.Size.fromSize(abi_size),
	9476	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
	9477	}),
	9478	) else try self.asmRegisterRegister(
	9479	mir_tag,
	9480	dst_reg,
	9481	registerAlias(if (src_mcv.isRegister())
	9482	src_mcv.getReg().?
	9483	else
	9484	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
	9485	);
	9486	},
	9487	.cmp => {
	9488	const imm = Immediate.u(switch (air_tag) {
	9489	.cmp_eq => 0,
	9490	.cmp_lt, .cmp_gt => 1,
	9491	.cmp_lte, .cmp_gte => 2,
	9492	.cmp_neq => 4,
	9493	else => unreachable,
	9494	});
	9495	if (self.hasFeature(.avx)) {
	9496	const lhs_reg =
	9497	if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
	9498	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
	9499	mir_tag,
	9500	dst_reg,
	9501	lhs_reg,
	9502	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
	9503	else => Memory.Size.fromSize(abi_size),
	9504	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
	9505	}),
	9506	imm,
	9507	) else try self.asmRegisterRegisterRegisterImmediate(
	9508	mir_tag,
	9509	dst_reg,
	9510	lhs_reg,
	9511	registerAlias(if (src_mcv.isRegister())
	9512	src_mcv.getReg().?
	9513	else
	9514	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
	9515	imm,
	9516	);
	9517	} else {
	9518	assert(copied_to_dst);
	9519	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
	9520	mir_tag,
	9521	dst_reg,
	9522	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
	9523	else => Memory.Size.fromSize(abi_size),
	9524	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
	9525	}),
	9526	imm,
	9527	) else try self.asmRegisterRegisterImmediate(
	9528	mir_tag,
	9529	dst_reg,
	9530	registerAlias(if (src_mcv.isRegister())
	9531	src_mcv.getReg().?
	9532	else
	9533	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
	9534	imm,
	9535	);
	9536	}
	9537	},
9044	9538	}
9045	9539
9046	9540	switch (air_tag) {
...	...	@@ -9281,48 +9775,46 @@ fn genBinOp(
9281	9775	);
9282	9776	}
9283	9777	},
9284		.cmp_lt,
9285		.cmp_lte,
9286		.cmp_eq,
9287		.cmp_gte,
9288		.cmp_gt,
9289		.cmp_neq,
9290		=> {
9291		switch (air_tag) {
9292		.cmp_lt,
9293		.cmp_eq,
9294		.cmp_gt,
9295		=> {},
9296		.cmp_lte,
9297		.cmp_gte,
9298		.cmp_neq,
9299		=> {
9300		const unsigned_ty = try lhs_ty.toUnsigned(mod);
9301		const not_mcv = try self.genTypedValue(.{
9302		.ty = lhs_ty,
9303		.val = try unsigned_ty.maxInt(mod, unsigned_ty),
9304		});
9305		const not_mem: Memory = if (not_mcv.isMemory())
9306		not_mcv.mem(Memory.Size.fromSize(abi_size))
9307		else
9308		.{ .base = .{
9309		.reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
9310		}, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };
9311		switch (mir_tag[0]) {
9312		.vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
9313		.{ .vp_, .xor },
9314		dst_reg,
9315		dst_reg,
9316		not_mem,
9317		),
9318		.p_b, .p_d, .p_q, .p_w => try self.asmRegisterMemory(
9319		.{ .p_, .xor },
9320		dst_reg,
9321		not_mem,
9322		),
9323		else => unreachable,
9324		}
	9778	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => {
	9779	switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
	9780	.Int => switch (air_tag) {
	9781	.cmp_lt,
	9782	.cmp_eq,
	9783	.cmp_gt,
	9784	=> {},
	9785	.cmp_lte,
	9786	.cmp_gte,
	9787	.cmp_neq,
	9788	=> {
	9789	const unsigned_ty = try lhs_ty.toUnsigned(mod);
	9790	const not_mcv = try self.genTypedValue(.{
	9791	.ty = lhs_ty,
	9792	.val = try unsigned_ty.maxInt(mod, unsigned_ty),
	9793	});
	9794	const not_mem: Memory = if (not_mcv.isMemory())
	9795	try not_mcv.mem(self, Memory.Size.fromSize(abi_size))
	9796	else
	9797	.{ .base = .{
	9798	.reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
	9799	}, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };
	9800	switch (mir_tag[0]) {
	9801	.vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
	9802	.{ .vp_, .xor },
	9803	dst_reg,
	9804	dst_reg,
	9805	not_mem,
	9806	),
	9807	.p_b, .p_d, .p_q, .p_w => try self.asmRegisterMemory(
	9808	.{ .p_, .xor },
	9809	dst_reg,
	9810	not_mem,
	9811	),
	9812	else => unreachable,
	9813	}
	9814	},
	9815	else => unreachable,
9325	9816	},
	9817	.Float => {},
9326	9818	else => unreachable,
9327	9819	}
9328	9820
...	...	@@ -9331,8 +9823,12 @@ fn genBinOp(
9331	9823	defer self.register_manager.unlockReg(gp_lock);
9332	9824
9333	9825	try self.asmRegisterRegister(switch (mir_tag[0]) {
9334		.vp_b, .vp_d, .vp_q, .vp_w => .{ .vp_b, .movmsk },
	9826	._pd, ._sd => .{ ._pd, .movmsk },
	9827	._ps, ._ss => .{ ._ps, .movmsk },
9335	9828	.p_b, .p_d, .p_q, .p_w => .{ .p_b, .movmsk },
	9829	.v_pd, .v_sd => .{ .v_pd, .movmsk },
	9830	.v_ps, .v_ss => .{ .v_ps, .movmsk },
	9831	.vp_b, .vp_d, .vp_q, .vp_w => .{ .vp_b, .movmsk },
9336	9832	else => unreachable,
9337	9833	}, gp_reg.to32(), dst_reg);
9338	9834	return .{ .register = gp_reg };
...	...	@@ -9459,13 +9955,13 @@ fn genBinOpMir(
9459	9955	.load_frame,
9460	9956	.lea_frame,
9461	9957	=> {
9462		blk: {
9463		return self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) {
	9958	direct: {
	9959	try self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) {
9464	9960	.memory => \|addr\| .{
9465	9961	.base = .{ .reg = .ds },
9466	9962	.mod = .{ .rm = .{
9467	9963	.size = Memory.Size.fromSize(limb_abi_size),
9468		.disp = math.cast(i32, addr + off) orelse break :blk,
	9964	.disp = math.cast(i32, addr + off) orelse break :direct,
9469	9965	} },
9470	9966	},
9471	9967	.indirect => \|reg_off\| .{
...	...	@@ -9482,8 +9978,9 @@ fn genBinOpMir(
9482	9978	.disp = frame_addr.off + off,
9483	9979	} },
9484	9980	},
9485		else => break :blk,
	9981	else => break :direct,
9486	9982	});
	9983	continue;
9487	9984	}
9488	9985
9489	9986	switch (src_mcv) {
...	...	@@ -10180,7 +10677,7 @@ fn genCall(self: *Self, info: union(enum) {
10180	10677	.none, .unreach => {},
10181	10678	.indirect => \|reg_off\| {
10182	10679	const ret_ty = fn_info.return_type.toType();
10183		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ret_ty, mod));
	10680	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ret_ty, mod));
10184	10681	try self.genSetReg(reg_off.reg, Type.usize, .{
10185	10682	.lea_frame = .{ .index = frame_index, .off = -reg_off.off },
10186	10683	});
...	...	@@ -10306,19 +10803,20 @@ fn genCall(self: *Self, info: union(enum) {
10306	10803	fn airRet(self: *Self, inst: Air.Inst.Index) !void {
10307	10804	const mod = self.bin_file.options.module.?;
10308	10805	const un_op = self.air.instructions.items(.data)[inst].un_op;
10309		const operand = try self.resolveInst(un_op);
10310	10806
10311	10807	const ret_ty = self.fn_type.fnReturnType(mod);
10312	10808	switch (self.ret_mcv.short) {
10313	10809	.none => {},
10314		.register, .register_pair => try self.genCopy(ret_ty, self.ret_mcv.short, operand),
	10810	.register,
	10811	.register_pair,
	10812	=> try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }),
10315	10813	.indirect => \|reg_off\| {
10316	10814	try self.register_manager.getReg(reg_off.reg, null);
10317	10815	const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
10318	10816	defer self.register_manager.unlockReg(lock);
10319	10817
10320	10818	try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long);
10321		try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, operand);
	10819	try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, .{ .air_ref = un_op });
10322	10820	},
10323	10821	else => unreachable,
10324	10822	}
...	...	@@ -10593,7 +11091,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
10593	11091	const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
10594	11092	defer for (locks) \|lock\| self.register_manager.unlockReg(lock);
10595	11093
10596		const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable;
	11094	const limbs_len = math.divCeil(u16, abi_size, 8) catch unreachable;
10597	11095	var limb_i: u16 = 0;
10598	11096	while (limb_i < limbs_len) : (limb_i += 1) {
10599	11097	const off = limb_i * 8;
...	...	@@ -10688,7 +11186,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
10688	11186	.{ .vp_w, .insr },
10689	11187	tmp1_reg,
10690	11188	dst_reg.to128(),
10691		src_mcv.mem(.word),
	11189	try src_mcv.mem(self, .word),
10692	11190	Immediate.u(1),
10693	11191	) else try self.asmRegisterRegisterRegister(
10694	11192	.{ .vp_, .unpcklwd },
...	...	@@ -10892,8 +11390,8 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
10892	11390	},
10893	11391	.register => \|reg\| {
10894	11392	try self.spillEflagsIfOccupied();
10895		try self.asmRegisterImmediate(.{ ._, .@"test" }, reg, Immediate.u(1));
10896		return self.asmJccReloc(.e, undefined);
	11393	try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), Immediate.u(1));
	11394	return self.asmJccReloc(.z, undefined);
10897	11395	},
10898	11396	.immediate,
10899	11397	.load_frame,
...	...	@@ -11433,12 +11931,12 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
11433	11931	if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
11434	11932	if (first_br) break :result src_mcv;
11435	11933
11436		if (block_tracking.getReg()) \|block_reg\|
	11934	for (block_tracking.getRegs()) \|block_reg\|
11437	11935	try self.register_manager.getReg(block_reg, br.block_inst);
11438	11936	// .long = .none to avoid merging operand and block result stack frames.
11439	11937	var current_tracking = InstTracking{ .long = .none, .short = src_mcv };
11440	11938	try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*);
11441		if (src_mcv.getReg()) \|src_reg\| self.register_manager.freeReg(src_reg);
	11939	for (src_mcv.getRegs()) \|src_reg\| self.register_manager.freeReg(src_reg);
11442	11940	break :result block_tracking.short;
11443	11941	}
11444	11942
...	...	@@ -12177,16 +12675,87 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
12177	12675	.general_purpose, .segment => return .{ .move = .{ ._, .mov } },
12178	12676	.x87 => return .x87_load_store,
12179	12677	.mmx => {},
12180		.sse => {
12181		switch (ty.zigTypeTag(mod)) {
12182		else => {
12183		const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
12184		assert(std.mem.indexOfNone(abi.Class, classes, &.{
12185		.integer, .sse, .float, .float_combine,
12186		}) == null);
12187		const abi_size = ty.abiSize(mod);
12188		if (abi_size < 4 or
12189		std.mem.indexOfScalar(abi.Class, classes, .integer) != null) switch (abi_size) {
	12678	.sse => switch (ty.zigTypeTag(mod)) {
	12679	else => {
	12680	const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
	12681	assert(std.mem.indexOfNone(abi.Class, classes, &.{
	12682	.integer, .sse, .float, .float_combine,
	12683	}) == null);
	12684	const abi_size = ty.abiSize(mod);
	12685	if (abi_size < 4 or
	12686	std.mem.indexOfScalar(abi.Class, classes, .integer) != null) switch (abi_size) {
	12687	1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
	12688	.insert = .{ .vp_b, .insr },
	12689	.extract = .{ .vp_b, .extr },
	12690	} } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
	12691	.insert = .{ .p_b, .insr },
	12692	.extract = .{ .p_b, .extr },
	12693	} },
	12694	2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	12695	.insert = .{ .vp_w, .insr },
	12696	.extract = .{ .vp_w, .extr },
	12697	} } else .{ .insert_extract = .{
	12698	.insert = .{ .p_w, .insr },
	12699	.extract = .{ .p_w, .extr },
	12700	} },
	12701	3...4 => return .{ .move = if (self.hasFeature(.avx))
	12702	.{ .v_d, .mov }
	12703	else
	12704	.{ ._d, .mov } },
	12705	5...8 => return .{ .move = if (self.hasFeature(.avx))
	12706	.{ .v_q, .mov }
	12707	else
	12708	.{ ._q, .mov } },
	12709	9...16 => return .{ .move = if (self.hasFeature(.avx))
	12710	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12711	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12712	17...32 => if (self.hasFeature(.avx))
	12713	return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
	12714	else => {},
	12715	} else switch (abi_size) {
	12716	4 => return .{ .move = if (self.hasFeature(.avx))
	12717	.{ .v_ss, .mov }
	12718	else
	12719	.{ ._ss, .mov } },
	12720	5...8 => return .{ .move = if (self.hasFeature(.avx))
	12721	.{ .v_sd, .mov }
	12722	else
	12723	.{ ._sd, .mov } },
	12724	9...16 => return .{ .move = if (self.hasFeature(.avx))
	12725	if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
	12726	else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
	12727	17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned)
	12728	.{ .v_pd, .mova }
	12729	else
	12730	.{ .v_pd, .movu } },
	12731	else => {},
	12732	}
	12733	},
	12734	.Float => switch (ty.floatBits(self.target.*)) {
	12735	16 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	12736	.insert = .{ .vp_w, .insr },
	12737	.extract = .{ .vp_w, .extr },
	12738	} } else .{ .insert_extract = .{
	12739	.insert = .{ .p_w, .insr },
	12740	.extract = .{ .p_w, .extr },
	12741	} },
	12742	32 => return .{ .move = if (self.hasFeature(.avx))
	12743	.{ .v_ss, .mov }
	12744	else
	12745	.{ ._ss, .mov } },
	12746	64 => return .{ .move = if (self.hasFeature(.avx))
	12747	.{ .v_sd, .mov }
	12748	else
	12749	.{ ._sd, .mov } },
	12750	128 => return .{ .move = if (self.hasFeature(.avx))
	12751	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12752	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12753	else => {},
	12754	},
	12755	.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
	12756	.Bool => {},
	12757	.Int => switch (ty.childType(mod).intInfo(mod).bits) {
	12758	8 => switch (ty.vectorLen(mod)) {
12190	12759	1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
12191	12760	.insert = .{ .vp_b, .insr },
12192	12761	.extract = .{ .vp_b, .extr },
...	...	@@ -12213,242 +12782,169 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
12213	12782	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12214	12783	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12215	12784	17...32 => if (self.hasFeature(.avx))
12216		return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
	12785	return .{ .move = if (aligned)
	12786	.{ .v_, .movdqa }
	12787	else
	12788	.{ .v_, .movdqu } },
12217	12789	else => {},
12218		} else switch (abi_size) {
12219		4 => return .{ .move = if (self.hasFeature(.avx))
12220		.{ .v_ss, .mov }
	12790	},
	12791	16 => switch (ty.vectorLen(mod)) {
	12792	1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	12793	.insert = .{ .vp_w, .insr },
	12794	.extract = .{ .vp_w, .extr },
	12795	} } else .{ .insert_extract = .{
	12796	.insert = .{ .p_w, .insr },
	12797	.extract = .{ .p_w, .extr },
	12798	} },
	12799	2 => return .{ .move = if (self.hasFeature(.avx))
	12800	.{ .v_d, .mov }
12221	12801	else
12222		.{ ._ss, .mov } },
	12802	.{ ._d, .mov } },
	12803	3...4 => return .{ .move = if (self.hasFeature(.avx))
	12804	.{ .v_q, .mov }
	12805	else
	12806	.{ ._q, .mov } },
12223	12807	5...8 => return .{ .move = if (self.hasFeature(.avx))
12224		.{ .v_sd, .mov }
	12808	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12809	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12810	9...16 => if (self.hasFeature(.avx))
	12811	return .{ .move = if (aligned)
	12812	.{ .v_, .movdqa }
	12813	else
	12814	.{ .v_, .movdqu } },
	12815	else => {},
	12816	},
	12817	32 => switch (ty.vectorLen(mod)) {
	12818	1 => return .{ .move = if (self.hasFeature(.avx))
	12819	.{ .v_d, .mov }
12225	12820	else
12226		.{ ._sd, .mov } },
12227		9...16 => return .{ .move = if (self.hasFeature(.avx))
12228		if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
12229		else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
12230		17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned)
12231		.{ .v_pd, .mova }
	12821	.{ ._d, .mov } },
	12822	2 => return .{ .move = if (self.hasFeature(.avx))
	12823	.{ .v_q, .mov }
12232	12824	else
12233		.{ .v_pd, .movu } },
12234		else => {},
12235		}
12236		},
12237		.Float => switch (ty.floatBits(self.target.*)) {
12238		16 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12239		.insert = .{ .vp_w, .insr },
12240		.extract = .{ .vp_w, .extr },
12241		} } else .{ .insert_extract = .{
12242		.insert = .{ .p_w, .insr },
12243		.extract = .{ .p_w, .extr },
12244		} },
12245		32 => return .{ .move = if (self.hasFeature(.avx))
12246		.{ .v_ss, .mov }
12247		else
12248		.{ ._ss, .mov } },
12249		64 => return .{ .move = if (self.hasFeature(.avx))
12250		.{ .v_sd, .mov }
12251		else
12252		.{ ._sd, .mov } },
12253		128 => return .{ .move = if (self.hasFeature(.avx))
12254		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12255		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12256		else => {},
12257		},
12258		.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
12259		.Bool => return .{ .move = .{ ._, .mov } },
12260		.Int => switch (ty.childType(mod).intInfo(mod).bits) {
12261		8 => switch (ty.vectorLen(mod)) {
12262		1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
12263		.insert = .{ .vp_b, .insr },
12264		.extract = .{ .vp_b, .extr },
12265		} } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
12266		.insert = .{ .p_b, .insr },
12267		.extract = .{ .p_b, .extr },
12268		} },
12269		2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12270		.insert = .{ .vp_w, .insr },
12271		.extract = .{ .vp_w, .extr },
12272		} } else .{ .insert_extract = .{
12273		.insert = .{ .p_w, .insr },
12274		.extract = .{ .p_w, .extr },
12275		} },
12276		3...4 => return .{ .move = if (self.hasFeature(.avx))
12277		.{ .v_d, .mov }
12278		else
12279		.{ ._d, .mov } },
12280		5...8 => return .{ .move = if (self.hasFeature(.avx))
12281		.{ .v_q, .mov }
12282		else
12283		.{ ._q, .mov } },
12284		9...16 => return .{ .move = if (self.hasFeature(.avx))
12285		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12286		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12287		17...32 => if (self.hasFeature(.avx))
12288		return .{ .move = if (aligned)
12289		.{ .v_, .movdqa }
12290		else
12291		.{ .v_, .movdqu } },
12292		else => {},
12293		},
12294		16 => switch (ty.vectorLen(mod)) {
12295		1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12296		.insert = .{ .vp_w, .insr },
12297		.extract = .{ .vp_w, .extr },
12298		} } else .{ .insert_extract = .{
12299		.insert = .{ .p_w, .insr },
12300		.extract = .{ .p_w, .extr },
12301		} },
12302		2 => return .{ .move = if (self.hasFeature(.avx))
12303		.{ .v_d, .mov }
12304		else
12305		.{ ._d, .mov } },
12306		3...4 => return .{ .move = if (self.hasFeature(.avx))
12307		.{ .v_q, .mov }
12308		else
12309		.{ ._q, .mov } },
12310		5...8 => return .{ .move = if (self.hasFeature(.avx))
12311		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12312		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12313		9...16 => if (self.hasFeature(.avx))
12314		return .{ .move = if (aligned)
12315		.{ .v_, .movdqa }
12316		else
12317		.{ .v_, .movdqu } },
12318		else => {},
12319		},
12320		32 => switch (ty.vectorLen(mod)) {
12321		1 => return .{ .move = if (self.hasFeature(.avx))
12322		.{ .v_d, .mov }
	12825	.{ ._q, .mov } },
	12826	3...4 => return .{ .move = if (self.hasFeature(.avx))
	12827	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12828	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12829	5...8 => if (self.hasFeature(.avx))
	12830	return .{ .move = if (aligned)
	12831	.{ .v_, .movdqa }
12323	12832	else
12324		.{ ._d, .mov } },
12325		2 => return .{ .move = if (self.hasFeature(.avx))
12326		.{ .v_q, .mov }
	12833	.{ .v_, .movdqu } },
	12834	else => {},
	12835	},
	12836	64 => switch (ty.vectorLen(mod)) {
	12837	1 => return .{ .move = if (self.hasFeature(.avx))
	12838	.{ .v_q, .mov }
	12839	else
	12840	.{ ._q, .mov } },
	12841	2 => return .{ .move = if (self.hasFeature(.avx))
	12842	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12843	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12844	3...4 => if (self.hasFeature(.avx))
	12845	return .{ .move = if (aligned)
	12846	.{ .v_, .movdqa }
12327	12847	else
12328		.{ ._q, .mov } },
12329		3...4 => return .{ .move = if (self.hasFeature(.avx))
12330		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12331		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12332		5...8 => if (self.hasFeature(.avx))
12333		return .{ .move = if (aligned)
12334		.{ .v_, .movdqa }
12335		else
12336		.{ .v_, .movdqu } },
12337		else => {},
12338		},
12339		64 => switch (ty.vectorLen(mod)) {
12340		1 => return .{ .move = if (self.hasFeature(.avx))
12341		.{ .v_q, .mov }
	12848	.{ .v_, .movdqu } },
	12849	else => {},
	12850	},
	12851	128 => switch (ty.vectorLen(mod)) {
	12852	1 => return .{ .move = if (self.hasFeature(.avx))
	12853	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12854	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12855	2 => if (self.hasFeature(.avx))
	12856	return .{ .move = if (aligned)
	12857	.{ .v_, .movdqa }
12342	12858	else
12343		.{ ._q, .mov } },
12344		2 => return .{ .move = if (self.hasFeature(.avx))
12345		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12346		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12347		3...4 => if (self.hasFeature(.avx))
12348		return .{ .move = if (aligned)
12349		.{ .v_, .movdqa }
12350		else
12351		.{ .v_, .movdqu } },
12352		else => {},
12353		},
12354		128 => switch (ty.vectorLen(mod)) {
12355		1 => return .{ .move = if (self.hasFeature(.avx))
12356		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12357		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12358		2 => if (self.hasFeature(.avx))
12359		return .{ .move = if (aligned)
12360		.{ .v_, .movdqa }
12361		else
12362		.{ .v_, .movdqu } },
12363		else => {},
12364		},
12365		256 => switch (ty.vectorLen(mod)) {
12366		1 => if (self.hasFeature(.avx))
12367		return .{ .move = if (aligned)
12368		.{ .v_, .movdqa }
12369		else
12370		.{ .v_, .movdqu } },
12371		else => {},
12372		},
	12859	.{ .v_, .movdqu } },
12373	12860	else => {},
12374	12861	},
12375		.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
12376		16 => switch (ty.vectorLen(mod)) {
12377		1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12378		.insert = .{ .vp_w, .insr },
12379		.extract = .{ .vp_w, .extr },
12380		} } else .{ .insert_extract = .{
12381		.insert = .{ .p_w, .insr },
12382		.extract = .{ .p_w, .extr },
12383		} },
12384		2 => return .{ .move = if (self.hasFeature(.avx))
12385		.{ .v_d, .mov }
	12862	256 => switch (ty.vectorLen(mod)) {
	12863	1 => if (self.hasFeature(.avx))
	12864	return .{ .move = if (aligned)
	12865	.{ .v_, .movdqa }
12386	12866	else
12387		.{ ._d, .mov } },
12388		3...4 => return .{ .move = if (self.hasFeature(.avx))
12389		.{ .v_q, .mov }
	12867	.{ .v_, .movdqu } },
	12868	else => {},
	12869	},
	12870	else => {},
	12871	},
	12872	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
	12873	16 => switch (ty.vectorLen(mod)) {
	12874	1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	12875	.insert = .{ .vp_w, .insr },
	12876	.extract = .{ .vp_w, .extr },
	12877	} } else .{ .insert_extract = .{
	12878	.insert = .{ .p_w, .insr },
	12879	.extract = .{ .p_w, .extr },
	12880	} },
	12881	2 => return .{ .move = if (self.hasFeature(.avx))
	12882	.{ .v_d, .mov }
	12883	else
	12884	.{ ._d, .mov } },
	12885	3...4 => return .{ .move = if (self.hasFeature(.avx))
	12886	.{ .v_q, .mov }
	12887	else
	12888	.{ ._q, .mov } },
	12889	5...8 => return .{ .move = if (self.hasFeature(.avx))
	12890	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12891	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12892	9...16 => if (self.hasFeature(.avx))
	12893	return .{ .move = if (aligned)
	12894	.{ .v_, .movdqa }
12390	12895	else
12391		.{ ._q, .mov } },
12392		5...8 => return .{ .move = if (self.hasFeature(.avx))
12393		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12394		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12395		9...16 => if (self.hasFeature(.avx))
12396		return .{ .move = if (aligned)
12397		.{ .v_, .movdqa }
12398		else
12399		.{ .v_, .movdqu } },
12400		else => {},
12401		},
12402		32 => switch (ty.vectorLen(mod)) {
12403		1 => return .{ .move = if (self.hasFeature(.avx))
12404		.{ .v_ss, .mov }
	12896	.{ .v_, .movdqu } },
	12897	else => {},
	12898	},
	12899	32 => switch (ty.vectorLen(mod)) {
	12900	1 => return .{ .move = if (self.hasFeature(.avx))
	12901	.{ .v_ss, .mov }
	12902	else
	12903	.{ ._ss, .mov } },
	12904	2 => return .{ .move = if (self.hasFeature(.avx))
	12905	.{ .v_sd, .mov }
	12906	else
	12907	.{ ._sd, .mov } },
	12908	3...4 => return .{ .move = if (self.hasFeature(.avx))
	12909	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
	12910	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
	12911	5...8 => if (self.hasFeature(.avx))
	12912	return .{ .move = if (aligned)
	12913	.{ .v_ps, .mova }
12405	12914	else
12406		.{ ._ss, .mov } },
12407		2 => return .{ .move = if (self.hasFeature(.avx))
12408		.{ .v_sd, .mov }
	12915	.{ .v_ps, .movu } },
	12916	else => {},
	12917	},
	12918	64 => switch (ty.vectorLen(mod)) {
	12919	1 => return .{ .move = if (self.hasFeature(.avx))
	12920	.{ .v_sd, .mov }
	12921	else
	12922	.{ ._sd, .mov } },
	12923	2 => return .{ .move = if (self.hasFeature(.avx))
	12924	if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
	12925	else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
	12926	3...4 => if (self.hasFeature(.avx))
	12927	return .{ .move = if (aligned)
	12928	.{ .v_pd, .mova }
12409	12929	else
12410		.{ ._sd, .mov } },
12411		3...4 => return .{ .move = if (self.hasFeature(.avx))
12412		if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
12413		else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
12414		5...8 => if (self.hasFeature(.avx))
12415		return .{ .move = if (aligned)
12416		.{ .v_ps, .mova }
12417		else
12418		.{ .v_ps, .movu } },
12419		else => {},
12420		},
12421		64 => switch (ty.vectorLen(mod)) {
12422		1 => return .{ .move = if (self.hasFeature(.avx))
12423		.{ .v_sd, .mov }
	12930	.{ .v_pd, .movu } },
	12931	else => {},
	12932	},
	12933	128 => switch (ty.vectorLen(mod)) {
	12934	1 => return .{ .move = if (self.hasFeature(.avx))
	12935	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	12936	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	12937	2 => if (self.hasFeature(.avx))
	12938	return .{ .move = if (aligned)
	12939	.{ .v_, .movdqa }
12424	12940	else
12425		.{ ._sd, .mov } },
12426		2 => return .{ .move = if (self.hasFeature(.avx))
12427		if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
12428		else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
12429		3...4 => if (self.hasFeature(.avx))
12430		return .{ .move = if (aligned)
12431		.{ .v_pd, .mova }
12432		else
12433		.{ .v_pd, .movu } },
12434		else => {},
12435		},
12436		128 => switch (ty.vectorLen(mod)) {
12437		1 => return .{ .move = if (self.hasFeature(.avx))
12438		if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12439		else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12440		2 => if (self.hasFeature(.avx))
12441		return .{ .move = if (aligned)
12442		.{ .v_, .movdqa }
12443		else
12444		.{ .v_, .movdqu } },
12445		else => {},
12446		},
	12941	.{ .v_, .movdqu } },
12447	12942	else => {},
12448	12943	},
12449	12944	else => {},
12450	12945	},
12451		}
	12946	else => {},
	12947	},
12452	12948	},
12453	12949	}
12454	12950	return self.fail("TODO moveStrategy for {}", .{ty.fmt(mod)});
...	...	@@ -12514,32 +13010,18 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
12514	13010	};
12515	13011	defer if (src_info) \|info\| self.register_manager.unlockReg(info.addr_lock);
12516	13012
12517		const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
12518		for (dst_regs, classes, 0..) \|dst_reg, class, dst_reg_i\| {
12519		const class_ty = switch (class) {
12520		.integer => Type.usize,
12521		.sse, .float, .float_combine => Type.f64,
12522		else => unreachable,
12523		};
12524		const off: i32 = @intCast(dst_reg_i * 8);
12525		switch (src_mcv) {
12526		.register_pair => \|src_regs\| try self.genSetReg(
12527		dst_reg,
12528		class_ty,
12529		.{ .register = src_regs[dst_reg_i] },
12530		),
12531		.memory, .indirect, .load_frame => try self.genSetReg(
12532		dst_reg,
12533		class_ty,
12534		src_mcv.address().offset(off).deref(),
12535		),
12536		.load_symbol, .load_direct, .load_got, .load_tlv => try self.genSetReg(
12537		dst_reg,
12538		class_ty,
12539		.{ .indirect = .{ .reg = src_info.?.addr_reg, .off = off } },
12540		),
	13013	var part_disp: i32 = 0;
	13014	for (dst_regs, try self.splitType(ty), 0..) \|dst_reg, dst_ty, part_i\| {
	13015	try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) {
	13016	.register_pair => \|src_regs\| .{ .register = src_regs[part_i] },
	13017	.memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
	13018	.load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{
	13019	.reg = src_info.?.addr_reg,
	13020	.off = part_disp,
	13021	} },
12541	13022	else => unreachable,
12542		}
	13023	});
	13024	part_disp += @intCast(dst_ty.abiSize(mod));
12543	13025	}
12544	13026	},
12545	13027	.indirect => \|reg_off\| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),
...	...	@@ -12584,6 +13066,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12584	13066	if (imm == 0) {
12585	13067	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
12586	13068	// register is the fastest way to zero a register.
	13069	try self.spillEflagsIfOccupied();
12587	13070	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
12588	13071	} else if (abi_size > 4 and math.cast(u32, imm) != null) {
12589	13072	// 32-bit moves zero-extend to 64-bit.
...	...	@@ -12933,44 +13416,65 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12933	13416	.eflags => \|cc\| try self.asmSetccMemory(cc, .{ .base = base, .mod = .{
12934	13417	.rm = .{ .size = .byte, .disp = disp },
12935	13418	} }),
12936		.register => \|src_reg\| try (try self.moveStrategy(ty, src_reg.class(), switch (base) {
12937		.none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12938		.reg => \|reg\| switch (reg) {
12939		.es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12940		else => false,
12941		},
12942		.frame => \|frame_index\| self.getFrameAddrAlignment(
12943		.{ .index = frame_index, .off = disp },
12944		).compare(.gte, ty.abiAlignment(mod)),
12945		.reloc => false,
12946		})).write(
12947		self,
12948		.{ .base = base, .mod = .{ .rm = .{
12949		.size = self.memSize(ty),
12950		.disp = disp,
12951		} } },
12952		registerAlias(src_reg, abi_size),
12953		),
12954		.register_pair => \|src_regs\| for (src_regs, 0..) \|src_reg, src_reg_i\| {
12955		const part_size: u16 = @min(abi_size - src_reg_i * 8, 8);
12956		try (try self.moveStrategy(
12957		try mod.intType(.unsigned, part_size * 8),
12958		src_reg.class(),
12959		switch (base) {
12960		.none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12961		.reg => \|reg\| switch (reg) {
12962		.es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12963		else => false,
12964		},
12965		.frame => \|frame_index\| self.getFrameAddrAlignment(
12966		.{ .index = frame_index, .off = disp },
12967		).compare(.gte, ty.abiAlignment(mod)),
12968		.reloc => false,
	13419	.register => \|src_reg\| {
	13420	const mem_size = switch (base) {
	13421	.frame => \|base_fi\| mem_size: {
	13422	assert(disp >= 0);
	13423	const frame_abi_size = self.frame_allocs.items(.abi_size)[@intFromEnum(base_fi)];
	13424	const frame_spill_pad = self.frame_allocs.items(.spill_pad)[@intFromEnum(base_fi)];
	13425	assert(frame_abi_size - frame_spill_pad - disp >= abi_size);
	13426	break :mem_size if (frame_abi_size - frame_spill_pad - disp == abi_size)
	13427	frame_abi_size
	13428	else
	13429	abi_size;
	13430	},
	13431	else => abi_size,
	13432	};
	13433	const src_alias = registerAlias(src_reg, abi_size);
	13434	const src_size: u32 = @intCast(switch (src_alias.class()) {
	13435	.general_purpose, .segment, .x87 => @divExact(src_alias.bitSize(), 8),
	13436	.mmx, .sse => abi_size,
	13437	});
	13438	if (src_size > mem_size) {
	13439	const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
	13440	.size = src_size,
	13441	.alignment = Alignment.fromNonzeroByteUnits(src_size),
	13442	}));
	13443	const frame_mcv: MCValue = .{ .load_frame = .{ .index = frame_index } };
	13444	try (try self.moveStrategy(ty, src_alias.class(), true)).write(
	13445	self,
	13446	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{
	13447	.size = Memory.Size.fromSize(src_size),
	13448	} } },
	13449	src_alias,
	13450	);
	13451	try self.genSetMem(base, disp, ty, frame_mcv);
	13452	try self.freeValue(frame_mcv);
	13453	} else try (try self.moveStrategy(ty, src_alias.class(), switch (base) {
	13454	.none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
	13455	.reg => \|reg\| switch (reg) {
	13456	.es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
	13457	else => false,
12969	13458	},
12970		)).write(self, .{ .base = base, .mod = .{ .rm = .{
12971		.size = Memory.Size.fromSize(part_size),
12972		.disp = disp + @as(i32, @intCast(src_reg_i * 8)),
12973		} } }, registerAlias(src_reg, part_size));
	13459	.frame => \|frame_index\| self.getFrameAddrAlignment(
	13460	.{ .index = frame_index, .off = disp },
	13461	).compare(.gte, ty.abiAlignment(mod)),
	13462	.reloc => false,
	13463	})).write(
	13464	self,
	13465	.{ .base = base, .mod = .{ .rm = .{
	13466	.size = self.memSize(ty),
	13467	.disp = disp,
	13468	} } },
	13469	src_alias,
	13470	);
	13471	},
	13472	.register_pair => \|src_regs\| {
	13473	var part_disp: i32 = disp;
	13474	for (try self.splitType(ty), src_regs) \|src_ty, src_reg\| {
	13475	try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg });
	13476	part_disp += @intCast(src_ty.abiSize(mod));
	13477	}
12974	13478	},
12975	13479	.register_overflow => \|ro\| switch (ty.zigTypeTag(mod)) {
12976	13480	.Struct => {
...	...	@@ -13226,50 +13730,43 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
13226	13730	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
13227	13731	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
13228	13732
13229		const dst_mcv = if (dst_rc.supersetOf(src_rc) and
13230		self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
13231		src_mcv
13232		else dst: {
	13733	const dst_mcv = if (dst_rc.supersetOf(src_rc) and dst_ty.abiSize(mod) <= src_ty.abiSize(mod) and
	13734	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
13233	13735	const dst_mcv = try self.allocRegOrMem(inst, true);
13234		try self.genCopy(
13235		if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
13236		dst_mcv,
13237		src_mcv,
13238		);
	13736	try self.genCopy(switch (math.order(dst_ty.abiSize(mod), src_ty.abiSize(mod))) {
	13737	.lt => dst_ty,
	13738	.eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
	13739	.gt => src_ty,
	13740	}, dst_mcv, src_mcv);
13239	13741	break :dst dst_mcv;
13240	13742	};
13241	13743
13242	13744	if (dst_ty.isRuntimeFloat()) break :result dst_mcv;
13243	13745
13244		const dst_signedness =
13245		if (dst_ty.isAbiInt(mod)) dst_ty.intInfo(mod).signedness else .unsigned;
13246		if (!src_ty.isRuntimeFloat() or src_ty.floatBits(self.target.*) != 80) {
13247		const src_signedness =
13248		if (src_ty.isAbiInt(mod)) src_ty.intInfo(mod).signedness else .unsigned;
13249		if (dst_signedness == src_signedness) break :result dst_mcv;
13250		}
	13746	if (dst_ty.isAbiInt(mod) and src_ty.isAbiInt(mod) and
	13747	dst_ty.intInfo(mod).signedness == src_ty.intInfo(mod).signedness) break :result dst_mcv;
13251	13748
13252		const abi_size: u16 = @intCast(dst_ty.abiSize(mod));
13253		const bit_size: u16 = @intCast(dst_ty.bitSize(mod));
13254		if (abi_size * 8 <= bit_size) break :result dst_mcv;
	13749	const abi_size = dst_ty.abiSize(mod);
	13750	const bit_size = dst_ty.bitSize(mod);
	13751	if (abi_size * 8 <= bit_size or dst_ty.isVector(mod)) break :result dst_mcv;
13255	13752
13256		const dst_limbs_len = math.divCeil(i32, bit_size, 64) catch unreachable;
13257		const high_reg = if (dst_mcv.isRegister())
13258		dst_mcv.getReg().?
	13753	const dst_limbs_len = math.divCeil(i32, @intCast(bit_size), 64) catch unreachable;
	13754	const high_mcv: MCValue = switch (dst_mcv) {
	13755	.register => \|dst_reg\| .{ .register = dst_reg },
	13756	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
	13757	else => dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
	13758	};
	13759	const high_reg = if (high_mcv.isRegister())
	13760	high_mcv.getReg().?
13259	13761	else
13260		try self.copyToTmpRegister(
13261		Type.usize,
13262		dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
13263		);
	13762	try self.copyToTmpRegister(Type.usize, high_mcv);
13264	13763	const high_lock = self.register_manager.lockReg(high_reg);
13265	13764	defer if (high_lock) \|lock\| self.register_manager.unlockReg(lock);
13266	13765
13267		const high_ty = try mod.intType(dst_signedness, bit_size % 64);
13268
13269		try self.truncateRegister(high_ty, high_reg);
13270		if (!dst_mcv.isRegister()) try self.genCopy(
13271		Type.usize,
13272		dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
	13766	try self.truncateRegister(dst_ty, high_reg);
	13767	if (!high_mcv.isRegister()) try self.genCopy(
	13768	if (abi_size <= 8) dst_ty else Type.usize,
	13769	high_mcv,
13273	13770	.{ .register = high_reg },
13274	13771	);
13275	13772	break :result dst_mcv;
...	...	@@ -13287,7 +13784,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
13287	13784	const array_ty = ptr_ty.childType(mod);
13288	13785	const array_len = array_ty.arrayLen(mod);
13289	13786
13290		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, mod));
	13787	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod));
13291	13788	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
13292	13789	try self.genSetMem(
13293	13790	.{ .frame = frame_index },
...	...	@@ -13497,7 +13994,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
13497	13994	const ptr_mcv = try self.resolveInst(extra.ptr);
13498	13995	const mem_size = Memory.Size.fromSize(val_abi_size);
13499	13996	const ptr_mem: Memory = switch (ptr_mcv) {
13500		.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),
	13997	.immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
13501	13998	else => .{
13502	13999	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13503	14000	.mod = .{ .rm = .{ .size = mem_size } },
...	...	@@ -13563,7 +14060,7 @@ fn atomicOp(
13563	14060	const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
13564	14061	const mem_size = Memory.Size.fromSize(val_abi_size);
13565	14062	const ptr_mem: Memory = switch (ptr_mcv) {
13566		.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),
	14063	.immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
13567	14064	else => .{
13568	14065	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13569	14066	.mod = .{ .rm = .{ .size = mem_size } },
...	...	@@ -13671,27 +14168,41 @@ fn atomicOp(
13671	14168	},
13672	14169	};
13673	14170
13674		try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
13675	14171	const cmov_abi_size = @max(val_abi_size, 2);
13676	14172	switch (val_mcv) {
13677		.register => \|val_reg\| try self.asmCmovccRegisterRegister(
13678		cc,
13679		registerAlias(tmp_reg, cmov_abi_size),
13680		registerAlias(val_reg, cmov_abi_size),
13681		),
13682		.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
13683		cc,
13684		registerAlias(tmp_reg, cmov_abi_size),
13685		val_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),
13686		),
13687		else => {
13688		const val_reg = try self.copyToTmpRegister(val_ty, val_mcv);
	14173	.register => \|val_reg\| {
	14174	try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
13689	14175	try self.asmCmovccRegisterRegister(
13690	14176	cc,
13691	14177	registerAlias(tmp_reg, cmov_abi_size),
13692	14178	registerAlias(val_reg, cmov_abi_size),
13693	14179	);
13694	14180	},
	14181	.memory, .indirect, .load_frame => {
	14182	try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
	14183	try self.asmCmovccRegisterMemory(
	14184	cc,
	14185	registerAlias(tmp_reg, cmov_abi_size),
	14186	try val_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
	14187	);
	14188	},
	14189	else => {
	14190	const mat_reg = try self.copyToTmpRegister(val_ty, val_mcv);
	14191	const mat_lock = self.register_manager.lockRegAssumeUnused(mat_reg);
	14192	defer self.register_manager.unlockReg(mat_lock);
	14193
	14194	try self.genBinOpMir(
	14195	.{ ._, .cmp },
	14196	val_ty,
	14197	tmp_mcv,
	14198	.{ .register = mat_reg },
	14199	);
	14200	try self.asmCmovccRegisterRegister(
	14201	cc,
	14202	registerAlias(tmp_reg, cmov_abi_size),
	14203	registerAlias(mat_reg, cmov_abi_size),
	14204	);
	14205	},
13695	14206	}
13696	14207	},
13697	14208	};
...	...	@@ -13728,8 +14239,8 @@ fn atomicOp(
13728	14239	.reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),
13729	14240	} },
13730	14241	};
13731		const val_lo_mem = val_mem_mcv.mem(.qword);
13732		const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);
	14242	const val_lo_mem = try val_mem_mcv.mem(self, .qword);
	14243	const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .qword);
13733	14244	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
13734	14245	try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
13735	14246	try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
...	...	@@ -14000,7 +14511,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
14000	14511	try self.asmRegisterMemoryImmediate(
14001	14512	.{ .i_, .mul },
14002	14513	len_reg,
14003		dst_ptr.address().offset(8).deref().mem(.qword),
	14514	try dst_ptr.address().offset(8).deref().mem(self, .qword),
14004	14515	Immediate.s(@intCast(dst_ptr_ty.childType(mod).abiSize(mod))),
14005	14516	);
14006	14517	break :len .{ .register = len_reg };
...	...	@@ -14171,28 +14682,162 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14171	14682	const mod = self.bin_file.options.module.?;
14172	14683	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
14173	14684	const vector_ty = self.typeOfIndex(inst);
	14685	const vector_len = vector_ty.vectorLen(mod);
14174	14686	const dst_rc = self.regClassForType(vector_ty);
14175		const scalar_ty = vector_ty.scalarType(mod);
	14687	const scalar_ty = self.typeOf(ty_op.operand);
14176	14688
14177		const src_mcv = try self.resolveInst(ty_op.operand);
14178	14689	const result: MCValue = result: {
14179	14690	switch (scalar_ty.zigTypeTag(mod)) {
14180	14691	else => {},
	14692	.Bool => {
	14693	const regs =
	14694	try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp);
	14695	const reg_locks = self.register_manager.lockRegsAssumeUnused(2, regs);
	14696	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
	14697
	14698	try self.genSetReg(regs[1], vector_ty, .{ .immediate = 0 });
	14699	try self.genSetReg(
	14700	regs[1],
	14701	vector_ty,
	14702	.{ .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - vector_len) },
	14703	);
	14704	const src_mcv = try self.resolveInst(ty_op.operand);
	14705	const abi_size = @max(math.divCeil(u32, vector_len, 8) catch unreachable, 4);
	14706	try self.asmCmovccRegisterRegister(
	14707	switch (src_mcv) {
	14708	.eflags => \|cc\| cc,
	14709	.register => \|src_reg\| cc: {
	14710	try self.asmRegisterImmediate(
	14711	.{ ._, .@"test" },
	14712	src_reg.to8(),
	14713	Immediate.u(1),
	14714	);
	14715	break :cc .nz;
	14716	},
	14717	else => cc: {
	14718	try self.asmMemoryImmediate(
	14719	.{ ._, .@"test" },
	14720	try src_mcv.mem(self, .byte),
	14721	Immediate.u(1),
	14722	);
	14723	break :cc .nz;
	14724	},
	14725	},
	14726	registerAlias(regs[0], abi_size),
	14727	registerAlias(regs[1], abi_size),
	14728	);
	14729	break :result .{ .register = regs[0] };
	14730	},
	14731	.Int => if (self.hasFeature(.avx2)) avx2: {
	14732	const mir_tag = @as(?Mir.Inst.FixedTag, switch (scalar_ty.intInfo(mod).bits) {
	14733	else => null,
	14734	1...8 => switch (vector_len) {
	14735	else => null,
	14736	1...32 => .{ .vp_b, .broadcast },
	14737	},
	14738	9...16 => switch (vector_len) {
	14739	else => null,
	14740	1...16 => .{ .vp_w, .broadcast },
	14741	},
	14742	17...32 => switch (vector_len) {
	14743	else => null,
	14744	1...8 => .{ .vp_d, .broadcast },
	14745	},
	14746	33...64 => switch (vector_len) {
	14747	else => null,
	14748	1...4 => .{ .vp_q, .broadcast },
	14749	},
	14750	65...128 => switch (vector_len) {
	14751	else => null,
	14752	1...2 => .{ .vp_i128, .broadcast },
	14753	},
	14754	}) orelse break :avx2;
	14755
	14756	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
	14757	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
	14758	defer self.register_manager.unlockReg(dst_lock);
	14759
	14760	const src_mcv = try self.resolveInst(ty_op.operand);
	14761	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	14762	mir_tag,
	14763	registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
	14764	try src_mcv.mem(self, self.memSize(scalar_ty)),
	14765	) else {
	14766	if (mir_tag[0] == .vp_i128) break :avx2;
	14767	try self.genSetReg(dst_reg, scalar_ty, src_mcv);
	14768	try self.asmRegisterRegister(
	14769	mir_tag,
	14770	registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
	14771	registerAlias(dst_reg, @intCast(scalar_ty.abiSize(mod))),
	14772	);
	14773	}
	14774	break :result .{ .register = dst_reg };
	14775	} else {
	14776	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
	14777	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
	14778	defer self.register_manager.unlockReg(dst_lock);
	14779
	14780	try self.genSetReg(dst_reg, scalar_ty, .{ .air_ref = ty_op.operand });
	14781	if (vector_len == 1) break :result .{ .register = dst_reg };
	14782
	14783	const dst_alias = registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod)));
	14784	const scalar_bits = scalar_ty.intInfo(mod).bits;
	14785	if (switch (scalar_bits) {
	14786	1...8 => true,
	14787	9...128 => false,
	14788	else => unreachable,
	14789	}) if (self.hasFeature(.avx)) try self.asmRegisterRegisterRegister(
	14790	.{ .vp_, .unpcklbw },
	14791	dst_alias,
	14792	dst_alias,
	14793	dst_alias,
	14794	) else try self.asmRegisterRegister(
	14795	.{ .p_, .unpcklbw },
	14796	dst_alias,
	14797	dst_alias,
	14798	);
	14799	if (switch (scalar_bits) {
	14800	1...8 => vector_len > 2,
	14801	9...16 => true,
	14802	17...128 => false,
	14803	else => unreachable,
	14804	}) try self.asmRegisterRegisterImmediate(
	14805	.{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
	14806	dst_alias,
	14807	dst_alias,
	14808	Immediate.u(0),
	14809	);
	14810	if (switch (scalar_bits) {
	14811	1...8 => vector_len > 4,
	14812	9...16 => vector_len > 2,
	14813	17...64 => true,
	14814	65...128 => false,
	14815	else => unreachable,
	14816	}) try self.asmRegisterRegisterImmediate(
	14817	.{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf },
	14818	dst_alias,
	14819	dst_alias,
	14820	Immediate.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),
	14821	);
	14822	break :result .{ .register = dst_reg };
	14823	},
14181	14824	.Float => switch (scalar_ty.floatBits(self.target.*)) {
14182		32 => switch (vector_ty.vectorLen(mod)) {
	14825	32 => switch (vector_len) {
14183	14826	1 => {
	14827	const src_mcv = try self.resolveInst(ty_op.operand);
14184	14828	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14185	14829	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14186	14830	try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14187	14831	break :result .{ .register = dst_reg };
14188	14832	},
14189	14833	2...4 => {
	14834	const src_mcv = try self.resolveInst(ty_op.operand);
14190	14835	if (self.hasFeature(.avx)) {
14191	14836	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14192	14837	if (src_mcv.isMemory()) try self.asmRegisterMemory(
14193	14838	.{ .v_ss, .broadcast },
14194	14839	dst_reg.to128(),
14195		src_mcv.mem(.dword),
	14840	try src_mcv.mem(self, .dword),
14196	14841	) else {
14197	14842	const src_reg = if (src_mcv.isRegister())
14198	14843	src_mcv.getReg().?
...	...	@@ -14224,11 +14869,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14224	14869	}
14225	14870	},
14226	14871	5...8 => if (self.hasFeature(.avx)) {
	14872	const src_mcv = try self.resolveInst(ty_op.operand);
14227	14873	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14228	14874	if (src_mcv.isMemory()) try self.asmRegisterMemory(
14229	14875	.{ .v_ss, .broadcast },
14230	14876	dst_reg.to256(),
14231		src_mcv.mem(.dword),
	14877	try src_mcv.mem(self, .dword),
14232	14878	) else {
14233	14879	const src_reg = if (src_mcv.isRegister())
14234	14880	src_mcv.getReg().?
...	...	@@ -14259,20 +14905,22 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14259	14905	},
14260	14906	else => {},
14261	14907	},
14262		64 => switch (vector_ty.vectorLen(mod)) {
	14908	64 => switch (vector_len) {
14263	14909	1 => {
	14910	const src_mcv = try self.resolveInst(ty_op.operand);
14264	14911	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14265	14912	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14266	14913	try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14267	14914	break :result .{ .register = dst_reg };
14268	14915	},
14269	14916	2 => {
	14917	const src_mcv = try self.resolveInst(ty_op.operand);
14270	14918	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14271	14919	if (self.hasFeature(.sse3)) {
14272	14920	if (src_mcv.isMemory()) try self.asmRegisterMemory(
14273	14921	if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
14274	14922	dst_reg.to128(),
14275		src_mcv.mem(.qword),
	14923	try src_mcv.mem(self, .qword),
14276	14924	) else try self.asmRegisterRegister(
14277	14925	if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
14278	14926	dst_reg.to128(),
...	...	@@ -14292,11 +14940,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14292	14940	);
14293	14941	},
14294	14942	3...4 => if (self.hasFeature(.avx)) {
	14943	const src_mcv = try self.resolveInst(ty_op.operand);
14295	14944	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14296	14945	if (src_mcv.isMemory()) try self.asmRegisterMemory(
14297	14946	.{ .v_sd, .broadcast },
14298	14947	dst_reg.to256(),
14299		src_mcv.mem(.qword),
	14948	try src_mcv.mem(self, .qword),
14300	14949	) else {
14301	14950	const src_reg = if (src_mcv.isRegister())
14302	14951	src_mcv.getReg().?
...	...	@@ -14325,19 +14974,21 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14325	14974	},
14326	14975	else => {},
14327	14976	},
14328		128 => switch (vector_ty.vectorLen(mod)) {
	14977	128 => switch (vector_len) {
14329	14978	1 => {
	14979	const src_mcv = try self.resolveInst(ty_op.operand);
14330	14980	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14331	14981	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14332	14982	try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14333	14983	break :result .{ .register = dst_reg };
14334	14984	},
14335	14985	2 => if (self.hasFeature(.avx)) {
	14986	const src_mcv = try self.resolveInst(ty_op.operand);
14336	14987	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14337	14988	if (src_mcv.isMemory()) try self.asmRegisterMemory(
14338	14989	.{ .v_f128, .broadcast },
14339	14990	dst_reg.to256(),
14340		src_mcv.mem(.xword),
	14991	try src_mcv.mem(self, .xword),
14341	14992	) else {
14342	14993	const src_reg = if (src_mcv.isRegister())
14343	14994	src_mcv.getReg().?
...	...	@@ -14389,7 +15040,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
14389	15040	try self.spillEflagsIfOccupied();
14390	15041
14391	15042	const operand_mcv = try self.resolveInst(reduce.operand);
14392		const mask_len = (std.math.cast(u6, operand_ty.vectorLen(mod)) orelse
	15043	const mask_len = (math.cast(u6, operand_ty.vectorLen(mod)) orelse
14393	15044	return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(mod)}));
14394	15045	const mask = (@as(u64, 1) << mask_len) - 1;
14395	15046	const abi_size: u32 = @intCast(operand_ty.abiSize(mod));
...	...	@@ -14397,7 +15048,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
14397	15048	.Or => {
14398	15049	if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
14399	15050	.{ ._, .@"test" },
14400		operand_mcv.mem(Memory.Size.fromSize(abi_size)),
	15051	try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)),
14401	15052	Immediate.u(mask),
14402	15053	) else {
14403	15054	const operand_reg = registerAlias(if (operand_mcv.isRegister())
...	...	@@ -14445,8 +15096,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
14445	15096	const result: MCValue = result: {
14446	15097	switch (result_ty.zigTypeTag(mod)) {
14447	15098	.Struct => {
14448		const frame_index =
14449		try self.allocFrameIndex(FrameAlloc.initType(result_ty, mod));
	15099	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, mod));
14450	15100	if (result_ty.containerLayout(mod) == .Packed) {
14451	15101	const struct_type = mod.typeToStruct(result_ty).?;
14452	15102	try self.genInlineMemset(
...	...	@@ -14542,8 +15192,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
14542	15192	break :result .{ .load_frame = .{ .index = frame_index } };
14543	15193	},
14544	15194	.Array => {
14545		const frame_index =
14546		try self.allocFrameIndex(FrameAlloc.initType(result_ty, mod));
	15195	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, mod));
14547	15196	const elem_ty = result_ty.childType(mod);
14548	15197	const elem_size: u32 = @intCast(elem_ty.abiSize(mod));
14549	15198
...	...	@@ -14789,7 +15438,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
14789	15438	mir_tag,
14790	15439	mop1_reg,
14791	15440	mop2_reg,
14792		mops[2].mem(Memory.Size.fromSize(abi_size)),
	15441	try mops[2].mem(self, Memory.Size.fromSize(abi_size)),
14793	15442	);
14794	15443	break :result mops[0];
14795	15444	};
...	...	@@ -14807,7 +15456,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
14807	15456	)) {
14808	15457	.SysV => result: {
14809	15458	const info = self.va_info.sysv;
14810		const dst_fi = try self.allocFrameIndex(FrameAlloc.initType(va_list_ty, mod));
	15459	const dst_fi = try self.allocFrameIndex(FrameAlloc.initSpill(va_list_ty, mod));
14811	15460	var field_off: u31 = 0;
14812	15461	// gp_offset: c_uint,
14813	15462	try self.genSetMem(
...	...	@@ -15015,7 +15664,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
15015	15664	.{ .v_ss, .cvtsd2 },
15016	15665	dst_reg,
15017	15666	dst_reg,
15018		promote_mcv.mem(.qword),
	15667	try promote_mcv.mem(self, .qword),
15019	15668	) else try self.asmRegisterRegisterRegister(
15020	15669	.{ .v_ss, .cvtsd2 },
15021	15670	dst_reg,
...	...	@@ -15027,7 +15676,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
15027	15676	) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(
15028	15677	.{ ._ss, .cvtsd2 },
15029	15678	dst_reg,
15030		promote_mcv.mem(.qword),
	15679	try promote_mcv.mem(self, .qword),
15031	15680	) else try self.asmRegisterRegister(
15032	15681	.{ ._ss, .cvtsd2 },
15033	15682	dst_reg,
...	...	@@ -15473,6 +16122,33 @@ fn memSize(self: *Self, ty: Type) Memory.Size {
15473	16122	};
15474	16123	}
15475	16124
	16125	fn splitType(self: *Self, ty: Type) ![2]Type {
	16126	const mod = self.bin_file.options.module.?;
	16127	const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
	16128	var parts: [2]Type = undefined;
	16129	if (classes.len == 2) for (&parts, classes, 0..) \|*part, class, part_i\| {
	16130	part.* = switch (class) {
	16131	.integer => switch (part_i) {
	16132	0 => Type.u64,
	16133	1 => part: {
	16134	const elem_size = ty.abiAlignment(mod).minStrict(.@"8").toByteUnitsOptional().?;
	16135	const elem_ty = try mod.intType(.unsigned, @intCast(elem_size * 8));
	16136	break :part switch (@divExact(ty.abiSize(mod) - 8, elem_size)) {
	16137	1 => elem_ty,
	16138	else => \|len\| try mod.arrayType(.{ .len = len, .child = elem_ty.toIntern() }),
	16139	};
	16140	},
	16141	else => unreachable,
	16142	},
	16143	.float => Type.f32,
	16144	.float_combine => try mod.vectorType(.{ .len = 2, .child = .f32_type }),
	16145	.sse => Type.f64,
	16146	else => break,
	16147	};
	16148	} else if (parts[0].abiSize(mod) + parts[1].abiSize(mod) == ty.abiSize(mod)) return parts;
	16149	return self.fail("TODO implement splitType for {}", .{ty.fmt(mod)});
	16150	}
	16151
15476	16152	/// Truncates the value in the register in place.
15477	16153	/// Clobbers any remaining bits.
15478	16154	fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {

src/arch/x86_64/Encoding.zig+10-8

...	...	@@ -410,6 +410,8 @@ pub const Mnemonic = enum {
410	410	vfmadd132ps, vfmadd213ps, vfmadd231ps,
411	411	vfmadd132sd, vfmadd213sd, vfmadd231sd,
412	412	vfmadd132ss, vfmadd213ss, vfmadd231ss,
	413	// AVX2
	414	vpbroadcastb, vpbroadcastd, vpbroadcasti128, vpbroadcastq, vpbroadcastw,
413	415	// zig fmt: on
414	416	};
415	417
...	...	@@ -444,7 +446,7 @@ pub const Op = enum {
444	446	moffs,
445	447	sreg,
446	448	st, mm, mm_m64,
447		xmm0, xmm, xmm_m32, xmm_m64, xmm_m128,
	449	xmm0, xmm, xmm_m8, xmm_m16, xmm_m32, xmm_m64, xmm_m128,
448	450	ymm, ymm_m256,
449	451	// zig fmt: on
450	452
...	...	@@ -534,7 +536,7 @@ pub const Op = enum {
534	536	.eax, .r32, .rm32, .r32_m16 => unreachable,
535	537	.rax, .r64, .rm64, .r64_m16 => unreachable,
536	538	.st, .mm, .mm_m64 => unreachable,
537		.xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
	539	.xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
538	540	.ymm, .ymm_m256 => unreachable,
539	541	.m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
540	542	.unity => 1,
...	...	@@ -556,7 +558,7 @@ pub const Op = enum {
556	558	.eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,
557	559	.rax, .r64, .rm64, .r64_m16, .mm, .mm_m64 => 64,
558	560	.st => 80,
559		.xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
	561	.xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
560	562	.ymm, .ymm_m256 => 256,
561	563	};
562	564	}
...	...	@@ -568,8 +570,8 @@ pub const Op = enum {
568	570	.rel8, .rel16, .rel32 => unreachable,
569	571	.al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64 => unreachable,
570	572	.st, .mm, .xmm0, .xmm, .ymm => unreachable,
571		.m8, .rm8, .r32_m8 => 8,
572		.m16, .rm16, .r32_m16, .r64_m16 => 16,
	573	.m8, .rm8, .r32_m8, .xmm_m8 => 8,
	574	.m16, .rm16, .r32_m16, .r64_m16, .xmm_m16 => 16,
573	575	.m32, .rm32, .xmm_m32 => 32,
574	576	.m64, .rm64, .mm_m64, .xmm_m64 => 64,
575	577	.m80 => 80,
...	...	@@ -600,7 +602,7 @@ pub const Op = enum {
600	602	.rm8, .rm16, .rm32, .rm64,
601	603	.r32_m8, .r32_m16, .r64_m16,
602	604	.st, .mm, .mm_m64,
603		.xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128,
	605	.xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128,
604	606	.ymm, .ymm_m256,
605	607	=> true,
606	608	else => false,
...	...	@@ -629,7 +631,7 @@ pub const Op = enum {
629	631	.m8, .m16, .m32, .m64, .m80, .m128, .m256,
630	632	.m,
631	633	.mm_m64,
632		.xmm_m32, .xmm_m64, .xmm_m128,
	634	.xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128,
633	635	.ymm_m256,
634	636	=> true,
635	637	else => false,
...	...	@@ -654,7 +656,7 @@ pub const Op = enum {
654	656	.sreg => .segment,
655	657	.st => .x87,
656	658	.mm, .mm_m64 => .mmx,
657		.xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .sse,
	659	.xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => .sse,
658	660	.ymm, .ymm_m256 => .sse,
659	661	};
660	662	}

src/arch/x86_64/Mir.zig+2

...	...	@@ -255,6 +255,8 @@ pub const Inst = struct {
255	255	vp_q,
256	256	/// VEX-Encoded Packed ___ Double Quadword
257	257	vp_dq,
	258	/// VEX-Encoded Packed ___ Integer Data
	259	vp_i128,
258	260	/// VEX-Encoded ___ Scalar Single-Precision Values
259	261	v_ss,
260	262	/// VEX-Encoded ___ Packed Single-Precision Values

src/arch/x86_64/bits.zig+1-1

...	...	@@ -237,7 +237,7 @@ pub const Register = enum(u7) {
237	237	return @intCast(@intFromEnum(reg) - base);
238	238	}
239	239
240		pub fn bitSize(reg: Register) u64 {
	240	pub fn bitSize(reg: Register) u10 {
241	241	return switch (@intFromEnum(reg)) {
242	242	// zig fmt: off
243	243	@intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => 64,

src/arch/x86_64/encodings.zig+10

...	...	@@ -1742,6 +1742,16 @@ pub const table = [_]Entry{
1742	1742
1743	1743	.{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },
1744	1744
	1745	.{ .vpbroadcastb, .rm, &.{ .xmm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 },
	1746	.{ .vpbroadcastb, .rm, &.{ .ymm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 },
	1747	.{ .vpbroadcastw, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 },
	1748	.{ .vpbroadcastw, .rm, &.{ .ymm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_256_w0, .avx2 },
	1749	.{ .vpbroadcastd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_128_w0, .avx2 },
	1750	.{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 },
	1751	.{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 },
	1752	.{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 },
	1753	.{ .vpbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
	1754
1745	1755	.{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },
1746	1756	.{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },
1747	1757	.{ .vpcmpeqd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x76 }, 0, .vex_256_wig, .avx2 },

src/codegen.zig+4-1

...	...	@@ -376,7 +376,10 @@ pub fn generateSymbol(
376	376	.val = switch (aggregate.storage) {
377	377	.bytes => unreachable,
378	378	.elems => \|elems\| elems[@as(usize, @intCast(index))],
379		.repeated_elem => \|elem\| elem,
	379	.repeated_elem => \|elem\| if (index < array_type.len)
	380	elem
	381	else
	382	array_type.sentinel,
380	383	}.toValue(),
381	384	}, code, debug_output, reloc_info)) {
382	385	.ok => {},

src/link/Coff.zig+1

...	...	@@ -388,6 +388,7 @@ fn populateMissingMetadata(self: *Coff) !void {
388	388	self.rdata_section_index = try self.allocateSection(".rdata", file_size, .{
389	389	.CNT_INITIALIZED_DATA = 1,
390	390	.MEM_READ = 1,
	391	.MEM_WRITE = 1,
391	392	});
392	393	}
393	394

test/behavior/vector.zig-1

...	...	@@ -1260,7 +1260,6 @@ test "zero multiplicand" {
1260	1260
1261	1261	test "@intCast to u0" {
1262	1262	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1263		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1264	1263	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1265	1264	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1266	1265	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO