Commit fa022d1ecc148280a3b6e95312087b4e8c0c6166 | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-10-29 13:10:51-04:00
committer	GitHub <noreply@github.com> 2023-10-29 13:10:51-04:00
log	fa022d1ecc148280a3b6e95312087b4e8c0c6166
tree	576863cc60c8626c8b89b01621d0538a4923a9ce
parent	37295696ec5b1350333b88173187a3db8b199925
parent	f6f2708d8289e1cd6c24b954c261d478ee73ccfc
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17759 from jacobly0/x86_64

x86_64: pass more tests

37 files changed, 1482 insertions(+), 1405 deletions(-)

lib
- compiler_rt
- std
  - Thread
    Condition.zig
  - crypto
    phc_encoding.zig
  - dwarf
    expressions.zig
  - fmt.zig
  - hash
    auto_hash.zig
    wyhash.zig
  - io
    bit_reader.zig
    test.zig
  - leb128.zig
  - math
    big
    int_test.zig
    ilogb.zig
    ldexp.zig
src
- arch
  - x86_64
    CodeGen.zig
    Disassembler.zig
    Emit.zig
    Encoding.zig
    Lower.zig
    Mir.zig
    bits.zig
    encoder.zig
test
- behavior
- tests.zig

lib/compiler_rt/addf3_test.zig-4

...	...	@@ -35,8 +35,6 @@ fn test__addtf3(a: f128, b: f128, expected_hi: u64, expected_lo: u64) !void {
35	35	}
36	36
37	37	test "addtf3" {
38		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
39
40	38	try test__addtf3(qnan128, 0x1.23456789abcdefp+5, 0x7fff800000000000, 0x0);
41	39
42	40	// NaN + any = NaN
...	...	@@ -106,8 +104,6 @@ fn test__addxf3(a: f80, b: f80, expected: u80) !void {
106	104	}
107	105
108	106	test "addxf3" {
109		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
110
111	107	// NaN + any = NaN
112	108	try test__addxf3(qnan80, 0x1.23456789abcdefp+5, @as(u80, @bitCast(qnan80)));
113	109	try test__addxf3(@as(f80, @bitCast(@as(u80, 0x7fff_8000_8000_3000_0000))), 0x1.23456789abcdefp+5, @as(u80, @bitCast(qnan80)));

lib/compiler_rt/addoti4_test.zig-2

...	...	@@ -23,8 +23,6 @@ fn simple_addoti4(a: i128, b: i128, overflow: *c_int) i128 {
23	23	}
24	24
25	25	test "addoti4" {
26		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
27
28	26	const min: i128 = math.minInt(i128);
29	27	const max: i128 = math.maxInt(i128);
30	28	var i: i128 = 1;

lib/compiler_rt/comparef.zig-2

...	...	@@ -98,8 +98,6 @@ pub inline fn cmp_f80(comptime RT: type, a: f80, b: f80) RT {
98	98	}
99	99
100	100	test "cmp_f80" {
101		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
102
103	101	inline for (.{ LE, GE }) \|RT\| {
104	102	try std.testing.expect(cmp_f80(RT, 1.0, 1.0) == RT.Equal);
105	103	try std.testing.expect(cmp_f80(RT, 0.0, -0.0) == RT.Equal);

lib/compiler_rt/cos.zig-2

...	...	@@ -134,8 +134,6 @@ pub fn cosl(x: c_longdouble) callconv(.C) c_longdouble {
134	134	}
135	135
136	136	test "cos32" {
137		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
138
139	137	const epsilon = 0.00001;
140	138
141	139	try expect(math.approxEqAbs(f32, cosf(0.0), 1.0, epsilon));

lib/compiler_rt/divxf3_test.zig-2

...	...	@@ -39,8 +39,6 @@ fn test__divxf3(a: f80, b: f80) !void {
39	39	}
40	40
41	41	test "divxf3" {
42		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
43
44	42	// NaN / any = NaN
45	43	try expect__divxf3_result(math.nan(f80), 0x1.23456789abcdefp+5, 0x7fffC000000000000000);
46	44	// inf / any(except inf and nan) = inf

lib/compiler_rt/fmodx_test.zig-3

...	...	@@ -23,9 +23,6 @@ fn test_fmodx_infs() !void {
23	23	}
24	24
25	25	test "fmodx" {
26		if (builtin.zig_backend == .stage2_x86_64 and
27		!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .lzcnt)) return error.SkipZigTest;
28
29	26	try test_fmodx(6.4, 4.0, 2.4);
30	27	try test_fmodx(6.4, -4.0, 2.4);
31	28	try test_fmodx(-6.4, 4.0, -2.4);

lib/compiler_rt/int_from_float_test.zig+1-36

...	...	@@ -41,8 +41,6 @@ fn test__fixunssfsi(a: f32, expected: u32) !void {
41	41	}
42	42
43	43	test "fixsfsi" {
44		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
45
46	44	try test__fixsfsi(-math.floatMax(f32), math.minInt(i32));
47	45
48	46	try test__fixsfsi(-0x1.FFFFFFFFFFFFFp+1023, math.minInt(i32));
...	...	@@ -71,6 +69,7 @@ test "fixsfsi" {
71	69	try test__fixsfsi(-1.0, -1);
72	70	try test__fixsfsi(-0.99, 0);
73	71	try test__fixsfsi(-0.5, 0);
	72
74	73	try test__fixsfsi(-math.floatMin(f32), 0);
75	74	try test__fixsfsi(0.0, 0);
76	75	try test__fixsfsi(math.floatMin(f32), 0);
...	...	@@ -106,8 +105,6 @@ test "fixsfsi" {
106	105	}
107	106
108	107	test "fixunssfsi" {
109		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
110
111	108	try test__fixunssfsi(0.0, 0);
112	109
113	110	try test__fixunssfsi(0.5, 0);
...	...	@@ -147,8 +144,6 @@ fn test__fixunssfdi(a: f32, expected: u64) !void {
147	144	}
148	145
149	146	test "fixsfdi" {
150		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
151
152	147	try test__fixsfdi(-math.floatMax(f32), math.minInt(i64));
153	148
154	149	try test__fixsfdi(-0x1.FFFFFFFFFFFFFp+1023, math.minInt(i64));
...	...	@@ -204,8 +199,6 @@ test "fixsfdi" {
204	199	}
205	200
206	201	test "fixunssfdi" {
207		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
208
209	202	try test__fixunssfdi(0.0, 0);
210	203
211	204	try test__fixunssfdi(0.5, 0);
...	...	@@ -244,8 +237,6 @@ fn test__fixunssfti(a: f32, expected: u128) !void {
244	237	}
245	238
246	239	test "fixsfti" {
247		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
248
249	240	try test__fixsfti(-math.floatMax(f32), math.minInt(i128));
250	241
251	242	try test__fixsfti(-0x1.FFFFFFFFFFFFFp+1023, math.minInt(i128));
...	...	@@ -317,8 +308,6 @@ test "fixsfti" {
317	308	}
318	309
319	310	test "fixunssfti" {
320		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
321
322	311	try test__fixunssfti(0.0, 0);
323	312
324	313	try test__fixunssfti(0.5, 0);
...	...	@@ -365,8 +354,6 @@ fn test__fixunsdfsi(a: f64, expected: u32) !void {
365	354	}
366	355
367	356	test "fixdfsi" {
368		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
369
370	357	try test__fixdfsi(-math.floatMax(f64), math.minInt(i32));
371	358
372	359	try test__fixdfsi(-0x1.FFFFFFFFFFFFFp+1023, math.minInt(i32));
...	...	@@ -428,8 +415,6 @@ test "fixdfsi" {
428	415	}
429	416
430	417	test "fixunsdfsi" {
431		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
432
433	418	try test__fixunsdfsi(0.0, 0);
434	419
435	420	try test__fixunsdfsi(0.5, 0);
...	...	@@ -472,8 +457,6 @@ fn test__fixunsdfdi(a: f64, expected: u64) !void {
472	457	}
473	458
474	459	test "fixdfdi" {
475		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
476
477	460	try test__fixdfdi(-math.floatMax(f64), math.minInt(i64));
478	461
479	462	try test__fixdfdi(-0x1.FFFFFFFFFFFFFp+1023, math.minInt(i64));
...	...	@@ -527,8 +510,6 @@ test "fixdfdi" {
527	510	}
528	511
529	512	test "fixunsdfdi" {
530		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
531
532	513	try test__fixunsdfdi(0.0, 0);
533	514	try test__fixunsdfdi(0.5, 0);
534	515	try test__fixunsdfdi(0.99, 0);
...	...	@@ -571,8 +552,6 @@ fn test__fixunsdfti(a: f64, expected: u128) !void {
571	552	}
572	553
573	554	test "fixdfti" {
574		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
575
576	555	try test__fixdfti(-math.floatMax(f64), math.minInt(i128));
577	556
578	557	try test__fixdfti(-0x1.FFFFFFFFFFFFFp+1023, math.minInt(i128));
...	...	@@ -626,8 +605,6 @@ test "fixdfti" {
626	605	}
627	606
628	607	test "fixunsdfti" {
629		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
630
631	608	try test__fixunsdfti(0.0, 0);
632	609
633	610	try test__fixunsdfti(0.5, 0);
...	...	@@ -677,8 +654,6 @@ fn test__fixunstfsi(a: f128, expected: u32) !void {
677	654	}
678	655
679	656	test "fixtfsi" {
680		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
681
682	657	try test__fixtfsi(-math.floatMax(f128), math.minInt(i32));
683	658
684	659	try test__fixtfsi(-0x1.FFFFFFFFFFFFFp+1023, math.minInt(i32));
...	...	@@ -742,8 +717,6 @@ test "fixtfsi" {
742	717	}
743	718
744	719	test "fixunstfsi" {
745		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
746
747	720	try test__fixunstfsi(math.inf(f128), 0xffffffff);
748	721	try test__fixunstfsi(0, 0x0);
749	722	try test__fixunstfsi(0x1.23456789abcdefp+5, 0x24);
...	...	@@ -767,8 +740,6 @@ fn test__fixunstfdi(a: f128, expected: u64) !void {
767	740	}
768	741
769	742	test "fixtfdi" {
770		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
771
772	743	try test__fixtfdi(-math.floatMax(f128), math.minInt(i64));
773	744
774	745	try test__fixtfdi(-0x1.FFFFFFFFFFFFFp+1023, math.minInt(i64));
...	...	@@ -832,8 +803,6 @@ test "fixtfdi" {
832	803	}
833	804
834	805	test "fixunstfdi" {
835		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
836
837	806	try test__fixunstfdi(0.0, 0);
838	807
839	808	try test__fixunstfdi(0.5, 0);
...	...	@@ -886,8 +855,6 @@ fn test__fixunstfti(a: f128, expected: u128) !void {
886	855	}
887	856
888	857	test "fixtfti" {
889		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
890
891	858	try test__fixtfti(-math.floatMax(f128), math.minInt(i128));
892	859
893	860	try test__fixtfti(-0x1.FFFFFFFFFFFFFp+1023, math.minInt(i128));
...	...	@@ -969,8 +936,6 @@ fn test__fixunshfti(a: f16, expected: u128) !void {
969	936	}
970	937
971	938	test "fixunshfti for f16" {
972		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
973
974	939	try test__fixunshfti(math.inf(f16), math.maxInt(u128));
975	940	try test__fixunshfti(math.floatMax(f16), 65504);
976	941	}

lib/compiler_rt/negti2_test.zig-2

...	...	@@ -7,8 +7,6 @@ fn test__negti2(a: i128, expected: i128) !void {
7	7	}
8	8
9	9	test "negti2" {
10		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
11
12	10	// TODO ensuring that math.minInt(i128); returns error
13	11
14	12	try test__negti2(-3, 3);

lib/compiler_rt/powiXf2_test.zig-7

...	...	@@ -34,9 +34,6 @@ fn test__powixf2(a: f80, b: i32, expected: f80) !void {
34	34	}
35	35
36	36	test "powihf2" {
37		if (builtin.zig_backend == .stage2_x86_64 and
38		!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .f16c)) return error.SkipZigTest;
39
40	37	const inf_f16 = math.inf(f16);
41	38	try test__powisf2(0, 0, 1);
42	39	try test__powihf2(1, 0, 1);
...	...	@@ -356,8 +353,6 @@ test "powidf2" {
356	353	}
357	354
358	355	test "powitf2" {
359		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
360
361	356	const inf_f128 = math.inf(f128);
362	357	try test__powitf2(0, 0, 1);
363	358	try test__powitf2(1, 0, 1);
...	...	@@ -463,8 +458,6 @@ test "powitf2" {
463	458	}
464	459
465	460	test "powixf2" {
466		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
467
468	461	const inf_f80 = math.inf(f80);
469	462	try test__powixf2(0, 0, 1);
470	463	try test__powixf2(1, 0, 1);

lib/compiler_rt/sin.zig-2

...	...	@@ -140,8 +140,6 @@ pub fn sinl(x: c_longdouble) callconv(.C) c_longdouble {
140	140	}
141	141
142	142	test "sin32" {
143		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
144
145	143	const epsilon = 0.00001;
146	144
147	145	try expect(math.approxEqAbs(f32, sinf(0.0), 0.0, epsilon));

lib/compiler_rt/suboti4_test.zig-2

...	...	@@ -27,8 +27,6 @@ pub fn simple_suboti4(a: i128, b: i128, overflow: *c_int) i128 {
27	27	}
28	28
29	29	test "suboti3" {
30		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
31
32	30	const min: i128 = math.minInt(i128);
33	31	const max: i128 = math.maxInt(i128);
34	32	var i: i128 = 1;

lib/compiler_rt/tan.zig-2

...	...	@@ -131,8 +131,6 @@ test "tan" {
131	131	}
132	132
133	133	test "tan32" {
134		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
135
136	134	const epsilon = 0.00001;
137	135
138	136	try expect(math.approxEqAbs(f32, tanf(0.0), 0.0, epsilon));

lib/std/Thread/Condition.zig+2

...	...	@@ -440,6 +440,8 @@ test "Condition - multi signal" {
440	440	return error.SkipZigTest;
441	441	}
442	442
	443	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	444
443	445	const num_threads = 4;
444	446	const num_iterations = 4;
445	447

lib/std/crypto/phc_encoding.zig-2

...	...	@@ -363,8 +363,6 @@ test "phc format - hash without salt" {
363	363	}
364	364
365	365	test "phc format - calcSize" {
366		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
367
368	366	const s = "$scrypt$v=1$ln=15,r=8,p=1$c2FsdHNhbHQ$dGVzdHBhc3M";
369	367	const v = try deserialize(struct {
370	368	alg_id: []const u8,

lib/std/dwarf/expressions.zig-2

...	...	@@ -1054,8 +1054,6 @@ fn isOpcodeRegisterLocation(opcode: u8) bool {
1054	1054
1055	1055	const testing = std.testing;
1056	1056	test "DWARF expressions" {
1057		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1058
1059	1057	const allocator = std.testing.allocator;
1060	1058
1061	1059	const options = ExpressionOptions{};

lib/std/fmt.zig-2

...	...	@@ -2435,8 +2435,6 @@ test "float.hexadecimal" {
2435	2435	}
2436	2436
2437	2437	test "float.hexadecimal.precision" {
2438		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2439
2440	2438	try expectFmt("f16: 0x1.5p-2", "f16: {x:.1}", .{@as(f16, 1.0 / 3.0)});
2441	2439	try expectFmt("f32: 0x1.555p-2", "f32: {x:.3}", .{@as(f32, 1.0 / 3.0)});
2442	2440	try expectFmt("f64: 0x1.55555p-2", "f64: {x:.5}", .{@as(f64, 1.0 / 3.0)});

lib/std/hash/auto_hash.zig-21

...	...	@@ -1,6 +1,5 @@
1	1	const std = @import("std");
2	2	const assert = std.debug.assert;
3		const builtin = @import("builtin");
4	3	const mem = std.mem;
5	4	const meta = std.meta;
6	5
...	...	@@ -253,8 +252,6 @@ test "typeContainsSlice" {
253	252	}
254	253
255	254	test "hash pointer" {
256		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
257
258	255	const array = [_]u32{ 123, 123, 123 };
259	256	const a = &array[0];
260	257	const b = &array[1];
...	...	@@ -275,8 +272,6 @@ test "hash pointer" {
275	272	}
276	273
277	274	test "hash slice shallow" {
278		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
279
280	275	// Allocate one array dynamically so that we're assured it is not merged
281	276	// with the other by the optimization passes.
282	277	const array1 = try std.testing.allocator.create([6]u32);
...	...	@@ -295,8 +290,6 @@ test "hash slice shallow" {
295	290	}
296	291
297	292	test "hash slice deep" {
298		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
299
300	293	// Allocate one array dynamically so that we're assured it is not merged
301	294	// with the other by the optimization passes.
302	295	const array1 = try std.testing.allocator.create([6]u32);
...	...	@@ -313,8 +306,6 @@ test "hash slice deep" {
313	306	}
314	307
315	308	test "hash struct deep" {
316		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
317
318	309	const Foo = struct {
319	310	a: u32,
320	311	b: u16,
...	...	@@ -354,8 +345,6 @@ test "hash struct deep" {
354	345	}
355	346
356	347	test "testHash optional" {
357		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
358
359	348	const a: ?u32 = 123;
360	349	const b: ?u32 = null;
361	350	try testing.expectEqual(testHash(a), testHash(@as(u32, 123)));
...	...	@@ -364,8 +353,6 @@ test "testHash optional" {
364	353	}
365	354
366	355	test "testHash array" {
367		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
368
369	356	const a = [_]u32{ 1, 2, 3 };
370	357	const h = testHash(a);
371	358	var hasher = Wyhash.init(0);
...	...	@@ -382,8 +369,6 @@ test "testHash multi-dimensional array" {
382	369	}
383	370
384	371	test "testHash struct" {
385		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
386
387	372	const Foo = struct {
388	373	a: u32 = 1,
389	374	b: u32 = 2,
...	...	@@ -399,8 +384,6 @@ test "testHash struct" {
399	384	}
400	385
401	386	test "testHash union" {
402		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
403
404	387	const Foo = union(enum) {
405	388	A: u32,
406	389	B: bool,
...	...	@@ -425,8 +408,6 @@ test "testHash union" {
425	408	}
426	409
427	410	test "testHash vector" {
428		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
429
430	411	const a: @Vector(4, u32) = [_]u32{ 1, 2, 3, 4 };
431	412	const b: @Vector(4, u32) = [_]u32{ 1, 2, 3, 5 };
432	413	try testing.expect(testHash(a) == testHash(a));
...	...	@@ -439,8 +420,6 @@ test "testHash vector" {
439	420	}
440	421
441	422	test "testHash error union" {
442		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
443
444	423	const Errors = error{Test};
445	424	const Foo = struct {
446	425	a: u32 = 1,

lib/std/hash/wyhash.zig-2

...	...	@@ -242,8 +242,6 @@ test "smhasher" {
242	242	}
243	243
244	244	test "iterative api" {
245		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
246
247	245	const Test = struct {
248	246	fn do() !void {
249	247	try verify.iterativeApi(Wyhash);

lib/std/io/bit_reader.zig-2

...	...	@@ -164,8 +164,6 @@ pub fn bitReader(
164	164	}
165	165
166	166	test "api coverage" {
167		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
168
169	167	const mem_be = [_]u8{ 0b11001101, 0b00001011 };
170	168	const mem_le = [_]u8{ 0b00011101, 0b10010101 };
171	169

lib/std/io/test.zig+1-4

...	...	@@ -1,5 +1,4 @@
1	1	const std = @import("std");
2		const builtin = @import("builtin");
3	2	const io = std.io;
4	3	const meta = std.meta;
5	4	const trait = std.trait;
...	...	@@ -10,7 +9,7 @@ const expectError = std.testing.expectError;
10	9	const mem = std.mem;
11	10	const fs = std.fs;
12	11	const File = std.fs.File;
13		const native_endian = builtin.target.cpu.arch.endian();
	12	const native_endian = @import("builtin").target.cpu.arch.endian();
14	13
15	14	const tmpDir = std.testing.tmpDir;
16	15
...	...	@@ -61,8 +60,6 @@ test "write a file, read it, then delete it" {
61	60	}
62	61
63	62	test "BitStreams with File Stream" {
64		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
65
66	63	var tmp = tmpDir(.{});
67	64	defer tmp.cleanup();
68	65

lib/std/leb128.zig-5

...	...	@@ -1,4 +1,3 @@
1		const builtin = @import("builtin");
2	1	const std = @import("std");
3	2	const testing = std.testing;
4	3
...	...	@@ -215,8 +214,6 @@ fn test_read_uleb128_seq(comptime T: type, comptime N: usize, encoded: []const u
215	214	}
216	215
217	216	test "deserialize signed LEB128" {
218		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
219
220	217	// Truncated
221	218	try testing.expectError(error.EndOfStream, test_read_stream_ileb128(i64, "\x80"));
222	219
...	...	@@ -363,8 +360,6 @@ test "serialize unsigned LEB128" {
363	360	}
364	361
365	362	test "serialize signed LEB128" {
366		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
367
368	363	// explicitly test i0 because starting `t` at 0
369	364	// will break the while loop
370	365	try test_write_leb128(@as(i0, 0));

lib/std/math/big/int_test.zig-4

...	...	@@ -990,8 +990,6 @@ test "big.int mul 0*0" {
990	990	}
991	991
992	992	test "big.int mul large" {
993		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
994
995	993	var a = try Managed.initCapacity(testing.allocator, 50);
996	994	defer a.deinit();
997	995	var b = try Managed.initCapacity(testing.allocator, 100);
...	...	@@ -1075,8 +1073,6 @@ test "big.int mulWrap multi-multi signed" {
1075	1073	}
1076	1074
1077	1075	test "big.int mulWrap large" {
1078		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1079
1080	1076	var a = try Managed.initCapacity(testing.allocator, 50);
1081	1077	defer a.deinit();
1082	1078	var b = try Managed.initCapacity(testing.allocator, 100);

lib/std/math/ilogb.zig-2

...	...	@@ -108,8 +108,6 @@ test "64" {
108	108	}
109	109
110	110	test "80" {
111		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
112
113	111	try expect(ilogbX(f80, 0.0) == fp_ilogb0);
114	112	try expect(ilogbX(f80, 0.5) == -1);
115	113	try expect(ilogbX(f80, 0.8923) == -1);

lib/std/math/ldexp.zig-2

...	...	@@ -67,8 +67,6 @@ pub fn ldexp(x: anytype, n: i32) @TypeOf(x) {
67	67	}
68	68
69	69	test "math.ldexp" {
70		if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
71
72	70	// subnormals
73	71	try expect(ldexp(@as(f16, 0x1.1FFp14), -14 - 9 - 15) == math.floatTrueMin(f16));
74	72	try expect(ldexp(@as(f32, 0x1.3FFFFFp-1), -126 - 22) == math.floatTrueMin(f32));

src/arch/x86_64/CodeGen.zig+997-703

...	...	@@ -115,6 +115,7 @@ const mir_to_air_map_init = if (builtin.mode == .Debug) std.AutoHashMapUnmanaged
115	115
116	116	const FrameAddr = struct { index: FrameIndex, off: i32 = 0 };
117	117	const RegisterOffset = struct { reg: Register, off: i32 = 0 };
	118	const SymbolOffset = struct { sym: u32, off: i32 = 0 };
118	119
119	120	const Owner = union(enum) {
120	121	func_index: InternPool.Index,
...	...	@@ -195,9 +196,9 @@ pub const MCValue = union(enum) {
195	196	memory: u64,
196	197	/// The value is in memory at an address not-yet-allocated by the linker.
197	198	/// This traditionally corresponds to a relocation emitted in a relocatable object file.
198		load_symbol: u32,
	199	load_symbol: SymbolOffset,
199	200	/// The address of the memory location not-yet-allocated by the linker.
200		lea_symbol: u32,
	201	lea_symbol: SymbolOffset,
201	202	/// The value is in memory at a constant offset from the address in a register.
202	203	indirect: RegisterOffset,
203	204	/// The value is in memory.
...	...	@@ -315,7 +316,7 @@ pub const MCValue = union(enum) {
315	316	.load_got => \|sym_index\| .{ .lea_got = sym_index },
316	317	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
317	318	.load_frame => \|frame_addr\| .{ .lea_frame = frame_addr },
318		.load_symbol => \|sym_index\| .{ .lea_symbol = sym_index },
	319	.load_symbol => \|sym_off\| .{ .lea_symbol = sym_off },
319	320	};
320	321	}
321	322
...	...	@@ -387,7 +388,7 @@ pub const MCValue = union(enum) {
387	388	};
388	389	}
389	390
390		fn mem(mcv: MCValue, ptr_size: Memory.PtrSize) Memory {
	391	fn mem(mcv: MCValue, size: Memory.Size) Memory {
391	392	return switch (mcv) {
392	393	.none,
393	394	.unreach,
...	...	@@ -411,18 +412,27 @@ pub const MCValue = union(enum) {
411	412	.load_symbol,
412	413	.lea_symbol,
413	414	=> unreachable,
414		.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
415		Memory.sib(ptr_size, .{ .base = .{ .reg = .ds }, .disp = small_addr })
416		else
417		Memory.moffs(.ds, addr),
418		.indirect => \|reg_off\| Memory.sib(ptr_size, .{
	415	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\| .{
	416	.base = .{ .reg = .ds },
	417	.mod = .{ .rm = .{
	418	.size = size,
	419	.disp = small_addr,
	420	} },
	421	} else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } },
	422	.indirect => \|reg_off\| .{
419	423	.base = .{ .reg = reg_off.reg },
420		.disp = reg_off.off,
421		}),
422		.load_frame => \|frame_addr\| Memory.sib(ptr_size, .{
	424	.mod = .{ .rm = .{
	425	.size = size,
	426	.disp = reg_off.off,
	427	} },
	428	},
	429	.load_frame => \|frame_addr\| .{
423	430	.base = .{ .frame = frame_addr.index },
424		.disp = frame_addr.off,
425		}),
	431	.mod = .{ .rm = .{
	432	.size = size,
	433	.disp = frame_addr.off,
	434	} },
	435	},
426	436	};
427	437	}
428	438
...	...	@@ -442,6 +452,8 @@ pub const MCValue = union(enum) {
442	452	.register_overflow => \|pl\| try writer.print("{s}:{s}", .{
443	453	@tagName(pl.eflags), @tagName(pl.reg),
444	454	}),
	455	.load_symbol => \|pl\| try writer.print("[{} + 0x{x}]", .{ pl.sym, pl.off }),
	456	.lea_symbol => \|pl\| try writer.print("{} + 0x{x}", .{ pl.sym, pl.off }),
445	457	.indirect => \|pl\| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
446	458	.load_direct => \|pl\| try writer.print("[direct:{d}]", .{pl}),
447	459	.lea_direct => \|pl\| try writer.print("direct:{d}", .{pl}),
...	...	@@ -453,8 +465,6 @@ pub const MCValue = union(enum) {
453	465	.lea_frame => \|pl\| try writer.print("{} + 0x{x}", .{ pl.index, pl.off }),
454	466	.reserved_frame => \|pl\| try writer.print("(dead:{})", .{pl}),
455	467	.air_ref => \|pl\| try writer.print("(air:0x{x})", .{@intFromEnum(pl)}),
456		.load_symbol => \|pl\| try writer.print("[symbol:{d}]", .{pl}),
457		.lea_symbol => \|pl\| try writer.print("symbol:{d}", .{pl}),
458	468	}
459	469	}
460	470	};
...	...	@@ -888,7 +898,6 @@ pub fn generate(
888	898	.cc = cc,
889	899	.src_loc = src_loc,
890	900	},
891		.bin_file = bin_file,
892	901	.debug_output = debug_output,
893	902	.code = code,
894	903	.prev_di_pc = 0,
...	...	@@ -976,7 +985,6 @@ pub fn generateLazy(
976	985	.cc = abi.resolveCallingConvention(.Unspecified, function.target.*),
977	986	.src_loc = src_loc,
978	987	},
979		.bin_file = bin_file,
980	988	.debug_output = debug_output,
981	989	.code = code,
982	990	.prev_di_pc = undefined, // no debug info yet
...	...	@@ -1139,7 +1147,7 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1139	1147	inline for (fields) \|field\| {
1140	1148	self.mir_extra.appendAssumeCapacity(switch (field.type) {
1141	1149	u32 => @field(extra, field.name),
1142		i32 => @bitCast(@field(extra, field.name)),
	1150	i32, Mir.Memory.Info => @bitCast(@field(extra, field.name)),
1143	1151	else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)),
1144	1152	});
1145	1153	}
...	...	@@ -1178,17 +1186,9 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)
1178	1186	.nz_or_p => .pseudo,
1179	1187	},
1180	1188	.ops = switch (cc) {
1181		else => switch (m) {
1182		.sib => .rm_sib,
1183		.rip => .rm_rip,
1184		else => unreachable,
1185		},
	1189	else => .rm,
1186	1190	.z_and_np => unreachable,
1187		.nz_or_p => switch (m) {
1188		.sib => .pseudo_cmov_nz_or_p_rm_sib,
1189		.rip => .pseudo_cmov_nz_or_p_rm_rip,
1190		else => unreachable,
1191		},
	1191	.nz_or_p => .pseudo_cmov_nz_or_p_rm,
1192	1192	},
1193	1193	.data = .{ .rx = .{
1194	1194	.fixes = switch (cc) {
...	...	@@ -1197,11 +1197,7 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)
1197	1197	.nz_or_p => ._,
1198	1198	},
1199	1199	.r1 = reg,
1200		.payload = switch (m) {
1201		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1202		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1203		else => unreachable,
1204		},
	1200	.payload = try self.addExtra(Mir.Memory.encode(m)),
1205	1201	} },
1206	1202	});
1207	1203	}
...	...	@@ -1231,32 +1227,16 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
1231	1227	}
1232	1228
1233	1229	fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
1234		const payload = switch (m) {
1235		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1236		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1237		else => unreachable,
1238		};
	1230	const payload = try self.addExtra(Mir.Memory.encode(m));
1239	1231	_ = try self.addInst(.{
1240	1232	.tag = switch (cc) {
1241	1233	else => .set,
1242	1234	.z_and_np, .nz_or_p => .pseudo,
1243	1235	},
1244	1236	.ops = switch (cc) {
1245		else => switch (m) {
1246		.sib => .m_sib,
1247		.rip => .m_rip,
1248		else => unreachable,
1249		},
1250		.z_and_np => switch (m) {
1251		.sib => .pseudo_set_z_and_np_m_sib,
1252		.rip => .pseudo_set_z_and_np_m_rip,
1253		else => unreachable,
1254		},
1255		.nz_or_p => switch (m) {
1256		.sib => .pseudo_set_nz_or_p_m_sib,
1257		.rip => .pseudo_set_nz_or_p_m_rip,
1258		else => unreachable,
1259		},
	1237	else => .m,
	1238	.z_and_np => .pseudo_set_z_and_np_m,
	1239	.nz_or_p => .pseudo_set_nz_or_p_m,
1260	1240	},
1261	1241	.data = switch (cc) {
1262	1242	else => .{ .x = .{
...	...	@@ -1504,20 +1484,12 @@ fn asmRegisterRegisterMemory(
1504	1484	) !void {
1505	1485	_ = try self.addInst(.{
1506	1486	.tag = tag[1],
1507		.ops = switch (m) {
1508		.sib => .rrm_sib,
1509		.rip => .rrm_rip,
1510		else => unreachable,
1511		},
	1487	.ops = .rrm,
1512	1488	.data = .{ .rrx = .{
1513	1489	.fixes = tag[0],
1514	1490	.r1 = reg1,
1515	1491	.r2 = reg2,
1516		.payload = switch (m) {
1517		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1518		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1519		else => unreachable,
1520		},
	1492	.payload = try self.addExtra(Mir.Memory.encode(m)),
1521	1493	} },
1522	1494	});
1523	1495	}
...	...	@@ -1525,18 +1497,10 @@ fn asmRegisterRegisterMemory(
1525	1497	fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1526	1498	_ = try self.addInst(.{
1527	1499	.tag = tag[1],
1528		.ops = switch (m) {
1529		.sib => .m_sib,
1530		.rip => .m_rip,
1531		else => unreachable,
1532		},
	1500	.ops = .m,
1533	1501	.data = .{ .x = .{
1534	1502	.fixes = tag[0],
1535		.payload = switch (m) {
1536		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1537		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1538		else => unreachable,
1539		},
	1503	.payload = try self.addExtra(Mir.Memory.encode(m)),
1540	1504	} },
1541	1505	});
1542	1506	}
...	...	@@ -1544,19 +1508,11 @@ fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1544	1508	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
1545	1509	_ = try self.addInst(.{
1546	1510	.tag = tag[1],
1547		.ops = switch (m) {
1548		.sib => .rm_sib,
1549		.rip => .rm_rip,
1550		else => unreachable,
1551		},
	1511	.ops = .rm,
1552	1512	.data = .{ .rx = .{
1553	1513	.fixes = tag[0],
1554	1514	.r1 = reg,
1555		.payload = switch (m) {
1556		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1557		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1558		else => unreachable,
1559		},
	1515	.payload = try self.addExtra(Mir.Memory.encode(m)),
1560	1516	} },
1561	1517	});
1562	1518	}
...	...	@@ -1574,20 +1530,12 @@ fn asmRegisterMemoryImmediate(
1574	1530	}) \|small_imm\| {
1575	1531	_ = try self.addInst(.{
1576	1532	.tag = tag[1],
1577		.ops = switch (m) {
1578		.sib => .rmi_sib,
1579		.rip => .rmi_rip,
1580		else => unreachable,
1581		},
	1533	.ops = .rmi,
1582	1534	.data = .{ .rix = .{
1583	1535	.fixes = tag[0],
1584	1536	.r1 = reg,
1585	1537	.i = small_imm,
1586		.payload = switch (m) {
1587		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1588		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1589		else => unreachable,
1590		},
	1538	.payload = try self.addExtra(Mir.Memory.encode(m)),
1591	1539	} },
1592	1540	});
1593	1541	} else {
...	...	@@ -1595,23 +1543,12 @@ fn asmRegisterMemoryImmediate(
1595	1543	.signed => \|s\| @bitCast(s),
1596	1544	.unsigned => unreachable,
1597	1545	} });
1598		assert(payload + 1 == switch (m) {
1599		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1600		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1601		else => unreachable,
1602		});
	1546	assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m)));
1603	1547	_ = try self.addInst(.{
1604	1548	.tag = tag[1],
1605		.ops = switch (m) {
1606		.sib => switch (imm) {
1607		.signed => .rmi_sib_s,
1608		.unsigned => .rmi_sib_u,
1609		},
1610		.rip => switch (imm) {
1611		.signed => .rmi_rip_s,
1612		.unsigned => .rmi_rip_u,
1613		},
1614		else => unreachable,
	1549	.ops = switch (imm) {
	1550	.signed => .rmi_s,
	1551	.unsigned => .rmi_u,
1615	1552	},
1616	1553	.data = .{ .rx = .{
1617	1554	.fixes = tag[0],
...	...	@@ -1632,21 +1569,13 @@ fn asmRegisterRegisterMemoryImmediate(
1632	1569	) !void {
1633	1570	_ = try self.addInst(.{
1634	1571	.tag = tag[1],
1635		.ops = switch (m) {
1636		.sib => .rrmi_sib,
1637		.rip => .rrmi_rip,
1638		else => unreachable,
1639		},
	1572	.ops = .rrmi,
1640	1573	.data = .{ .rrix = .{
1641	1574	.fixes = tag[0],
1642	1575	.r1 = reg1,
1643	1576	.r2 = reg2,
1644	1577	.i = @intCast(imm.unsigned),
1645		.payload = switch (m) {
1646		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1647		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1648		else => unreachable,
1649		},
	1578	.payload = try self.addExtra(Mir.Memory.encode(m)),
1650	1579	} },
1651	1580	});
1652	1581	}
...	...	@@ -1654,19 +1583,11 @@ fn asmRegisterRegisterMemoryImmediate(
1654	1583	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
1655	1584	_ = try self.addInst(.{
1656	1585	.tag = tag[1],
1657		.ops = switch (m) {
1658		.sib => .mr_sib,
1659		.rip => .mr_rip,
1660		else => unreachable,
1661		},
	1586	.ops = .mr,
1662	1587	.data = .{ .rx = .{
1663	1588	.fixes = tag[0],
1664	1589	.r1 = reg,
1665		.payload = switch (m) {
1666		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1667		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1668		else => unreachable,
1669		},
	1590	.payload = try self.addExtra(Mir.Memory.encode(m)),
1670	1591	} },
1671	1592	});
1672	1593	}
...	...	@@ -1676,23 +1597,12 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed
1676	1597	.signed => \|s\| @bitCast(s),
1677	1598	.unsigned => \|u\| @intCast(u),
1678	1599	} });
1679		assert(payload + 1 == switch (m) {
1680		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1681		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1682		else => unreachable,
1683		});
	1600	assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m)));
1684	1601	_ = try self.addInst(.{
1685	1602	.tag = tag[1],
1686		.ops = switch (m) {
1687		.sib => switch (imm) {
1688		.signed => .mi_sib_s,
1689		.unsigned => .mi_sib_u,
1690		},
1691		.rip => switch (imm) {
1692		.signed => .mi_rip_s,
1693		.unsigned => .mi_rip_u,
1694		},
1695		else => unreachable,
	1603	.ops = switch (imm) {
	1604	.signed => .mi_s,
	1605	.unsigned => .mi_u,
1696	1606	},
1697	1607	.data = .{ .x = .{
1698	1608	.fixes = tag[0],
...	...	@@ -1710,20 +1620,12 @@ fn asmMemoryRegisterRegister(
1710	1620	) !void {
1711	1621	_ = try self.addInst(.{
1712	1622	.tag = tag[1],
1713		.ops = switch (m) {
1714		.sib => .mrr_sib,
1715		.rip => .mrr_rip,
1716		else => unreachable,
1717		},
	1623	.ops = .mrr,
1718	1624	.data = .{ .rrx = .{
1719	1625	.fixes = tag[0],
1720	1626	.r1 = reg1,
1721	1627	.r2 = reg2,
1722		.payload = switch (m) {
1723		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1724		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1725		else => unreachable,
1726		},
	1628	.payload = try self.addExtra(Mir.Memory.encode(m)),
1727	1629	} },
1728	1630	});
1729	1631	}
...	...	@@ -1737,20 +1639,12 @@ fn asmMemoryRegisterImmediate(
1737	1639	) !void {
1738	1640	_ = try self.addInst(.{
1739	1641	.tag = tag[1],
1740		.ops = switch (m) {
1741		.sib => .mri_sib,
1742		.rip => .mri_rip,
1743		else => unreachable,
1744		},
	1642	.ops = .mri,
1745	1643	.data = .{ .rix = .{
1746	1644	.fixes = tag[0],
1747	1645	.r1 = reg,
1748	1646	.i = @intCast(imm.unsigned),
1749		.payload = switch (m) {
1750		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1751		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1752		else => unreachable,
1753		},
	1647	.payload = try self.addExtra(Mir.Memory.encode(m)),
1754	1648	} },
1755	1649	});
1756	1650	}
...	...	@@ -1916,13 +1810,16 @@ fn gen(self: *Self) InnerError!void {
1916	1810	if (need_frame_align or need_stack_adjust) {
1917	1811	self.mir_instructions.set(backpatch_stack_dealloc, .{
1918	1812	.tag = .lea,
1919		.ops = .rm_sib,
	1813	.ops = .rm,
1920	1814	.data = .{ .rx = .{
1921	1815	.r1 = .rsp,
1922		.payload = try self.addExtra(Mir.MemorySib.encode(Memory.sib(.qword, .{
	1816	.payload = try self.addExtra(Mir.Memory.encode(.{
1923	1817	.base = .{ .reg = .rbp },
1924		.disp = -frame_layout.save_reg_list.size(),
1925		}))),
	1818	.mod = .{ .rm = .{
	1819	.size = .qword,
	1820	.disp = -frame_layout.save_reg_list.size(),
	1821	} },
	1822	})),
1926	1823	} },
1927	1824	});
1928	1825	}
...	...	@@ -3098,14 +2995,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3098	2995	.{ .vp_, .@"and" },
3099	2996	dst_reg,
3100	2997	dst_reg,
3101		splat_addr_mcv.deref().mem(Memory.PtrSize.fromSize(splat_abi_size)),
	2998	splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),
3102	2999	);
3103	3000	try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);
3104	3001	} else {
3105	3002	try self.asmRegisterMemory(
3106	3003	.{ .p_, .@"and" },
3107	3004	dst_reg,
3108		splat_addr_mcv.deref().mem(Memory.PtrSize.fromSize(splat_abi_size)),
	3005	splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),
3109	3006	);
3110	3007	try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);
3111	3008	}
...	...	@@ -3254,7 +3151,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3254	3151	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(Type.usize, mod));
3255	3152	try self.asmMemoryImmediate(
3256	3153	.{ ._, .mov },
3257		Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
	3154	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
3258	3155	Immediate.u(0),
3259	3156	);
3260	3157
...	...	@@ -3338,9 +3235,10 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3338	3235	call_mcv.register_pair[0],
3339	3236	call_mcv.register_pair[1],
3340	3237	);
3341		try self.asmSetccMemory(.nz, Memory.sib(.byte, .{
	3238	try self.asmSetccMemory(.nz, .{
3342	3239	.base = .{ .frame = signed_div_floor_state.frame_index },
3343		}));
	3240	.mod = .{ .rm = .{ .size = .byte } },
	3241	});
3344	3242	try self.performReloc(signed_div_floor_state.reloc);
3345	3243	const dst_mcv = try self.genCall(
3346	3244	.{ .lib = .{
...	...	@@ -3356,9 +3254,10 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3356	3254	try self.asmRegisterMemory(
3357	3255	.{ ._, .sub },
3358	3256	dst_mcv.register_pair[0],
3359		Memory.sib(.qword, .{
	3257	.{
3360	3258	.base = .{ .frame = signed_div_floor_state.frame_index },
3361		}),
	3259	.mod = .{ .rm = .{ .size = .qword } },
	3260	},
3362	3261	);
3363	3262	try self.asmRegisterImmediate(
3364	3263	.{ ._, .sbb },
...	...	@@ -3596,59 +3495,140 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3596	3495	const mod = self.bin_file.options.module.?;
3597	3496	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
3598	3497	const ty = self.typeOf(bin_op.lhs);
3599		if (ty.zigTypeTag(mod) == .Vector or ty.abiSize(mod) > 8) return self.fail(
3600		"TODO implement airMulSat for {}",
3601		.{ty.fmt(mod)},
3602		);
3603	3498
3604		try self.spillRegisters(&.{ .rax, .rdx });
3605		const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx });
3606		defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
	3499	const result = result: {
	3500	if (ty.toIntern() == .i128_type) {
	3501	const ptr_c_int = try mod.singleMutPtrType(Type.c_int);
	3502	const overflow = try self.allocTempRegOrMem(Type.c_int, false);
	3503
	3504	const dst_mcv = try self.genCall(.{ .lib = .{
	3505	.return_type = .i128_type,
	3506	.param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() },
	3507	.callee = "__muloti4",
	3508	} }, &.{ Type.i128, Type.i128, ptr_c_int }, &.{
	3509	.{ .air_ref = bin_op.lhs },
	3510	.{ .air_ref = bin_op.rhs },
	3511	overflow.address(),
	3512	});
	3513	const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_mcv.register_pair);
	3514	defer for (dst_locks) \|lock\| self.register_manager.unlockReg(lock);
3607	3515
3608		const lhs_mcv = try self.resolveInst(bin_op.lhs);
3609		const lhs_lock = switch (lhs_mcv) {
3610		.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
3611		else => null,
3612		};
3613		defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	3516	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	3517	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	3518	defer self.register_manager.unlockReg(tmp_lock);
3614	3519
3615		const rhs_mcv = try self.resolveInst(bin_op.rhs);
3616		const rhs_lock = switch (rhs_mcv) {
3617		.register => \|reg\| self.register_manager.lockReg(reg),
3618		else => null,
3619		};
3620		defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	3520	const lhs_mcv = try self.resolveInst(bin_op.lhs);
	3521	const mat_lhs_mcv = switch (lhs_mcv) {
	3522	.load_symbol => mat_lhs_mcv: {
	3523	// TODO clean this up!
	3524	const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());
	3525	break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
	3526	},
	3527	else => lhs_mcv,
	3528	};
	3529	const mat_lhs_lock = switch (mat_lhs_mcv) {
	3530	.indirect => \|reg_off\| self.register_manager.lockReg(reg_off.reg),
	3531	else => null,
	3532	};
	3533	defer if (mat_lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	3534	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
	3535	.{ ._, .mov },
	3536	tmp_reg,
	3537	mat_lhs_mcv.address().offset(8).deref().mem(.qword),
	3538	) else try self.asmRegisterRegister(
	3539	.{ ._, .mov },
	3540	tmp_reg,
	3541	mat_lhs_mcv.register_pair[1],
	3542	);
3621	3543
3622		const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
3623		const limit_mcv = MCValue{ .register = limit_reg };
3624		const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
3625		defer self.register_manager.unlockReg(limit_lock);
	3544	const rhs_mcv = try self.resolveInst(bin_op.rhs);
	3545	const mat_rhs_mcv = switch (rhs_mcv) {
	3546	.load_symbol => mat_rhs_mcv: {
	3547	// TODO clean this up!
	3548	const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());
	3549	break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
	3550	},
	3551	else => rhs_mcv,
	3552	};
	3553	const mat_rhs_lock = switch (mat_rhs_mcv) {
	3554	.indirect => \|reg_off\| self.register_manager.lockReg(reg_off.reg),
	3555	else => null,
	3556	};
	3557	defer if (mat_rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	3558	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
	3559	.{ ._, .xor },
	3560	tmp_reg,
	3561	mat_rhs_mcv.address().offset(8).deref().mem(.qword),
	3562	) else try self.asmRegisterRegister(
	3563	.{ ._, .xor },
	3564	tmp_reg,
	3565	mat_rhs_mcv.register_pair[1],
	3566	);
3626	3567
3627		const reg_bits = self.regBitSize(ty);
3628		const cc: Condition = if (ty.isSignedInt(mod)) cc: {
3629		try self.genSetReg(limit_reg, ty, lhs_mcv);
3630		try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
3631		try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
3632		try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
3633		.immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
3634		});
3635		break :cc .o;
3636		} else cc: {
3637		try self.genSetReg(limit_reg, ty, .{
3638		.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - reg_bits),
3639		});
3640		break :cc .c;
3641		};
	3568	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));
	3569	try self.asmRegister(.{ ._, .not }, tmp_reg);
	3570	try self.asmMemoryImmediate(.{ ._, .cmp }, overflow.mem(.dword), Immediate.s(0));
	3571	try self.freeValue(overflow);
	3572	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
	3573	try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63));
	3574	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[1], tmp_reg);
	3575	break :result dst_mcv;
	3576	}
3642	3577
3643		const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv);
3644		const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2);
3645		try self.asmCmovccRegisterRegister(
3646		cc,
3647		registerAlias(dst_mcv.register, cmov_abi_size),
3648		registerAlias(limit_reg, cmov_abi_size),
3649		);
	3578	if (ty.zigTypeTag(mod) == .Vector or ty.abiSize(mod) > 8) return self.fail(
	3579	"TODO implement airMulSat for {}",
	3580	.{ty.fmt(mod)},
	3581	);
3650	3582
3651		return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
	3583	try self.spillRegisters(&.{ .rax, .rdx });
	3584	const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx });
	3585	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
	3586
	3587	const lhs_mcv = try self.resolveInst(bin_op.lhs);
	3588	const lhs_lock = switch (lhs_mcv) {
	3589	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
	3590	else => null,
	3591	};
	3592	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	3593
	3594	const rhs_mcv = try self.resolveInst(bin_op.rhs);
	3595	const rhs_lock = switch (rhs_mcv) {
	3596	.register => \|reg\| self.register_manager.lockReg(reg),
	3597	else => null,
	3598	};
	3599	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	3600
	3601	const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	3602	const limit_mcv = MCValue{ .register = limit_reg };
	3603	const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
	3604	defer self.register_manager.unlockReg(limit_lock);
	3605
	3606	const reg_bits = self.regBitSize(ty);
	3607	const cc: Condition = if (ty.isSignedInt(mod)) cc: {
	3608	try self.genSetReg(limit_reg, ty, lhs_mcv);
	3609	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
	3610	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
	3611	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
	3612	.immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
	3613	});
	3614	break :cc .o;
	3615	} else cc: {
	3616	try self.genSetReg(limit_reg, ty, .{
	3617	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - reg_bits),
	3618	});
	3619	break :cc .c;
	3620	};
	3621
	3622	const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv);
	3623	const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2);
	3624	try self.asmCmovccRegisterRegister(
	3625	cc,
	3626	registerAlias(dst_mcv.register, cmov_abi_size),
	3627	registerAlias(limit_reg, cmov_abi_size),
	3628	);
	3629	break :result dst_mcv;
	3630	};
	3631	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3652	3632	}
3653	3633
3654	3634	fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -3802,10 +3782,13 @@ fn genSetFrameTruncatedOverflowCompare(
3802	3782	const ty = tuple_ty.structFieldType(0, mod);
3803	3783	const int_info = ty.intInfo(mod);
3804	3784
3805		const hi_limb_bits = (int_info.bits - 1) % 64 + 1;
3806		const hi_limb_ty = try mod.intType(int_info.signedness, hi_limb_bits);
	3785	const hi_bits = (int_info.bits - 1) % 64 + 1;
	3786	const hi_ty = try mod.intType(int_info.signedness, hi_bits);
	3787
	3788	const limb_bits: u16 = @intCast(if (int_info.bits <= 64) self.regBitSize(ty) else 64);
	3789	const limb_ty = try mod.intType(int_info.signedness, limb_bits);
3807	3790
3808		const rest_ty = try mod.intType(.unsigned, int_info.bits - hi_limb_bits);
	3791	const rest_ty = try mod.intType(.unsigned, int_info.bits - hi_bits);
3809	3792
3810	3793	const temp_regs =
3811	3794	try self.register_manager.allocRegs(3, .{ null, null, null }, abi.RegisterClass.gp);
...	...	@@ -3821,9 +3804,9 @@ fn genSetFrameTruncatedOverflowCompare(
3821	3804	src_mcv.address().offset(int_info.bits / 64 * 8).deref()
3822	3805	else
3823	3806	src_mcv;
3824		try self.genSetReg(scratch_reg, hi_limb_ty, hi_limb_mcv);
3825		try self.truncateRegister(hi_limb_ty, scratch_reg);
3826		try self.genBinOpMir(.{ ._, .cmp }, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
	3807	try self.genSetReg(scratch_reg, limb_ty, hi_limb_mcv);
	3808	try self.truncateRegister(hi_ty, scratch_reg);
	3809	try self.genBinOpMir(.{ ._, .cmp }, limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
3827	3810
3828	3811	const eq_reg = temp_regs[2];
3829	3812	if (overflow_cc) \|_\| {
...	...	@@ -3841,7 +3824,7 @@ fn genSetFrameTruncatedOverflowCompare(
3841	3824	try self.genSetMem(
3842	3825	.{ .frame = frame_index },
3843	3826	payload_off + hi_limb_off,
3844		hi_limb_ty,
	3827	limb_ty,
3845	3828	.{ .register = scratch_reg },
3846	3829	);
3847	3830	try self.genSetMem(
...	...	@@ -3891,7 +3874,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3891	3874	);
3892	3875	try self.asmMemoryImmediate(
3893	3876	.{ ._, .cmp },
3894		overflow.mem(self.memPtrSize(Type.c_int)),
	3877	overflow.mem(self.memSize(Type.c_int)),
3895	3878	Immediate.s(0),
3896	3879	);
3897	3880	try self.genSetMem(
...	...	@@ -4123,7 +4106,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
4123	4106	.register => \|reg\| try self.asmRegister(tag, registerAlias(reg, abi_size)),
4124	4107	.memory, .indirect, .load_frame => try self.asmMemory(
4125	4108	tag,
4126		mat_rhs.mem(Memory.PtrSize.fromSize(abi_size)),
	4109	mat_rhs.mem(Memory.Size.fromSize(abi_size)),
4127	4110	),
4128	4111	else => unreachable,
4129	4112	}
...	...	@@ -4185,6 +4168,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
4185	4168	}
4186	4169
4187	4170	fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
	4171	const mod = self.bin_file.options.module.?;
4188	4172	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
4189	4173
4190	4174	try self.spillRegisters(&.{.rcx});
...	...	@@ -4196,9 +4180,38 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
4196	4180	const lhs_ty = self.typeOf(bin_op.lhs);
4197	4181	const rhs_ty = self.typeOf(bin_op.rhs);
4198	4182
4199		const result = try self.genShiftBinOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
	4183	const dst_mcv = try self.genShiftBinOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
	4184	switch (tag) {
	4185	.shr, .shr_exact, .shl_exact => {},
	4186	.shl => switch (dst_mcv) {
	4187	.register => \|dst_reg\| try self.truncateRegister(lhs_ty, dst_reg),
	4188	.register_pair => \|dst_regs\| try self.truncateRegister(lhs_ty, dst_regs[1]),
	4189	.load_frame => \|frame_addr\| {
	4190	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	4191	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	4192	defer self.register_manager.unlockReg(tmp_lock);
4200	4193
4201		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	4194	const lhs_bits: u31 = @intCast(lhs_ty.bitSize(mod));
	4195	const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;
	4196	const off = frame_addr.off + lhs_bits / 64 * 8;
	4197	try self.genSetReg(
	4198	tmp_reg,
	4199	tmp_ty,
	4200	.{ .load_frame = .{ .index = frame_addr.index, .off = off } },
	4201	);
	4202	try self.truncateRegister(lhs_ty, tmp_reg);
	4203	try self.genSetMem(
	4204	.{ .frame = frame_addr.index },
	4205	off,
	4206	tmp_ty,
	4207	.{ .register = tmp_reg },
	4208	);
	4209	},
	4210	else => {},
	4211	},
	4212	else => unreachable,
	4213	}
	4214	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
4202	4215	}
4203	4216
4204	4217	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -4360,10 +4373,13 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
4360	4373	try self.asmRegisterMemory(
4361	4374	.{ ._, .mov },
4362	4375	registerAlias(dst_reg, err_abi_size),
4363		Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
	4376	.{
4364	4377	.base = .{ .reg = src_reg },
4365		.disp = err_off,
4366		}),
	4378	.mod = .{ .rm = .{
	4379	.size = Memory.Size.fromSize(err_abi_size),
	4380	.disp = err_off,
	4381	} },
	4382	},
4367	4383	);
4368	4384
4369	4385	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...	...	@@ -4398,10 +4414,13 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
4398	4414	const err_abi_size: u32 = @intCast(err_ty.abiSize(mod));
4399	4415	try self.asmMemoryImmediate(
4400	4416	.{ ._, .mov },
4401		Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
	4417	.{
4402	4418	.base = .{ .reg = src_reg },
4403		.disp = err_off,
4404		}),
	4419	.mod = .{ .rm = .{
	4420	.size = Memory.Size.fromSize(err_abi_size),
	4421	.disp = err_off,
	4422	} },
	4423	},
4405	4424	Immediate.u(0),
4406	4425	);
4407	4426
...	...	@@ -4420,7 +4439,10 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
4420	4439	try self.asmRegisterMemory(
4421	4440	.{ ._, .lea },
4422	4441	registerAlias(dst_reg, dst_abi_size),
4423		Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
	4442	.{
	4443	.base = .{ .reg = src_reg },
	4444	.mod = .{ .rm = .{ .size = .qword, .disp = pl_off } },
	4445	},
4424	4446	);
4425	4447	break :result .{ .register = dst_reg };
4426	4448	};
...	...	@@ -4542,10 +4564,13 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
4542	4564
4543	4565	.load_frame => \|frame_addr\| try self.asmMemoryImmediate(
4544	4566	.{ ._, .mov },
4545		Memory.sib(.byte, .{
	4567	.{
4546	4568	.base = .{ .frame = frame_addr.index },
4547		.disp = frame_addr.off + pl_abi_size,
4548		}),
	4569	.mod = .{ .rm = .{
	4570	.size = .byte,
	4571	.disp = frame_addr.off + pl_abi_size,
	4572	} },
	4573	},
4549	4574	Immediate.u(1),
4550	4575	),
4551	4576	}
...	...	@@ -4618,18 +4643,24 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
4618	4643	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
4619	4644	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
4620	4645
4621		const operand = try self.resolveInst(ty_op.operand);
4622		const dst_mcv: MCValue = blk: {
4623		switch (operand) {
4624		.load_frame => \|frame_addr\| break :blk .{ .load_frame = .{
4625		.index = frame_addr.index,
4626		.off = frame_addr.off + 8,
4627		} },
4628		else => return self.fail("TODO implement slice_len for {}", .{operand}),
	4646	const result: MCValue = result: {
	4647	const src_mcv = try self.resolveInst(ty_op.operand);
	4648	switch (src_mcv) {
	4649	.load_frame => \|frame_addr\| {
	4650	const len_mcv: MCValue = .{ .load_frame = .{
	4651	.index = frame_addr.index,
	4652	.off = frame_addr.off + 8,
	4653	} };
	4654	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;
	4655
	4656	const dst_mcv = try self.allocRegOrMem(inst, true);
	4657	try self.genCopy(Type.usize, dst_mcv, len_mcv);
	4658	break :result dst_mcv;
	4659	},
	4660	else => return self.fail("TODO implement slice_len for {}", .{src_mcv}),
4629	4661	}
4630	4662	};
4631
4632		return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
	4663	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4633	4664	}
4634	4665
4635	4666	fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -4658,10 +4689,10 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
4658	4689	try self.asmRegisterMemory(
4659	4690	.{ ._, .lea },
4660	4691	registerAlias(dst_reg, dst_abi_size),
4661		Memory.sib(.qword, .{
	4692	.{
4662	4693	.base = .{ .reg = src_reg },
4663		.disp = @divExact(self.target.ptrBitWidth(), 8),
4664		}),
	4694	.mod = .{ .rm = .{ .size = .qword, .disp = 8 } },
	4695	},
4665	4696	);
4666	4697
4667	4698	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...	...	@@ -4791,13 +4822,16 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
4791	4822	try self.asmRegisterMemory(
4792	4823	.{ ._, .lea },
4793	4824	addr_reg,
4794		Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
	4825	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
4795	4826	);
4796	4827	},
4797	4828	.load_frame => \|frame_addr\| try self.asmRegisterMemory(
4798	4829	.{ ._, .lea },
4799	4830	addr_reg,
4800		Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),
	4831	.{
	4832	.base = .{ .frame = frame_addr.index },
	4833	.mod = .{ .rm = .{ .size = .qword, .disp = frame_addr.off } },
	4834	},
4801	4835	),
4802	4836	.memory,
4803	4837	.load_symbol,
...	...	@@ -5019,13 +5053,19 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
5019	5053	const has_lzcnt = self.hasFeature(.lzcnt);
5020	5054	if (src_bits > 64 and !has_lzcnt) {
5021	5055	var callee_buf: ["__clz?i2".len]u8 = undefined;
5022		break :result try self.genCall(.{ .lib = .{
	5056	const result = try self.genCall(.{ .lib = .{
5023	5057	.return_type = .i32_type,
5024	5058	.param_types = &.{src_ty.toIntern()},
5025	5059	.callee = std.fmt.bufPrint(&callee_buf, "__clz{c}i2", .{
5026	5060	intCompilerRtAbiName(src_bits),
5027	5061	}) catch unreachable,
5028	5062	} }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
	5063	if (src_bits < 128) try self.asmRegisterImmediate(
	5064	.{ ._, .sub },
	5065	result.register,
	5066	Immediate.u(128 - src_bits),
	5067	);
	5068	break :result result;
5029	5069	}
5030	5070
5031	5071	const src_mcv = try self.resolveInst(ty_op.operand);
...	...	@@ -5610,10 +5650,14 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
5610	5650	try self.asmRegisterMemory(
5611	5651	.{ ._, .lea },
5612	5652	if (limb_abi_size > 4) tmp.to64() else tmp.to32(),
5613		Memory.sib(.qword, .{
	5653	.{
5614	5654	.base = .{ .reg = dst.to64() },
5615		.scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 },
5616		}),
	5655	.mod = .{ .rm = .{
	5656	.size = .qword,
	5657	.index = tmp.to64(),
	5658	.scale = .@"4",
	5659	} },
	5660	},
5617	5661	);
5618	5662	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
5619	5663	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
...	...	@@ -5633,10 +5677,14 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
5633	5677	try self.asmRegisterMemory(
5634	5678	.{ ._, .lea },
5635	5679	if (limb_abi_size > 4) dst.to64() else dst.to32(),
5636		Memory.sib(.qword, .{
	5680	.{
5637	5681	.base = .{ .reg = tmp.to64() },
5638		.scale_index = .{ .index = dst.to64(), .scale = 1 << 1 },
5639		}),
	5682	.mod = .{ .rm = .{
	5683	.size = .qword,
	5684	.index = dst.to64(),
	5685	.scale = .@"2",
	5686	} },
	5687	},
5640	5688	);
5641	5689	// dst = ((temp3 >> 1) & 0x55...55) + ((temp3 & 0x55...55) << 1)
5642	5690	}
...	...	@@ -5668,7 +5716,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
5668	5716	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
5669	5717
5670	5718	const dst_mcv: MCValue = .{ .register = .st0 };
5671		if (std.meta.eql(src_mcv, dst_mcv) and self.reuseOperand(inst, operand, 0, src_mcv))
	5719	if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv))
5672	5720	try self.register_manager.getReg(.st0, inst);
5673	5721
5674	5722	try self.genCopy(ty, dst_mcv, src_mcv);
...	...	@@ -5713,12 +5761,13 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
5713	5761	.abs => try vec_ty.maxInt(mod, vec_ty),
5714	5762	else => unreachable,
5715	5763	} });
5716		const sign_mem = if (sign_mcv.isMemory())
5717		sign_mcv.mem(Memory.PtrSize.fromSize(abi_size))
	5764	const sign_mem: Memory = if (sign_mcv.isMemory())
	5765	sign_mcv.mem(Memory.Size.fromSize(abi_size))
5718	5766	else
5719		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	5767	.{
5720	5768	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
5721		});
	5769	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } },
	5770	};
5722	5771
5723	5772	if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(
5724	5773	switch (scalar_bits) {
...	...	@@ -5896,7 +5945,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5896	5945	mir_tag,
5897	5946	dst_alias,
5898	5947	dst_alias,
5899		src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	5948	src_mcv.mem(Memory.Size.fromSize(abi_size)),
5900	5949	Immediate.u(@as(u5, @bitCast(mode))),
5901	5950	) else try self.asmRegisterRegisterRegisterImmediate(
5902	5951	mir_tag,
...	...	@@ -5911,7 +5960,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5911	5960	else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
5912	5961	mir_tag,
5913	5962	dst_alias,
5914		src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	5963	src_mcv.mem(Memory.Size.fromSize(abi_size)),
5915	5964	Immediate.u(@as(u5, @bitCast(mode))),
5916	5965	) else try self.asmRegisterRegisterImmediate(
5917	5966	mir_tag,
...	...	@@ -5951,7 +6000,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
5951	6000	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
5952	6001	.l,
5953	6002	registerAlias(dst_mcv.register, cmov_abi_size),
5954		src_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)),
	6003	src_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),
5955	6004	),
5956	6005	else => {
5957	6006	const val_reg = try self.copyToTmpRegister(ty, src_mcv);
...	...	@@ -6051,7 +6100,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6051	6100	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6052	6101	mir_tag,
6053	6102	dst_alias,
6054		src_mcv.mem(self.memPtrSize(ty)),
	6103	src_mcv.mem(self.memSize(ty)),
6055	6104	) else try self.asmRegisterRegister(
6056	6105	mir_tag,
6057	6106	dst_alias,
...	...	@@ -6157,7 +6206,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6157	6206	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6158	6207	.{ .v_ps, .cvtph2 },
6159	6208	wide_reg,
6160		src_mcv.mem(Memory.PtrSize.fromSize(
	6209	src_mcv.mem(Memory.Size.fromSize(
6161	6210	@intCast(@divExact(wide_reg.bitSize(), 16)),
6162	6211	)),
6163	6212	) else try self.asmRegisterRegister(
...	...	@@ -6205,7 +6254,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6205	6254	mir_tag,
6206	6255	dst_reg,
6207	6256	dst_reg,
6208		src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	6257	src_mcv.mem(Memory.Size.fromSize(abi_size)),
6209	6258	) else try self.asmRegisterRegisterRegister(
6210	6259	mir_tag,
6211	6260	dst_reg,
...	...	@@ -6218,7 +6267,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6218	6267	else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
6219	6268	mir_tag,
6220	6269	dst_reg,
6221		src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	6270	src_mcv.mem(Memory.Size.fromSize(abi_size)),
6222	6271	) else try self.asmRegisterRegister(
6223	6272	mir_tag,
6224	6273	dst_reg,
...	...	@@ -6337,14 +6386,13 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6337	6386	if (val_bit_off < val_extra_bits) val_abi_size else val_abi_size * 2;
6338	6387	if (load_abi_size <= 8) {
6339	6388	const load_reg = registerAlias(dst_reg, load_abi_size);
6340		try self.asmRegisterMemory(
6341		.{ ._, .mov },
6342		load_reg,
6343		Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
6344		.base = .{ .reg = ptr_reg },
	6389	try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
	6390	.base = .{ .reg = ptr_reg },
	6391	.mod = .{ .rm = .{
	6392	.size = Memory.Size.fromSize(load_abi_size),
6345	6393	.disp = val_byte_off,
6346		}),
6347		);
	6394	} },
	6395	});
6348	6396	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
6349	6397	} else {
6350	6398	const tmp_reg =
...	...	@@ -6353,22 +6401,20 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6353	6401	defer self.register_manager.unlockReg(tmp_lock);
6354	6402
6355	6403	const dst_alias = registerAlias(dst_reg, val_abi_size);
6356		try self.asmRegisterMemory(
6357		.{ ._, .mov },
6358		dst_alias,
6359		Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
6360		.base = .{ .reg = ptr_reg },
	6404	try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{
	6405	.base = .{ .reg = ptr_reg },
	6406	.mod = .{ .rm = .{
	6407	.size = Memory.Size.fromSize(val_abi_size),
6361	6408	.disp = val_byte_off,
6362		}),
6363		);
6364		try self.asmRegisterMemory(
6365		.{ ._, .mov },
6366		tmp_reg,
6367		Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
6368		.base = .{ .reg = ptr_reg },
	6409	} },
	6410	});
	6411	try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
	6412	.base = .{ .reg = ptr_reg },
	6413	.mod = .{ .rm = .{
	6414	.size = Memory.Size.fromSize(val_abi_size),
6369	6415	.disp = val_byte_off + 1,
6370		}),
6371		);
	6416	} },
	6417	});
6372	6418	try self.asmRegisterRegisterImmediate(
6373	6419	.{ ._rd, .sh },
6374	6420	dst_alias,
...	...	@@ -6480,10 +6526,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
6480	6526	const part_bit_off = if (limb_i == 0) src_bit_off else 0;
6481	6527	const part_bit_size =
6482	6528	@min(src_bit_off + src_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off;
6483		const limb_mem = Memory.sib(Memory.PtrSize.fromSize(limb_abi_size), .{
	6529	const limb_mem: Memory = .{
6484	6530	.base = .{ .reg = ptr_reg },
6485		.disp = src_byte_off + limb_i * limb_abi_bits,
6486		});
	6531	.mod = .{ .rm = .{
	6532	.size = Memory.Size.fromSize(limb_abi_size),
	6533	.disp = src_byte_off + limb_i * limb_abi_bits,
	6534	} },
	6535	};
6487	6536
6488	6537	const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(64 - part_bit_size)) <<
6489	6538	@intCast(part_bit_off);
...	...	@@ -6814,7 +6863,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6814	6863	break :result dst_mcv;
6815	6864	}
6816	6865
6817		const limb_abi_size: u32 = @min(field_abi_size, 8);
	6866	const limb_abi_size: u31 = @min(field_abi_size, 8);
6818	6867	const limb_abi_bits = limb_abi_size * 8;
6819	6868	const field_byte_off: i32 = @intCast(field_off / limb_abi_bits * limb_abi_size);
6820	6869	const field_bit_off = field_off % limb_abi_bits;
...	...	@@ -6832,14 +6881,13 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6832	6881	if (field_bit_off < field_extra_bits) field_abi_size else field_abi_size * 2;
6833	6882	if (load_abi_size <= 8) {
6834	6883	const load_reg = registerAlias(dst_reg, load_abi_size);
6835		try self.asmRegisterMemory(
6836		.{ ._, .mov },
6837		load_reg,
6838		Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
6839		.base = .{ .frame = frame_addr.index },
	6884	try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
	6885	.base = .{ .frame = frame_addr.index },
	6886	.mod = .{ .rm = .{
	6887	.size = Memory.Size.fromSize(load_abi_size),
6840	6888	.disp = frame_addr.off + field_byte_off,
6841		}),
6842		);
	6889	} },
	6890	});
6843	6891	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
6844	6892	} else {
6845	6893	const tmp_reg = registerAlias(
...	...	@@ -6853,19 +6901,21 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6853	6901	try self.asmRegisterMemory(
6854	6902	.{ ._, .mov },
6855	6903	dst_alias,
6856		Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
6857		.base = .{ .frame = frame_addr.index },
6858		.disp = frame_addr.off + field_byte_off,
6859		}),
6860		);
6861		try self.asmRegisterMemory(
6862		.{ ._, .mov },
6863		tmp_reg,
6864		Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
	6904	.{
6865	6905	.base = .{ .frame = frame_addr.index },
6866		.disp = frame_addr.off + field_byte_off + @as(i32, @intCast(limb_abi_size)),
6867		}),
	6906	.mod = .{ .rm = .{
	6907	.size = Memory.Size.fromSize(field_abi_size),
	6908	.disp = frame_addr.off + field_byte_off,
	6909	} },
	6910	},
6868	6911	);
	6912	try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
	6913	.base = .{ .frame = frame_addr.index },
	6914	.mod = .{ .rm = .{
	6915	.size = Memory.Size.fromSize(field_abi_size),
	6916	.disp = frame_addr.off + field_byte_off + limb_abi_size,
	6917	} },
	6918	});
6869	6919	try self.asmRegisterRegisterImmediate(
6870	6920	.{ ._rd, .sh },
6871	6921	dst_alias,
...	...	@@ -6998,14 +7048,13 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
6998	7048	defer self.register_manager.unlockReg(addr_reg_lock);
6999	7049
7000	7050	try self.genSetReg(addr_reg, Type.usize, dst_mcv.address());
7001		try self.asmMemory(
7002		mir_tag,
7003		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
7004		);
	7051	try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{
	7052	.size = Memory.Size.fromSize(abi_size),
	7053	} } });
7005	7054	},
7006	7055	.indirect, .load_frame => try self.asmMemory(
7007	7056	mir_tag,
7008		dst_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	7057	dst_mcv.mem(Memory.Size.fromSize(abi_size)),
7009	7058	),
7010	7059	}
7011	7060	}
...	...	@@ -7053,25 +7102,34 @@ fn genShiftBinOpMir(
7053	7102	}),
7054	7103	},
7055	7104	.memory, .indirect, .load_frame => {
7056		const lhs_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (lhs_mcv) {
	7105	const lhs_mem: Memory = switch (lhs_mcv) {
7057	7106	.memory => \|addr\| .{
7058	7107	.base = .{ .reg = .ds },
7059		.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
7060		return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
7061		@tagName(lhs_mcv),
7062		@tagName(rhs_mcv),
7063		}),
	7108	.mod = .{ .rm = .{
	7109	.size = Memory.Size.fromSize(abi_size),
	7110	.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
	7111	return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
	7112	@tagName(lhs_mcv),
	7113	@tagName(rhs_mcv),
	7114	}),
	7115	} },
7064	7116	},
7065	7117	.indirect => \|reg_off\| .{
7066	7118	.base = .{ .reg = reg_off.reg },
7067		.disp = reg_off.off,
	7119	.mod = .{ .rm = .{
	7120	.size = Memory.Size.fromSize(abi_size),
	7121	.disp = reg_off.off,
	7122	} },
7068	7123	},
7069	7124	.load_frame => \|frame_addr\| .{
7070	7125	.base = .{ .frame = frame_addr.index },
7071		.disp = frame_addr.off,
	7126	.mod = .{ .rm = .{
	7127	.size = Memory.Size.fromSize(abi_size),
	7128	.disp = frame_addr.off,
	7129	} },
7072	7130	},
7073	7131	else => unreachable,
7074		});
	7132	};
7075	7133	switch (rhs_mcv) {
7076	7134	.immediate => \|rhs_imm\| try self.asmMemoryImmediate(
7077	7135	tag,
...	...	@@ -7187,26 +7245,35 @@ fn genShiftBinOpMir(
7187	7245	try self.asmRegisterMemory(
7188	7246	.{ ._, .mov },
7189	7247	tmp_reg,
7190		Memory.sib(.qword, .{
	7248	.{
7191	7249	.base = .{ .frame = dst_frame_addr.index },
7192		.disp = dst_frame_addr.off + info.indices[0] * 8,
7193		}),
	7250	.mod = .{ .rm = .{
	7251	.size = .qword,
	7252	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7253	} },
	7254	},
7194	7255	);
7195	7256	try self.asmMemoryRegisterImmediate(
7196	7257	info.double_tag,
7197		Memory.sib(.qword, .{
	7258	.{
7198	7259	.base = .{ .frame = dst_frame_addr.index },
7199		.disp = dst_frame_addr.off + info.indices[1] * 8,
7200		}),
	7260	.mod = .{ .rm = .{
	7261	.size = .qword,
	7262	.disp = dst_frame_addr.off + info.indices[1] * 8,
	7263	} },
	7264	},
7201	7265	tmp_reg,
7202	7266	Immediate.u(rhs_imm),
7203	7267	);
7204	7268	try self.asmMemoryImmediate(
7205	7269	tag,
7206		Memory.sib(.qword, .{
	7270	.{
7207	7271	.base = .{ .frame = dst_frame_addr.index },
7208		.disp = dst_frame_addr.off + info.indices[0] * 8,
7209		}),
	7272	.mod = .{ .rm = .{
	7273	.size = .qword,
	7274	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7275	} },
	7276	},
7210	7277	Immediate.u(rhs_imm),
7211	7278	);
7212	7279	} else {
...	...	@@ -7214,10 +7281,13 @@ fn genShiftBinOpMir(
7214	7281	try self.asmRegisterMemory(
7215	7282	.{ ._, .mov },
7216	7283	tmp_reg,
7217		Memory.sib(.qword, .{
	7284	.{
7218	7285	.base = .{ .frame = dst_frame_addr.index },
7219		.disp = dst_frame_addr.off + info.indices[0] * 8,
7220		}),
	7286	.mod = .{ .rm = .{
	7287	.size = .qword,
	7288	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7289	} },
	7290	},
7221	7291	);
7222	7292	if (rhs_imm > 64) try self.asmRegisterImmediate(
7223	7293	tag,
...	...	@@ -7226,27 +7296,36 @@ fn genShiftBinOpMir(
7226	7296	);
7227	7297	try self.asmMemoryRegister(
7228	7298	.{ ._, .mov },
7229		Memory.sib(.qword, .{
	7299	.{
7230	7300	.base = .{ .frame = dst_frame_addr.index },
7231		.disp = dst_frame_addr.off + info.indices[1] * 8,
7232		}),
	7301	.mod = .{ .rm = .{
	7302	.size = .qword,
	7303	.disp = dst_frame_addr.off + info.indices[1] * 8,
	7304	} },
	7305	},
7233	7306	tmp_reg,
7234	7307	);
7235	7308	if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryImmediate(
7236	7309	tag,
7237		Memory.sib(.qword, .{
	7310	.{
7238	7311	.base = .{ .frame = dst_frame_addr.index },
7239		.disp = dst_frame_addr.off + info.indices[0] * 8,
7240		}),
	7312	.mod = .{ .rm = .{
	7313	.size = .qword,
	7314	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7315	} },
	7316	},
7241	7317	Immediate.u(63),
7242	7318	) else {
7243	7319	try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());
7244	7320	try self.asmMemoryRegister(
7245	7321	.{ ._, .mov },
7246		Memory.sib(.qword, .{
	7322	.{
7247	7323	.base = .{ .frame = dst_frame_addr.index },
7248		.disp = dst_frame_addr.off + info.indices[0] * 8,
7249		}),
	7324	.mod = .{ .rm = .{
	7325	.size = .qword,
	7326	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7327	} },
	7328	},
7250	7329	tmp_reg,
7251	7330	);
7252	7331	}
...	...	@@ -7265,18 +7344,24 @@ fn genShiftBinOpMir(
7265	7344	try self.asmRegisterMemory(
7266	7345	.{ ._, .mov },
7267	7346	first_reg,
7268		Memory.sib(.qword, .{
	7347	.{
7269	7348	.base = .{ .frame = dst_frame_addr.index },
7270		.disp = dst_frame_addr.off + info.indices[0] * 8,
7271		}),
	7349	.mod = .{ .rm = .{
	7350	.size = .qword,
	7351	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7352	} },
	7353	},
7272	7354	);
7273	7355	try self.asmRegisterMemory(
7274	7356	.{ ._, .mov },
7275	7357	second_reg,
7276		Memory.sib(.qword, .{
	7358	.{
7277	7359	.base = .{ .frame = dst_frame_addr.index },
7278		.disp = dst_frame_addr.off + info.indices[1] * 8,
7279		}),
	7360	.mod = .{ .rm = .{
	7361	.size = .qword,
	7362	.disp = dst_frame_addr.off + info.indices[1] * 8,
	7363	} },
	7364	},
7280	7365	);
7281	7366	if (tag[0] == ._r and tag[1] == .sa) {
7282	7367	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);
...	...	@@ -7302,18 +7387,24 @@ fn genShiftBinOpMir(
7302	7387	try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg);
7303	7388	try self.asmMemoryRegister(
7304	7389	.{ ._, .mov },
7305		Memory.sib(.qword, .{
	7390	.{
7306	7391	.base = .{ .frame = dst_frame_addr.index },
7307		.disp = dst_frame_addr.off + info.indices[1] * 8,
7308		}),
	7392	.mod = .{ .rm = .{
	7393	.size = .qword,
	7394	.disp = dst_frame_addr.off + info.indices[1] * 8,
	7395	} },
	7396	},
7309	7397	second_reg,
7310	7398	);
7311	7399	try self.asmMemoryRegister(
7312	7400	.{ ._, .mov },
7313		Memory.sib(.qword, .{
	7401	.{
7314	7402	.base = .{ .frame = dst_frame_addr.index },
7315		.disp = dst_frame_addr.off + info.indices[0] * 8,
7316		}),
	7403	.mod = .{ .rm = .{
	7404	.size = .qword,
	7405	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7406	} },
	7407	},
7317	7408	first_reg,
7318	7409	);
7319	7410	},
...	...	@@ -7352,6 +7443,7 @@ fn genShiftBinOp(
7352	7443	});
7353	7444
7354	7445	assert(rhs_ty.abiSize(mod) == 1);
	7446	try self.spillEflagsIfOccupied();
7355	7447
7356	7448	const lhs_abi_size = lhs_ty.abiSize(mod);
7357	7449	if (lhs_abi_size > 16) return self.fail("TODO implement genShiftBinOp for {}", .{
...	...	@@ -7541,22 +7633,20 @@ fn genMulDivBinOp(
7541	7633	}, dst_abi_size) };
7542	7634
7543	7635	const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);
7544		try self.asmMemoryRegister(
7545		.{ ._, .mov },
7546		Memory.sib(.qword, .{
7547		.base = .{ .frame = dst_mcv.load_frame.index },
	7636	try self.asmMemoryRegister(.{ ._, .mov }, .{
	7637	.base = .{ .frame = dst_mcv.load_frame.index },
	7638	.mod = .{ .rm = .{
	7639	.size = .qword,
7548	7640	.disp = dst_mcv.load_frame.off,
7549		}),
7550		.rax,
7551		);
7552		try self.asmMemoryRegister(
7553		.{ ._, .mov },
7554		Memory.sib(.qword, .{
7555		.base = .{ .frame = dst_mcv.load_frame.index },
	7641	} },
	7642	}, .rax);
	7643	try self.asmMemoryRegister(.{ ._, .mov }, .{
	7644	.base = .{ .frame = dst_mcv.load_frame.index },
	7645	.mod = .{ .rm = .{
	7646	.size = .qword,
7556	7647	.disp = dst_mcv.load_frame.off + 8,
7557		}),
7558		.rdx,
7559		);
	7648	} },
	7649	}, .rdx);
7560	7650	return dst_mcv;
7561	7651	},
7562	7652
...	...	@@ -7768,7 +7858,7 @@ fn genBinOp(
7768	7858	mir_tag,
7769	7859	dst_reg,
7770	7860	dst_reg,
7771		src_mcv.mem(Memory.PtrSize.fromBitSize(float_bits)),
	7861	src_mcv.mem(Memory.Size.fromBitSize(float_bits)),
7772	7862	) else try self.asmRegisterRegisterRegister(
7773	7863	mir_tag,
7774	7864	dst_reg,
...	...	@@ -7787,7 +7877,7 @@ fn genBinOp(
7787	7877	if (src_mcv.isMemory()) try self.asmRegisterMemory(
7788	7878	mir_tag,
7789	7879	dst_reg,
7790		src_mcv.mem(Memory.PtrSize.fromBitSize(float_bits)),
	7880	src_mcv.mem(Memory.Size.fromBitSize(float_bits)),
7791	7881	) else try self.asmRegisterRegister(
7792	7882	mir_tag,
7793	7883	dst_reg,
...	...	@@ -7858,7 +7948,7 @@ fn genBinOp(
7858	7948	}) .{ .lhs = rhs_air, .rhs = lhs_air } else .{ .lhs = lhs_air, .rhs = rhs_air };
7859	7949
7860	7950	const lhs_mcv = try self.resolveInst(ordered_air.lhs);
7861		const rhs_mcv = try self.resolveInst(ordered_air.rhs);
	7951	var rhs_mcv = try self.resolveInst(ordered_air.rhs);
7862	7952	switch (lhs_mcv) {
7863	7953	.immediate => \|imm\| switch (imm) {
7864	7954	0 => switch (air_tag) {
...	...	@@ -7930,6 +8020,7 @@ fn genBinOp(
7930	8020	copied_to_dst = false
7931	8021	else
7932	8022	try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
	8023	rhs_mcv = try self.resolveInst(ordered_air.rhs);
7933	8024	break :dst dst_mcv;
7934	8025	};
7935	8026	const dst_locks: [2]?RegisterLock = switch (dst_mcv) {
...	...	@@ -8149,21 +8240,30 @@ fn genBinOp(
8149	8240	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
8150	8241	cc,
8151	8242	registerAlias(tmp_reg, cmov_abi_size),
8152		Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), switch (mat_src_mcv) {
	8243	switch (mat_src_mcv) {
8153	8244	.memory => \|addr\| .{
8154	8245	.base = .{ .reg = .ds },
8155		.disp = @intCast(@as(i64, @bitCast(addr))),
	8246	.mod = .{ .rm = .{
	8247	.size = Memory.Size.fromSize(cmov_abi_size),
	8248	.disp = @intCast(@as(i64, @bitCast(addr))),
	8249	} },
8156	8250	},
8157	8251	.indirect => \|reg_off\| .{
8158	8252	.base = .{ .reg = reg_off.reg },
8159		.disp = reg_off.off,
	8253	.mod = .{ .rm = .{
	8254	.size = Memory.Size.fromSize(cmov_abi_size),
	8255	.disp = reg_off.off,
	8256	} },
8160	8257	},
8161	8258	.load_frame => \|frame_addr\| .{
8162	8259	.base = .{ .frame = frame_addr.index },
8163		.disp = frame_addr.off,
	8260	.mod = .{ .rm = .{
	8261	.size = Memory.Size.fromSize(cmov_abi_size),
	8262	.disp = frame_addr.off,
	8263	} },
8164	8264	},
8165	8265	else => unreachable,
8166		}),
	8266	},
8167	8267	),
8168	8268	}
8169	8269	try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
...	...	@@ -8912,8 +9012,8 @@ fn genBinOp(
8912	9012	dst_reg,
8913	9013	lhs_reg,
8914	9014	src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {
8915		else => Memory.PtrSize.fromSize(abi_size),
8916		.Vector => Memory.PtrSize.fromBitSize(dst_reg.bitSize()),
	9015	else => Memory.Size.fromSize(abi_size),
	9016	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
8917	9017	}),
8918	9018	) else try self.asmRegisterRegisterRegister(
8919	9019	mir_tag,
...	...	@@ -8930,8 +9030,8 @@ fn genBinOp(
8930	9030	mir_tag,
8931	9031	dst_reg,
8932	9032	src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {
8933		else => Memory.PtrSize.fromSize(abi_size),
8934		.Vector => Memory.PtrSize.fromBitSize(dst_reg.bitSize()),
	9033	else => Memory.Size.fromSize(abi_size),
	9034	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
8935	9035	}),
8936	9036	) else try self.asmRegisterRegister(
8937	9037	mir_tag,
...	...	@@ -9202,12 +9302,12 @@ fn genBinOp(
9202	9302	.ty = lhs_ty,
9203	9303	.val = try unsigned_ty.maxInt(mod, unsigned_ty),
9204	9304	});
9205		const not_mem = if (not_mcv.isMemory())
9206		not_mcv.mem(Memory.PtrSize.fromSize(abi_size))
	9305	const not_mem: Memory = if (not_mcv.isMemory())
	9306	not_mcv.mem(Memory.Size.fromSize(abi_size))
9207	9307	else
9208		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{
	9308	.{ .base = .{
9209	9309	.reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
9210		} });
	9310	}, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };
9211	9311	switch (mir_tag[0]) {
9212	9312	.vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
9213	9313	.{ .vp_, .xor },
...	...	@@ -9252,6 +9352,7 @@ fn genBinOpMir(
9252	9352	) !void {
9253	9353	const mod = self.bin_file.options.module.?;
9254	9354	const abi_size: u32 = @intCast(ty.abiSize(mod));
	9355	try self.spillEflagsIfOccupied();
9255	9356	switch (dst_mcv) {
9256	9357	.none,
9257	9358	.unreach,
...	...	@@ -9359,25 +9460,30 @@ fn genBinOpMir(
9359	9460	.lea_frame,
9360	9461	=> {
9361	9462	blk: {
9362		return self.asmRegisterMemory(
9363		mir_limb_tag,
9364		dst_alias,
9365		Memory.sib(Memory.PtrSize.fromSize(limb_abi_size), switch (src_mcv) {
9366		.memory => \|addr\| .{
9367		.base = .{ .reg = .ds },
	9463	return self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) {
	9464	.memory => \|addr\| .{
	9465	.base = .{ .reg = .ds },
	9466	.mod = .{ .rm = .{
	9467	.size = Memory.Size.fromSize(limb_abi_size),
9368	9468	.disp = math.cast(i32, addr + off) orelse break :blk,
9369		},
9370		.indirect => \|reg_off\| .{
9371		.base = .{ .reg = reg_off.reg },
	9469	} },
	9470	},
	9471	.indirect => \|reg_off\| .{
	9472	.base = .{ .reg = reg_off.reg },
	9473	.mod = .{ .rm = .{
	9474	.size = Memory.Size.fromSize(limb_abi_size),
9372	9475	.disp = reg_off.off + off,
9373		},
9374		.load_frame => \|frame_addr\| .{
9375		.base = .{ .frame = frame_addr.index },
	9476	} },
	9477	},
	9478	.load_frame => \|frame_addr\| .{
	9479	.base = .{ .frame = frame_addr.index },
	9480	.mod = .{ .rm = .{
	9481	.size = Memory.Size.fromSize(limb_abi_size),
9376	9482	.disp = frame_addr.off + off,
9377		},
9378		else => break :blk,
9379		}),
9380		);
	9483	} },
	9484	},
	9485	else => break :blk,
	9486	});
9381	9487	}
9382	9488
9383	9489	switch (src_mcv) {
...	...	@@ -9510,26 +9616,35 @@ fn genBinOpMir(
9510	9616	}),
9511	9617	},
9512	9618	};
9513		const dst_limb_mem = Memory.sib(
9514		Memory.PtrSize.fromSize(limb_abi_size),
9515		switch (dst_mcv) {
9516		.memory,
9517		.load_symbol,
9518		.load_got,
9519		.load_direct,
9520		.load_tlv,
9521		=> .{ .base = .{ .reg = dst_info.?.addr_reg }, .disp = off },
9522		.indirect => \|reg_off\| .{
9523		.base = .{ .reg = reg_off.reg },
	9619	const dst_limb_mem: Memory = switch (dst_mcv) {
	9620	.memory,
	9621	.load_symbol,
	9622	.load_got,
	9623	.load_direct,
	9624	.load_tlv,
	9625	=> .{
	9626	.base = .{ .reg = dst_info.?.addr_reg },
	9627	.mod = .{ .rm = .{
	9628	.size = Memory.Size.fromSize(limb_abi_size),
	9629	.disp = off,
	9630	} },
	9631	},
	9632	.indirect => \|reg_off\| .{
	9633	.base = .{ .reg = reg_off.reg },
	9634	.mod = .{ .rm = .{
	9635	.size = Memory.Size.fromSize(limb_abi_size),
9524	9636	.disp = reg_off.off + off,
9525		},
9526		.load_frame => \|frame_addr\| .{
9527		.base = .{ .frame = frame_addr.index },
	9637	} },
	9638	},
	9639	.load_frame => \|frame_addr\| .{
	9640	.base = .{ .frame = frame_addr.index },
	9641	.mod = .{ .rm = .{
	9642	.size = Memory.Size.fromSize(limb_abi_size),
9528	9643	.disp = frame_addr.off + off,
9529		},
9530		else => unreachable,
	9644	} },
9531	9645	},
9532		);
	9646	else => unreachable,
	9647	};
9533	9648	switch (resolved_src_mcv) {
9534	9649	.none,
9535	9650	.unreach,
...	...	@@ -9730,29 +9845,38 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
9730	9845	.memory, .indirect, .load_frame => try self.asmRegisterMemory(
9731	9846	.{ .i_, .mul },
9732	9847	dst_alias,
9733		Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (resolved_src_mcv) {
	9848	switch (resolved_src_mcv) {
9734	9849	.memory => \|addr\| .{
9735	9850	.base = .{ .reg = .ds },
9736		.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
9737		return self.asmRegisterRegister(
9738		.{ .i_, .mul },
9739		dst_alias,
9740		registerAlias(
9741		try self.copyToTmpRegister(dst_ty, resolved_src_mcv),
9742		abi_size,
	9851	.mod = .{ .rm = .{
	9852	.size = Memory.Size.fromSize(abi_size),
	9853	.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
	9854	return self.asmRegisterRegister(
	9855	.{ .i_, .mul },
	9856	dst_alias,
	9857	registerAlias(
	9858	try self.copyToTmpRegister(dst_ty, resolved_src_mcv),
	9859	abi_size,
	9860	),
9743	9861	),
9744		),
	9862	} },
9745	9863	},
9746	9864	.indirect => \|reg_off\| .{
9747	9865	.base = .{ .reg = reg_off.reg },
9748		.disp = reg_off.off,
	9866	.mod = .{ .rm = .{
	9867	.size = Memory.Size.fromSize(abi_size),
	9868	.disp = reg_off.off,
	9869	} },
9749	9870	},
9750	9871	.load_frame => \|frame_addr\| .{
9751	9872	.base = .{ .frame = frame_addr.index },
9752		.disp = frame_addr.off,
	9873	.mod = .{ .rm = .{
	9874	.size = Memory.Size.fromSize(abi_size),
	9875	.disp = frame_addr.off,
	9876	} },
9753	9877	},
9754	9878	else => unreachable,
9755		}),
	9879	},
9756	9880	),
9757	9881	}
9758	9882	},
...	...	@@ -9857,9 +9981,14 @@ fn genVarDbgInfo(
9857	9981	// .offset = -off,
9858	9982	//} },
9859	9983	.memory => \|address\| .{ .memory = address },
9860		.load_symbol => \|sym_index\| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } }, // TODO
	9984	.load_symbol => \|sym_off\| loc: {
	9985	assert(sym_off.off == 0);
	9986	break :loc .{ .linker_load = .{ .type = .direct, .sym_index = sym_off.sym } };
	9987	}, // TODO
9861	9988	.load_got => \|sym_index\| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } },
9862		.load_direct => \|sym_index\| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } },
	9989	.load_direct => \|sym_index\| .{
	9990	.linker_load = .{ .type = .direct, .sym_index = sym_index },
	9991	},
9863	9992	.immediate => \|x\| .{ .immediate = x },
9864	9993	.undef => .undef,
9865	9994	.none => .none,
...	...	@@ -9971,6 +10100,7 @@ fn genCall(self: *Self, info: union(enum) {
9971	10100	}),
9972	10101	};
9973	10102	const fn_info = mod.typeToFunc(fn_ty).?;
	10103	const resolved_cc = abi.resolveCallingConvention(fn_info.cc, self.target.*);
9974	10104
9975	10105	const ExpectedContents = extern struct {
9976	10106	var_args: [16][@sizeOf(Type)]u8 align(@alignOf(Type)),
...	...	@@ -10012,9 +10142,7 @@ fn genCall(self: *Self, info: union(enum) {
10012	10142	}
10013	10143
10014	10144	try self.spillEflagsIfOccupied();
10015		try self.spillRegisters(abi.getCallerPreservedRegs(
10016		abi.resolveCallingConvention(fn_info.cc, self.target.*),
10017		));
	10145	try self.spillRegisters(abi.getCallerPreservedRegs(resolved_cc));
10018	10146
10019	10147	// set stack arguments first because this can clobber registers
10020	10148	// also clobber spill arguments as we go
...	...	@@ -10109,18 +10237,31 @@ fn genCall(self: *Self, info: union(enum) {
10109	10237	const sym = elf_file.symbol(sym_index);
10110	10238	_ = try sym.getOrCreateZigGotEntry(sym_index, elf_file);
10111	10239	if (self.bin_file.options.pic) {
10112		try self.genSetReg(.rax, Type.usize, .{ .load_symbol = sym.esym_index });
10113		try self.asmRegister(.{ ._, .call }, .rax);
10114		} else {
10115		_ = try self.addInst(.{
10116		.tag = .call,
10117		.ops = .linker_reloc,
10118		.data = .{ .reloc = .{
10119		.atom_index = try self.owner.getSymbolIndex(self),
10120		.sym_index = sym.esym_index,
10121		} },
10122		});
10123		}
	10240	const callee_reg: Register = switch (resolved_cc) {
	10241	.SysV => callee: {
	10242	if (!fn_info.is_var_args) break :callee .rax;
	10243	const param_regs = abi.getCAbiIntParamRegs(resolved_cc);
	10244	break :callee if (call_info.gp_count < param_regs.len)
	10245	param_regs[call_info.gp_count]
	10246	else
	10247	.r10;
	10248	},
	10249	.Win64 => .rax,
	10250	else => unreachable,
	10251	};
	10252	try self.genSetReg(
	10253	callee_reg,
	10254	Type.usize,
	10255	.{ .load_symbol = .{ .sym = sym.esym_index } },
	10256	);
	10257	try self.asmRegister(.{ ._, .call }, callee_reg);
	10258	} else try self.asmMemory(.{ ._, .call }, .{
	10259	.base = .{ .reloc = .{
	10260	.atom_index = try self.owner.getSymbolIndex(self),
	10261	.sym_index = sym.esym_index,
	10262	} },
	10263	.mod = .{ .rm = .{ .size = .qword } },
	10264	});
10124	10265	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
10125	10266	const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
10126	10267	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
...	...	@@ -10134,10 +10275,13 @@ fn genCall(self: *Self, info: union(enum) {
10134	10275	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
10135	10276	const atom_index = try p9.seeDecl(func.owner_decl);
10136	10277	const atom = p9.getAtom(atom_index);
10137		try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
	10278	try self.asmMemory(.{ ._, .call }, .{
10138	10279	.base = .{ .reg = .ds },
10139		.disp = @intCast(atom.getOffsetTableAddress(p9)),
10140		}));
	10280	.mod = .{ .rm = .{
	10281	.size = .qword,
	10282	.disp = @intCast(atom.getOffsetTableAddress(p9)),
	10283	} },
	10284	});
10141	10285	} else unreachable;
10142	10286	},
10143	10287	.extern_func => \|extern_func\| {
...	...	@@ -10622,7 +10766,10 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
10622	10766	try self.asmRegisterMemory(
10623	10767	.{ ._, .cmp },
10624	10768	registerAlias(dst_reg, op_abi_size),
10625		Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),
	10769	.{
	10770	.base = .{ .reg = addr_reg },
	10771	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(op_abi_size) } },
	10772	},
10626	10773	);
10627	10774
10628	10775	self.eflags_inst = inst;
...	...	@@ -10876,10 +11023,13 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
10876	11023	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(mod));
10877	11024	try self.asmMemoryImmediate(
10878	11025	.{ ._, .cmp },
10879		Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
	11026	.{
10880	11027	.base = .{ .reg = addr_reg },
10881		.disp = some_info.off,
10882		}),
	11028	.mod = .{ .rm = .{
	11029	.size = Memory.Size.fromSize(some_abi_size),
	11030	.disp = some_info.off,
	11031	} },
	11032	},
10883	11033	Immediate.u(0),
10884	11034	);
10885	11035	return .{ .eflags = .e };
...	...	@@ -10889,17 +11039,23 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
10889	11039	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(mod));
10890	11040	try self.asmMemoryImmediate(
10891	11041	.{ ._, .cmp },
10892		Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
	11042	switch (opt_mcv) {
10893	11043	.indirect => \|reg_off\| .{
10894	11044	.base = .{ .reg = reg_off.reg },
10895		.disp = reg_off.off + some_info.off,
	11045	.mod = .{ .rm = .{
	11046	.size = Memory.Size.fromSize(some_abi_size),
	11047	.disp = reg_off.off + some_info.off,
	11048	} },
10896	11049	},
10897	11050	.load_frame => \|frame_addr\| .{
10898	11051	.base = .{ .frame = frame_addr.index },
10899		.disp = frame_addr.off + some_info.off,
	11052	.mod = .{ .rm = .{
	11053	.size = Memory.Size.fromSize(some_abi_size),
	11054	.disp = frame_addr.off + some_info.off,
	11055	} },
10900	11056	},
10901	11057	else => unreachable,
10902		}),
	11058	},
10903	11059	Immediate.u(0),
10904	11060	);
10905	11061	return .{ .eflags = .e };
...	...	@@ -10929,10 +11085,13 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
10929	11085	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(mod));
10930	11086	try self.asmMemoryImmediate(
10931	11087	.{ ._, .cmp },
10932		Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
	11088	.{
10933	11089	.base = .{ .reg = ptr_reg },
10934		.disp = some_info.off,
10935		}),
	11090	.mod = .{ .rm = .{
	11091	.size = Memory.Size.fromSize(some_abi_size),
	11092	.disp = some_info.off,
	11093	} },
	11094	},
10936	11095	Immediate.u(0),
10937	11096	);
10938	11097
...	...	@@ -11005,10 +11164,13 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV
11005	11164	const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(mod), mod));
11006	11165	try self.asmMemoryImmediate(
11007	11166	.{ ._, .cmp },
11008		Memory.sib(self.memPtrSize(Type.anyerror), .{
	11167	.{
11009	11168	.base = .{ .reg = ptr_reg },
11010		.disp = err_off,
11011		}),
	11169	.mod = .{ .rm = .{
	11170	.size = self.memSize(Type.anyerror),
	11171	.disp = err_off,
	11172	} },
	11173	},
11012	11174	Immediate.u(0),
11013	11175	);
11014	11176
...	...	@@ -11121,11 +11283,11 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
11121	11283	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
11122	11284	const loop = self.air.extraData(Air.Block, ty_pl.payload);
11123	11285	const body = self.air.extra[loop.end..][0..loop.data.body_len];
11124		const jmp_target: Mir.Inst.Index = @intCast(self.mir_instructions.len);
11125	11286
11126	11287	self.scope_generation += 1;
11127	11288	const state = try self.saveState();
11128	11289
	11290	const jmp_target: Mir.Inst.Index = @intCast(self.mir_instructions.len);
11129	11291	try self.genBody(body);
11130	11292	try self.restoreState(state, &.{}, .{
11131	11293	.emit_instructions = true,
...	...	@@ -11303,14 +11465,14 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
11303	11465	.close_scope = false,
11304	11466	});
11305	11467
11306		// Stop tracking block result without forgetting tracking info
11307		try self.freeValue(block_tracking.short);
11308
11309	11468	// Emit a jump with a relocation. It will be patched up after the block ends.
11310	11469	// Leave the jump offset undefined
11311	11470	const jmp_reloc = try self.asmJmpReloc(undefined);
11312	11471	try block_data.relocs.append(self.gpa, jmp_reloc);
11313	11472
	11473	// Stop tracking block result without forgetting tracking info
	11474	try self.freeValue(block_tracking.short);
	11475
11314	11476	self.finishAirBookkeeping();
11315	11477	}
11316	11478
...	...	@@ -11586,7 +11748,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11586	11748	label_gop.value_ptr.target = @intCast(self.mir_instructions.len);
11587	11749	} else continue;
11588	11750
11589		var mnem_size: ?Memory.PtrSize = null;
	11751	var mnem_size: ?Memory.Size = null;
11590	11752	const mnem_tag = mnem: {
11591	11753	mnem_size = if (mem.endsWith(u8, mnem_str, "b"))
11592	11754	.byte
...	...	@@ -11605,7 +11767,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11605	11767	mnem_size = null;
11606	11768	break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str);
11607	11769	} orelse return self.fail("invalid mnemonic: '{s}'", .{mnem_str});
11608		if (@as(?Memory.PtrSize, switch (mnem_tag) {
	11770	if (@as(?Memory.Size, switch (mnem_tag) {
11609	11771	.fldenv, .fnstenv, .fstenv => .none,
11610	11772	.ldmxcsr, .stmxcsr, .vldmxcsr, .vstmxcsr => .dword,
11611	11773	else => null,
...	...	@@ -11670,10 +11832,13 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11670	11832	if (colon) \|colon_pos\| {
11671	11833	const disp = std.fmt.parseInt(i32, op_str[colon_pos + ":".len ..], 0) catch
11672	11834	return self.fail("invalid displacement: '{s}'", .{op_str});
11673		op.* = .{ .mem = Memory.sib(
11674		mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
11675		.{ .base = .{ .reg = reg }, .disp = disp },
11676		) };
	11835	op.* = .{ .mem = .{
	11836	.base = .{ .reg = reg },
	11837	.mod = .{ .rm = .{
	11838	.size = mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
	11839	.disp = disp,
	11840	} },
	11841	} };
11677	11842	} else {
11678	11843	if (mnem_size) \|size\| if (reg.bitSize() != size.bitSize())
11679	11844	return self.fail("invalid register size: '{s}'", .{op_str});
...	...	@@ -11697,34 +11862,45 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11697	11862	.{ .reg = reg }
11698	11863	else
11699	11864	return self.fail("invalid modifier: '{s}'", .{modifier}),
11700		.memory => \|addr\| if (mem.eql(u8, modifier, "") or
11701		mem.eql(u8, modifier, "P"))
11702		.{ .mem = Memory.sib(
11703		mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
11704		.{ .base = .{ .reg = .ds }, .disp = @intCast(@as(i64, @bitCast(addr))) },
11705		) }
	11865	.memory => \|addr\| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "P"))
	11866	.{ .mem = .{
	11867	.base = .{ .reg = .ds },
	11868	.mod = .{ .rm = .{
	11869	.size = mnem_size orelse
	11870	return self.fail("unknown size: '{s}'", .{op_str}),
	11871	.disp = @intCast(@as(i64, @bitCast(addr))),
	11872	} },
	11873	} }
11706	11874	else
11707	11875	return self.fail("invalid modifier: '{s}'", .{modifier}),
11708	11876	.indirect => \|reg_off\| if (mem.eql(u8, modifier, ""))
11709		.{ .mem = Memory.sib(
11710		mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
11711		.{ .base = .{ .reg = reg_off.reg }, .disp = reg_off.off },
11712		) }
	11877	.{ .mem = .{
	11878	.base = .{ .reg = reg_off.reg },
	11879	.mod = .{ .rm = .{
	11880	.size = mnem_size orelse
	11881	return self.fail("unknown size: '{s}'", .{op_str}),
	11882	.disp = reg_off.off,
	11883	} },
	11884	} }
11713	11885	else
11714	11886	return self.fail("invalid modifier: '{s}'", .{modifier}),
11715	11887	.load_frame => \|frame_addr\| if (mem.eql(u8, modifier, ""))
11716		.{ .mem = Memory.sib(
11717		mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
11718		.{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off },
11719		) }
	11888	.{ .mem = .{
	11889	.base = .{ .frame = frame_addr.index },
	11890	.mod = .{ .rm = .{
	11891	.size = mnem_size orelse
	11892	return self.fail("unknown size: '{s}'", .{op_str}),
	11893	.disp = frame_addr.off,
	11894	} },
	11895	} }
11720	11896	else
11721	11897	return self.fail("invalid modifier: '{s}'", .{modifier}),
11722	11898	.lea_got => \|sym_index\| if (mem.eql(u8, modifier, "P"))
11723	11899	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_got = sym_index }) }
11724	11900	else
11725	11901	return self.fail("invalid modifier: '{s}'", .{modifier}),
11726		.load_symbol => \|sym_index\| if (mem.eql(u8, modifier, "P"))
11727		.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .load_symbol = sym_index }) }
	11902	.load_symbol => \|sym_off\| if (mem.eql(u8, modifier, "P"))
	11903	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .load_symbol = sym_off }) }
11728	11904	else
11729	11905	return self.fail("invalid modifier: '{s}'", .{modifier}),
11730	11906	else => return self.fail("invalid constraint: '{s}'", .{op_str}),
...	...	@@ -11759,47 +11935,61 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11759	11935	const scale_str = sib_it.next() orelse "";
11760	11936	if (index_str.len == 0 and scale_str.len > 0)
11761	11937	return self.fail("invalid memory operand: '{s}'", .{op_str});
11762		const scale = if (scale_str.len > 0) switch (std.fmt.parseInt(u4, scale_str, 10) catch
11763		return self.fail("invalid scale: '{s}'", .{op_str})) {
11764		1, 2, 4, 8 => \|scale\| scale,
11765		else => return self.fail("invalid scale: '{s}'", .{op_str}),
11766		} else 1;
	11938	const scale: Memory.Scale = if (scale_str.len > 0)
	11939	switch (std.fmt.parseInt(u4, scale_str, 10) catch
	11940	return self.fail("invalid scale: '{s}'", .{op_str})) {
	11941	1 => .@"1",
	11942	2 => .@"2",
	11943	4 => .@"4",
	11944	8 => .@"8",
	11945	else => return self.fail("invalid scale: '{s}'", .{op_str}),
	11946	}
	11947	else
	11948	.@"1";
11767	11949	if (sib_it.next()) \|_\| return self.fail("invalid memory operand: '{s}'", .{op_str});
11768		op.* = .{ .mem = Memory.sib(mnem_size orelse
11769		return self.fail("unknown size: '{s}'", .{op_str}), .{
11770		.disp = if (mem.startsWith(u8, op_str[0..open], "%[") and
11771		mem.endsWith(u8, op_str[0..open], "]"))
11772		disp: {
11773		const colon = mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');
11774		const modifier = if (colon) \|colon_pos\|
11775		op_str[colon_pos + ":".len .. open - "]".len]
	11950	op.* = .{
	11951	.mem = .{
	11952	.base = if (base_str.len > 0)
	11953	.{ .reg = parseRegName(base_str["%%".len..]) orelse
	11954	return self.fail("invalid base register: '{s}'", .{base_str}) }
11776	11955	else
11777		"";
11778		break :disp switch (args.items[
11779		arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse
11780		return self.fail("no matching constraint: '{s}'", .{op_str})
11781		]) {
11782		.immediate => \|imm\| if (mem.eql(u8, modifier, "") or
11783		mem.eql(u8, modifier, "c"))
11784		math.cast(i32, @as(i64, @bitCast(imm))) orelse
	11956	.none,
	11957	.mod = .{ .rm = .{
	11958	.size = mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
	11959	.index = if (index_str.len > 0)
	11960	parseRegName(index_str["%%".len..]) orelse
	11961	return self.fail("invalid index register: '{s}'", .{op_str})
	11962	else
	11963	.none,
	11964	.scale = scale,
	11965	.disp = if (mem.startsWith(u8, op_str[0..open], "%[") and
	11966	mem.endsWith(u8, op_str[0..open], "]"))
	11967	disp: {
	11968	const colon = mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');
	11969	const modifier = if (colon) \|colon_pos\|
	11970	op_str[colon_pos + ":".len .. open - "]".len]
	11971	else
	11972	"";
	11973	break :disp switch (args.items[
	11974	arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse
	11975	return self.fail("no matching constraint: '{s}'", .{op_str})
	11976	]) {
	11977	.immediate => \|imm\| if (mem.eql(u8, modifier, "") or
	11978	mem.eql(u8, modifier, "c"))
	11979	math.cast(i32, @as(i64, @bitCast(imm))) orelse
	11980	return self.fail("invalid displacement: '{s}'", .{op_str})
	11981	else
	11982	return self.fail("invalid modifier: '{s}'", .{modifier}),
	11983	else => return self.fail("invalid constraint: '{s}'", .{op_str}),
	11984	};
	11985	} else if (open > 0)
	11986	std.fmt.parseInt(i32, op_str[0..open], 0) catch
11785	11987	return self.fail("invalid displacement: '{s}'", .{op_str})
11786	11988	else
11787		return self.fail("invalid modifier: '{s}'", .{modifier}),
11788		else => return self.fail("invalid constraint: '{s}'", .{op_str}),
11789		};
11790		} else if (open > 0)
11791		std.fmt.parseInt(i32, op_str[0..open], 0) catch
11792		return self.fail("invalid displacement: '{s}'", .{op_str})
11793		else
11794		0,
11795		.base = if (base_str.len > 0) .{ .reg = parseRegName(base_str["%%".len..]) orelse
11796		return self.fail("invalid base register: '{s}'", .{base_str}) } else .none,
11797		.scale_index = if (index_str.len > 0) .{
11798		.index = parseRegName(index_str["%%".len..]) orelse
11799		return self.fail("invalid index register: '{s}'", .{op_str}),
11800		.scale = scale,
11801		} else null,
11802		}) };
	11989	0,
	11990	} },
	11991	},
	11992	};
11803	11993	} else if (Label.isValid(.reference, op_str)) {
11804	11994	const anon = std.ascii.isDigit(op_str[0]);
11805	11995	const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else op_str.len]);
...	...	@@ -12519,20 +12709,23 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12519	12709	),
12520	12710	.lea_frame => .{ .move = .{ ._, .lea } },
12521	12711	else => unreachable,
12522		}).read(self, registerAlias(dst_reg, abi_size), Memory.sib(
12523		self.memPtrSize(ty),
12524		switch (src_mcv) {
12525		.register_offset, .indirect => \|reg_off\| .{
12526		.base = .{ .reg = reg_off.reg },
	12712	}).read(self, registerAlias(dst_reg, abi_size), switch (src_mcv) {
	12713	.register_offset, .indirect => \|reg_off\| .{
	12714	.base = .{ .reg = reg_off.reg },
	12715	.mod = .{ .rm = .{
	12716	.size = self.memSize(ty),
12527	12717	.disp = reg_off.off,
12528		},
12529		.load_frame, .lea_frame => \|frame_addr\| .{
12530		.base = .{ .frame = frame_addr.index },
	12718	} },
	12719	},
	12720	.load_frame, .lea_frame => \|frame_addr\| .{
	12721	.base = .{ .frame = frame_addr.index },
	12722	.mod = .{ .rm = .{
	12723	.size = self.memSize(ty),
12531	12724	.disp = frame_addr.off,
12532		},
12533		else => unreachable,
	12725	} },
12534	12726	},
12535		)),
	12727	else => unreachable,
	12728	}),
12536	12729	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {
12537	12730	switch (src_mcv) {
12538	12731	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
...	...	@@ -12540,20 +12733,40 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12540	12733	ty,
12541	12734	dst_reg.class(),
12542	12735	ty.abiAlignment(mod).check(@as(u32, @bitCast(small_addr))),
12543		)).read(self, registerAlias(dst_reg, abi_size), Memory.sib(
12544		self.memPtrSize(ty),
12545		.{ .base = .{ .reg = .ds }, .disp = small_addr },
12546		)),
	12736	)).read(self, registerAlias(dst_reg, abi_size), .{
	12737	.base = .{ .reg = .ds },
	12738	.mod = .{ .rm = .{
	12739	.size = self.memSize(ty),
	12740	.disp = small_addr,
	12741	} },
	12742	}),
	12743	.load_symbol => \|sym_off\| switch (dst_reg.class()) {
	12744	.general_purpose => {
	12745	assert(sym_off.off == 0);
	12746	try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(dst_reg, abi_size), .{
	12747	.base = .{ .reloc = .{
	12748	.atom_index = try self.owner.getSymbolIndex(self),
	12749	.sym_index = sym_off.sym,
	12750	} },
	12751	.mod = .{ .rm = .{
	12752	.size = self.memSize(ty),
	12753	.disp = sym_off.off,
	12754	} },
	12755	});
	12756	return;
	12757	},
	12758	.segment, .mmx => unreachable,
	12759	.x87, .sse => {},
	12760	},
12547	12761	.load_direct => \|sym_index\| switch (dst_reg.class()) {
12548	12762	.general_purpose => {
12549		const atom_index = try self.owner.getSymbolIndex(self);
12550	12763	_ = try self.addInst(.{
12551	12764	.tag = .mov,
12552	12765	.ops = .direct_reloc,
12553	12766	.data = .{ .rx = .{
12554		.r1 = dst_reg.to64(),
12555		.payload = try self.addExtra(Mir.Reloc{
12556		.atom_index = atom_index,
	12767	.r1 = registerAlias(dst_reg, abi_size),
	12768	.payload = try self.addExtra(bits.Symbol{
	12769	.atom_index = try self.owner.getSymbolIndex(self),
12557	12770	.sym_index = sym_index,
12558	12771	}),
12559	12772	} },
...	...	@@ -12563,7 +12776,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12563	12776	.segment, .mmx => unreachable,
12564	12777	.x87, .sse => {},
12565	12778	},
12566		.load_symbol, .load_got, .load_tlv => {},
	12779	.load_got, .load_tlv => {},
12567	12780	else => unreachable,
12568	12781	}
12569	12782
...	...	@@ -12574,22 +12787,42 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12574	12787	try (try self.moveStrategy(ty, dst_reg.class(), false)).read(
12575	12788	self,
12576	12789	registerAlias(dst_reg, abi_size),
12577		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
	12790	.{
	12791	.base = .{ .reg = addr_reg },
	12792	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
	12793	},
12578	12794	);
12579	12795	},
12580		.lea_symbol, .lea_direct, .lea_got => \|sym_index\| {
	12796	.lea_symbol => \|sym_index\| {
12581	12797	const atom_index = try self.owner.getSymbolIndex(self);
12582	12798	if (self.bin_file.cast(link.File.Elf)) \|_\| {
12583		_ = try self.addInst(.{
12584		.tag = .lea,
12585		.ops = .linker_reloc,
12586		.data = .{ .rx = .{
12587		.r1 = dst_reg.to64(),
12588		.payload = try self.addExtra(Mir.Reloc{
	12799	try self.asmRegisterMemory(
	12800	.{ ._, .lea },
	12801	dst_reg.to64(),
	12802	.{
	12803	.base = .{ .reloc = .{
12589	12804	.atom_index = atom_index,
12590		.sym_index = sym_index,
12591		}),
	12805	.sym_index = sym_index.sym,
	12806	} },
	12807	.mod = .{ .rm = .{
	12808	.size = .qword,
	12809	.disp = sym_index.off,
	12810	} },
	12811	},
	12812	);
	12813	} else return self.fail("TODO emit symbol sequence on {s}", .{
	12814	@tagName(self.bin_file.tag),
	12815	});
	12816	},
	12817	.lea_direct, .lea_got => \|sym_index\| {
	12818	const atom_index = try self.owner.getSymbolIndex(self);
	12819	if (self.bin_file.cast(link.File.Elf)) \|_\| {
	12820	try self.asmRegisterMemory(.{ ._, .lea }, dst_reg.to64(), .{
	12821	.base = .{ .reloc = .{
	12822	.atom_index = atom_index,
	12823	.sym_index = sym_index,
12592	12824	} },
	12825	.mod = .{ .rm = .{ .size = .qword } },
12593	12826	});
12594	12827	} else {
12595	12828	_ = try self.addInst(.{
...	...	@@ -12605,7 +12838,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12605	12838	},
12606	12839	.data = .{ .rx = .{
12607	12840	.r1 = dst_reg.to64(),
12608		.payload = try self.addExtra(Mir.Reloc{
	12841	.payload = try self.addExtra(bits.Symbol{
12609	12842	.atom_index = atom_index,
12610	12843	.sym_index = sym_index,
12611	12844	}),
...	...	@@ -12621,14 +12854,17 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12621	12854	.ops = .tlv_reloc,
12622	12855	.data = .{ .rx = .{
12623	12856	.r1 = .rdi,
12624		.payload = try self.addExtra(Mir.Reloc{
	12857	.payload = try self.addExtra(bits.Symbol{
12625	12858	.atom_index = atom_index,
12626	12859	.sym_index = sym_index,
12627	12860	}),
12628	12861	} },
12629	12862	});
12630	12863	// TODO: spill registers before calling
12631		try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
	12864	try self.asmMemory(.{ ._, .call }, .{
	12865	.base = .{ .reg = .rdi },
	12866	.mod = .{ .rm = .{ .size = .qword } },
	12867	});
12632	12868	try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });
12633	12869	} else return self.fail("TODO emit ptr to TLV sequence on {s}", .{
12634	12870	@tagName(self.bin_file.tag),
...	...	@@ -12645,6 +12881,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12645	12881	.none => .{ .immediate = @bitCast(@as(i64, disp)) },
12646	12882	.reg => \|base_reg\| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
12647	12883	.frame => \|base_frame_index\| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
	12884	.reloc => \|base_symbol\| .{ .lea_symbol = .{ .sym = base_symbol.sym_index, .off = disp } },
12648	12885	};
12649	12886	switch (src_mcv) {
12650	12887	.none, .unreach, .dead, .reserved_frame => unreachable,
...	...	@@ -12660,7 +12897,10 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12660	12897	};
12661	12898	try self.asmMemoryImmediate(
12662	12899	.{ ._, .mov },
12663		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
	12900	.{ .base = base, .mod = .{ .rm = .{
	12901	.size = Memory.Size.fromSize(abi_size),
	12902	.disp = disp,
	12903	} } },
12664	12904	immediate,
12665	12905	);
12666	12906	},
...	...	@@ -12668,27 +12908,31 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12668	12908	else => if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\| {
12669	12909	try self.asmMemoryImmediate(
12670	12910	.{ ._, .mov },
12671		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
	12911	.{ .base = base, .mod = .{ .rm = .{
	12912	.size = Memory.Size.fromSize(abi_size),
	12913	.disp = disp,
	12914	} } },
12672	12915	Immediate.s(small),
12673	12916	);
12674	12917	} else {
12675	12918	var offset: i32 = 0;
12676	12919	while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
12677	12920	.{ ._, .mov },
12678		Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
12679		if (ty.isSignedInt(mod))
12680		Immediate.s(
12681		@truncate(@as(i64, @bitCast(imm)) >> (math.cast(u6, offset * 8) orelse 63)),
12682		)
12683		else
12684		Immediate.u(@as(
12685		u32,
12686		@truncate(if (math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0),
12687		)),
	12921	.{ .base = base, .mod = .{ .rm = .{
	12922	.size = .dword,
	12923	.disp = disp + offset,
	12924	} } },
	12925	if (ty.isSignedInt(mod)) Immediate.s(
	12926	@truncate(@as(i64, @bitCast(imm)) >> (math.cast(u6, offset * 8) orelse 63)),
	12927	) else Immediate.u(
	12928	@as(u32, @truncate(if (math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0)),
	12929	),
12688	12930	);
12689	12931	},
12690	12932	},
12691		.eflags => \|cc\| try self.asmSetccMemory(cc, Memory.sib(.byte, .{ .base = base, .disp = disp })),
	12933	.eflags => \|cc\| try self.asmSetccMemory(cc, .{ .base = base, .mod = .{
	12934	.rm = .{ .size = .byte, .disp = disp },
	12935	} }),
12692	12936	.register => \|src_reg\| try (try self.moveStrategy(ty, src_reg.class(), switch (base) {
12693	12937	.none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12694	12938	.reg => \|reg\| switch (reg) {
...	...	@@ -12698,9 +12942,13 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12698	12942	.frame => \|frame_index\| self.getFrameAddrAlignment(
12699	12943	.{ .index = frame_index, .off = disp },
12700	12944	).compare(.gte, ty.abiAlignment(mod)),
	12945	.reloc => false,
12701	12946	})).write(
12702	12947	self,
12703		Memory.sib(self.memPtrSize(ty), .{ .base = base, .disp = disp }),
	12948	.{ .base = base, .mod = .{ .rm = .{
	12949	.size = self.memSize(ty),
	12950	.disp = disp,
	12951	} } },
12704	12952	registerAlias(src_reg, abi_size),
12705	12953	),
12706	12954	.register_pair => \|src_regs\| for (src_regs, 0..) \|src_reg, src_reg_i\| {
...	...	@@ -12717,11 +12965,12 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12717	12965	.frame => \|frame_index\| self.getFrameAddrAlignment(
12718	12966	.{ .index = frame_index, .off = disp },
12719	12967	).compare(.gte, ty.abiAlignment(mod)),
	12968	.reloc => false,
12720	12969	},
12721		)).write(self, Memory.sib(
12722		Memory.PtrSize.fromSize(part_size),
12723		.{ .base = base, .disp = disp + @as(i32, @intCast(src_reg_i * 8)) },
12724		), registerAlias(src_reg, part_size));
	12970	)).write(self, .{ .base = base, .mod = .{ .rm = .{
	12971	.size = Memory.Size.fromSize(part_size),
	12972	.disp = disp + @as(i32, @intCast(src_reg_i * 8)),
	12973	} } }, registerAlias(src_reg, part_size));
12725	12974	},
12726	12975	.register_overflow => \|ro\| switch (ty.zigTypeTag(mod)) {
12727	12976	.Struct => {
...	...	@@ -12819,7 +13068,7 @@ fn genExternSymbolRef(
12819	13068	.ops = .import_reloc,
12820	13069	.data = .{ .rx = .{
12821	13070	.r1 = .rax,
12822		.payload = try self.addExtra(Mir.Reloc{
	13071	.payload = try self.addExtra(bits.Symbol{
12823	13072	.atom_index = atom_index,
12824	13073	.sym_index = try coff_file.getGlobalSymbol(callee, lib),
12825	13074	}),
...	...	@@ -12856,8 +13105,12 @@ fn genLazySymbolRef(
12856	13105
12857	13106	if (self.bin_file.options.pic) {
12858	13107	switch (tag) {
12859		.lea, .call => try self.genSetReg(reg, Type.usize, .{ .load_symbol = sym.esym_index }),
12860		.mov => try self.genSetReg(reg, Type.usize, .{ .load_symbol = sym.esym_index }),
	13108	.lea, .call => try self.genSetReg(reg, Type.usize, .{
	13109	.load_symbol = .{ .sym = sym.esym_index },
	13110	}),
	13111	.mov => try self.genSetReg(reg, Type.usize, .{
	13112	.load_symbol = .{ .sym = sym.esym_index },
	13113	}),
12861	13114	else => unreachable,
12862	13115	}
12863	13116	switch (tag) {
...	...	@@ -12866,23 +13119,18 @@ fn genLazySymbolRef(
12866	13119	else => unreachable,
12867	13120	}
12868	13121	} else {
12869		const reloc = Mir.Reloc{
	13122	const reloc = bits.Symbol{
12870	13123	.atom_index = try self.owner.getSymbolIndex(self),
12871	13124	.sym_index = sym.esym_index,
12872	13125	};
12873	13126	switch (tag) {
12874		.lea, .mov => _ = try self.addInst(.{
12875		.tag = .mov,
12876		.ops = .linker_reloc,
12877		.data = .{ .rx = .{
12878		.r1 = reg.to64(),
12879		.payload = try self.addExtra(reloc),
12880		} },
	13127	.lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), .{
	13128	.base = .{ .reloc = reloc },
	13129	.mod = .{ .rm = .{ .size = .qword } },
12881	13130	}),
12882		.call => _ = try self.addInst(.{
12883		.tag = .call,
12884		.ops = .linker_reloc,
12885		.data = .{ .reloc = reloc },
	13131	.call => try self.asmMemory(.{ ._, .call }, .{
	13132	.base = .{ .reloc = reloc },
	13133	.mod = .{ .rm = .{ .size = .qword } },
12886	13134	}),
12887	13135	else => unreachable,
12888	13136	}
...	...	@@ -12893,8 +13141,13 @@ fn genLazySymbolRef(
12893	13141	var atom = p9_file.getAtom(atom_index);
12894	13142	_ = atom.getOrCreateOffsetTableEntry(p9_file);
12895	13143	const got_addr = atom.getOffsetTableAddress(p9_file);
12896		const got_mem =
12897		Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(got_addr) });
	13144	const got_mem: Memory = .{
	13145	.base = .{ .reg = .ds },
	13146	.mod = .{ .rm = .{
	13147	.size = .qword,
	13148	.disp = @intCast(got_addr),
	13149	} },
	13150	};
12898	13151	switch (tag) {
12899	13152	.lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), got_mem),
12900	13153	.call => try self.asmMemory(.{ ._, .call }, got_mem),
...	...	@@ -13240,19 +13493,20 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
13240	13493	defer if (new_lock) \|lock\| self.register_manager.unlockReg(lock);
13241	13494
13242	13495	const ptr_mcv = try self.resolveInst(extra.ptr);
13243		const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
13244		const ptr_mem = switch (ptr_mcv) {
13245		.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(ptr_size),
13246		else => Memory.sib(ptr_size, .{
	13496	const mem_size = Memory.Size.fromSize(val_abi_size);
	13497	const ptr_mem: Memory = switch (ptr_mcv) {
	13498	.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),
	13499	else => .{
13247	13500	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13248		}),
	13501	.mod = .{ .rm = .{ .size = mem_size } },
	13502	},
13249	13503	};
13250		switch (ptr_mem) {
13251		.sib, .rip => {},
13252		.moffs => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
	13504	switch (ptr_mem.mod) {
	13505	.rm => {},
	13506	.off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
13253	13507	}
13254		const ptr_lock = switch (ptr_mem.base()) {
13255		.none, .frame => null,
	13508	const ptr_lock = switch (ptr_mem.base) {
	13509	.none, .frame, .reloc => null,
13256	13510	.reg => \|reg\| self.register_manager.lockReg(reg),
13257	13511	};
13258	13512	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -13305,19 +13559,20 @@ fn atomicOp(
13305	13559	defer if (val_lock) \|lock\| self.register_manager.unlockReg(lock);
13306	13560
13307	13561	const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
13308		const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
13309		const ptr_mem = switch (ptr_mcv) {
13310		.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(ptr_size),
13311		else => Memory.sib(ptr_size, .{
	13562	const mem_size = Memory.Size.fromSize(val_abi_size);
	13563	const ptr_mem: Memory = switch (ptr_mcv) {
	13564	.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),
	13565	else => .{
13312	13566	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13313		}),
	13567	.mod = .{ .rm = .{ .size = mem_size } },
	13568	},
13314	13569	};
13315		switch (ptr_mem) {
13316		.sib, .rip => {},
13317		.moffs => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
	13570	switch (ptr_mem.mod) {
	13571	.rm => {},
	13572	.off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
13318	13573	}
13319		const mem_lock = switch (ptr_mem.base()) {
13320		.none, .frame => null,
	13574	const mem_lock = switch (ptr_mem.base) {
	13575	.none, .frame, .reloc => null,
13321	13576	.reg => \|reg\| self.register_manager.lockReg(reg),
13322	13577	};
13323	13578	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -13425,7 +13680,7 @@ fn atomicOp(
13425	13680	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
13426	13681	cc,
13427	13682	registerAlias(tmp_reg, cmov_abi_size),
13428		val_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)),
	13683	val_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),
13429	13684	),
13430	13685	else => {
13431	13686	const val_reg = try self.copyToTmpRegister(val_ty, val_mcv);
...	...	@@ -13446,16 +13701,24 @@ fn atomicOp(
13446	13701	_ = try self.asmJccReloc(.ne, loop);
13447	13702	return if (unused) .unreach else .{ .register = .rax };
13448	13703	} else {
13449		try self.asmRegisterMemory(.{ ._, .mov }, .rax, Memory.sib(.qword, .{
13450		.base = ptr_mem.sib.base,
13451		.scale_index = ptr_mem.scaleIndex(),
13452		.disp = ptr_mem.sib.disp + 0,
13453		}));
13454		try self.asmRegisterMemory(.{ ._, .mov }, .rdx, Memory.sib(.qword, .{
13455		.base = ptr_mem.sib.base,
13456		.scale_index = ptr_mem.scaleIndex(),
13457		.disp = ptr_mem.sib.disp + 8,
13458		}));
	13704	try self.asmRegisterMemory(.{ ._, .mov }, .rax, .{
	13705	.base = ptr_mem.base,
	13706	.mod = .{ .rm = .{
	13707	.size = .qword,
	13708	.index = ptr_mem.mod.rm.index,
	13709	.scale = ptr_mem.mod.rm.scale,
	13710	.disp = ptr_mem.mod.rm.disp + 0,
	13711	} },
	13712	});
	13713	try self.asmRegisterMemory(.{ ._, .mov }, .rdx, .{
	13714	.base = ptr_mem.base,
	13715	.mod = .{ .rm = .{
	13716	.size = .qword,
	13717	.index = ptr_mem.mod.rm.index,
	13718	.scale = ptr_mem.mod.rm.scale,
	13719	.disp = ptr_mem.mod.rm.disp + 8,
	13720	} },
	13721	});
13459	13722	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
13460	13723	const val_mem_mcv: MCValue = switch (val_mcv) {
13461	13724	.memory, .indirect, .load_frame => val_mcv,
...	...	@@ -13509,22 +13772,20 @@ fn atomicOp(
13509	13772
13510	13773	if (unused) return .unreach;
13511	13774	const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
13512		try self.asmMemoryRegister(
13513		.{ ._, .mov },
13514		Memory.sib(.qword, .{
13515		.base = .{ .frame = dst_mcv.load_frame.index },
	13775	try self.asmMemoryRegister(.{ ._, .mov }, .{
	13776	.base = .{ .frame = dst_mcv.load_frame.index },
	13777	.mod = .{ .rm = .{
	13778	.size = .qword,
13516	13779	.disp = dst_mcv.load_frame.off + 0,
13517		}),
13518		.rax,
13519		);
13520		try self.asmMemoryRegister(
13521		.{ ._, .mov },
13522		Memory.sib(.qword, .{
13523		.base = .{ .frame = dst_mcv.load_frame.index },
	13780	} },
	13781	}, .rax);
	13782	try self.asmMemoryRegister(.{ ._, .mov }, .{
	13783	.base = .{ .frame = dst_mcv.load_frame.index },
	13784	.mod = .{ .rm = .{
	13785	.size = .qword,
13524	13786	.disp = dst_mcv.load_frame.off + 8,
13525		}),
13526		.rdx,
13527		);
	13787	} },
	13788	}, .rdx);
13528	13789	return dst_mcv;
13529	13790	},
13530	13791	.libcall => return self.fail("TODO implement x86 atomic libcall", .{}),
...	...	@@ -13830,55 +14091,74 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
13830	14091	try self.asmRegisterMemory(
13831	14092	.{ ._, .mov },
13832	14093	start_reg.to32(),
13833		Memory.sib(.dword, .{
	14094	.{
13834	14095	.base = .{ .reg = addr_reg.to64() },
13835		.scale_index = .{ .scale = 4, .index = err_reg.to64() },
13836		.disp = 4,
13837		}),
	14096	.mod = .{ .rm = .{
	14097	.size = .dword,
	14098	.index = err_reg.to64(),
	14099	.scale = .@"4",
	14100	.disp = 4,
	14101	} },
	14102	},
13838	14103	);
13839	14104	try self.asmRegisterMemory(
13840	14105	.{ ._, .mov },
13841	14106	end_reg.to32(),
13842		Memory.sib(.dword, .{
	14107	.{
13843	14108	.base = .{ .reg = addr_reg.to64() },
13844		.scale_index = .{ .scale = 4, .index = err_reg.to64() },
13845		.disp = 8,
13846		}),
	14109	.mod = .{ .rm = .{
	14110	.size = .dword,
	14111	.index = err_reg.to64(),
	14112	.scale = .@"4",
	14113	.disp = 8,
	14114	} },
	14115	},
13847	14116	);
13848	14117	try self.asmRegisterRegister(.{ ._, .sub }, end_reg.to32(), start_reg.to32());
13849	14118	try self.asmRegisterMemory(
13850	14119	.{ ._, .lea },
13851	14120	start_reg.to64(),
13852		Memory.sib(.byte, .{
	14121	.{
13853	14122	.base = .{ .reg = addr_reg.to64() },
13854		.scale_index = .{ .scale = 1, .index = start_reg.to64() },
13855		.disp = 0,
13856		}),
	14123	.mod = .{ .rm = .{
	14124	.size = .dword,
	14125	.index = start_reg.to64(),
	14126	} },
	14127	},
13857	14128	);
13858	14129	try self.asmRegisterMemory(
13859	14130	.{ ._, .lea },
13860	14131	end_reg.to32(),
13861		Memory.sib(.byte, .{
	14132	.{
13862	14133	.base = .{ .reg = end_reg.to64() },
13863		.disp = -1,
13864		}),
	14134	.mod = .{ .rm = .{
	14135	.size = .byte,
	14136	.disp = -1,
	14137	} },
	14138	},
13865	14139	);
13866	14140
13867	14141	const dst_mcv = try self.allocRegOrMem(inst, false);
13868	14142	try self.asmMemoryRegister(
13869	14143	.{ ._, .mov },
13870		Memory.sib(.qword, .{
	14144	.{
13871	14145	.base = .{ .frame = dst_mcv.load_frame.index },
13872		.disp = dst_mcv.load_frame.off,
13873		}),
	14146	.mod = .{ .rm = .{
	14147	.size = .qword,
	14148	.disp = dst_mcv.load_frame.off,
	14149	} },
	14150	},
13874	14151	start_reg.to64(),
13875	14152	);
13876	14153	try self.asmMemoryRegister(
13877	14154	.{ ._, .mov },
13878		Memory.sib(.qword, .{
	14155	.{
13879	14156	.base = .{ .frame = dst_mcv.load_frame.index },
13880		.disp = dst_mcv.load_frame.off + 8,
13881		}),
	14157	.mod = .{ .rm = .{
	14158	.size = .qword,
	14159	.disp = dst_mcv.load_frame.off + 8,
	14160	} },
	14161	},
13882	14162	end_reg.to64(),
13883	14163	);
13884	14164
...	...	@@ -14115,7 +14395,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
14115	14395	.Or => {
14116	14396	if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
14117	14397	.{ ._, .@"test" },
14118		operand_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	14398	operand_mcv.mem(Memory.Size.fromSize(abi_size)),
14119	14399	Immediate.u(mask),
14120	14400	) else {
14121	14401	const operand_reg = registerAlias(if (operand_mcv.isRegister())
...	...	@@ -14507,7 +14787,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
14507	14787	mir_tag,
14508	14788	mop1_reg,
14509	14789	mop2_reg,
14510		mops[2].mem(Memory.PtrSize.fromSize(abi_size)),
	14790	mops[2].mem(Memory.Size.fromSize(abi_size)),
14511	14791	);
14512	14792	break :result mops[0];
14513	14793	};
...	...	@@ -14618,24 +14898,32 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
14618	14898	const mem_reloc = try self.asmJccReloc(.ae, undefined);
14619	14899
14620	14900	try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area);
14621		if (!unused)
14622		try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, Memory.sib(.qword, .{
14623		.base = .{ .reg = addr_reg },
14624		.scale_index = .{ .scale = 1, .index = offset_reg.to64() },
14625		}));
14626		try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, Memory.sib(.qword, .{
	14901	if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
	14902	.base = .{ .reg = addr_reg },
	14903	.mod = .{ .rm = .{
	14904	.size = .qword,
	14905	.index = offset_reg.to64(),
	14906	} },
	14907	});
	14908	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
14627	14909	.base = .{ .reg = offset_reg.to64() },
14628		.disp = 8,
14629		}));
	14910	.mod = .{ .rm = .{
	14911	.size = .qword,
	14912	.disp = 8,
	14913	} },
	14914	});
14630	14915	try self.genCopy(Type.c_uint, gp_offset, .{ .register = offset_reg });
14631	14916	const done_reloc = try self.asmJmpReloc(undefined);
14632	14917
14633	14918	try self.performReloc(mem_reloc);
14634	14919	try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area);
14635		try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), Memory.sib(.qword, .{
	14920	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
14636	14921	.base = .{ .reg = addr_reg },
14637		.disp = @intCast(@max(promote_ty.abiSize(mod), 8)),
14638		}));
	14922	.mod = .{ .rm = .{
	14923	.size = .qword,
	14924	.disp = @intCast(@max(promote_ty.abiSize(mod), 8)),
	14925	} },
	14926	});
14639	14927	try self.genCopy(
14640	14928	ptr_anyopaque_ty,
14641	14929	overflow_arg_area,
...	...	@@ -14657,24 +14945,32 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
14657	14945	const mem_reloc = try self.asmJccReloc(.ae, undefined);
14658	14946
14659	14947	try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area);
14660		if (!unused)
14661		try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, Memory.sib(.qword, .{
14662		.base = .{ .reg = addr_reg },
14663		.scale_index = .{ .scale = 1, .index = offset_reg.to64() },
14664		}));
14665		try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, Memory.sib(.qword, .{
	14948	if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
	14949	.base = .{ .reg = addr_reg },
	14950	.mod = .{ .rm = .{
	14951	.size = .qword,
	14952	.index = offset_reg.to64(),
	14953	} },
	14954	});
	14955	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
14666	14956	.base = .{ .reg = offset_reg.to64() },
14667		.disp = 16,
14668		}));
	14957	.mod = .{ .rm = .{
	14958	.size = .qword,
	14959	.disp = 16,
	14960	} },
	14961	});
14669	14962	try self.genCopy(Type.c_uint, fp_offset, .{ .register = offset_reg });
14670	14963	const done_reloc = try self.asmJmpReloc(undefined);
14671	14964
14672	14965	try self.performReloc(mem_reloc);
14673	14966	try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area);
14674		try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), Memory.sib(.qword, .{
	14967	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
14675	14968	.base = .{ .reg = addr_reg },
14676		.disp = @intCast(@max(promote_ty.abiSize(mod), 8)),
14677		}));
	14969	.mod = .{ .rm = .{
	14970	.size = .qword,
	14971	.disp = @intCast(@max(promote_ty.abiSize(mod), 8)),
	14972	} },
	14973	});
14678	14974	try self.genCopy(
14679	14975	ptr_anyopaque_ty,
14680	14976	overflow_arg_area,
...	...	@@ -14823,7 +15119,7 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
14823	15119	.undef => .undef,
14824	15120	.immediate => \|imm\| .{ .immediate = imm },
14825	15121	.memory => \|addr\| .{ .memory = addr },
14826		.load_symbol => \|sym_index\| .{ .load_symbol = sym_index },
	15122	.load_symbol => \|sym_index\| .{ .load_symbol = .{ .sym = sym_index } },
14827	15123	.load_direct => \|sym_index\| .{ .load_direct = sym_index },
14828	15124	.load_got => \|sym_index\| .{ .lea_got = sym_index },
14829	15125	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
...	...	@@ -15167,11 +15463,11 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
15167	15463	};
15168	15464	}
15169	15465
15170		fn memPtrSize(self: *Self, ty: Type) Memory.PtrSize {
	15466	fn memSize(self: *Self, ty: Type) Memory.Size {
15171	15467	const mod = self.bin_file.options.module.?;
15172	15468	return switch (ty.zigTypeTag(mod)) {
15173		.Float => Memory.PtrSize.fromBitSize(ty.floatBits(self.target.*)),
15174		else => Memory.PtrSize.fromSize(@intCast(ty.abiSize(mod))),
	15469	.Float => Memory.Size.fromBitSize(ty.floatBits(self.target.*)),
	15470	else => Memory.Size.fromSize(@intCast(ty.abiSize(mod))),
15175	15471	};
15176	15472	}
15177	15473
...	...	@@ -15183,11 +15479,10 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
15183	15479	.signedness = .unsigned,
15184	15480	.bits = @intCast(ty.bitSize(mod)),
15185	15481	};
15186		const max_reg_bit_width = Register.rax.bitSize();
	15482	const shift = math.cast(u6, 64 - int_info.bits % 64) orelse return;
15187	15483	try self.spillEflagsIfOccupied();
15188	15484	switch (int_info.signedness) {
15189	15485	.signed => {
15190		const shift: u6 = @intCast(max_reg_bit_width - int_info.bits);
15191	15486	try self.genShiftBinOpMir(
15192	15487	.{ ._l, .sa },
15193	15488	Type.isize,
...	...	@@ -15202,7 +15497,6 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
15202	15497	);
15203	15498	},
15204	15499	.unsigned => {
15205		const shift: u6 = @intCast(max_reg_bit_width - int_info.bits);
15206	15500	const mask = ~@as(u64, 0) >> shift;
15207	15501	if (int_info.bits <= 32) {
15208	15502	try self.genBinOpMir(

src/arch/x86_64/Disassembler.zig+1-1

...	...	@@ -11,7 +11,7 @@ const Encoding = @import("Encoding.zig");
11	11	const Immediate = bits.Immediate;
12	12	const Instruction = encoder.Instruction;
13	13	const LegacyPrefixes = encoder.LegacyPrefixes;
14		const Memory = bits.Memory;
	14	const Memory = Instruction.Memory;
15	15	const Register = bits.Register;
16	16	const Rex = encoder.Rex;
17	17

src/arch/x86_64/Emit.zig+15-14

...	...	@@ -1,7 +1,6 @@
1	1	//! This file contains the functionality for emitting x86_64 MIR as machine code
2	2
3	3	lower: Lower,
4		bin_file: *link.File,
5	4	debug_output: DebugInfoOutput,
6	5	code: *std.ArrayList(u8),
7	6
...	...	@@ -41,7 +40,7 @@ pub fn emitMir(emit: *Emit) Error!void {
41	40	.offset = end_offset - 4,
42	41	.length = @intCast(end_offset - start_offset),
43	42	}),
44		.linker_extern_fn => \|symbol\| if (emit.bin_file.cast(link.File.Elf)) \|elf_file\| {
	43	.linker_extern_fn => \|symbol\| if (emit.lower.bin_file.cast(link.File.Elf)) \|elf_file\| {
45	44	// Add relocation to the decl.
46	45	const atom_ptr = elf_file.symbol(symbol.atom_index).atom(elf_file).?;
47	46	try atom_ptr.addReloc(elf_file, .{
...	...	@@ -49,9 +48,10 @@ pub fn emitMir(emit: *Emit) Error!void {
49	48	.r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) \| std.elf.R_X86_64_PLT32,
50	49	.r_addend = -4,
51	50	});
52		} else if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	51	} else if (emit.lower.bin_file.cast(link.File.MachO)) \|macho_file\| {
53	52	// Add relocation to the decl.
54		const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = symbol.atom_index }).?;
	53	const atom_index =
	54	macho_file.getAtomIndexForSymbol(.{ .sym_index = symbol.atom_index }).?;
55	55	const target = macho_file.getGlobalByIndex(symbol.sym_index);
56	56	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
57	57	.type = .branch,
...	...	@@ -61,7 +61,7 @@ pub fn emitMir(emit: *Emit) Error!void {
61	61	.pcrel = true,
62	62	.length = 2,
63	63	});
64		} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
	64	} else if (emit.lower.bin_file.cast(link.File.Coff)) \|coff_file\| {
65	65	// Add relocation to the decl.
66	66	const atom_index = coff_file.getAtomIndexForSymbol(
67	67	.{ .sym_index = symbol.atom_index, .file = null },
...	...	@@ -76,12 +76,12 @@ pub fn emitMir(emit: *Emit) Error!void {
76	76	.length = 2,
77	77	});
78	78	} else return emit.fail("TODO implement extern reloc for {s}", .{
79		@tagName(emit.bin_file.tag),
	79	@tagName(emit.lower.bin_file.tag),
80	80	}),
81		.linker_reloc => \|data\| if (emit.bin_file.cast(link.File.Elf)) \|elf_file\| {
	81	.linker_reloc => \|data\| if (emit.lower.bin_file.cast(link.File.Elf)) \|elf_file\| {
82	82	const atom = elf_file.symbol(data.atom_index).atom(elf_file).?;
83	83	const sym = elf_file.symbol(elf_file.zigModulePtr().symbol(data.sym_index));
84		if (emit.bin_file.options.pic) {
	84	if (emit.lower.bin_file.options.pic) {
85	85	const r_type: u32 = if (sym.flags.has_zig_got)
86	86	link.File.Elf.R_X86_64_ZIG_GOTPCREL
87	87	else if (sym.flags.needs_got)
...	...	@@ -111,10 +111,11 @@ pub fn emitMir(emit: *Emit) Error!void {
111	111	.linker_direct,
112	112	.linker_import,
113	113	.linker_tlv,
114		=> \|symbol\| if (emit.bin_file.cast(link.File.Elf)) \|_\| {
	114	=> \|symbol\| if (emit.lower.bin_file.cast(link.File.Elf)) \|_\| {
115	115	unreachable;
116		} else if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
117		const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = symbol.atom_index }).?;
	116	} else if (emit.lower.bin_file.cast(link.File.MachO)) \|macho_file\| {
	117	const atom_index =
	118	macho_file.getAtomIndexForSymbol(.{ .sym_index = symbol.atom_index }).?;
118	119	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
119	120	.type = switch (lowered_relocs[0].target) {
120	121	.linker_got => .got,
...	...	@@ -128,7 +129,7 @@ pub fn emitMir(emit: *Emit) Error!void {
128	129	.pcrel = true,
129	130	.length = 2,
130	131	});
131		} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
	132	} else if (emit.lower.bin_file.cast(link.File.Coff)) \|coff_file\| {
132	133	const atom_index = coff_file.getAtomIndexForSymbol(.{
133	134	.sym_index = symbol.atom_index,
134	135	.file = null,
...	...	@@ -152,7 +153,7 @@ pub fn emitMir(emit: *Emit) Error!void {
152	153	.pcrel = true,
153	154	.length = 2,
154	155	});
155		} else if (emit.bin_file.cast(link.File.Plan9)) \|p9_file\| {
	156	} else if (emit.lower.bin_file.cast(link.File.Plan9)) \|p9_file\| {
156	157	const atom_index = symbol.atom_index;
157	158	try p9_file.addReloc(atom_index, .{ // TODO we may need to add a .type field to the relocs if they are .linker_got instead of just .linker_direct
158	159	.target = symbol.sym_index, // we set sym_index to just be the atom index
...	...	@@ -161,7 +162,7 @@ pub fn emitMir(emit: *Emit) Error!void {
161	162	.type = .pcrel,
162	163	});
163	164	} else return emit.fail("TODO implement linker reloc for {s}", .{
164		@tagName(emit.bin_file.tag),
	165	@tagName(emit.lower.bin_file.tag),
165	166	}),
166	167	};
167	168	}

src/arch/x86_64/Encoding.zig+4-1

...	...	@@ -803,7 +803,10 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
803	803	@memcpy(inst.ops[0..ops.len], ops);
804	804
805	805	var cwriter = std.io.countingWriter(std.io.null_writer);
806		inst.encode(cwriter.writer(), .{ .allow_frame_loc = true }) catch unreachable; // Not allowed to fail here unless OOM.
	806	inst.encode(cwriter.writer(), .{
	807	.allow_frame_locs = true,
	808	.allow_symbols = true,
	809	}) catch unreachable; // Not allowed to fail here unless OOM.
807	810	return @as(usize, @intCast(cwriter.bytes_written));
808	811	}
809	812

src/arch/x86_64/Lower.zig+90-181

...	...	@@ -50,12 +50,12 @@ pub const Reloc = struct {
50	50
51	51	const Target = union(enum) {
52	52	inst: Mir.Inst.Index,
53		linker_reloc: Mir.Reloc,
54		linker_extern_fn: Mir.Reloc,
55		linker_got: Mir.Reloc,
56		linker_direct: Mir.Reloc,
57		linker_import: Mir.Reloc,
58		linker_tlv: Mir.Reloc,
	53	linker_reloc: bits.Symbol,
	54	linker_extern_fn: bits.Symbol,
	55	linker_got: bits.Symbol,
	56	linker_direct: bits.Symbol,
	57	linker_import: bits.Symbol,
	58	linker_tlv: bits.Symbol,
59	59	};
60	60	};
61	61
...	...	@@ -99,17 +99,15 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
99	99	.{ .reg = inst.data.rr.r2 },
100	100	});
101	101	},
102		.pseudo_cmov_nz_or_p_rm_sib,
103		.pseudo_cmov_nz_or_p_rm_rip,
104		=> {
	102	.pseudo_cmov_nz_or_p_rm => {
105	103	assert(inst.data.rx.fixes == ._);
106	104	try lower.emit(.none, .cmovnz, &.{
107	105	.{ .reg = inst.data.rx.r1 },
108		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	106	.{ .mem = lower.mem(inst.data.rx.payload) },
109	107	});
110	108	try lower.emit(.none, .cmovp, &.{
111	109	.{ .reg = inst.data.rx.r1 },
112		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	110	.{ .mem = lower.mem(inst.data.rx.payload) },
113	111	});
114	112	},
115	113	.pseudo_set_z_and_np_r => {
...	...	@@ -125,18 +123,16 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
125	123	.{ .reg = inst.data.rr.r2 },
126	124	});
127	125	},
128		.pseudo_set_z_and_np_m_sib,
129		.pseudo_set_z_and_np_m_rip,
130		=> {
	126	.pseudo_set_z_and_np_m => {
131	127	assert(inst.data.rx.fixes == ._);
132	128	try lower.emit(.none, .setz, &.{
133		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	129	.{ .mem = lower.mem(inst.data.rx.payload) },
134	130	});
135	131	try lower.emit(.none, .setnp, &.{
136	132	.{ .reg = inst.data.rx.r1 },
137	133	});
138	134	try lower.emit(.none, .@"and", &.{
139		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	135	.{ .mem = lower.mem(inst.data.rx.payload) },
140	136	.{ .reg = inst.data.rx.r1 },
141	137	});
142	138	},
...	...	@@ -153,18 +149,16 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
153	149	.{ .reg = inst.data.rr.r2 },
154	150	});
155	151	},
156		.pseudo_set_nz_or_p_m_sib,
157		.pseudo_set_nz_or_p_m_rip,
158		=> {
	152	.pseudo_set_nz_or_p_m => {
159	153	assert(inst.data.rx.fixes == ._);
160	154	try lower.emit(.none, .setnz, &.{
161		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	155	.{ .mem = lower.mem(inst.data.rx.payload) },
162	156	});
163	157	try lower.emit(.none, .setp, &.{
164	158	.{ .reg = inst.data.rx.r1 },
165	159	});
166	160	try lower.emit(.none, .@"or", &.{
167		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	161	.{ .mem = lower.mem(inst.data.rx.payload) },
168	162	.{ .reg = inst.data.rx.r1 },
169	163	});
170	164	},
...	...	@@ -289,28 +283,20 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
289	283	.rri_s,
290	284	.ri_s,
291	285	.i_s,
292		.mi_sib_s,
293		.mi_rip_s,
294		.rmi_sib_s,
295		.rmi_rip_s,
	286	.mi_s,
	287	.rmi_s,
296	288	=> Immediate.s(@bitCast(i)),
297	289
298	290	.rrri,
299	291	.rri_u,
300	292	.ri_u,
301	293	.i_u,
302		.mi_sib_u,
303		.mi_rip_u,
304		.rmi_sib,
305		.rmi_rip,
306		.rmi_sib_u,
307		.rmi_rip_u,
308		.mri_sib,
309		.mri_rip,
310		.rrm_sib,
311		.rrm_rip,
312		.rrmi_sib,
313		.rrmi_rip,
	294	.mi_u,
	295	.rmi,
	296	.rmi_u,
	297	.mri,
	298	.rrm,
	299	.rrmi,
314	300	=> Immediate.u(i),
315	301
316	302	.ri64 => Immediate.u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
...	...	@@ -319,50 +305,8 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
319	305	};
320	306	}
321	307
322		fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
323		return lower.mir.resolveFrameLoc(switch (ops) {
324		.rm_sib,
325		.rmi_sib,
326		.rmi_sib_s,
327		.rmi_sib_u,
328		.m_sib,
329		.mi_sib_u,
330		.mi_sib_s,
331		.mr_sib,
332		.mrr_sib,
333		.mri_sib,
334		.rrm_sib,
335		.rrmi_sib,
336
337		.pseudo_cmov_nz_or_p_rm_sib,
338		.pseudo_set_z_and_np_m_sib,
339		.pseudo_set_nz_or_p_m_sib,
340		=> lower.mir.extraData(Mir.MemorySib, payload).data.decode(),
341
342		.rm_rip,
343		.rmi_rip,
344		.rmi_rip_s,
345		.rmi_rip_u,
346		.m_rip,
347		.mi_rip_u,
348		.mi_rip_s,
349		.mr_rip,
350		.mrr_rip,
351		.mri_rip,
352		.rrm_rip,
353		.rrmi_rip,
354
355		.pseudo_cmov_nz_or_p_rm_rip,
356		.pseudo_set_z_and_np_m_rip,
357		.pseudo_set_nz_or_p_m_rip,
358		=> lower.mir.extraData(Mir.MemoryRip, payload).data.decode(),
359
360		.rax_moffs,
361		.moffs_rax,
362		=> lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),
363
364		else => unreachable,
365		});
	308	fn mem(lower: Lower, payload: u32) Memory {
	309	return lower.mir.resolveFrameLoc(lower.mir.extraData(Mir.Memory, payload).data).decode();
366	310	}
367	311
368	312	fn reloc(lower: *Lower, target: Reloc.Target) Immediate {
...	...	@@ -375,7 +319,42 @@ fn reloc(lower: *Lower, target: Reloc.Target) Immediate {
375	319	}
376	320
377	321	fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
378		lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
	322	var emit_prefix = prefix;
	323	var emit_mnemonic = mnemonic;
	324	var emit_ops_storage: [4]Operand = undefined;
	325	const emit_ops = emit_ops_storage[0..ops.len];
	326	for (emit_ops, ops) \|*emit_op, op\| {
	327	emit_op.* = switch (op) {
	328	else => op,
	329	.mem => \|mem_op\| switch (mem_op.base()) {
	330	else => op,
	331	.reloc => \|sym\| op: {
	332	assert(prefix == .none);
	333	assert(mem_op.sib.disp == 0);
	334	assert(mem_op.sib.scale_index.scale == 0);
	335	_ = lower.reloc(.{ .linker_reloc = sym });
	336	break :op if (lower.bin_file.options.pic) switch (mnemonic) {
	337	.mov, .lea => .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) },
	338	else => unreachable,
	339	} else switch (mnemonic) {
	340	.call => .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{
	341	.base = .{ .reg = .ds },
	342	}) },
	343	.lea => {
	344	emit_mnemonic = .mov;
	345	break :op .{ .imm = Immediate.s(0) };
	346	},
	347	.mov => .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{
	348	.base = .{ .reg = .ds },
	349	}) },
	350	else => unreachable,
	351	};
	352	},
	353	},
	354	};
	355	}
	356	lower.result_insts[lower.result_insts_len] =
	357	try Instruction.new(emit_prefix, emit_mnemonic, emit_ops);
379	358	lower.result_insts_len += 1;
380	359	}
381	360
...	...	@@ -391,74 +370,13 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
391	370	.rrri => inst.data.rrri.fixes,
392	371	.rri_s, .rri_u => inst.data.rri.fixes,
393	372	.ri_s, .ri_u => inst.data.ri.fixes,
394		.ri64,
395		.rm_sib,
396		.rm_rip,
397		.rmi_sib_s,
398		.rmi_sib_u,
399		.rmi_rip_s,
400		.rmi_rip_u,
401		.mr_sib,
402		.mr_rip,
403		=> inst.data.rx.fixes,
404		.mrr_sib, .mrr_rip, .rrm_sib, .rrm_rip => inst.data.rrx.fixes,
405		.rmi_sib, .rmi_rip, .mri_sib, .mri_rip => inst.data.rix.fixes,
406		.rrmi_sib, .rrmi_rip => inst.data.rrix.fixes,
407		.mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => inst.data.x.fixes,
408		.m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes,
409		.extern_fn_reloc,
410		.got_reloc,
411		.direct_reloc,
412		.import_reloc,
413		.tlv_reloc,
414		=> ._,
415		.linker_reloc => {
416		if (lower.bin_file.options.pic) {
417		assert(inst.data.rx.fixes == ._);
418		const reg = inst.data.rx.r1;
419		const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
420		_ = lower.reloc(.{ .linker_reloc = extra });
421		const mnemonic: Mnemonic = switch (inst.tag) {
422		.mov => .mov,
423		.lea => .lea,
424		else => unreachable,
425		};
426		try lower.emit(.none, mnemonic, &.{
427		.{ .reg = reg },
428		.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
429		});
430		} else {
431		switch (inst.tag) {
432		.call => {
433		_ = lower.reloc(.{ .linker_reloc = inst.data.reloc });
434		try lower.emit(.none, .call, &.{
435		.{ .mem = Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = 0 }) },
436		});
437		},
438		.lea => {
439		assert(inst.data.rx.fixes == ._);
440		const reg = inst.data.rx.r1;
441		const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
442		try lower.emit(.none, .mov, &.{
443		.{ .reg = reg },
444		.{ .imm = lower.reloc(.{ .linker_reloc = extra }) },
445		});
446		},
447		.mov => {
448		assert(inst.data.rx.fixes == ._);
449		const reg = inst.data.rx.r1;
450		const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
451		_ = lower.reloc(.{ .linker_reloc = extra });
452		try lower.emit(.none, .mov, &.{
453		.{ .reg = reg },
454		.{ .mem = Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = 0 }) },
455		});
456		},
457		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
458		}
459		}
460		return;
461		},
	373	.ri64, .rm, .rmi_s, .mr => inst.data.rx.fixes,
	374	.mrr, .rrm => inst.data.rrx.fixes,
	375	.rmi, .mri => inst.data.rix.fixes,
	376	.rrmi => inst.data.rrix.fixes,
	377	.mi_u, .mi_s => inst.data.x.fixes,
	378	.m => inst.data.x.fixes,
	379	.extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._,
462	380	else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),
463	381	};
464	382	try lower.emit(switch (fixes) {
...	...	@@ -527,73 +445,64 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
527	445	.{ .reg = inst.data.rri.r2 },
528	446	.{ .imm = lower.imm(inst.ops, inst.data.rri.i) },
529	447	},
530		.m_sib, .m_rip => &.{
531		.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
	448	.m => &.{
	449	.{ .mem = lower.mem(inst.data.x.payload) },
532	450	},
533		.mi_sib_s, .mi_sib_u, .mi_rip_s, .mi_rip_u => &.{
534		.{ .mem = lower.mem(inst.ops, inst.data.x.payload + 1) },
	451	.mi_s, .mi_u => &.{
	452	.{ .mem = lower.mem(inst.data.x.payload + 1) },
535	453	.{ .imm = lower.imm(
536	454	inst.ops,
537	455	lower.mir.extraData(Mir.Imm32, inst.data.x.payload).data.imm,
538	456	) },
539	457	},
540		.rm_sib, .rm_rip => &.{
	458	.rm => &.{
541	459	.{ .reg = inst.data.rx.r1 },
542		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	460	.{ .mem = lower.mem(inst.data.rx.payload) },
543	461	},
544		.rmi_sib, .rmi_rip => &.{
	462	.rmi => &.{
545	463	.{ .reg = inst.data.rix.r1 },
546		.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
	464	.{ .mem = lower.mem(inst.data.rix.payload) },
547	465	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
548	466	},
549		.rmi_sib_s, .rmi_sib_u, .rmi_rip_s, .rmi_rip_u => &.{
	467	.rmi_s, .rmi_u => &.{
550	468	.{ .reg = inst.data.rx.r1 },
551		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload + 1) },
	469	.{ .mem = lower.mem(inst.data.rx.payload + 1) },
552	470	.{ .imm = lower.imm(
553	471	inst.ops,
554	472	lower.mir.extraData(Mir.Imm32, inst.data.rx.payload).data.imm,
555	473	) },
556	474	},
557		.mr_sib, .mr_rip => &.{
558		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	475	.mr => &.{
	476	.{ .mem = lower.mem(inst.data.rx.payload) },
559	477	.{ .reg = inst.data.rx.r1 },
560	478	},
561		.mrr_sib, .mrr_rip => &.{
562		.{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
	479	.mrr => &.{
	480	.{ .mem = lower.mem(inst.data.rrx.payload) },
563	481	.{ .reg = inst.data.rrx.r1 },
564	482	.{ .reg = inst.data.rrx.r2 },
565	483	},
566		.mri_sib, .mri_rip => &.{
567		.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
	484	.mri => &.{
	485	.{ .mem = lower.mem(inst.data.rix.payload) },
568	486	.{ .reg = inst.data.rix.r1 },
569	487	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
570	488	},
571		.rrm_sib, .rrm_rip => &.{
	489	.rrm => &.{
572	490	.{ .reg = inst.data.rrx.r1 },
573	491	.{ .reg = inst.data.rrx.r2 },
574		.{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
	492	.{ .mem = lower.mem(inst.data.rrx.payload) },
575	493	},
576		.rrmi_sib, .rrmi_rip => &.{
	494	.rrmi => &.{
577	495	.{ .reg = inst.data.rrix.r1 },
578	496	.{ .reg = inst.data.rrix.r2 },
579		.{ .mem = lower.mem(inst.ops, inst.data.rrix.payload) },
	497	.{ .mem = lower.mem(inst.data.rrix.payload) },
580	498	.{ .imm = lower.imm(inst.ops, inst.data.rrix.i) },
581	499	},
582		.rax_moffs => &.{
583		.{ .reg = .rax },
584		.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
585		},
586		.moffs_rax => &.{
587		.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
588		.{ .reg = .rax },
589		},
590	500	.extern_fn_reloc => &.{
591	501	.{ .imm = lower.reloc(.{ .linker_extern_fn = inst.data.reloc }) },
592	502	},
593		.linker_reloc => unreachable,
594	503	.got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ops: {
595	504	const reg = inst.data.rx.r1;
596		const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
	505	const extra = lower.mir.extraData(bits.Symbol, inst.data.rx.payload).data;
597	506	_ = lower.reloc(switch (inst.ops) {
598	507	.got_reloc => .{ .linker_got = extra },
599	508	.direct_reloc => .{ .linker_direct = extra },
...	...	@@ -635,7 +544,7 @@ const ErrorMsg = Module.ErrorMsg;
635	544	const Immediate = bits.Immediate;
636	545	const Instruction = encoder.Instruction;
637	546	const Lower = @This();
638		const Memory = bits.Memory;
	547	const Memory = Instruction.Memory;
639	548	const Mir = @import("Mir.zig");
640	549	const Mnemonic = Instruction.Mnemonic;
641	550	const Module = @import("../../Module.zig");

src/arch/x86_64/Mir.zig+136-203

...	...	@@ -17,7 +17,6 @@ const encoder = @import("encoder.zig");
17	17	const Air = @import("../../Air.zig");
18	18	const CodeGen = @import("CodeGen.zig");
19	19	const IntegerBitSet = std.bit_set.IntegerBitSet;
20		const Memory = bits.Memory;
21	20	const Register = bits.Register;
22	21
23	22	instructions: std.MultiArrayList(Inst).Slice,
...	...	@@ -767,84 +766,42 @@ pub const Inst = struct {
767	766	/// Relative displacement operand.
768	767	/// Uses `imm` payload.
769	768	rel,
770		/// Register, memory (SIB) operands.
	769	/// Register, memory operands.
771	770	/// Uses `rx` payload.
772		rm_sib,
773		/// Register, memory (RIP) operands.
774		/// Uses `rx` payload.
775		rm_rip,
776		/// Register, memory (SIB), immediate (word) operands.
777		/// Uses `rix` payload with extra data of type `MemorySib`.
778		rmi_sib,
779		/// Register, memory (RIP), immediate (word) operands.
780		/// Uses `rix` payload with extra data of type `MemoryRip`.
781		rmi_rip,
782		/// Register, memory (SIB), immediate (signed) operands.
783		/// Uses `rx` payload with extra data of type `Imm32` followed by `MemorySib`.
784		rmi_sib_s,
785		/// Register, memory (SIB), immediate (unsigned) operands.
786		/// Uses `rx` payload with extra data of type `Imm32` followed by `MemorySib`.
787		rmi_sib_u,
788		/// Register, memory (RIP), immediate (signed) operands.
789		/// Uses `rx` payload with extra data of type `Imm32` followed by `MemoryRip`.
790		rmi_rip_s,
791		/// Register, memory (RIP), immediate (unsigned) operands.
792		/// Uses `rx` payload with extra data of type `Imm32` followed by `MemoryRip`.
793		rmi_rip_u,
794		/// Register, register, memory (RIP).
795		/// Uses `rrix` payload with extra data of type `MemoryRip`.
796		rrm_rip,
797		/// Register, register, memory (SIB).
798		/// Uses `rrix` payload with extra data of type `MemorySib`.
799		rrm_sib,
800		/// Register, register, memory (RIP), immediate (byte) operands.
801		/// Uses `rrix` payload with extra data of type `MemoryRip`.
802		rrmi_rip,
803		/// Register, register, memory (SIB), immediate (byte) operands.
804		/// Uses `rrix` payload with extra data of type `MemorySib`.
805		rrmi_sib,
806		/// Single memory (SIB) operand.
807		/// Uses `x` with extra data of type `MemorySib`.
808		m_sib,
809		/// Single memory (RIP) operand.
810		/// Uses `x` with extra data of type `MemoryRip`.
811		m_rip,
812		/// Memory (SIB), immediate (sign-extend) operands.
813		/// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
814		mi_sib_s,
815		/// Memory (SIB), immediate (unsigned) operands.
816		/// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
817		mi_sib_u,
818		/// Memory (RIP), immediate (sign-extend) operands.
819		/// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
820		mi_rip_s,
821		/// Memory (RIP), immediate (unsigned) operands.
822		/// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
823		mi_rip_u,
824		/// Memory (SIB), register operands.
825		/// Uses `rx` payload with extra data of type `MemorySib`.
826		mr_sib,
827		/// Memory (RIP), register operands.
828		/// Uses `rx` payload with extra data of type `MemoryRip`.
829		mr_rip,
830		/// Memory (SIB), register, register operands.
831		/// Uses `rrx` payload with extra data of type `MemorySib`.
832		mrr_sib,
833		/// Memory (RIP), register, register operands.
834		/// Uses `rrx` payload with extra data of type `MemoryRip`.
835		mrr_rip,
836		/// Memory (SIB), register, immediate (word) operands.
837		/// Uses `rix` payload with extra data of type `MemorySib`.
838		mri_sib,
839		/// Memory (RIP), register, immediate (word) operands.
840		/// Uses `rix` payload with extra data of type `MemoryRip`.
841		mri_rip,
842		/// Rax, Memory moffs.
843		/// Uses `x` with extra data of type `MemoryMoffs`.
844		rax_moffs,
845		/// Memory moffs, rax.
846		/// Uses `x` with extra data of type `MemoryMoffs`.
847		moffs_rax,
	771	rm,
	772	/// Register, memory, immediate (word) operands.
	773	/// Uses `rix` payload with extra data of type `Memory`.
	774	rmi,
	775	/// Register, memory, immediate (signed) operands.
	776	/// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`.
	777	rmi_s,
	778	/// Register, memory, immediate (unsigned) operands.
	779	/// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`.
	780	rmi_u,
	781	/// Register, register, memory.
	782	/// Uses `rrix` payload with extra data of type `Memory`.
	783	rrm,
	784	/// Register, register, memory, immediate (byte) operands.
	785	/// Uses `rrix` payload with extra data of type `Memory`.
	786	rrmi,
	787	/// Single memory operand.
	788	/// Uses `x` with extra data of type `Memory`.
	789	m,
	790	/// Memory, immediate (sign-extend) operands.
	791	/// Uses `x` payload with extra data of type `Imm32` followed by `Memory`.
	792	mi_s,
	793	/// Memory, immediate (unsigned) operands.
	794	/// Uses `x` payload with extra data of type `Imm32` followed by `Memory`.
	795	mi_u,
	796	/// Memory, register operands.
	797	/// Uses `rx` payload with extra data of type `Memory`.
	798	mr,
	799	/// Memory, register, register operands.
	800	/// Uses `rrx` payload with extra data of type `Memory`.
	801	mrr,
	802	/// Memory, register, immediate (word) operands.
	803	/// Uses `rix` payload with extra data of type `Memory`.
	804	mri,
848	805	/// References another Mir instruction directly.
849	806	/// Uses `inst` payload.
850	807	inst,
...	...	@@ -852,20 +809,17 @@ pub const Inst = struct {
852	809	/// Uses `reloc` payload.
853	810	extern_fn_reloc,
854	811	/// Linker relocation - GOT indirection.
855		/// Uses `rx` payload with extra data of type `Reloc`.
	812	/// Uses `rx` payload with extra data of type `bits.Symbol`.
856	813	got_reloc,
857	814	/// Linker relocation - direct reference.
858		/// Uses `rx` payload with extra data of type `Reloc`.
	815	/// Uses `rx` payload with extra data of type `bits.Symbol`.
859	816	direct_reloc,
860	817	/// Linker relocation - imports table indirection (binding).
861		/// Uses `rx` payload with extra data of type `Reloc`.
	818	/// Uses `rx` payload with extra data of type `bits.Symbol`.
862	819	import_reloc,
863	820	/// Linker relocation - threadlocal variable via GOT indirection.
864		/// Uses `rx` payload with extra data of type `Reloc`.
	821	/// Uses `rx` payload with extra data of type `bits.Symbol`.
865	822	tlv_reloc,
866		/// Linker relocation.
867		/// Uses `rx` payload with extra data of type `Reloc`.
868		linker_reloc,
869	823
870	824	// Pseudo instructions:
871	825
...	...	@@ -878,10 +832,7 @@ pub const Inst = struct {
878	832	pseudo_cmov_nz_or_p_rr,
879	833	/// Conditional move if zero flag not set or parity flag set
880	834	/// Uses `rx` payload.
881		pseudo_cmov_nz_or_p_rm_sib,
882		/// Conditional move if zero flag not set or parity flag set
883		/// Uses `rx` payload.
884		pseudo_cmov_nz_or_p_rm_rip,
	835	pseudo_cmov_nz_or_p_rm,
885	836	/// Set byte if zero flag set and parity flag not set
886	837	/// Requires a scratch register!
887	838	/// Uses `rr` payload.
...	...	@@ -889,11 +840,7 @@ pub const Inst = struct {
889	840	/// Set byte if zero flag set and parity flag not set
890	841	/// Requires a scratch register!
891	842	/// Uses `rx` payload.
892		pseudo_set_z_and_np_m_sib,
893		/// Set byte if zero flag set and parity flag not set
894		/// Requires a scratch register!
895		/// Uses `rx` payload.
896		pseudo_set_z_and_np_m_rip,
	843	pseudo_set_z_and_np_m,
897	844	/// Set byte if zero flag not set or parity flag set
898	845	/// Requires a scratch register!
899	846	/// Uses `rr` payload.
...	...	@@ -901,11 +848,7 @@ pub const Inst = struct {
901	848	/// Set byte if zero flag not set or parity flag set
902	849	/// Requires a scratch register!
903	850	/// Uses `rx` payload.
904		pseudo_set_nz_or_p_m_sib,
905		/// Set byte if zero flag not set or parity flag set
906		/// Requires a scratch register!
907		/// Uses `rx` payload.
908		pseudo_set_nz_or_p_m_rip,
	851	pseudo_set_nz_or_p_m,
909	852	/// Jump if zero flag set and parity flag not set
910	853	/// Uses `inst` payload.
911	854	pseudo_j_z_and_np_inst,
...	...	@@ -1036,7 +979,7 @@ pub const Inst = struct {
1036	979	/// Relocation for the linker where:
1037	980	/// * `atom_index` is the index of the source
1038	981	/// * `sym_index` is the index of the target
1039		reloc: Reloc,
	982	reloc: bits.Symbol,
1040	983	/// Debug line and column position
1041	984	line_column: struct {
1042	985	line: u32,
...	...	@@ -1055,14 +998,6 @@ pub const Inst = struct {
1055	998	}
1056	999	};
1057	1000
1058		/// A linker symbol not yet allocated in VM.
1059		pub const Reloc = struct {
1060		/// Index of the containing atom.
1061		atom_index: u32,
1062		/// Index into the linker's symbol table.
1063		sym_index: u32,
1064		};
1065
1066	1001	/// Used in conjunction with payload to transfer a list of used registers in a compact manner.
1067	1002	pub const RegisterList = struct {
1068	1003	bitset: BitSet = BitSet.initEmpty(),
...	...	@@ -1123,100 +1058,94 @@ pub const Imm64 = struct {
1123	1058	}
1124	1059	};
1125	1060
1126		// TODO this can be further compacted using packed struct
1127		pub const MemorySib = struct {
1128		/// Size of the pointer.
1129		ptr_size: u32,
1130		/// Base register tag of type Memory.Base.Tag
1131		base_tag: u32,
1132		/// Base register of type Register or FrameIndex
	1061	pub const Memory = struct {
	1062	info: Info,
1133	1063	base: u32,
1134		/// Scale starting at bit 0 and index register starting at bit 4.
1135		scale_index: u32,
1136		/// Displacement value.
1137		disp: i32,
	1064	off: u32,
	1065	extra: u32,
	1066
	1067	pub const Info = packed struct(u32) {
	1068	base: @typeInfo(bits.Memory.Base).Union.tag_type.?,
	1069	mod: @typeInfo(bits.Memory.Mod).Union.tag_type.?,
	1070	size: bits.Memory.Size,
	1071	index: Register,
	1072	scale: bits.Memory.Scale,
	1073	_: u16 = undefined,
	1074	};
1138	1075
1139		pub fn encode(mem: Memory) MemorySib {
1140		const sib = mem.sib;
1141		assert(sib.scale_index.scale == 0 or std.math.isPowerOfTwo(sib.scale_index.scale));
	1076	pub fn encode(mem: bits.Memory) Memory {
	1077	assert(mem.base != .reloc or mem.mod != .off);
1142	1078	return .{
1143		.ptr_size = @intFromEnum(sib.ptr_size),
1144		.base_tag = @intFromEnum(sib.base),
1145		.base = switch (sib.base) {
	1079	.info = .{
	1080	.base = mem.base,
	1081	.mod = mem.mod,
	1082	.size = switch (mem.mod) {
	1083	.rm => \|rm\| rm.size,
	1084	.off => undefined,
	1085	},
	1086	.index = switch (mem.mod) {
	1087	.rm => \|rm\| rm.index,
	1088	.off => undefined,
	1089	},
	1090	.scale = switch (mem.mod) {
	1091	.rm => \|rm\| rm.scale,
	1092	.off => undefined,
	1093	},
	1094	},
	1095	.base = switch (mem.base) {
1146	1096	.none => undefined,
1147		.reg => \|r\| @intFromEnum(r),
1148		.frame => \|fi\| @intFromEnum(fi),
	1097	.reg => \|reg\| @intFromEnum(reg),
	1098	.frame => \|frame_index\| @intFromEnum(frame_index),
	1099	.reloc => \|symbol\| symbol.sym_index,
	1100	},
	1101	.off = switch (mem.mod) {
	1102	.rm => \|rm\| @bitCast(rm.disp),
	1103	.off => \|off\| @truncate(off),
1149	1104	},
1150		.scale_index = @as(u32, sib.scale_index.scale) << 0 \|
1151		@as(u32, if (sib.scale_index.scale > 0)
1152		@intFromEnum(sib.scale_index.index)
	1105	.extra = if (mem.base == .reloc)
	1106	mem.base.reloc.atom_index
	1107	else if (mem.mod == .off)
	1108	@intCast(mem.mod.off >> 32)
1153	1109	else
1154		undefined) << 4,
1155		.disp = sib.disp,
	1110	undefined,
1156	1111	};
1157	1112	}
1158	1113
1159		pub fn decode(msib: MemorySib) Memory {
1160		const scale: u4 = @truncate(msib.scale_index);
1161		assert(scale == 0 or std.math.isPowerOfTwo(scale));
1162		return .{ .sib = .{
1163		.ptr_size = @enumFromInt(msib.ptr_size),
1164		.base = switch (@as(Memory.Base.Tag, @enumFromInt(msib.base_tag))) {
1165		.none => .none,
1166		.reg => .{ .reg = @enumFromInt(msib.base) },
1167		.frame => .{ .frame = @enumFromInt(msib.base) },
	1114	pub fn decode(mem: Memory) encoder.Instruction.Memory {
	1115	switch (mem.info.mod) {
	1116	.rm => {
	1117	if (mem.info.base == .reg and @as(Register, @enumFromInt(mem.base)) == .rip) {
	1118	assert(mem.info.index == .none and mem.info.scale == .@"1");
	1119	return encoder.Instruction.Memory.rip(mem.info.size, @bitCast(mem.off));
	1120	}
	1121	return encoder.Instruction.Memory.sib(mem.info.size, .{
	1122	.disp = @bitCast(mem.off),
	1123	.base = switch (mem.info.base) {
	1124	.none => .none,
	1125	.reg => .{ .reg = @enumFromInt(mem.base) },
	1126	.frame => .{ .frame = @enumFromInt(mem.base) },
	1127	.reloc => .{ .reloc = .{ .atom_index = mem.extra, .sym_index = mem.base } },
	1128	},
	1129	.scale_index = switch (mem.info.index) {
	1130	.none => null,
	1131	else => \|index\| .{ .scale = switch (mem.info.scale) {
	1132	inline else => \|scale\| comptime std.fmt.parseInt(
	1133	u4,
	1134	@tagName(scale),
	1135	10,
	1136	) catch unreachable,
	1137	}, .index = index },
	1138	},
	1139	});
1168	1140	},
1169		.scale_index = .{
1170		.scale = scale,
1171		.index = if (scale > 0) @enumFromInt(msib.scale_index >> 4) else undefined,
	1141	.off => {
	1142	assert(mem.info.base == .reg);
	1143	return encoder.Instruction.Memory.moffs(
	1144	@enumFromInt(mem.base),
	1145	@as(u64, mem.extra) << 32 \| mem.off,
	1146	);
1172	1147	},
1173		.disp = msib.disp,
1174		} };
1175		}
1176		};
1177
1178		pub const MemoryRip = struct {
1179		/// Size of the pointer.
1180		ptr_size: u32,
1181		/// Displacement value.
1182		disp: i32,
1183
1184		pub fn encode(mem: Memory) MemoryRip {
1185		return .{
1186		.ptr_size = @intFromEnum(mem.rip.ptr_size),
1187		.disp = mem.rip.disp,
1188		};
1189		}
1190
1191		pub fn decode(mrip: MemoryRip) Memory {
1192		return .{ .rip = .{
1193		.ptr_size = @enumFromInt(mrip.ptr_size),
1194		.disp = mrip.disp,
1195		} };
1196		}
1197		};
1198
1199		pub const MemoryMoffs = struct {
1200		/// Segment register.
1201		seg: u32,
1202		/// Absolute offset wrt to the segment register split between MSB and LSB parts much like
1203		/// `Imm64` payload.
1204		msb: u32,
1205		lsb: u32,
1206
1207		pub fn encode(seg: Register, offset: u64) MemoryMoffs {
1208		return .{
1209		.seg = @intFromEnum(seg),
1210		.msb = @truncate(offset >> 32),
1211		.lsb = @truncate(offset >> 0),
1212		};
1213		}
1214
1215		pub fn decode(moffs: MemoryMoffs) Memory {
1216		return .{ .moffs = .{
1217		.seg = @enumFromInt(moffs.seg),
1218		.offset = @as(u64, moffs.msb) << 32 \| @as(u64, moffs.lsb) << 0,
1219		} };
	1148	}
1220	1149	}
1221	1150	};
1222	1151
...	...	@@ -1234,8 +1163,8 @@ pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end:
1234	1163	inline for (fields) \|field\| {
1235	1164	@field(result, field.name) = switch (field.type) {
1236	1165	u32 => mir.extra[i],
1237		i32 => @bitCast(mir.extra[i]),
1238		else => @compileError("bad field type"),
	1166	i32, Memory.Info => @bitCast(mir.extra[i]),
	1167	else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)),
1239	1168	};
1240	1169	i += 1;
1241	1170	}
...	...	@@ -1251,15 +1180,19 @@ pub const FrameLoc = struct {
1251	1180	};
1252	1181
1253	1182	pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory {
1254		return switch (mem) {
1255		.sib => \|sib\| switch (sib.base) {
1256		.none, .reg => mem,
1257		.frame => \|index\| if (mir.frame_locs.len > 0) Memory.sib(sib.ptr_size, .{
1258		.base = .{ .reg = mir.frame_locs.items(.base)[@intFromEnum(index)] },
1259		.disp = mir.frame_locs.items(.disp)[@intFromEnum(index)] + sib.disp,
1260		.scale_index = mem.scaleIndex(),
1261		}) else mem,
1262		},
1263		.rip, .moffs => mem,
	1183	return switch (mem.info.base) {
	1184	.none, .reg, .reloc => mem,
	1185	.frame => if (mir.frame_locs.len > 0) Memory{
	1186	.info = .{
	1187	.base = .reg,
	1188	.mod = mem.info.mod,
	1189	.size = mem.info.size,
	1190	.index = mem.info.index,
	1191	.scale = mem.info.scale,
	1192	},
	1193	.base = @intFromEnum(mir.frame_locs.items(.base)[mem.base]),
	1194	.off = @bitCast(mir.frame_locs.items(.disp)[mem.base] + @as(i32, @bitCast(mem.off))),
	1195	.extra = mem.extra,
	1196	} else mem,
1264	1197	};
1265	1198	}

src/arch/x86_64/bits.zig+43-89

...	...	@@ -181,6 +181,8 @@ pub const Register = enum(u7) {
181	181
182	182	es, cs, ss, ds, fs, gs,
183	183
	184	rip, eip, ip,
	185
184	186	none,
185	187	// zig fmt: on
186	188
...	...	@@ -442,34 +444,58 @@ pub const FrameIndex = enum(u32) {
442	444	}
443	445	};
444	446
445		pub const Memory = union(enum) {
446		sib: Sib,
447		rip: Rip,
448		moffs: Moffs,
	447	/// A linker symbol not yet allocated in VM.
	448	pub const Symbol = struct {
	449	/// Index of the containing atom.
	450	atom_index: u32,
	451	/// Index into the linker's symbol table.
	452	sym_index: u32,
	453
	454	pub fn format(
	455	sym: Symbol,
	456	comptime fmt: []const u8,
	457	options: std.fmt.FormatOptions,
	458	writer: anytype,
	459	) @TypeOf(writer).Error!void {
	460	try writer.writeAll("Symbol(");
	461	try std.fmt.formatType(sym.atom_index, fmt, options, writer, 0);
	462	try writer.writeAll(", ");
	463	try std.fmt.formatType(sym.sym_index, fmt, options, writer, 0);
	464	try writer.writeByte(')');
	465	}
	466	};
	467
	468	pub const Memory = struct {
	469	base: Base,
	470	mod: Mod,
449	471
450		pub const Base = union(enum) {
	472	pub const Base = union(enum(u2)) {
451	473	none,
452	474	reg: Register,
453	475	frame: FrameIndex,
	476	reloc: Symbol,
454	477
455	478	pub const Tag = @typeInfo(Base).Union.tag_type.?;
456	479
457	480	pub fn isExtended(self: Base) bool {
458	481	return switch (self) {
459		.none, .frame => false, // neither rsp nor rbp are extended
	482	.none, .frame, .reloc => false, // rsp, rbp, and rip are not extended
460	483	.reg => \|reg\| reg.isExtended(),
461	484	};
462	485	}
463	486	};
464	487
465		pub const ScaleIndex = struct {
466		scale: u4,
467		index: Register,
468
469		const none = ScaleIndex{ .scale = 0, .index = undefined };
	488	pub const Mod = union(enum(u1)) {
	489	rm: struct {
	490	size: Size,
	491	index: Register = .none,
	492	scale: Scale = .@"1",
	493	disp: i32 = 0,
	494	},
	495	off: u64,
470	496	};
471	497
472		pub const PtrSize = enum {
	498	pub const Size = enum(u4) {
473	499	none,
474	500	byte,
475	501	word,
...	...	@@ -480,7 +506,7 @@ pub const Memory = union(enum) {
480	506	yword,
481	507	zword,
482	508
483		pub fn fromSize(size: u32) PtrSize {
	509	pub fn fromSize(size: u32) Size {
484	510	return switch (size) {
485	511	1...1 => .byte,
486	512	2...2 => .word,
...	...	@@ -493,7 +519,7 @@ pub const Memory = union(enum) {
493	519	};
494	520	}
495	521
496		pub fn fromBitSize(bit_size: u64) PtrSize {
	522	pub fn fromBitSize(bit_size: u64) Size {
497	523	return switch (bit_size) {
498	524	8 => .byte,
499	525	16 => .word,
...	...	@@ -507,7 +533,7 @@ pub const Memory = union(enum) {
507	533	};
508	534	}
509	535
510		pub fn bitSize(s: PtrSize) u64 {
	536	pub fn bitSize(s: Size) u64 {
511	537	return switch (s) {
512	538	.none => 0,
513	539	.byte => 8,
...	...	@@ -522,7 +548,7 @@ pub const Memory = union(enum) {
522	548	}
523	549
524	550	pub fn format(
525		s: PtrSize,
	551	s: Size,
526	552	comptime _: []const u8,
527	553	_: std.fmt.FormatOptions,
528	554	writer: anytype,
...	...	@@ -533,79 +559,7 @@ pub const Memory = union(enum) {
533	559	}
534	560	};
535	561
536		pub const Sib = struct {
537		ptr_size: PtrSize,
538		base: Base,
539		scale_index: ScaleIndex,
540		disp: i32,
541		};
542
543		pub const Rip = struct {
544		ptr_size: PtrSize,
545		disp: i32,
546		};
547
548		pub const Moffs = struct {
549		seg: Register,
550		offset: u64,
551		};
552
553		pub fn moffs(reg: Register, offset: u64) Memory {
554		assert(reg.class() == .segment);
555		return .{ .moffs = .{ .seg = reg, .offset = offset } };
556		}
557
558		pub fn sib(ptr_size: PtrSize, args: struct {
559		disp: i32 = 0,
560		base: Base = .none,
561		scale_index: ?ScaleIndex = null,
562		}) Memory {
563		if (args.scale_index) \|si\| assert(std.math.isPowerOfTwo(si.scale));
564		return .{ .sib = .{
565		.base = args.base,
566		.disp = args.disp,
567		.ptr_size = ptr_size,
568		.scale_index = if (args.scale_index) \|si\| si else ScaleIndex.none,
569		} };
570		}
571
572		pub fn rip(ptr_size: PtrSize, disp: i32) Memory {
573		return .{ .rip = .{ .ptr_size = ptr_size, .disp = disp } };
574		}
575
576		pub fn isSegmentRegister(mem: Memory) bool {
577		return switch (mem) {
578		.moffs => true,
579		.rip => false,
580		.sib => \|s\| switch (s.base) {
581		.none, .frame => false,
582		.reg => \|reg\| reg.class() == .segment,
583		},
584		};
585		}
586
587		pub fn base(mem: Memory) Base {
588		return switch (mem) {
589		.moffs => \|m\| .{ .reg = m.seg },
590		.sib => \|s\| s.base,
591		.rip => .none,
592		};
593		}
594
595		pub fn scaleIndex(mem: Memory) ?ScaleIndex {
596		return switch (mem) {
597		.moffs, .rip => null,
598		.sib => \|s\| if (s.scale_index.scale > 0) s.scale_index else null,
599		};
600		}
601
602		pub fn bitSize(mem: Memory) u64 {
603		return switch (mem) {
604		.rip => \|r\| r.ptr_size.bitSize(),
605		.sib => \|s\| s.ptr_size.bitSize(),
606		.moffs => 64,
607		};
608		}
	562	pub const Scale = enum(u2) { @"1", @"2", @"4", @"8" };
609	563	};
610	564
611	565	pub const Immediate = union(enum) {

src/arch/x86_64/encoder.zig+181-85

...	...	@@ -6,9 +6,10 @@ const testing = std.testing;
6	6
7	7	const bits = @import("bits.zig");
8	8	const Encoding = @import("Encoding.zig");
	9	const FrameIndex = bits.FrameIndex;
9	10	const Immediate = bits.Immediate;
10		const Memory = bits.Memory;
11	11	const Register = bits.Register;
	12	const Symbol = bits.Symbol;
12	13
13	14	pub const Instruction = struct {
14	15	prefix: Prefix = .none,
...	...	@@ -27,6 +28,97 @@ pub const Instruction = struct {
27	28	repnz,
28	29	};
29	30
	31	pub const Memory = union(enum) {
	32	sib: Sib,
	33	rip: Rip,
	34	moffs: Moffs,
	35
	36	pub const Base = bits.Memory.Base;
	37
	38	pub const ScaleIndex = struct {
	39	scale: u4,
	40	index: Register,
	41
	42	const none = ScaleIndex{ .scale = 0, .index = undefined };
	43	};
	44
	45	pub const PtrSize = bits.Memory.Size;
	46
	47	pub const Sib = struct {
	48	ptr_size: PtrSize,
	49	base: Base,
	50	scale_index: ScaleIndex,
	51	disp: i32,
	52	};
	53
	54	pub const Rip = struct {
	55	ptr_size: PtrSize,
	56	disp: i32,
	57	};
	58
	59	pub const Moffs = struct {
	60	seg: Register,
	61	offset: u64,
	62	};
	63
	64	pub fn moffs(reg: Register, offset: u64) Memory {
	65	assert(reg.class() == .segment);
	66	return .{ .moffs = .{ .seg = reg, .offset = offset } };
	67	}
	68
	69	pub fn sib(ptr_size: PtrSize, args: struct {
	70	disp: i32 = 0,
	71	base: Base = .none,
	72	scale_index: ?ScaleIndex = null,
	73	}) Memory {
	74	if (args.scale_index) \|si\| assert(std.math.isPowerOfTwo(si.scale));
	75	return .{ .sib = .{
	76	.base = args.base,
	77	.disp = args.disp,
	78	.ptr_size = ptr_size,
	79	.scale_index = if (args.scale_index) \|si\| si else ScaleIndex.none,
	80	} };
	81	}
	82
	83	pub fn rip(ptr_size: PtrSize, disp: i32) Memory {
	84	return .{ .rip = .{ .ptr_size = ptr_size, .disp = disp } };
	85	}
	86
	87	pub fn isSegmentRegister(mem: Memory) bool {
	88	return switch (mem) {
	89	.moffs => true,
	90	.rip => false,
	91	.sib => \|s\| switch (s.base) {
	92	.none, .frame, .reloc => false,
	93	.reg => \|reg\| reg.class() == .segment,
	94	},
	95	};
	96	}
	97
	98	pub fn base(mem: Memory) Base {
	99	return switch (mem) {
	100	.moffs => \|m\| .{ .reg = m.seg },
	101	.sib => \|s\| s.base,
	102	.rip => .none,
	103	};
	104	}
	105
	106	pub fn scaleIndex(mem: Memory) ?ScaleIndex {
	107	return switch (mem) {
	108	.moffs, .rip => null,
	109	.sib => \|s\| if (s.scale_index.scale > 0) s.scale_index else null,
	110	};
	111	}
	112
	113	pub fn bitSize(mem: Memory) u64 {
	114	return switch (mem) {
	115	.rip => \|r\| r.ptr_size.bitSize(),
	116	.sib => \|s\| s.ptr_size.bitSize(),
	117	.moffs => 64,
	118	};
	119	}
	120	};
	121
30	122	pub const Operand = union(enum) {
31	123	none,
32	124	reg: Register,
...	...	@@ -125,8 +217,8 @@ pub const Instruction = struct {
125	217	try writer.print("{s}", .{@tagName(reg)});
126	218	any = true;
127	219	},
128		.frame => \|frame\| {
129		try writer.print("{}", .{frame});
	220	inline .frame, .reloc => \|payload\| {
	221	try writer.print("{}", .{payload});
130	222	any = true;
131	223	},
132	224	}
...	...	@@ -498,7 +590,11 @@ pub const Instruction = struct {
498	590	}
499	591	}
500	592	},
501		.frame => if (@TypeOf(encoder).options.allow_frame_loc) {
	593	.frame => if (@TypeOf(encoder).options.allow_frame_locs) {
	594	try encoder.modRm_indirectDisp32(operand_enc, undefined);
	595	try encoder.disp32(undefined);
	596	} else return error.CannotEncode,
	597	.reloc => if (@TypeOf(encoder).options.allow_symbols) {
502	598	try encoder.modRm_indirectDisp32(operand_enc, undefined);
503	599	try encoder.disp32(undefined);
504	600	} else return error.CannotEncode,
...	...	@@ -570,7 +666,7 @@ pub const LegacyPrefixes = packed struct {
570	666	}
571	667	};
572	668
573		pub const Options = struct { allow_frame_loc: bool = false };
	669	pub const Options = struct { allow_frame_locs: bool = false, allow_symbols: bool = false };
574	670
575	671	fn Encoder(comptime T: type, comptime opts: Options) type {
576	672	return struct {
...	...	@@ -1085,7 +1181,7 @@ test "lower MI encoding" {
1085	1181	try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
1086	1182
1087	1183	try enc.encode(.mov, &.{
1088		.{ .mem = Memory.sib(.byte, .{ .base = .r12 }) },
	1184	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .r12 }) },
1089	1185	.{ .imm = Immediate.u(0x10) },
1090	1186	});
1091	1187	try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10");
...	...	@@ -1109,13 +1205,13 @@ test "lower MI encoding" {
1109	1205	try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10");
1110	1206
1111	1207	try enc.encode(.mov, &.{
1112		.{ .mem = Memory.sib(.dword, .{ .base = .r11 }) },
	1208	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .r11 }) },
1113	1209	.{ .imm = Immediate.u(0x10) },
1114	1210	});
1115	1211	try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10");
1116	1212
1117	1213	try enc.encode(.mov, &.{
1118		.{ .mem = Memory.rip(.qword, 0x10) },
	1214	.{ .mem = Instruction.Memory.rip(.qword, 0x10) },
1119	1215	.{ .imm = Immediate.u(0x10) },
1120	1216	});
1121	1217	try expectEqualHexStrings(
...	...	@@ -1125,25 +1221,25 @@ test "lower MI encoding" {
1125	1221	);
1126	1222
1127	1223	try enc.encode(.mov, &.{
1128		.{ .mem = Memory.sib(.qword, .{ .base = .rbp, .disp = -8 }) },
	1224	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp, .disp = -8 }) },
1129	1225	.{ .imm = Immediate.u(0x10) },
1130	1226	});
1131	1227	try expectEqualHexStrings("\x48\xc7\x45\xf8\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rbp - 8], 0x10");
1132	1228
1133	1229	try enc.encode(.mov, &.{
1134		.{ .mem = Memory.sib(.word, .{ .base = .rbp, .disp = -2 }) },
	1230	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp, .disp = -2 }) },
1135	1231	.{ .imm = Immediate.s(-16) },
1136	1232	});
1137	1233	try expectEqualHexStrings("\x66\xC7\x45\xFE\xF0\xFF", enc.code(), "mov WORD PTR [rbp - 2], -16");
1138	1234
1139	1235	try enc.encode(.mov, &.{
1140		.{ .mem = Memory.sib(.byte, .{ .base = .rbp, .disp = -1 }) },
	1236	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .rbp, .disp = -1 }) },
1141	1237	.{ .imm = Immediate.u(0x10) },
1142	1238	});
1143	1239	try expectEqualHexStrings("\xC6\x45\xFF\x10", enc.code(), "mov BYTE PTR [rbp - 1], 0x10");
1144	1240
1145	1241	try enc.encode(.mov, &.{
1146		.{ .mem = Memory.sib(.qword, .{
	1242	.{ .mem = Instruction.Memory.sib(.qword, .{
1147	1243	.base = .ds,
1148	1244	.disp = 0x10000000,
1149	1245	.scale_index = .{ .scale = 2, .index = .rcx },
...	...	@@ -1157,13 +1253,13 @@ test "lower MI encoding" {
1157	1253	);
1158	1254
1159	1255	try enc.encode(.adc, &.{
1160		.{ .mem = Memory.sib(.byte, .{ .base = .rbp, .disp = -0x10 }) },
	1256	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .rbp, .disp = -0x10 }) },
1161	1257	.{ .imm = Immediate.u(0x10) },
1162	1258	});
1163	1259	try expectEqualHexStrings("\x80\x55\xF0\x10", enc.code(), "adc BYTE PTR [rbp - 0x10], 0x10");
1164	1260
1165	1261	try enc.encode(.adc, &.{
1166		.{ .mem = Memory.rip(.qword, 0) },
	1262	.{ .mem = Instruction.Memory.rip(.qword, 0) },
1167	1263	.{ .imm = Immediate.u(0x10) },
1168	1264	});
1169	1265	try expectEqualHexStrings("\x48\x83\x15\x00\x00\x00\x00\x10", enc.code(), "adc QWORD PTR [rip], 0x10");
...	...	@@ -1175,7 +1271,7 @@ test "lower MI encoding" {
1175	1271	try expectEqualHexStrings("\x48\x83\xD0\x10", enc.code(), "adc rax, 0x10");
1176	1272
1177	1273	try enc.encode(.add, &.{
1178		.{ .mem = Memory.sib(.dword, .{ .base = .rdx, .disp = -8 }) },
	1274	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .rdx, .disp = -8 }) },
1179	1275	.{ .imm = Immediate.u(0x10) },
1180	1276	});
1181	1277	try expectEqualHexStrings("\x83\x42\xF8\x10", enc.code(), "add DWORD PTR [rdx - 8], 0x10");
...	...	@@ -1187,13 +1283,13 @@ test "lower MI encoding" {
1187	1283	try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10");
1188	1284
1189	1285	try enc.encode(.add, &.{
1190		.{ .mem = Memory.sib(.qword, .{ .base = .rbp, .disp = -0x10 }) },
	1286	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp, .disp = -0x10 }) },
1191	1287	.{ .imm = Immediate.s(-0x10) },
1192	1288	});
1193	1289	try expectEqualHexStrings("\x48\x83\x45\xF0\xF0", enc.code(), "add QWORD PTR [rbp - 0x10], -0x10");
1194	1290
1195	1291	try enc.encode(.@"and", &.{
1196		.{ .mem = Memory.sib(.dword, .{ .base = .ds, .disp = 0x10000000 }) },
	1292	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .ds, .disp = 0x10000000 }) },
1197	1293	.{ .imm = Immediate.u(0x10) },
1198	1294	});
1199	1295	try expectEqualHexStrings(
...	...	@@ -1203,7 +1299,7 @@ test "lower MI encoding" {
1203	1299	);
1204	1300
1205	1301	try enc.encode(.@"and", &.{
1206		.{ .mem = Memory.sib(.dword, .{ .base = .es, .disp = 0x10000000 }) },
	1302	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .es, .disp = 0x10000000 }) },
1207	1303	.{ .imm = Immediate.u(0x10) },
1208	1304	});
1209	1305	try expectEqualHexStrings(
...	...	@@ -1213,7 +1309,7 @@ test "lower MI encoding" {
1213	1309	);
1214	1310
1215	1311	try enc.encode(.@"and", &.{
1216		.{ .mem = Memory.sib(.dword, .{ .base = .r12, .disp = 0x10000000 }) },
	1312	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .r12, .disp = 0x10000000 }) },
1217	1313	.{ .imm = Immediate.u(0x10) },
1218	1314	});
1219	1315	try expectEqualHexStrings(
...	...	@@ -1223,7 +1319,7 @@ test "lower MI encoding" {
1223	1319	);
1224	1320
1225	1321	try enc.encode(.sub, &.{
1226		.{ .mem = Memory.sib(.dword, .{ .base = .r11, .disp = 0x10000000 }) },
	1322	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .r11, .disp = 0x10000000 }) },
1227	1323	.{ .imm = Immediate.u(0x10) },
1228	1324	});
1229	1325	try expectEqualHexStrings(
...	...	@@ -1238,25 +1334,25 @@ test "lower RM encoding" {
1238	1334
1239	1335	try enc.encode(.mov, &.{
1240	1336	.{ .reg = .rax },
1241		.{ .mem = Memory.sib(.qword, .{ .base = .r11 }) },
	1337	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .r11 }) },
1242	1338	});
1243	1339	try expectEqualHexStrings("\x49\x8b\x03", enc.code(), "mov rax, QWORD PTR [r11]");
1244	1340
1245	1341	try enc.encode(.mov, &.{
1246	1342	.{ .reg = .rbx },
1247		.{ .mem = Memory.sib(.qword, .{ .base = .ds, .disp = 0x10 }) },
	1343	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .ds, .disp = 0x10 }) },
1248	1344	});
1249	1345	try expectEqualHexStrings("\x48\x8B\x1C\x25\x10\x00\x00\x00", enc.code(), "mov rbx, QWORD PTR ds:0x10");
1250	1346
1251	1347	try enc.encode(.mov, &.{
1252	1348	.{ .reg = .rax },
1253		.{ .mem = Memory.sib(.qword, .{ .base = .rbp, .disp = -4 }) },
	1349	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp, .disp = -4 }) },
1254	1350	});
1255	1351	try expectEqualHexStrings("\x48\x8B\x45\xFC", enc.code(), "mov rax, QWORD PTR [rbp - 4]");
1256	1352
1257	1353	try enc.encode(.mov, &.{
1258	1354	.{ .reg = .rax },
1259		.{ .mem = Memory.sib(.qword, .{
	1355	.{ .mem = Instruction.Memory.sib(.qword, .{
1260	1356	.base = .rbp,
1261	1357	.scale_index = .{ .scale = 1, .index = .rcx },
1262	1358	.disp = -8,
...	...	@@ -1266,7 +1362,7 @@ test "lower RM encoding" {
1266	1362
1267	1363	try enc.encode(.mov, &.{
1268	1364	.{ .reg = .eax },
1269		.{ .mem = Memory.sib(.dword, .{
	1365	.{ .mem = Instruction.Memory.sib(.dword, .{
1270	1366	.base = .rbp,
1271	1367	.scale_index = .{ .scale = 4, .index = .rdx },
1272	1368	.disp = -4,
...	...	@@ -1276,7 +1372,7 @@ test "lower RM encoding" {
1276	1372
1277	1373	try enc.encode(.mov, &.{
1278	1374	.{ .reg = .rax },
1279		.{ .mem = Memory.sib(.qword, .{
	1375	.{ .mem = Instruction.Memory.sib(.qword, .{
1280	1376	.base = .rbp,
1281	1377	.scale_index = .{ .scale = 8, .index = .rcx },
1282	1378	.disp = -8,
...	...	@@ -1286,7 +1382,7 @@ test "lower RM encoding" {
1286	1382
1287	1383	try enc.encode(.mov, &.{
1288	1384	.{ .reg = .r8b },
1289		.{ .mem = Memory.sib(.byte, .{
	1385	.{ .mem = Instruction.Memory.sib(.byte, .{
1290	1386	.base = .rsi,
1291	1387	.scale_index = .{ .scale = 1, .index = .rcx },
1292	1388	.disp = -24,
...	...	@@ -1302,7 +1398,7 @@ test "lower RM encoding" {
1302	1398	try expectEqualHexStrings("\x48\x8C\xC8", enc.code(), "mov rax, cs");
1303	1399
1304	1400	try enc.encode(.mov, &.{
1305		.{ .mem = Memory.sib(.qword, .{ .base = .rbp, .disp = -16 }) },
	1401	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp, .disp = -16 }) },
1306	1402	.{ .reg = .fs },
1307	1403	});
1308	1404	try expectEqualHexStrings("\x48\x8C\x65\xF0", enc.code(), "mov QWORD PTR [rbp - 16], fs");
...	...	@@ -1314,7 +1410,7 @@ test "lower RM encoding" {
1314	1410	try expectEqualHexStrings("\x66\x41\x8C\xCC", enc.code(), "mov r12w, cs");
1315	1411
1316	1412	try enc.encode(.mov, &.{
1317		.{ .mem = Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
	1413	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
1318	1414	.{ .reg = .fs },
1319	1415	});
1320	1416	try expectEqualHexStrings("\x66\x8C\x65\xF0", enc.code(), "mov WORD PTR [rbp - 16], fs");
...	...	@@ -1339,19 +1435,19 @@ test "lower RM encoding" {
1339	1435
1340	1436	try enc.encode(.movsx, &.{
1341	1437	.{ .reg = .eax },
1342		.{ .mem = Memory.sib(.word, .{ .base = .rbp }) },
	1438	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp }) },
1343	1439	});
1344	1440	try expectEqualHexStrings("\x0F\xBF\x45\x00", enc.code(), "movsx eax, BYTE PTR [rbp]");
1345	1441
1346	1442	try enc.encode(.movsx, &.{
1347	1443	.{ .reg = .eax },
1348		.{ .mem = Memory.sib(.byte, .{ .scale_index = .{ .index = .rax, .scale = 2 } }) },
	1444	.{ .mem = Instruction.Memory.sib(.byte, .{ .scale_index = .{ .index = .rax, .scale = 2 } }) },
1349	1445	});
1350	1446	try expectEqualHexStrings("\x0F\xBE\x04\x45\x00\x00\x00\x00", enc.code(), "movsx eax, BYTE PTR [rax * 2]");
1351	1447
1352	1448	try enc.encode(.movsx, &.{
1353	1449	.{ .reg = .ax },
1354		.{ .mem = Memory.rip(.byte, 0x10) },
	1450	.{ .mem = Instruction.Memory.rip(.byte, 0x10) },
1355	1451	});
1356	1452	try expectEqualHexStrings("\x66\x0F\xBE\x05\x10\x00\x00\x00", enc.code(), "movsx ax, BYTE PTR [rip + 0x10]");
1357	1453
...	...	@@ -1369,37 +1465,37 @@ test "lower RM encoding" {
1369	1465
1370	1466	try enc.encode(.lea, &.{
1371	1467	.{ .reg = .rax },
1372		.{ .mem = Memory.rip(.qword, 0x10) },
	1468	.{ .mem = Instruction.Memory.rip(.qword, 0x10) },
1373	1469	});
1374	1470	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, QWORD PTR [rip + 0x10]");
1375	1471
1376	1472	try enc.encode(.lea, &.{
1377	1473	.{ .reg = .rax },
1378		.{ .mem = Memory.rip(.dword, 0x10) },
	1474	.{ .mem = Instruction.Memory.rip(.dword, 0x10) },
1379	1475	});
1380	1476	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, DWORD PTR [rip + 0x10]");
1381	1477
1382	1478	try enc.encode(.lea, &.{
1383	1479	.{ .reg = .eax },
1384		.{ .mem = Memory.rip(.dword, 0x10) },
	1480	.{ .mem = Instruction.Memory.rip(.dword, 0x10) },
1385	1481	});
1386	1482	try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, DWORD PTR [rip + 0x10]");
1387	1483
1388	1484	try enc.encode(.lea, &.{
1389	1485	.{ .reg = .eax },
1390		.{ .mem = Memory.rip(.word, 0x10) },
	1486	.{ .mem = Instruction.Memory.rip(.word, 0x10) },
1391	1487	});
1392	1488	try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, WORD PTR [rip + 0x10]");
1393	1489
1394	1490	try enc.encode(.lea, &.{
1395	1491	.{ .reg = .ax },
1396		.{ .mem = Memory.rip(.byte, 0x10) },
	1492	.{ .mem = Instruction.Memory.rip(.byte, 0x10) },
1397	1493	});
1398	1494	try expectEqualHexStrings("\x66\x8D\x05\x10\x00\x00\x00", enc.code(), "lea ax, BYTE PTR [rip + 0x10]");
1399	1495
1400	1496	try enc.encode(.lea, &.{
1401	1497	.{ .reg = .rsi },
1402		.{ .mem = Memory.sib(.qword, .{
	1498	.{ .mem = Instruction.Memory.sib(.qword, .{
1403	1499	.base = .rbp,
1404	1500	.scale_index = .{ .scale = 1, .index = .rcx },
1405	1501	}) },
...	...	@@ -1408,31 +1504,31 @@ test "lower RM encoding" {
1408	1504
1409	1505	try enc.encode(.add, &.{
1410	1506	.{ .reg = .r11 },
1411		.{ .mem = Memory.sib(.qword, .{ .base = .ds, .disp = 0x10000000 }) },
	1507	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .ds, .disp = 0x10000000 }) },
1412	1508	});
1413	1509	try expectEqualHexStrings("\x4C\x03\x1C\x25\x00\x00\x00\x10", enc.code(), "add r11, QWORD PTR ds:0x10000000");
1414	1510
1415	1511	try enc.encode(.add, &.{
1416	1512	.{ .reg = .r12b },
1417		.{ .mem = Memory.sib(.byte, .{ .base = .ds, .disp = 0x10000000 }) },
	1513	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .ds, .disp = 0x10000000 }) },
1418	1514	});
1419	1515	try expectEqualHexStrings("\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR ds:0x10000000");
1420	1516
1421	1517	try enc.encode(.add, &.{
1422	1518	.{ .reg = .r12b },
1423		.{ .mem = Memory.sib(.byte, .{ .base = .fs, .disp = 0x10000000 }) },
	1519	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .fs, .disp = 0x10000000 }) },
1424	1520	});
1425	1521	try expectEqualHexStrings("\x64\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR fs:0x10000000");
1426	1522
1427	1523	try enc.encode(.sub, &.{
1428	1524	.{ .reg = .r11 },
1429		.{ .mem = Memory.sib(.qword, .{ .base = .r13, .disp = 0x10000000 }) },
	1525	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .r13, .disp = 0x10000000 }) },
1430	1526	});
1431	1527	try expectEqualHexStrings("\x4D\x2B\x9D\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r13 + 0x10000000]");
1432	1528
1433	1529	try enc.encode(.sub, &.{
1434	1530	.{ .reg = .r11 },
1435		.{ .mem = Memory.sib(.qword, .{ .base = .r12, .disp = 0x10000000 }) },
	1531	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .r12, .disp = 0x10000000 }) },
1436	1532	});
1437	1533	try expectEqualHexStrings("\x4D\x2B\x9C\x24\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r12 + 0x10000000]");
1438	1534
...	...	@@ -1455,7 +1551,7 @@ test "lower RMI encoding" {
1455	1551
1456	1552	try enc.encode(.imul, &.{
1457	1553	.{ .reg = .r11 },
1458		.{ .mem = Memory.rip(.qword, -16) },
	1554	.{ .mem = Instruction.Memory.rip(.qword, -16) },
1459	1555	.{ .imm = Immediate.s(-1024) },
1460	1556	});
1461	1557	try expectEqualHexStrings(
...	...	@@ -1466,7 +1562,7 @@ test "lower RMI encoding" {
1466	1562
1467	1563	try enc.encode(.imul, &.{
1468	1564	.{ .reg = .bx },
1469		.{ .mem = Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
	1565	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
1470	1566	.{ .imm = Immediate.s(-1024) },
1471	1567	});
1472	1568	try expectEqualHexStrings(
...	...	@@ -1477,7 +1573,7 @@ test "lower RMI encoding" {
1477	1573
1478	1574	try enc.encode(.imul, &.{
1479	1575	.{ .reg = .bx },
1480		.{ .mem = Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
	1576	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
1481	1577	.{ .imm = Immediate.u(1024) },
1482	1578	});
1483	1579	try expectEqualHexStrings(
...	...	@@ -1497,19 +1593,19 @@ test "lower MR encoding" {
1497	1593	try expectEqualHexStrings("\x48\x89\xD8", enc.code(), "mov rax, rbx");
1498	1594
1499	1595	try enc.encode(.mov, &.{
1500		.{ .mem = Memory.sib(.qword, .{ .base = .rbp, .disp = -4 }) },
	1596	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp, .disp = -4 }) },
1501	1597	.{ .reg = .r11 },
1502	1598	});
1503	1599	try expectEqualHexStrings("\x4c\x89\x5d\xfc", enc.code(), "mov QWORD PTR [rbp - 4], r11");
1504	1600
1505	1601	try enc.encode(.mov, &.{
1506		.{ .mem = Memory.rip(.qword, 0x10) },
	1602	.{ .mem = Instruction.Memory.rip(.qword, 0x10) },
1507	1603	.{ .reg = .r12 },
1508	1604	});
1509	1605	try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rip + 0x10], r12");
1510	1606
1511	1607	try enc.encode(.mov, &.{
1512		.{ .mem = Memory.sib(.qword, .{
	1608	.{ .mem = Instruction.Memory.sib(.qword, .{
1513	1609	.base = .r11,
1514	1610	.scale_index = .{ .scale = 2, .index = .r12 },
1515	1611	.disp = 0x10,
...	...	@@ -1519,13 +1615,13 @@ test "lower MR encoding" {
1519	1615	try expectEqualHexStrings("\x4F\x89\x6C\x63\x10", enc.code(), "mov QWORD PTR [r11 + 2 * r12 + 0x10], r13");
1520	1616
1521	1617	try enc.encode(.mov, &.{
1522		.{ .mem = Memory.rip(.word, -0x10) },
	1618	.{ .mem = Instruction.Memory.rip(.word, -0x10) },
1523	1619	.{ .reg = .r12w },
1524	1620	});
1525	1621	try expectEqualHexStrings("\x66\x44\x89\x25\xF0\xFF\xFF\xFF", enc.code(), "mov WORD PTR [rip - 0x10], r12w");
1526	1622
1527	1623	try enc.encode(.mov, &.{
1528		.{ .mem = Memory.sib(.byte, .{
	1624	.{ .mem = Instruction.Memory.sib(.byte, .{
1529	1625	.base = .r11,
1530	1626	.scale_index = .{ .scale = 2, .index = .r12 },
1531	1627	.disp = 0x10,
...	...	@@ -1535,25 +1631,25 @@ test "lower MR encoding" {
1535	1631	try expectEqualHexStrings("\x47\x88\x6C\x63\x10", enc.code(), "mov BYTE PTR [r11 + 2 * r12 + 0x10], r13b");
1536	1632
1537	1633	try enc.encode(.add, &.{
1538		.{ .mem = Memory.sib(.byte, .{ .base = .ds, .disp = 0x10000000 }) },
	1634	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .ds, .disp = 0x10000000 }) },
1539	1635	.{ .reg = .r12b },
1540	1636	});
1541	1637	try expectEqualHexStrings("\x44\x00\x24\x25\x00\x00\x00\x10", enc.code(), "add BYTE PTR ds:0x10000000, r12b");
1542	1638
1543	1639	try enc.encode(.add, &.{
1544		.{ .mem = Memory.sib(.dword, .{ .base = .ds, .disp = 0x10000000 }) },
	1640	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .ds, .disp = 0x10000000 }) },
1545	1641	.{ .reg = .r12d },
1546	1642	});
1547	1643	try expectEqualHexStrings("\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [ds:0x10000000], r12d");
1548	1644
1549	1645	try enc.encode(.add, &.{
1550		.{ .mem = Memory.sib(.dword, .{ .base = .gs, .disp = 0x10000000 }) },
	1646	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .gs, .disp = 0x10000000 }) },
1551	1647	.{ .reg = .r12d },
1552	1648	});
1553	1649	try expectEqualHexStrings("\x65\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [gs:0x10000000], r12d");
1554	1650
1555	1651	try enc.encode(.sub, &.{
1556		.{ .mem = Memory.sib(.qword, .{ .base = .r11, .disp = 0x10000000 }) },
	1652	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .r11, .disp = 0x10000000 }) },
1557	1653	.{ .reg = .r12 },
1558	1654	});
1559	1655	try expectEqualHexStrings("\x4D\x29\xA3\x00\x00\x00\x10", enc.code(), "sub QWORD PTR [r11 + 0x10000000], r12");
...	...	@@ -1568,12 +1664,12 @@ test "lower M encoding" {
1568	1664	try expectEqualHexStrings("\x41\xFF\xD4", enc.code(), "call r12");
1569	1665
1570	1666	try enc.encode(.call, &.{
1571		.{ .mem = Memory.sib(.qword, .{ .base = .r12 }) },
	1667	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .r12 }) },
1572	1668	});
1573	1669	try expectEqualHexStrings("\x41\xFF\x14\x24", enc.code(), "call QWORD PTR [r12]");
1574	1670
1575	1671	try enc.encode(.call, &.{
1576		.{ .mem = Memory.sib(.qword, .{
	1672	.{ .mem = Instruction.Memory.sib(.qword, .{
1577	1673	.base = null,
1578	1674	.scale_index = .{ .index = .r11, .scale = 2 },
1579	1675	}) },
...	...	@@ -1581,7 +1677,7 @@ test "lower M encoding" {
1581	1677	try expectEqualHexStrings("\x42\xFF\x14\x5D\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r11 * 2]");
1582	1678
1583	1679	try enc.encode(.call, &.{
1584		.{ .mem = Memory.sib(.qword, .{
	1680	.{ .mem = Instruction.Memory.sib(.qword, .{
1585	1681	.base = null,
1586	1682	.scale_index = .{ .index = .r12, .scale = 2 },
1587	1683	}) },
...	...	@@ -1589,7 +1685,7 @@ test "lower M encoding" {
1589	1685	try expectEqualHexStrings("\x42\xFF\x14\x65\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r12 * 2]");
1590	1686
1591	1687	try enc.encode(.call, &.{
1592		.{ .mem = Memory.sib(.qword, .{ .base = .gs }) },
	1688	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .gs }) },
1593	1689	});
1594	1690	try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0");
1595	1691
...	...	@@ -1599,22 +1695,22 @@ test "lower M encoding" {
1599	1695	try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0");
1600	1696
1601	1697	try enc.encode(.push, &.{
1602		.{ .mem = Memory.sib(.qword, .{ .base = .rbp }) },
	1698	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp }) },
1603	1699	});
1604	1700	try expectEqualHexStrings("\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
1605	1701
1606	1702	try enc.encode(.push, &.{
1607		.{ .mem = Memory.sib(.word, .{ .base = .rbp }) },
	1703	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp }) },
1608	1704	});
1609	1705	try expectEqualHexStrings("\x66\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
1610	1706
1611	1707	try enc.encode(.pop, &.{
1612		.{ .mem = Memory.rip(.qword, 0) },
	1708	.{ .mem = Instruction.Memory.rip(.qword, 0) },
1613	1709	});
1614	1710	try expectEqualHexStrings("\x8F\x05\x00\x00\x00\x00", enc.code(), "pop QWORD PTR [rip]");
1615	1711
1616	1712	try enc.encode(.pop, &.{
1617		.{ .mem = Memory.rip(.word, 0) },
	1713	.{ .mem = Instruction.Memory.rip(.word, 0) },
1618	1714	});
1619	1715	try expectEqualHexStrings("\x66\x8F\x05\x00\x00\x00\x00", enc.code(), "pop WORD PTR [rbp]");
1620	1716
...	...	@@ -1695,48 +1791,48 @@ test "lower FD/TD encoding" {
1695	1791
1696	1792	try enc.encode(.mov, &.{
1697	1793	.{ .reg = .rax },
1698		.{ .mem = Memory.moffs(.cs, 0x10) },
	1794	.{ .mem = Instruction.Memory.moffs(.cs, 0x10) },
1699	1795	});
1700	1796	try expectEqualHexStrings("\x2E\x48\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs rax, cs:0x10");
1701	1797
1702	1798	try enc.encode(.mov, &.{
1703	1799	.{ .reg = .eax },
1704		.{ .mem = Memory.moffs(.fs, 0x10) },
	1800	.{ .mem = Instruction.Memory.moffs(.fs, 0x10) },
1705	1801	});
1706	1802	try expectEqualHexStrings("\x64\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs eax, fs:0x10");
1707	1803
1708	1804	try enc.encode(.mov, &.{
1709	1805	.{ .reg = .ax },
1710		.{ .mem = Memory.moffs(.gs, 0x10) },
	1806	.{ .mem = Instruction.Memory.moffs(.gs, 0x10) },
1711	1807	});
1712	1808	try expectEqualHexStrings("\x65\x66\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ax, gs:0x10");
1713	1809
1714	1810	try enc.encode(.mov, &.{
1715	1811	.{ .reg = .al },
1716		.{ .mem = Memory.moffs(.ds, 0x10) },
	1812	.{ .mem = Instruction.Memory.moffs(.ds, 0x10) },
1717	1813	});
1718	1814	try expectEqualHexStrings("\xA0\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs al, ds:0x10");
1719	1815
1720	1816	try enc.encode(.mov, &.{
1721		.{ .mem = Memory.moffs(.cs, 0x10) },
	1817	.{ .mem = Instruction.Memory.moffs(.cs, 0x10) },
1722	1818	.{ .reg = .rax },
1723	1819	});
1724	1820	try expectEqualHexStrings("\x2E\x48\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs cs:0x10, rax");
1725	1821
1726	1822	try enc.encode(.mov, &.{
1727		.{ .mem = Memory.moffs(.fs, 0x10) },
	1823	.{ .mem = Instruction.Memory.moffs(.fs, 0x10) },
1728	1824	.{ .reg = .eax },
1729	1825	});
1730	1826	try expectEqualHexStrings("\x64\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs fs:0x10, eax");
1731	1827
1732	1828	try enc.encode(.mov, &.{
1733		.{ .mem = Memory.moffs(.gs, 0x10) },
	1829	.{ .mem = Instruction.Memory.moffs(.gs, 0x10) },
1734	1830	.{ .reg = .ax },
1735	1831	});
1736	1832	try expectEqualHexStrings("\x65\x66\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs gs:0x10, ax");
1737	1833
1738	1834	try enc.encode(.mov, &.{
1739		.{ .mem = Memory.moffs(.ds, 0x10) },
	1835	.{ .mem = Instruction.Memory.moffs(.ds, 0x10) },
1740	1836	.{ .reg = .al },
1741	1837	});
1742	1838	try expectEqualHexStrings("\xA2\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ds:0x10, al");
...	...	@@ -1774,16 +1870,16 @@ test "invalid instruction" {
1774	1870	.{ .reg = .al },
1775	1871	});
1776	1872	try invalidInstruction(.call, &.{
1777		.{ .mem = Memory.rip(.dword, 0) },
	1873	.{ .mem = Instruction.Memory.rip(.dword, 0) },
1778	1874	});
1779	1875	try invalidInstruction(.call, &.{
1780		.{ .mem = Memory.rip(.word, 0) },
	1876	.{ .mem = Instruction.Memory.rip(.word, 0) },
1781	1877	});
1782	1878	try invalidInstruction(.call, &.{
1783		.{ .mem = Memory.rip(.byte, 0) },
	1879	.{ .mem = Instruction.Memory.rip(.byte, 0) },
1784	1880	});
1785	1881	try invalidInstruction(.mov, &.{
1786		.{ .mem = Memory.rip(.word, 0x10) },
	1882	.{ .mem = Instruction.Memory.rip(.word, 0x10) },
1787	1883	.{ .reg = .r12 },
1788	1884	});
1789	1885	try invalidInstruction(.lea, &.{
...	...	@@ -1792,7 +1888,7 @@ test "invalid instruction" {
1792	1888	});
1793	1889	try invalidInstruction(.lea, &.{
1794	1890	.{ .reg = .al },
1795		.{ .mem = Memory.rip(.byte, 0) },
	1891	.{ .mem = Instruction.Memory.rip(.byte, 0) },
1796	1892	});
1797	1893	try invalidInstruction(.pop, &.{
1798	1894	.{ .reg = .r12b },
...	...	@@ -1817,7 +1913,7 @@ fn cannotEncode(mnemonic: Instruction.Mnemonic, ops: []const Instruction.Operand
1817	1913
1818	1914	test "cannot encode" {
1819	1915	try cannotEncode(.@"test", &.{
1820		.{ .mem = Memory.sib(.byte, .{ .base = .r12 }) },
	1916	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .r12 }) },
1821	1917	.{ .reg = .ah },
1822	1918	});
1823	1919	try cannotEncode(.@"test", &.{
...	...	@@ -2149,8 +2245,8 @@ const Assembler = struct {
2149	2245	return null;
2150	2246	}
2151	2247
2152		fn parseMemory(as: *Assembler) ParseError!Memory {
2153		const ptr_size: ?Memory.PtrSize = blk: {
	2248	fn parseMemory(as: *Assembler) ParseError!Instruction.Memory {
	2249	const ptr_size: ?Instruction.Memory.PtrSize = blk: {
2154	2250	const pos = as.it.pos;
2155	2251	const ptr_size = as.parsePtrSize() catch \|err\| switch (err) {
2156	2252	error.UnexpectedToken => {
...	...	@@ -2194,7 +2290,7 @@ const Assembler = struct {
2194	2290	if (res.rip) {
2195	2291	if (res.base != null or res.scale_index != null or res.offset != null)
2196	2292	return error.InvalidMemoryOperand;
2197		return Memory.rip(ptr_size orelse .qword, res.disp orelse 0);
	2293	return Instruction.Memory.rip(ptr_size orelse .qword, res.disp orelse 0);
2198	2294	}
2199	2295	if (res.base) \|base\| {
2200	2296	if (res.rip)
...	...	@@ -2202,9 +2298,9 @@ const Assembler = struct {
2202	2298	if (res.offset) \|offset\| {
2203	2299	if (res.scale_index != null or res.disp != null)
2204	2300	return error.InvalidMemoryOperand;
2205		return Memory.moffs(base, offset);
	2301	return Instruction.Memory.moffs(base, offset);
2206	2302	}
2207		return Memory.sib(ptr_size orelse .qword, .{
	2303	return Instruction.Memory.sib(ptr_size orelse .qword, .{
2208	2304	.base = base,
2209	2305	.scale_index = res.scale_index,
2210	2306	.disp = res.disp orelse 0,
...	...	@@ -2222,12 +2318,12 @@ const Assembler = struct {
2222	2318	const MemoryParseResult = struct {
2223	2319	rip: bool = false,
2224	2320	base: ?Register = null,
2225		scale_index: ?Memory.ScaleIndex = null,
	2321	scale_index: ?Instruction.Memory.ScaleIndex = null,
2226	2322	disp: ?i32 = null,
2227	2323	offset: ?u64 = null,
2228	2324	};
2229	2325
2230		fn parseMemoryRule(as: *Assembler, rule: anytype) ParseError!MemoryParseResult {
	2326	fn parseMemoryRule(as: *Assembler, rule: anytype) ParseError!Instruction.MemoryParseResult {
2231	2327	var res: MemoryParseResult = .{};
2232	2328	inline for (rule, 0..) \|cond, i\| {
2233	2329	if (@typeInfo(@TypeOf(cond)) != .EnumLiteral) {
...	...	@@ -2294,7 +2390,7 @@ const Assembler = struct {
2294	2390	return res;
2295	2391	}
2296	2392
2297		fn parsePtrSize(as: *Assembler) ParseError!Memory.PtrSize {
	2393	fn parsePtrSize(as: *Assembler) ParseError!Instruction.Memory.PtrSize {
2298	2394	const size = try as.expect(.string);
2299	2395	try as.skip(1, .{.space});
2300	2396	const ptr = try as.expect(.string);

test/behavior/export_builtin.zig-3

...	...	@@ -4,7 +4,6 @@ const expect = std.testing.expect;
4	4
5	5	test "exporting enum type and value" {
6	6	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
7		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
8	7	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
9	8	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
10	9	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
...	...	@@ -21,7 +20,6 @@ test "exporting enum type and value" {
21	20
22	21	test "exporting with internal linkage" {
23	22	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
24		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
25	23	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
26	24	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
27	25	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
...	...	@@ -37,7 +35,6 @@ test "exporting with internal linkage" {
37	35
38	36	test "exporting using field access" {
39	37	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
40		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
41	38	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
42	39	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
43	40	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;

test/behavior/fn.zig-1

...	...	@@ -577,7 +577,6 @@ test "pass and return comptime-only types" {
577	577
578	578	test "pointer to alias behaves same as pointer to function" {
579	579	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
580		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
581	580	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
582	581	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
583	582	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;

test/behavior/globals.zig-2

...	...	@@ -6,7 +6,6 @@ var pos = [2]f32{ 0.0, 0.0 };
6	6	test "store to global array" {
7	7	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
8	8	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
9		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
10	9	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
11	10	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
12	11	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
...	...	@@ -20,7 +19,6 @@ var vpos = @Vector(2, f32){ 0.0, 0.0 };
20	19	test "store to global vector" {
21	20	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
22	21	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
23		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
24	22	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
25	23	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
26	24	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;

test/behavior/nan.zig-1

...	...	@@ -23,7 +23,6 @@ const snan_f128: f128 = math.snan(f128);
23	23
24	24	test "nan memory equality" {
25	25	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
26		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
27	26	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
28	27	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
29	28	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;

test/tests.zig+11-7

...	...	@@ -28,6 +28,7 @@ const TestTarget = struct {
28	28	single_threaded: ?bool = null,
29	29	use_llvm: ?bool = null,
30	30	use_lld: ?bool = null,
	31	force_pic: ?bool = null,
31	32	};
32	33
33	34	const test_targets = blk: {
...	...	@@ -103,6 +104,7 @@ const test_targets = blk: {
103	104	},
104	105	.use_llvm = false,
105	106	.use_lld = false,
	107	.force_pic = true,
106	108	},
107	109	.{
108	110	.target = .{
...	...	@@ -494,6 +496,7 @@ const CAbiTarget = struct {
494	496	target: CrossTarget = .{},
495	497	use_llvm: ?bool = null,
496	498	use_lld: ?bool = null,
	499	force_pic: ?bool = null,
497	500	c_defines: []const []const u8 = &.{},
498	501	};
499	502
...	...	@@ -536,6 +539,7 @@ const c_abi_targets = [_]CAbiTarget{
536	539	},
537	540	.use_llvm = false,
538	541	.use_lld = false,
	542	.force_pic = true,
539	543	.c_defines = &.{"ZIG_BACKEND_STAGE2_X86_64"},
540	544	},
541	545	.{
...	...	@@ -1118,6 +1122,7 @@ pub fn addModuleTests(b: std.Build, options: ModuleTestOptions) Step {
1118	1122	.use_lld = test_target.use_lld,
1119	1123	.zig_lib_dir = .{ .path = "lib" },
1120	1124	});
	1125	these_tests.force_pic = test_target.force_pic;
1121	1126	const single_threaded_suffix = if (test_target.single_threaded == true) "-single" else "";
1122	1127	const backend_suffix = if (test_target.use_llvm == true)
1123	1128	"-llvm"
...	...	@@ -1128,6 +1133,7 @@ pub fn addModuleTests(b: std.Build, options: ModuleTestOptions) Step {
1128	1133	else
1129	1134	"";
1130	1135	const use_lld = if (test_target.use_lld == false) "-no-lld" else "";
	1136	const use_pic = if (test_target.force_pic == true) "-pic" else "";
1131	1137
1132	1138	these_tests.addIncludePath(.{ .path = "test" });
1133	1139
...	...	@@ -1136,7 +1142,7 @@ pub fn addModuleTests(b: std.Build, options: ModuleTestOptions) Step {
1136	1142	these_tests.stack_size = 2 * 1024 * 1024;
1137	1143	}
1138	1144
1139		const qualified_name = b.fmt("{s}-{s}-{s}-{s}{s}{s}{s}{s}", .{
	1145	const qualified_name = b.fmt("{s}-{s}-{s}-{s}{s}{s}{s}{s}{s}", .{
1140	1146	options.name,
1141	1147	triple_txt,
1142	1148	model_txt,
...	...	@@ -1145,6 +1151,7 @@ pub fn addModuleTests(b: std.Build, options: ModuleTestOptions) Step {
1145	1151	single_threaded_suffix,
1146	1152	backend_suffix,
1147	1153	use_lld,
	1154	use_pic,
1148	1155	});
1149	1156
1150	1157	if (test_target.target.ofmt == std.Target.ObjectFormat.c) {
...	...	@@ -1234,13 +1241,8 @@ pub fn addCAbiTests(b: std.Build, skip_non_native: bool, skip_release: bool) S
1234	1241	continue;
1235	1242	}
1236	1243
1237		if (c_abi_target.use_llvm == false and optimize_mode == .ReleaseFast) {
1238		// panic: unrecognized command line argument
1239		continue;
1240		}
1241
1242	1244	const test_step = b.addTest(.{
1243		.name = b.fmt("test-c-abi-{s}-{s}-{s}{s}{s}", .{
	1245	.name = b.fmt("test-c-abi-{s}-{s}-{s}{s}{s}{s}", .{
1244	1246	c_abi_target.target.zigTriple(b.allocator) catch @panic("OOM"),
1245	1247	c_abi_target.target.getCpuModel().name,
1246	1248	@tagName(optimize_mode),
...	...	@@ -1253,6 +1255,7 @@ pub fn addCAbiTests(b: std.Build, skip_non_native: bool, skip_release: bool) S
1253	1255	else
1254	1256	"",
1255	1257	if (c_abi_target.use_lld == false) "-no-lld" else "",
	1258	if (c_abi_target.force_pic == true) "-pic" else "",
1256	1259	}),
1257	1260	.root_source_file = .{ .path = "test/c_abi/main.zig" },
1258	1261	.target = c_abi_target.target,
...	...	@@ -1261,6 +1264,7 @@ pub fn addCAbiTests(b: std.Build, skip_non_native: bool, skip_release: bool) S
1261	1264	.use_llvm = c_abi_target.use_llvm,
1262	1265	.use_lld = c_abi_target.use_lld,
1263	1266	});
	1267	test_step.force_pic = c_abi_target.force_pic;
1264	1268	if (c_abi_target.target.abi != null and c_abi_target.target.abi.?.isMusl()) {
1265	1269	// TODO NativeTargetInfo insists on dynamically linking musl
1266	1270	// for some reason?