authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-01-17 01:20:11-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-01-17 01:20:11-05:00
log4bace0f6212a3007247c42b0effcc40c6cfe61a8
treea5c1fee87a9266319d8ceffa7717ca5c70b61f7b
parent257054a1467b2612725bd66852d84496024cf66c
parent8c8dfb35f398407319764f0f8998de34c5247ed6
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #22386 from jacobly0/x86_64-rewrite

x86_64: begin rewriting instruction selection

75 files changed, 16973 insertions(+), 4863 deletions(-)

lib/std/Target/Query.zig+4-4
......@@ -6,13 +6,13 @@
66/// `null` means native.
77cpu_arch: ?Target.Cpu.Arch = null,
88
9cpu_model: CpuModel = CpuModel.determined_by_arch_os,
9cpu_model: CpuModel = .determined_by_arch_os,
1010
1111/// Sparse set of CPU features to add to the set from `cpu_model`.
12cpu_features_add: Target.Cpu.Feature.Set = Target.Cpu.Feature.Set.empty,
12cpu_features_add: Target.Cpu.Feature.Set = .empty,
1313
1414/// Sparse set of CPU features to remove from the set from `cpu_model`.
15cpu_features_sub: Target.Cpu.Feature.Set = Target.Cpu.Feature.Set.empty,
15cpu_features_sub: Target.Cpu.Feature.Set = .empty,
1616
1717/// `null` means native.
1818os_tag: ?Target.Os.Tag = null,
......@@ -38,7 +38,7 @@ abi: ?Target.Abi = null,
3838
3939/// When `os_tag` is `null`, then `null` means native. Otherwise it means the standard path
4040/// based on the `os_tag`.
41dynamic_linker: Target.DynamicLinker = Target.DynamicLinker.none,
41dynamic_linker: Target.DynamicLinker = .none,
4242
4343/// `null` means default for the cpu/arch/os combo.
4444ofmt: ?Target.ObjectFormat = null,
lib/std/Target/x86.zig+65
......@@ -47,6 +47,7 @@ pub const Feature = enum {
4747 bmi2,
4848 branch_hint,
4949 branchfusion,
50 bsf_bsr_0_clobbers_result,
5051 ccmp,
5152 cf,
5253 cldemote,
......@@ -167,6 +168,8 @@ pub const Feature = enum {
167168 slow_unaligned_mem_32,
168169 sm3,
169170 sm4,
171 smap,
172 smep,
170173 soft_float,
171174 sse,
172175 sse2,
......@@ -497,6 +500,11 @@ pub const all_features = blk: {
497500 .description = "CMP/TEST can be fused with conditional branches",
498501 .dependencies = featureSet(&[_]Feature{}),
499502 };
503 result[@intFromEnum(Feature.bsf_bsr_0_clobbers_result)] = .{
504 .llvm_name = null,
505 .description = "BSF/BSR may clobber the lower 32-bits of the result register when the source is zero",
506 .dependencies = featureSet(&[_]Feature{}),
507 };
500508 result[@intFromEnum(Feature.ccmp)] = .{
501509 .llvm_name = "ccmp",
502510 .description = "Support conditional cmp & test instructions",
......@@ -1127,6 +1135,16 @@ pub const all_features = blk: {
11271135 .avx2,
11281136 }),
11291137 };
1138 result[@intFromEnum(Feature.smap)] = .{
1139 .llvm_name = null,
1140 .description = "Enable Supervisor Mode Access Prevention",
1141 .dependencies = featureSet(&[_]Feature{}),
1142 };
1143 result[@intFromEnum(Feature.smep)] = .{
1144 .llvm_name = null,
1145 .description = "Enable Supervisor Mode Execution Prevention",
1146 .dependencies = featureSet(&[_]Feature{}),
1147 };
11301148 result[@intFromEnum(Feature.soft_float)] = .{
11311149 .llvm_name = "soft-float",
11321150 .description = "Use software floating point features",
......@@ -1371,6 +1389,8 @@ pub const cpu = struct {
13711389 .sha,
13721390 .shstk,
13731391 .slow_3ops_lea,
1392 .smap,
1393 .smep,
13741394 .tuning_fast_imm_vector_shift,
13751395 .vaes,
13761396 .vpclmulqdq,
......@@ -1467,6 +1487,8 @@ pub const cpu = struct {
14671487 .sha,
14681488 .shstk,
14691489 .slow_3ops_lea,
1490 .smap,
1491 .smep,
14701492 .tuning_fast_imm_vector_shift,
14711493 .uintr,
14721494 .vaes,
......@@ -1545,6 +1567,8 @@ pub const cpu = struct {
15451567 .slow_3ops_lea,
15461568 .sm3,
15471569 .sm4,
1570 .smap,
1571 .smep,
15481572 .tuning_fast_imm_vector_shift,
15491573 .uintr,
15501574 .vaes,
......@@ -1783,6 +1807,8 @@ pub const cpu = struct {
17831807 .sahf,
17841808 .sbb_dep_breaking,
17851809 .slow_shld,
1810 .smap,
1811 .smep,
17861812 .sse4a,
17871813 .vzeroupper,
17881814 .x87,
......@@ -1995,6 +2021,8 @@ pub const cpu = struct {
19952021 .rdseed,
19962022 .sahf,
19972023 .slow_3ops_lea,
2024 .smap,
2025 .smep,
19982026 .vzeroupper,
19992027 .x87,
20002028 .xsaveopt,
......@@ -2136,6 +2164,8 @@ pub const cpu = struct {
21362164 .sahf,
21372165 .sha,
21382166 .slow_3ops_lea,
2167 .smap,
2168 .smep,
21392169 .tuning_fast_imm_vector_shift,
21402170 .vzeroupper,
21412171 .x87,
......@@ -2195,6 +2225,8 @@ pub const cpu = struct {
21952225 .rdseed,
21962226 .sahf,
21972227 .slow_3ops_lea,
2228 .smap,
2229 .smep,
21982230 .tuning_fast_imm_vector_shift,
21992231 .vzeroupper,
22002232 .x87,
......@@ -2450,6 +2482,8 @@ pub const cpu = struct {
24502482 .serialize,
24512483 .sha,
24522484 .shstk,
2485 .smap,
2486 .smep,
24532487 .tsxldtrk,
24542488 .tuning_fast_imm_vector_shift,
24552489 .uintr,
......@@ -2519,6 +2553,8 @@ pub const cpu = struct {
25192553 .slow_incdec,
25202554 .slow_lea,
25212555 .slow_two_mem_ops,
2556 .smap,
2557 .smep,
25222558 .sse4_2,
25232559 .use_glm_div_sqrt_costs,
25242560 .vzeroupper,
......@@ -2898,6 +2934,7 @@ pub const cpu = struct {
28982934 .rdrnd,
28992935 .sahf,
29002936 .slow_3ops_lea,
2937 .smep,
29012938 .vzeroupper,
29022939 .x87,
29032940 .xsaveopt,
......@@ -2907,6 +2944,7 @@ pub const cpu = struct {
29072944 .name = "i386",
29082945 .llvm_name = "i386",
29092946 .features = featureSet(&[_]Feature{
2947 .bsf_bsr_0_clobbers_result,
29102948 .slow_unaligned_mem_16,
29112949 .vzeroupper,
29122950 .x87,
......@@ -2916,6 +2954,7 @@ pub const cpu = struct {
29162954 .name = "i486",
29172955 .llvm_name = "i486",
29182956 .features = featureSet(&[_]Feature{
2957 .bsf_bsr_0_clobbers_result,
29192958 .slow_unaligned_mem_16,
29202959 .vzeroupper,
29212960 .x87,
......@@ -3096,6 +3135,7 @@ pub const cpu = struct {
30963135 .sahf,
30973136 .slow_3ops_lea,
30983137 .slow_unaligned_mem_32,
3138 .smep,
30993139 .vzeroupper,
31003140 .x87,
31013141 .xsaveopt,
......@@ -3403,6 +3443,8 @@ pub const cpu = struct {
34033443 .sha,
34043444 .shstk,
34053445 .slow_3ops_lea,
3446 .smap,
3447 .smep,
34063448 .tuning_fast_imm_vector_shift,
34073449 .vaes,
34083450 .vpclmulqdq,
......@@ -3766,6 +3808,8 @@ pub const cpu = struct {
37663808 .sha,
37673809 .shstk,
37683810 .slow_3ops_lea,
3811 .smap,
3812 .smep,
37693813 .tuning_fast_imm_vector_shift,
37703814 .vaes,
37713815 .vpclmulqdq,
......@@ -3831,6 +3875,8 @@ pub const cpu = struct {
38313875 .rdseed,
38323876 .sahf,
38333877 .sha,
3878 .smap,
3879 .smep,
38343880 .tuning_fast_imm_vector_shift,
38353881 .vaes,
38363882 .vpclmulqdq,
......@@ -3939,6 +3985,8 @@ pub const cpu = struct {
39393985 .serialize,
39403986 .sha,
39413987 .shstk,
3988 .smap,
3989 .smep,
39423990 .tsxldtrk,
39433991 .tuning_fast_imm_vector_shift,
39443992 .uintr,
......@@ -4042,6 +4090,7 @@ pub const cpu = struct {
40424090 .slow_lea,
40434091 .slow_pmulld,
40444092 .slow_two_mem_ops,
4093 .smep,
40454094 .sse4_2,
40464095 .use_slm_arith_costs,
40474096 .vzeroupper,
......@@ -4098,6 +4147,8 @@ pub const cpu = struct {
40984147 .rdseed,
40994148 .sahf,
41004149 .slow_3ops_lea,
4150 .smap,
4151 .smep,
41014152 .tuning_fast_imm_vector_shift,
41024153 .vzeroupper,
41034154 .x87,
......@@ -4150,6 +4201,8 @@ pub const cpu = struct {
41504201 .rdseed,
41514202 .sahf,
41524203 .slow_3ops_lea,
4204 .smap,
4205 .smep,
41534206 .vzeroupper,
41544207 .x87,
41554208 .xsavec,
......@@ -4305,6 +4358,8 @@ pub const cpu = struct {
43054358 .sahf,
43064359 .sha,
43074360 .shstk,
4361 .smap,
4362 .smep,
43084363 .tuning_fast_imm_vector_shift,
43094364 .vaes,
43104365 .vpclmulqdq,
......@@ -4574,6 +4629,8 @@ pub const cpu = struct {
45744629 .sbb_dep_breaking,
45754630 .sha,
45764631 .slow_shld,
4632 .smap,
4633 .smep,
45774634 .sse4a,
45784635 .vzeroupper,
45794636 .x87,
......@@ -4629,6 +4686,8 @@ pub const cpu = struct {
46294686 .sbb_dep_breaking,
46304687 .sha,
46314688 .slow_shld,
4689 .smap,
4690 .smep,
46324691 .sse4a,
46334692 .vzeroupper,
46344693 .wbnoinvd,
......@@ -4686,6 +4745,8 @@ pub const cpu = struct {
46864745 .sbb_dep_breaking,
46874746 .sha,
46884747 .slow_shld,
4748 .smap,
4749 .smep,
46894750 .sse4a,
46904751 .vaes,
46914752 .vpclmulqdq,
......@@ -4757,6 +4818,8 @@ pub const cpu = struct {
47574818 .sha,
47584819 .shstk,
47594820 .slow_shld,
4821 .smap,
4822 .smep,
47604823 .sse4a,
47614824 .vaes,
47624825 .vpclmulqdq,
......@@ -4833,6 +4896,8 @@ pub const cpu = struct {
48334896 .sha,
48344897 .shstk,
48354898 .slow_shld,
4899 .smap,
4900 .smep,
48364901 .sse4a,
48374902 .vaes,
48384903 .vpclmulqdq,
lib/std/Thread.zig+3-1
......@@ -372,9 +372,11 @@ pub const SpawnConfig = struct {
372372 // https://github.com/ziglang/zig/issues/157
373373
374374 /// Size in bytes of the Thread's stack
375 stack_size: usize = 16 * 1024 * 1024,
375 stack_size: usize = default_stack_size,
376376 /// The allocator to be used to allocate memory for the to-be-spawned thread
377377 allocator: ?std.mem.Allocator = null,
378
379 pub const default_stack_size = 16 * 1024 * 1024;
378380};
379381
380382pub const SpawnError = error{
lib/std/Thread/Condition.zig+3-3
......@@ -161,17 +161,17 @@ const WindowsImpl = struct {
161161 }
162162 }
163163
164 if (comptime builtin.mode == .Debug) {
164 if (builtin.mode == .Debug) {
165165 // The internal state of the DebugMutex needs to be handled here as well.
166166 mutex.impl.locking_thread.store(0, .unordered);
167167 }
168168 const rc = os.windows.kernel32.SleepConditionVariableSRW(
169169 &self.condition,
170 if (comptime builtin.mode == .Debug) &mutex.impl.impl.srwlock else &mutex.impl.srwlock,
170 if (builtin.mode == .Debug) &mutex.impl.impl.srwlock else &mutex.impl.srwlock,
171171 timeout_ms,
172172 0, // the srwlock was assumed to acquired in exclusive mode not shared
173173 );
174 if (comptime builtin.mode == .Debug) {
174 if (builtin.mode == .Debug) {
175175 // The internal state of the DebugMutex needs to be handled here as well.
176176 mutex.impl.locking_thread.store(std.Thread.getCurrentId(), .unordered);
177177 }
lib/std/Thread/Mutex.zig+1-1
......@@ -158,7 +158,7 @@ const FutexImpl = struct {
158158 // On x86, use `lock bts` instead of `lock cmpxchg` as:
159159 // - they both seem to mark the cache-line as modified regardless: https://stackoverflow.com/a/63350048
160160 // - `lock bts` is smaller instruction-wise which makes it better for inlining
161 if (comptime builtin.target.cpu.arch.isX86()) {
161 if (builtin.target.cpu.arch.isX86()) {
162162 const locked_bit = @ctz(locked);
163163 return self.state.bitSet(locked_bit, .acquire) == 0;
164164 }
lib/std/Thread/Pool.zig+5-1
......@@ -27,6 +27,7 @@ pub const Options = struct {
2727 allocator: std.mem.Allocator,
2828 n_jobs: ?usize = null,
2929 track_ids: bool = false,
30 stack_size: usize = std.Thread.SpawnConfig.default_stack_size,
3031};
3132
3233pub fn init(pool: *Pool, options: Options) !void {
......@@ -54,7 +55,10 @@ pub fn init(pool: *Pool, options: Options) !void {
5455 errdefer pool.join(spawned);
5556
5657 for (pool.threads) |*thread| {
57 thread.* = try std.Thread.spawn(.{}, worker, .{pool});
58 thread.* = try std.Thread.spawn(.{
59 .stack_size = options.stack_size,
60 .allocator = allocator,
61 }, worker, .{pool});
5862 spawned += 1;
5963 }
6064}
lib/std/crypto/aes/aesni.zig+1-1
......@@ -4,7 +4,7 @@ const mem = std.mem;
44const debug = std.debug;
55
66const has_vaes = builtin.cpu.arch == .x86_64 and std.Target.x86.featureSetHas(builtin.cpu.features, .vaes);
7const has_avx512f = builtin.cpu.arch == .x86_64 and std.Target.x86.featureSetHas(builtin.cpu.features, .avx512f);
7const has_avx512f = builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_x86_64 and std.Target.x86.featureSetHas(builtin.cpu.features, .avx512f);
88
99/// A single AES block.
1010pub const Block = struct {
lib/std/crypto/chacha20.zig+1-3
......@@ -499,11 +499,9 @@ fn ChaChaNonVecImpl(comptime rounds_nb: usize) type {
499499fn ChaChaImpl(comptime rounds_nb: usize) type {
500500 switch (builtin.cpu.arch) {
501501 .x86_64 => {
502 if (builtin.zig_backend == .stage2_x86_64) return ChaChaNonVecImpl(rounds_nb);
503
504502 const has_avx2 = std.Target.x86.featureSetHas(builtin.cpu.features, .avx2);
505503 const has_avx512f = std.Target.x86.featureSetHas(builtin.cpu.features, .avx512f);
506 if (has_avx512f) return ChaChaVecImpl(rounds_nb, 4);
504 if (builtin.zig_backend != .stage2_x86_64 and has_avx512f) return ChaChaVecImpl(rounds_nb, 4);
507505 if (has_avx2) return ChaChaVecImpl(rounds_nb, 2);
508506 return ChaChaVecImpl(rounds_nb, 1);
509507 },
lib/std/crypto/tls/Client.zig+7-56
......@@ -356,14 +356,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
356356 if (ciphertext.len > cleartext_fragment_buf.len) return error.TlsRecordOverflow;
357357 const cleartext = cleartext_fragment_buf[0..ciphertext.len];
358358 const auth_tag = record_decoder.array(P.AEAD.tag_length).*;
359 const nonce = if (builtin.zig_backend == .stage2_x86_64 and
360 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
361 nonce: {
362 var nonce = pv.server_handshake_iv;
363 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
364 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ read_seq, .big);
365 break :nonce nonce;
366 } else nonce: {
359 const nonce = nonce: {
367360 const V = @Vector(P.AEAD.nonce_length, u8);
368361 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
369362 const operand: V = pad ++ @as([8]u8, @bitCast(big(read_seq)));
......@@ -400,14 +393,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
400393 const record_iv = record_decoder.array(P.record_iv_length).*;
401394 const masked_read_seq = read_seq &
402395 comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);
403 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and
404 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
405 nonce: {
406 var nonce = pv.app_cipher.server_write_IV ++ record_iv;
407 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
408 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ masked_read_seq, .big);
409 break :nonce nonce;
410 } else nonce: {
396 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
411397 const V = @Vector(P.AEAD.nonce_length, u8);
412398 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
413399 const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));
......@@ -750,14 +736,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
750736 .app_cipher = std.mem.bytesToValue(P.Tls_1_2, &key_block),
751737 } };
752738 const pv = &p.version.tls_1_2;
753 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and
754 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
755 nonce: {
756 var nonce = pv.app_cipher.client_write_IV ++ pv.app_cipher.client_salt;
757 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
758 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ write_seq, .big);
759 break :nonce nonce;
760 } else nonce: {
739 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
761740 const V = @Vector(P.AEAD.nonce_length, u8);
762741 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
763742 const operand: V = pad ++ @as([8]u8, @bitCast(big(write_seq)));
......@@ -1043,14 +1022,7 @@ fn prepareCiphertextRecord(
10431022 ciphertext_end += ciphertext_len;
10441023 const auth_tag = ciphertext_buf[ciphertext_end..][0..P.AEAD.tag_length];
10451024 ciphertext_end += auth_tag.len;
1046 const nonce = if (builtin.zig_backend == .stage2_x86_64 and
1047 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1048 nonce: {
1049 var nonce = pv.client_iv;
1050 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1051 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.write_seq, .big);
1052 break :nonce nonce;
1053 } else nonce: {
1025 const nonce = nonce: {
10541026 const V = @Vector(P.AEAD.nonce_length, u8);
10551027 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
10561028 const operand: V = pad ++ std.mem.toBytes(big(c.write_seq));
......@@ -1098,14 +1070,7 @@ fn prepareCiphertextRecord(
10981070 const ad = std.mem.toBytes(big(c.write_seq)) ++ record_header[0 .. 1 + 2] ++ int(u16, message_len);
10991071 const record_iv = ciphertext_buf[ciphertext_end..][0..P.record_iv_length];
11001072 ciphertext_end += P.record_iv_length;
1101 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and
1102 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1103 nonce: {
1104 var nonce = pv.client_write_IV ++ pv.client_salt;
1105 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1106 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.write_seq, .big);
1107 break :nonce nonce;
1108 } else nonce: {
1073 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
11091074 const V = @Vector(P.AEAD.nonce_length, u8);
11101075 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
11111076 const operand: V = pad ++ @as([8]u8, @bitCast(big(c.write_seq)));
......@@ -1374,14 +1339,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove
13741339 const ciphertext = frag[in..][0..ciphertext_len];
13751340 in += ciphertext_len;
13761341 const auth_tag = frag[in..][0..P.AEAD.tag_length].*;
1377 const nonce = if (builtin.zig_backend == .stage2_x86_64 and
1378 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1379 nonce: {
1380 var nonce = pv.server_iv;
1381 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1382 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.read_seq, .big);
1383 break :nonce nonce;
1384 } else nonce: {
1342 const nonce = nonce: {
13851343 const V = @Vector(P.AEAD.nonce_length, u8);
13861344 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
13871345 const operand: V = pad ++ std.mem.toBytes(big(c.read_seq));
......@@ -1409,14 +1367,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove
14091367 in += P.record_iv_length;
14101368 const masked_read_seq = c.read_seq &
14111369 comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);
1412 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and
1413 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1414 nonce: {
1415 var nonce = pv.server_write_IV ++ record_iv;
1416 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1417 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ masked_read_seq, .big);
1418 break :nonce nonce;
1419 } else nonce: {
1370 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
14201371 const V = @Vector(P.AEAD.nonce_length, u8);
14211372 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
14221373 const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));
lib/std/debug.zig+4-3
......@@ -23,6 +23,7 @@ pub const Coverage = @import("debug/Coverage.zig");
2323
2424pub const FormattedPanic = @import("debug/FormattedPanic.zig");
2525pub const SimplePanic = @import("debug/SimplePanic.zig");
26pub const NoPanic = @import("debug/NoPanic.zig");
2627
2728/// Unresolved source locations can be represented with a single `usize` that
2829/// corresponds to a virtual memory address of the program counter. Combined
......@@ -179,7 +180,7 @@ pub fn dumpHexFallible(bytes: []const u8) !void {
179180/// TODO multithreaded awareness
180181pub fn dumpCurrentStackTrace(start_addr: ?usize) void {
181182 nosuspend {
182 if (comptime builtin.target.isWasm()) {
183 if (builtin.target.isWasm()) {
183184 if (native_os == .wasi) {
184185 const stderr = io.getStdErr().writer();
185186 stderr.print("Unable to dump stack trace: not implemented for Wasm\n", .{}) catch return;
......@@ -267,7 +268,7 @@ pub inline fn getContext(context: *ThreadContext) bool {
267268/// TODO multithreaded awareness
268269pub fn dumpStackTraceFromBase(context: *ThreadContext) void {
269270 nosuspend {
270 if (comptime builtin.target.isWasm()) {
271 if (builtin.target.isWasm()) {
271272 if (native_os == .wasi) {
272273 const stderr = io.getStdErr().writer();
273274 stderr.print("Unable to dump stack trace: not implemented for Wasm\n", .{}) catch return;
......@@ -365,7 +366,7 @@ pub fn captureStackTrace(first_address: ?usize, stack_trace: *std.builtin.StackT
365366/// TODO multithreaded awareness
366367pub fn dumpStackTrace(stack_trace: std.builtin.StackTrace) void {
367368 nosuspend {
368 if (comptime builtin.target.isWasm()) {
369 if (builtin.target.isWasm()) {
369370 if (native_os == .wasi) {
370371 const stderr = io.getStdErr().writer();
371372 stderr.print("Unable to dump stack trace: not implemented for Wasm\n", .{}) catch return;
lib/std/debug/NoPanic.zig created+59
......@@ -0,0 +1,59 @@
1//! This namespace can be used with `pub const Panic = std.debug.NoPanic;` in the root file.
2//! It emits as little code as possible, for testing purposes.
3//!
4//! For a functional alternative, see `std.debug.FormattedPanic`.
5
6const std = @import("../std.zig");
7
8pub fn call(_: []const u8, _: ?*std.builtin.StackTrace, _: ?usize) noreturn {
9 @branchHint(.cold);
10 @trap();
11}
12
13pub inline fn sentinelMismatch(_: anytype, _: anytype) noreturn {
14 @branchHint(.cold);
15 @trap();
16}
17
18pub inline fn unwrapError(_: ?*std.builtin.StackTrace, _: anyerror) noreturn {
19 @branchHint(.cold);
20 @trap();
21}
22
23pub inline fn outOfBounds(_: usize, _: usize) noreturn {
24 @branchHint(.cold);
25 @trap();
26}
27
28pub inline fn startGreaterThanEnd(_: usize, _: usize) noreturn {
29 @branchHint(.cold);
30 @trap();
31}
32
33pub inline fn inactiveUnionField(_: anytype, _: anytype) noreturn {
34 @branchHint(.cold);
35 @trap();
36}
37
38pub const messages = struct {
39 pub const reached_unreachable = "";
40 pub const unwrap_null = "";
41 pub const cast_to_null = "";
42 pub const incorrect_alignment = "";
43 pub const invalid_error_code = "";
44 pub const cast_truncated_data = "";
45 pub const negative_to_unsigned = "";
46 pub const integer_overflow = "";
47 pub const shl_overflow = "";
48 pub const shr_overflow = "";
49 pub const divide_by_zero = "";
50 pub const exact_division_remainder = "";
51 pub const integer_part_out_of_bounds = "";
52 pub const corrupt_switch = "";
53 pub const shift_rhs_too_big = "";
54 pub const invalid_enum_value = "";
55 pub const for_len_mismatch = "";
56 pub const memcpy_len_mismatch = "";
57 pub const memcpy_alias = "";
58 pub const noreturn_returned = "";
59};
lib/std/debug/SelfInfo.zig+4-4
......@@ -121,13 +121,13 @@ pub fn deinit(self: *SelfInfo) void {
121121}
122122
123123pub fn getModuleForAddress(self: *SelfInfo, address: usize) !*Module {
124 if (comptime builtin.target.isDarwin()) {
124 if (builtin.target.isDarwin()) {
125125 return self.lookupModuleDyld(address);
126126 } else if (native_os == .windows) {
127127 return self.lookupModuleWin32(address);
128128 } else if (native_os == .haiku) {
129129 return self.lookupModuleHaiku(address);
130 } else if (comptime builtin.target.isWasm()) {
130 } else if (builtin.target.isWasm()) {
131131 return self.lookupModuleWasm(address);
132132 } else {
133133 return self.lookupModuleDl(address);
......@@ -138,13 +138,13 @@ pub fn getModuleForAddress(self: *SelfInfo, address: usize) !*Module {
138138// This can be called when getModuleForAddress fails, so implementations should provide
139139// a path that doesn't rely on any side-effects of a prior successful module lookup.
140140pub fn getModuleNameForAddress(self: *SelfInfo, address: usize) ?[]const u8 {
141 if (comptime builtin.target.isDarwin()) {
141 if (builtin.target.isDarwin()) {
142142 return self.lookupModuleNameDyld(address);
143143 } else if (native_os == .windows) {
144144 return self.lookupModuleNameWin32(address);
145145 } else if (native_os == .haiku) {
146146 return null;
147 } else if (comptime builtin.target.isWasm()) {
147 } else if (builtin.target.isWasm()) {
148148 return null;
149149 } else {
150150 return self.lookupModuleNameDl(address);
lib/std/heap.zig+1-1
......@@ -890,7 +890,7 @@ test {
890890 _ = @import("heap/memory_pool.zig");
891891 _ = ArenaAllocator;
892892 _ = GeneralPurposeAllocator;
893 if (comptime builtin.target.isWasm()) {
893 if (builtin.target.isWasm()) {
894894 _ = WasmAllocator;
895895 _ = WasmPageAllocator;
896896 }
lib/std/http/protocol.zig-1
......@@ -4,7 +4,6 @@ const testing = std.testing;
44const mem = std.mem;
55
66const assert = std.debug.assert;
7const use_vectors = builtin.zig_backend != .stage2_x86_64;
87
98pub const State = enum {
109 invalid,
lib/std/math/big/int.zig+8-5
......@@ -2520,12 +2520,13 @@ pub const Const = struct {
25202520 return order(a, b) == .eq;
25212521 }
25222522
2523 /// Returns the number of leading zeros in twos-complement form.
25232524 pub fn clz(a: Const, bits: Limb) Limb {
2524 // Limbs are stored in little-endian order but we need
2525 // to iterate big-endian.
2525 // Limbs are stored in little-endian order but we need to iterate big-endian.
2526 if (!a.positive and !a.eqlZero()) return 0;
25262527 var total_limb_lz: Limb = 0;
25272528 var i: usize = a.limbs.len;
2528 const bits_per_limb = @sizeOf(Limb) * 8;
2529 const bits_per_limb = @bitSizeOf(Limb);
25292530 while (i != 0) {
25302531 i -= 1;
25312532 const limb = a.limbs[i];
......@@ -2537,13 +2538,15 @@ pub const Const = struct {
25372538 return total_limb_lz + bits - total_limb_bits;
25382539 }
25392540
2541 /// Returns the number of trailing zeros in twos-complement form.
25402542 pub fn ctz(a: Const, bits: Limb) Limb {
2541 // Limbs are stored in little-endian order.
2543 // Limbs are stored in little-endian order. Converting a negative number to twos-complement
2544 // flips all bits above the lowest set bit, which does not affect the trailing zero count.
25422545 var result: Limb = 0;
25432546 for (a.limbs) |limb| {
25442547 const limb_tz = @ctz(limb);
25452548 result += limb_tz;
2546 if (limb_tz != @sizeOf(Limb) * 8) break;
2549 if (limb_tz != @bitSizeOf(Limb)) break;
25472550 }
25482551 return @min(result, bits);
25492552 }
lib/std/mem.zig+2-2
......@@ -883,8 +883,8 @@ fn SliceTo(comptime T: type, comptime end: std.meta.Elem(T)) type {
883883 @compileError("invalid type given to std.mem.sliceTo: " ++ @typeName(T));
884884}
885885
886/// Takes an array, a pointer to an array, a sentinel-terminated pointer, or a slice and
887/// iterates searching for the first occurrence of `end`, returning the scanned slice.
886/// Takes a pointer to an array, a sentinel-terminated pointer, or a slice and iterates searching for
887/// the first occurrence of `end`, returning the scanned slice.
888888/// If `end` is not found, the full length of the array/slice/sentinel terminated pointer is returned.
889889/// If the pointer type is sentinel terminated and `end` matches that terminator, the
890890/// resulting slice is also sentinel terminated.
lib/std/os.zig+1-1
......@@ -157,7 +157,7 @@ pub fn getFdPath(fd: std.posix.fd_t, out_buffer: *[max_path_bytes]u8) std.posix.
157157 return target;
158158 },
159159 .freebsd => {
160 if (comptime builtin.os.isAtLeast(.freebsd, .{ .major = 13, .minor = 0, .patch = 0 }) orelse false) {
160 if (builtin.os.isAtLeast(.freebsd, .{ .major = 13, .minor = 0, .patch = 0 }) orelse false) {
161161 var kfile: std.c.kinfo_file = undefined;
162162 kfile.structsize = std.c.KINFO_FILE_SIZE;
163163 switch (posix.errno(std.c.fcntl(fd, std.c.F.KINFO, @intFromPtr(&kfile)))) {
lib/std/os/windows.zig+1-1
......@@ -1061,7 +1061,7 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil
10611061 // us INVALID_PARAMETER.
10621062 // The same reasoning for win10_rs5 as in os.renameatW() applies (FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE requires >= win10_rs5).
10631063 var need_fallback = true;
1064 if (comptime builtin.target.os.version_range.windows.min.isAtLeast(.win10_rs5)) {
1064 if (builtin.target.os.version_range.windows.min.isAtLeast(.win10_rs5)) {
10651065 // Deletion with posix semantics if the filesystem supports it.
10661066 var info = FILE_DISPOSITION_INFORMATION_EX{
10671067 .Flags = FILE_DISPOSITION_DELETE |
lib/std/posix.zig+1-1
......@@ -6819,7 +6819,7 @@ pub fn memfd_createZ(name: [*:0]const u8, flags: u32) MemFdCreateError!fd_t {
68196819 }
68206820 },
68216821 .freebsd => {
6822 if (comptime builtin.os.version_range.semver.max.order(.{ .major = 13, .minor = 0, .patch = 0 }) == .lt)
6822 if (builtin.os.version_range.semver.max.order(.{ .major = 13, .minor = 0, .patch = 0 }) == .lt)
68236823 @compileError("memfd_create is unavailable on FreeBSD < 13.0");
68246824 const rc = system.memfd_create(name, flags);
68256825 switch (errno(rc)) {
lib/std/posix/test.zig+1-1
......@@ -804,7 +804,7 @@ test "getrlimit and setrlimit" {
804804 //
805805 // This happens for example if RLIMIT_MEMLOCK is bigger than ~2GiB.
806806 // In that case the following the limit would be RLIM_INFINITY and the following setrlimit fails with EPERM.
807 if (comptime builtin.cpu.arch.isMIPS() and builtin.link_libc) {
807 if (builtin.cpu.arch.isMIPS() and builtin.link_libc) {
808808 if (limit.cur != linux.RLIM.INFINITY) {
809809 try posix.setrlimit(resource, limit);
810810 }
lib/std/simd.zig+4-6
......@@ -163,7 +163,7 @@ pub fn interlace(vecs: anytype) @Vector(vectorLength(@TypeOf(vecs[0])) * vecs.le
163163 // The indices are correct. The problem seems to be with the @shuffle builtin.
164164 // On MIPS, the test that interlaces small_base gives { 0, 2, 0, 0, 64, 255, 248, 200, 0, 0 }.
165165 // Calling this with two inputs seems to work fine, but I'll let the compile error trigger for all inputs, just to be safe.
166 comptime if (builtin.cpu.arch.isMIPS()) @compileError("TODO: Find out why interlace() doesn't work on MIPS");
166 if (builtin.cpu.arch.isMIPS()) @compileError("TODO: Find out why interlace() doesn't work on MIPS");
167167
168168 const VecType = @TypeOf(vecs[0]);
169169 const vecs_arr = @as([vecs.len]VecType, vecs);
......@@ -248,7 +248,7 @@ test "vector patterns" {
248248 try std.testing.expectEqual([8]u32{ 10, 20, 30, 40, 55, 66, 77, 88 }, join(base, other_base));
249249 try std.testing.expectEqual([2]u32{ 20, 30 }, extract(base, 1, 2));
250250
251 if (comptime !builtin.cpu.arch.isMIPS()) {
251 if (!builtin.cpu.arch.isMIPS()) {
252252 try std.testing.expectEqual([8]u32{ 10, 55, 20, 66, 30, 77, 40, 88 }, interlace(.{ base, other_base }));
253253
254254 const small_braid = interlace(small_bases);
......@@ -390,7 +390,7 @@ pub fn prefixScanWithFunc(
390390 comptime identity: std.meta.Child(@TypeOf(vec)),
391391) if (ErrorType == void) @TypeOf(vec) else ErrorType!@TypeOf(vec) {
392392 // I haven't debugged this, but it might be a cousin of sorts to what's going on with interlace.
393 comptime if (builtin.cpu.arch.isMIPS()) @compileError("TODO: Find out why prefixScan doesn't work on MIPS");
393 if (builtin.cpu.arch.isMIPS()) @compileError("TODO: Find out why prefixScan doesn't work on MIPS");
394394
395395 const len = vectorLength(@TypeOf(vec));
396396
......@@ -465,9 +465,7 @@ test "vector prefix scan" {
465465 if ((builtin.cpu.arch == .armeb or builtin.cpu.arch == .thumbeb) and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22060
466466 if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21893
467467
468 if (comptime builtin.cpu.arch.isMIPS()) {
469 return error.SkipZigTest;
470 }
468 if (builtin.cpu.arch.isMIPS()) return error.SkipZigTest;
471469
472470 const int_base = @Vector(4, i32){ 11, 23, 9, -21 };
473471 const float_base = @Vector(4, f32){ 2, 0.5, -10, 6.54321 };
lib/std/zig/system/NativePaths.zig+1-1
......@@ -83,7 +83,7 @@ pub fn detect(arena: Allocator, native_target: std.Target) !NativePaths {
8383
8484 // TODO: consider also adding homebrew paths
8585 // TODO: consider also adding macports paths
86 if (comptime builtin.target.isDarwin()) {
86 if (builtin.target.isDarwin()) {
8787 if (std.zig.system.darwin.isSdkInstalled(arena)) sdk: {
8888 const sdk = std.zig.system.darwin.getSdk(arena, native_target) orelse break :sdk;
8989 try self.addLibDir(try std.fs.path.join(arena, &.{ sdk, "usr/lib" }));
lib/std/zig/system/x86.zig+2
......@@ -369,6 +369,7 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
369369 setFeature(cpu, .bmi, bit(leaf.ebx, 3));
370370 // AVX2 is only supported if we have the OS save support from AVX.
371371 setFeature(cpu, .avx2, bit(leaf.ebx, 5) and has_avx_save);
372 setFeature(cpu, .smep, bit(leaf.ebx, 7));
372373 setFeature(cpu, .bmi2, bit(leaf.ebx, 8));
373374 setFeature(cpu, .invpcid, bit(leaf.ebx, 10));
374375 setFeature(cpu, .rtm, bit(leaf.ebx, 11));
......@@ -377,6 +378,7 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
377378 setFeature(cpu, .avx512dq, bit(leaf.ebx, 17) and has_avx512_save);
378379 setFeature(cpu, .rdseed, bit(leaf.ebx, 18));
379380 setFeature(cpu, .adx, bit(leaf.ebx, 19));
381 setFeature(cpu, .smap, bit(leaf.ebx, 20));
380382 setFeature(cpu, .avx512ifma, bit(leaf.ebx, 21) and has_avx512_save);
381383 setFeature(cpu, .clflushopt, bit(leaf.ebx, 23));
382384 setFeature(cpu, .clwb, bit(leaf.ebx, 24));
src/Air.zig+35-11
......@@ -893,14 +893,38 @@ pub const Inst = struct {
893893 pub const Index = enum(u32) {
894894 _,
895895
896 pub fn toRef(i: Index) Inst.Ref {
897 assert(@intFromEnum(i) >> 31 == 0);
898 return @enumFromInt((1 << 31) | @intFromEnum(i));
896 pub fn unwrap(index: Index) union(enum) { ref: Inst.Ref, target: u31 } {
897 const low_index: u31 = @truncate(@intFromEnum(index));
898 return switch (@as(u1, @intCast(@intFromEnum(index) >> 31))) {
899 0 => .{ .ref = @enumFromInt(@as(u32, 1 << 31) | low_index) },
900 1 => .{ .target = low_index },
901 };
902 }
903
904 pub fn toRef(index: Index) Inst.Ref {
905 return index.unwrap().ref;
906 }
907
908 pub fn fromTargetIndex(index: u31) Index {
909 return @enumFromInt((1 << 31) | @as(u32, index));
910 }
911
912 pub fn toTargetIndex(index: Index) u31 {
913 return index.unwrap().target;
899914 }
900915
901 pub fn toTargetIndex(i: Index) u31 {
902 assert(@intFromEnum(i) >> 31 == 1);
903 return @truncate(@intFromEnum(i));
916 pub fn format(
917 index: Index,
918 comptime _: []const u8,
919 _: std.fmt.FormatOptions,
920 writer: anytype,
921 ) @TypeOf(writer).Error!void {
922 try writer.writeByte('%');
923 switch (index.unwrap()) {
924 .ref => {},
925 .target => try writer.writeByte('t'),
926 }
927 try writer.print("{d}", .{@as(u31, @truncate(@intFromEnum(index)))});
904928 }
905929 };
906930
......@@ -1205,7 +1229,7 @@ pub const VectorCmp = struct {
12051229 op: u32,
12061230
12071231 pub fn compareOperator(self: VectorCmp) std.math.CompareOperator {
1208 return @as(std.math.CompareOperator, @enumFromInt(@as(u3, @truncate(self.op))));
1232 return @enumFromInt(@as(u3, @intCast(self.op)));
12091233 }
12101234
12111235 pub fn encodeOp(compare_operator: std.math.CompareOperator) u32 {
......@@ -1250,11 +1274,11 @@ pub const Cmpxchg = struct {
12501274 flags: u32,
12511275
12521276 pub fn successOrder(self: Cmpxchg) std.builtin.AtomicOrder {
1253 return @as(std.builtin.AtomicOrder, @enumFromInt(@as(u3, @truncate(self.flags))));
1277 return @enumFromInt(@as(u3, @truncate(self.flags)));
12541278 }
12551279
12561280 pub fn failureOrder(self: Cmpxchg) std.builtin.AtomicOrder {
1257 return @as(std.builtin.AtomicOrder, @enumFromInt(@as(u3, @truncate(self.flags >> 3))));
1281 return @enumFromInt(@as(u3, @intCast(self.flags >> 3)));
12581282 }
12591283};
12601284
......@@ -1265,11 +1289,11 @@ pub const AtomicRmw = struct {
12651289 flags: u32,
12661290
12671291 pub fn ordering(self: AtomicRmw) std.builtin.AtomicOrder {
1268 return @as(std.builtin.AtomicOrder, @enumFromInt(@as(u3, @truncate(self.flags))));
1292 return @enumFromInt(@as(u3, @truncate(self.flags)));
12691293 }
12701294
12711295 pub fn op(self: AtomicRmw) std.builtin.AtomicRmwOp {
1272 return @as(std.builtin.AtomicRmwOp, @enumFromInt(@as(u4, @truncate(self.flags >> 3))));
1296 return @enumFromInt(@as(u4, @intCast(self.flags >> 3)));
12731297 }
12741298};
12751299
src/Compilation.zig+1
......@@ -3067,6 +3067,7 @@ pub fn saveState(comp: *Compilation) !void {
30673067 // linker state
30683068 switch (lf.tag) {
30693069 .wasm => {
3070 dev.check(link.File.Tag.wasm.devFeature());
30703071 const wasm = lf.cast(.wasm).?;
30713072 const is_obj = comp.config.output_mode == .Obj;
30723073 try bufs.ensureUnusedCapacity(85);
src/Liveness.zig+6-27
......@@ -202,14 +202,6 @@ pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool
202202 return (l.tomb_bits[usize_index] & mask) != 0;
203203}
204204
205pub fn clearOperandDeath(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) void {
206 assert(operand < bpi - 1);
207 const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
208 const mask = @as(usize, 1) <<
209 @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + operand));
210 l.tomb_bits[usize_index] &= ~mask;
211}
212
213205const OperandCategory = enum {
214206 /// The operand lives on, but this instruction cannot possibly mutate memory.
215207 none,
......@@ -727,32 +719,25 @@ pub const SwitchBrTable = struct {
727719
728720/// Caller owns the memory.
729721pub fn getSwitchBr(l: Liveness, gpa: Allocator, inst: Air.Inst.Index, cases_len: u32) Allocator.Error!SwitchBrTable {
730 var index: usize = l.special.get(inst) orelse return SwitchBrTable{
731 .deaths = &.{},
732 };
722 var index: usize = l.special.get(inst) orelse return .{ .deaths = &.{} };
733723 const else_death_count = l.extra[index];
734724 index += 1;
735725
736 var deaths = std.ArrayList([]const Air.Inst.Index).init(gpa);
737 defer deaths.deinit();
738 try deaths.ensureTotalCapacity(cases_len + 1);
726 var deaths = try gpa.alloc([]const Air.Inst.Index, cases_len);
727 errdefer gpa.free(deaths);
739728
740729 var case_i: u32 = 0;
741730 while (case_i < cases_len - 1) : (case_i += 1) {
742731 const case_death_count: u32 = l.extra[index];
743732 index += 1;
744 const case_deaths: []const Air.Inst.Index = @ptrCast(l.extra[index..][0..case_death_count]);
733 deaths[case_i] = @ptrCast(l.extra[index..][0..case_death_count]);
745734 index += case_death_count;
746 deaths.appendAssumeCapacity(case_deaths);
747735 }
748736 {
749737 // Else
750 const else_deaths: []const Air.Inst.Index = @ptrCast(l.extra[index..][0..else_death_count]);
751 deaths.appendAssumeCapacity(else_deaths);
738 deaths[case_i] = @ptrCast(l.extra[index..][0..else_death_count]);
752739 }
753 return SwitchBrTable{
754 .deaths = try deaths.toOwnedSlice(),
755 };
740 return .{ .deaths = deaths };
756741}
757742
758743/// Note that this information is technically redundant, but is useful for
......@@ -844,12 +829,6 @@ const Analysis = struct {
844829 special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
845830 extra: std.ArrayListUnmanaged(u32),
846831
847 fn storeTombBits(a: *Analysis, inst: Air.Inst.Index, tomb_bits: Bpi) void {
848 const usize_index = (inst * bpi) / @bitSizeOf(usize);
849 a.tomb_bits[usize_index] |= @as(usize, tomb_bits) <<
850 @as(Log2Int(usize), @intCast((inst % (@bitSizeOf(usize) / bpi)) * bpi));
851 }
852
853832 fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 {
854833 const fields = std.meta.fields(@TypeOf(extra));
855834 try a.extra.ensureUnusedCapacity(a.gpa, fields.len);
src/Type.zig+25-22
......@@ -962,7 +962,6 @@ pub fn abiAlignmentInner(
962962) SemaError!AbiAlignmentInner {
963963 const pt = strat.pt(zcu, tid);
964964 const target = zcu.getTarget();
965 const use_llvm = zcu.comp.config.use_llvm;
966965 const ip = &zcu.intern_pool;
967966
968967 switch (ty.toIntern()) {
......@@ -970,7 +969,7 @@ pub fn abiAlignmentInner(
970969 else => switch (ip.indexToKey(ty.toIntern())) {
971970 .int_type => |int_type| {
972971 if (int_type.bits == 0) return .{ .scalar = .@"1" };
973 return .{ .scalar = intAbiAlignment(int_type.bits, target, use_llvm) };
972 return .{ .scalar = intAbiAlignment(int_type.bits, target) };
974973 },
975974 .ptr_type, .anyframe_type => {
976975 return .{ .scalar = ptrAbiAlignment(target) };
......@@ -1023,7 +1022,7 @@ pub fn abiAlignmentInner(
10231022 .error_set_type, .inferred_error_set_type => {
10241023 const bits = zcu.errorSetBits();
10251024 if (bits == 0) return .{ .scalar = .@"1" };
1026 return .{ .scalar = intAbiAlignment(bits, target, use_llvm) };
1025 return .{ .scalar = intAbiAlignment(bits, target) };
10271026 },
10281027
10291028 // represents machine code; not a pointer
......@@ -1036,7 +1035,7 @@ pub fn abiAlignmentInner(
10361035
10371036 .usize,
10381037 .isize,
1039 => return .{ .scalar = intAbiAlignment(target.ptrBitWidth(), target, use_llvm) },
1038 => return .{ .scalar = intAbiAlignment(target.ptrBitWidth(), target) },
10401039
10411040 .c_char => return .{ .scalar = cTypeAlign(target, .char) },
10421041 .c_short => return .{ .scalar = cTypeAlign(target, .short) },
......@@ -1067,7 +1066,7 @@ pub fn abiAlignmentInner(
10671066 .anyerror, .adhoc_inferred_error_set => {
10681067 const bits = zcu.errorSetBits();
10691068 if (bits == 0) return .{ .scalar = .@"1" };
1070 return .{ .scalar = intAbiAlignment(bits, target, use_llvm) };
1069 return .{ .scalar = intAbiAlignment(bits, target) };
10711070 },
10721071
10731072 .void,
......@@ -1291,7 +1290,6 @@ pub fn abiSizeInner(
12911290 tid: strat.Tid(),
12921291) SemaError!AbiSizeInner {
12931292 const target = zcu.getTarget();
1294 const use_llvm = zcu.comp.config.use_llvm;
12951293 const ip = &zcu.intern_pool;
12961294
12971295 switch (ty.toIntern()) {
......@@ -1300,7 +1298,7 @@ pub fn abiSizeInner(
13001298 else => switch (ip.indexToKey(ty.toIntern())) {
13011299 .int_type => |int_type| {
13021300 if (int_type.bits == 0) return .{ .scalar = 0 };
1303 return .{ .scalar = intAbiSize(int_type.bits, target, use_llvm) };
1301 return .{ .scalar = intAbiSize(int_type.bits, target) };
13041302 },
13051303 .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
13061304 .slice => return .{ .scalar = @divExact(target.ptrBitWidth(), 8) * 2 },
......@@ -1362,7 +1360,7 @@ pub fn abiSizeInner(
13621360 .error_set_type, .inferred_error_set_type => {
13631361 const bits = zcu.errorSetBits();
13641362 if (bits == 0) return .{ .scalar = 0 };
1365 return .{ .scalar = intAbiSize(bits, target, use_llvm) };
1363 return .{ .scalar = intAbiSize(bits, target) };
13661364 },
13671365
13681366 .error_union_type => |error_union_type| {
......@@ -1455,7 +1453,7 @@ pub fn abiSizeInner(
14551453 .anyerror, .adhoc_inferred_error_set => {
14561454 const bits = zcu.errorSetBits();
14571455 if (bits == 0) return .{ .scalar = 0 };
1458 return .{ .scalar = intAbiSize(bits, target, use_llvm) };
1456 return .{ .scalar = intAbiSize(bits, target) };
14591457 },
14601458
14611459 .noreturn => unreachable,
......@@ -1609,11 +1607,11 @@ pub fn ptrAbiAlignment(target: Target) Alignment {
16091607 return Alignment.fromNonzeroByteUnits(@divExact(target.ptrBitWidth(), 8));
16101608}
16111609
1612pub fn intAbiSize(bits: u16, target: Target, use_llvm: bool) u64 {
1613 return intAbiAlignment(bits, target, use_llvm).forward(@as(u16, @intCast((@as(u17, bits) + 7) / 8)));
1610pub fn intAbiSize(bits: u16, target: Target) u64 {
1611 return intAbiAlignment(bits, target).forward(@as(u16, @intCast((@as(u17, bits) + 7) / 8)));
16141612}
16151613
1616pub fn intAbiAlignment(bits: u16, target: Target, use_llvm: bool) Alignment {
1614pub fn intAbiAlignment(bits: u16, target: Target) Alignment {
16171615 return switch (target.cpu.arch) {
16181616 .x86 => switch (bits) {
16191617 0 => .none,
......@@ -1632,19 +1630,16 @@ pub fn intAbiAlignment(bits: u16, target: Target, use_llvm: bool) Alignment {
16321630 9...16 => .@"2",
16331631 17...32 => .@"4",
16341632 33...64 => .@"8",
1635 else => switch (target_util.zigBackend(target, use_llvm)) {
1636 .stage2_x86_64 => .@"8",
1637 else => .@"16",
1638 },
1633 else => .@"16",
16391634 },
16401635 else => return Alignment.fromByteUnits(@min(
16411636 std.math.ceilPowerOfTwoPromote(u16, @as(u16, @intCast((@as(u17, bits) + 7) / 8))),
1642 maxIntAlignment(target, use_llvm),
1637 maxIntAlignment(target),
16431638 )),
16441639 };
16451640}
16461641
1647pub fn maxIntAlignment(target: std.Target, use_llvm: bool) u16 {
1642pub fn maxIntAlignment(target: std.Target) u16 {
16481643 return switch (target.cpu.arch) {
16491644 .avr => 1,
16501645 .msp430 => 2,
......@@ -1685,10 +1680,7 @@ pub fn maxIntAlignment(target: std.Target, use_llvm: bool) u16 {
16851680 else => 8,
16861681 },
16871682
1688 .x86_64 => switch (target_util.zigBackend(target, use_llvm)) {
1689 .stage2_x86_64 => 8,
1690 else => 16,
1691 },
1683 .x86_64 => 16,
16921684
16931685 // Even LLVMABIAlignmentOfType(i128) agrees on these targets.
16941686 .x86,
......@@ -1928,6 +1920,17 @@ pub fn isSlice(ty: Type, zcu: *const Zcu) bool {
19281920 };
19291921}
19301922
1923pub fn isSliceAtRuntime(ty: Type, zcu: *const Zcu) bool {
1924 return switch (zcu.intern_pool.indexToKey(ty.toIntern())) {
1925 .ptr_type => |ptr_type| ptr_type.flags.size == .slice,
1926 .opt_type => |child| switch (zcu.intern_pool.indexToKey(child)) {
1927 .ptr_type => |ptr_type| !ptr_type.flags.is_allowzero and ptr_type.flags.size == .slice,
1928 else => false,
1929 },
1930 else => false,
1931 };
1932}
1933
19311934pub fn slicePtrFieldType(ty: Type, zcu: *const Zcu) Type {
19321935 return Type.fromInterned(zcu.intern_pool.slicePtrType(ty.toIntern()));
19331936}
src/arch/aarch64/CodeGen.zig+10-10
......@@ -71,6 +71,8 @@ end_di_column: u32,
7171/// which is a relative jump, based on the address following the reloc.
7272exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
7373
74reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
75
7476/// We postpone the creation of debug info for function args and locals
7577/// until after all Mir instructions have been generated. Only then we
7678/// will know saved_regs_stack_space which is necessary in order to
......@@ -646,6 +648,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
646648 const old_air_bookkeeping = self.air_bookkeeping;
647649 try self.ensureProcessDeathCapacity(Liveness.bpi);
648650
651 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
649652 switch (air_tags[@intFromEnum(inst)]) {
650653 // zig fmt: off
651654 .add => try self.airBinOp(inst, .add),
......@@ -927,16 +930,13 @@ fn finishAirBookkeeping(self: *Self) void {
927930}
928931
929932fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
930 var tomb_bits = self.liveness.getTombBits(inst);
931 for (operands) |op| {
932 const dies = @as(u1, @truncate(tomb_bits)) != 0;
933 tomb_bits >>= 1;
934 if (!dies) continue;
935 const op_index = op.toIndex() orelse continue;
936 self.processDeath(op_index);
933 const tomb_bits = self.liveness.getTombBits(inst);
934 for (0.., operands) |op_index, op| {
935 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
936 if (self.reused_operands.isSet(op_index)) continue;
937 self.processDeath(op.toIndexAllowNone() orelse continue);
937938 }
938 const is_used = @as(u1, @truncate(tomb_bits)) == 0;
939 if (is_used) {
939 if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
940940 log.debug("%{d} => {}", .{ inst, result });
941941 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
942942 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
......@@ -3614,7 +3614,7 @@ fn reuseOperand(
36143614 }
36153615
36163616 // Prevent the operand deaths processing code from deallocating it.
3617 self.liveness.clearOperandDeath(inst, op_index);
3617 self.reused_operands.set(op_index);
36183618
36193619 // That makes us responsible for doing the rest of the stuff that processDeath would have done.
36203620 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
src/arch/arm/CodeGen.zig+10-10
......@@ -72,6 +72,8 @@ end_di_column: u32,
7272/// which is a relative jump, based on the address following the reloc.
7373exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
7474
75reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
76
7577/// We postpone the creation of debug info for function args and locals
7678/// until after all Mir instructions have been generated. Only then we
7779/// will know saved_regs_stack_space which is necessary in order to
......@@ -635,6 +637,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
635637 const old_air_bookkeeping = self.air_bookkeeping;
636638 try self.ensureProcessDeathCapacity(Liveness.bpi);
637639
640 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
638641 switch (air_tags[@intFromEnum(inst)]) {
639642 // zig fmt: off
640643 .add, => try self.airBinOp(inst, .add),
......@@ -918,16 +921,13 @@ fn finishAirBookkeeping(self: *Self) void {
918921}
919922
920923fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
921 var tomb_bits = self.liveness.getTombBits(inst);
922 for (operands) |op| {
923 const dies = @as(u1, @truncate(tomb_bits)) != 0;
924 tomb_bits >>= 1;
925 if (!dies) continue;
926 const op_index = op.toIndex() orelse continue;
927 self.processDeath(op_index);
924 const tomb_bits = self.liveness.getTombBits(inst);
925 for (0.., operands) |op_index, op| {
926 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
927 if (self.reused_operands.isSet(op_index)) continue;
928 self.processDeath(op.toIndexAllowNone() orelse continue);
928929 }
929 const is_used = @as(u1, @truncate(tomb_bits)) == 0;
930 if (is_used) {
930 if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
931931 log.debug("%{d} => {}", .{ inst, result });
932932 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
933933 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
......@@ -2650,7 +2650,7 @@ fn reuseOperand(
26502650 }
26512651
26522652 // Prevent the operand deaths processing code from deallocating it.
2653 self.liveness.clearOperandDeath(inst, op_index);
2653 self.reused_operands.set(op_index);
26542654
26552655 // That makes us responsible for doing the rest of the stuff that processDeath would have done.
26562656 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
src/arch/riscv64/CodeGen.zig+11-7
......@@ -82,6 +82,8 @@ scope_generation: u32,
8282/// which is a relative jump, based on the address following the reloc.
8383exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
8484
85reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
86
8587/// Whenever there is a runtime branch, we push a Branch onto this stack,
8688/// and pop it off when the runtime branch joins. This provides an "overlay"
8789/// of the table of mappings from instructions to `MCValue` from within the branch.
......@@ -1443,8 +1445,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
14431445 verbose_tracking_log.debug("{}", .{func.fmtTracking()});
14441446
14451447 const old_air_bookkeeping = func.air_bookkeeping;
1448 try func.ensureProcessDeathCapacity(Liveness.bpi);
1449
1450 func.reused_operands = @TypeOf(func.reused_operands).initEmpty();
14461451 try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1);
1447 const tag: Air.Inst.Tag = air_tags[@intFromEnum(inst)];
1452 const tag = air_tags[@intFromEnum(inst)];
14481453 switch (tag) {
14491454 // zig fmt: off
14501455 .add,
......@@ -1783,11 +1788,10 @@ fn finishAir(
17831788 result: MCValue,
17841789 operands: [Liveness.bpi - 1]Air.Inst.Ref,
17851790) !void {
1786 var tomb_bits = func.liveness.getTombBits(inst);
1787 for (operands) |op| {
1788 const dies = @as(u1, @truncate(tomb_bits)) != 0;
1789 tomb_bits >>= 1;
1790 if (!dies) continue;
1791 const tomb_bits = func.liveness.getTombBits(inst);
1792 for (0.., operands) |op_index, op| {
1793 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
1794 if (func.reused_operands.isSet(op_index)) continue;
17911795 try func.processDeath(op.toIndexAllowNone() orelse continue);
17921796 }
17931797 func.finishAirResult(inst, result);
......@@ -4424,7 +4428,7 @@ fn reuseOperandAdvanced(
44244428 }
44254429
44264430 // Prevent the operand deaths processing code from deallocating it.
4427 func.liveness.clearOperandDeath(inst, op_index);
4431 func.reused_operands.set(op_index);
44284432 const op_inst = operand.toIndex().?;
44294433 func.getResolvedInstValue(op_inst).reuse(func, maybe_tracked_inst, op_inst);
44304434
src/arch/sparc64/CodeGen.zig+10-10
......@@ -78,6 +78,8 @@ end_di_column: u32,
7878/// which is a relative jump, based on the address following the reloc.
7979exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
8080
81reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
82
8183/// Whenever there is a runtime branch, we push a Branch onto this stack,
8284/// and pop it off when the runtime branch joins. This provides an "overlay"
8385/// of the table of mappings from instructions to `MCValue` from within the branch.
......@@ -493,6 +495,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
493495 const old_air_bookkeeping = self.air_bookkeeping;
494496 try self.ensureProcessDeathCapacity(Liveness.bpi);
495497
498 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
496499 switch (air_tags[@intFromEnum(inst)]) {
497500 // zig fmt: off
498501 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
......@@ -3523,16 +3526,13 @@ fn finishAirBookkeeping(self: *Self) void {
35233526}
35243527
35253528fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
3526 var tomb_bits = self.liveness.getTombBits(inst);
3527 for (operands) |op| {
3528 const dies = @as(u1, @truncate(tomb_bits)) != 0;
3529 tomb_bits >>= 1;
3530 if (!dies) continue;
3531 const op_index = op.toIndex() orelse continue;
3532 self.processDeath(op_index);
3529 const tomb_bits = self.liveness.getTombBits(inst);
3530 for (0.., operands) |op_index, op| {
3531 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
3532 if (self.reused_operands.isSet(op_index)) continue;
3533 self.processDeath(op.toIndexAllowNone() orelse continue);
35333534 }
3534 const is_used = @as(u1, @truncate(tomb_bits)) == 0;
3535 if (is_used) {
3535 if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
35363536 log.debug("%{d} => {}", .{ inst, result });
35373537 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
35383538 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
......@@ -4568,7 +4568,7 @@ fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_ind
45684568 }
45694569
45704570 // Prevent the operand deaths processing code from deallocating it.
4571 self.liveness.clearOperandDeath(inst, op_index);
4571 self.reused_operands.set(op_index);
45724572
45734573 // That makes us responsible for doing the rest of the stuff that processDeath would have done.
45744574 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
src/arch/x86_64/CodeGen.zig+14357-4049
......@@ -1,41 +1,26 @@
11const std = @import("std");
2const build_options = @import("build_options");
3const builtin = @import("builtin");
42const assert = std.debug.assert;
53const codegen = @import("../../codegen.zig");
6const leb128 = std.leb;
74const link = @import("../../link.zig");
85const log = std.log.scoped(.codegen);
96const tracking_log = std.log.scoped(.tracking);
107const verbose_tracking_log = std.log.scoped(.verbose_tracking);
118const wip_mir_log = std.log.scoped(.wip_mir);
12const math = std.math;
13const mem = std.mem;
14const target_util = @import("../../target.zig");
15const trace = @import("../../tracy.zig").trace;
169
1710const Air = @import("../../Air.zig");
18const Allocator = mem.Allocator;
19const CodeGenError = codegen.CodeGenError;
20const Compilation = @import("../../Compilation.zig");
21const ErrorMsg = Zcu.ErrorMsg;
11const Allocator = std.mem.Allocator;
2212const Emit = @import("Emit.zig");
2313const Liveness = @import("../../Liveness.zig");
2414const Lower = @import("Lower.zig");
2515const Mir = @import("Mir.zig");
26const Package = @import("../../Package.zig");
2716const Zcu = @import("../../Zcu.zig");
17const Module = @import("../../Package/Module.zig");
2818const InternPool = @import("../../InternPool.zig");
29const Alignment = InternPool.Alignment;
30const Target = std.Target;
3119const Type = @import("../../Type.zig");
3220const Value = @import("../../Value.zig");
33const Instruction = @import("encoder.zig").Instruction;
3421
3522const abi = @import("abi.zig");
3623const bits = @import("bits.zig");
37const errUnionErrorOffset = codegen.errUnionErrorOffset;
38const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
3924const encoder = @import("encoder.zig");
4025
4126const Condition = bits.Condition;
......@@ -46,7 +31,7 @@ const RegisterManager = abi.RegisterManager;
4631const RegisterLock = RegisterManager.RegisterLock;
4732const FrameIndex = bits.FrameIndex;
4833
49const InnerError = CodeGenError || error{OutOfRegisters};
34const InnerError = codegen.CodeGenError || error{OutOfRegisters};
5035
5136gpa: Allocator,
5237pt: Zcu.PerThread,
......@@ -57,7 +42,7 @@ debug_output: link.File.DebugInfoOutput,
5742target: *const std.Target,
5843owner: Owner,
5944inline_func: InternPool.Index,
60mod: *Package.Module,
45mod: *Module,
6146arg_index: u32,
6247args: []MCValue,
6348va_info: union {
......@@ -76,9 +61,10 @@ src_loc: Zcu.LazySrcLoc,
7661eflags_inst: ?Air.Inst.Index = null,
7762
7863/// MIR Instructions
79mir_instructions: std.MultiArrayList(Mir.Inst) = .{},
64mir_instructions: std.MultiArrayList(Mir.Inst) = .empty,
8065/// MIR extra data
8166mir_extra: std.ArrayListUnmanaged(u32) = .empty,
67mir_table: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
8268
8369/// Byte offset within the source file of the ending curly.
8470end_di_line: u32,
......@@ -87,10 +73,11 @@ end_di_column: u32,
8773/// The value is an offset into the `Function` `code` from the beginning.
8874/// To perform the reloc, write 32-bit signed little-endian integer
8975/// which is a relative jump, based on the address following the reloc.
90exitlude_jump_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
76epilogue_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
9177
92const_tracking: ConstTrackingMap = .{},
93inst_tracking: InstTrackingMap = .{},
78reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
79const_tracking: ConstTrackingMap = .empty,
80inst_tracking: InstTrackingMap = .empty,
9481
9582// Key is the block instruction
9683blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty,
......@@ -100,27 +87,35 @@ register_manager: RegisterManager = .{},
10087/// Generation of the current scope, increments by 1 for every entered scope.
10188scope_generation: u32 = 0,
10289
103frame_allocs: std.MultiArrayList(FrameAlloc) = .{},
90frame_allocs: std.MultiArrayList(FrameAlloc) = .empty,
10491free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .empty,
105frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{},
92frame_locs: std.MultiArrayList(Mir.FrameLoc) = .empty,
10693
10794loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
10895 /// The state to restore before branching.
10996 state: State,
11097 /// The branch target.
111 jmp_target: Mir.Inst.Index,
112}) = .{},
113
114/// Debug field, used to find bugs in the compiler.
115air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
98 target: Mir.Inst.Index,
99}) = .empty,
100loop_switches: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
101 start: u31,
102 len: u11,
103 min: Value,
104 else_relocs: union(enum) {
105 @"unreachable",
106 forward: std.ArrayListUnmanaged(Mir.Inst.Index),
107 backward: Mir.Inst.Index,
108 },
109}) = .empty,
116110
117const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
111next_temp_index: Temp.Index = @enumFromInt(0),
112temp_type: [Temp.Index.max]Type = undefined,
118113
119114const Owner = union(enum) {
120115 nav_index: InternPool.Nav.Index,
121116 lazy_sym: link.File.LazySymbol,
122117
123 fn getSymbolIndex(owner: Owner, ctx: *Self) !u32 {
118 fn getSymbolIndex(owner: Owner, ctx: *CodeGen) !u32 {
124119 const pt = ctx.pt;
125120 switch (owner) {
126121 .nav_index => |nav_index| if (ctx.bin_file.cast(.elf)) |elf_file| {
......@@ -151,6 +146,9 @@ const Owner = union(enum) {
151146 }
152147};
153148
149const MaskKind = enum(u1) { sign, all };
150const MaskInfo = packed struct { kind: MaskKind, inverted: bool, scalar: Memory.Size };
151
154152pub const MCValue = union(enum) {
155153 /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
156154 /// TODO Look into deleting this tag and using `dead` instead, since every use
......@@ -172,10 +170,16 @@ pub const MCValue = union(enum) {
172170 register: Register,
173171 /// The value is split across two registers.
174172 register_pair: [2]Register,
173 /// The value is split across three registers.
174 register_triple: [3]Register,
175 /// The value is split across four registers.
176 register_quadruple: [4]Register,
175177 /// The value is a constant offset from the value in a register.
176178 register_offset: bits.RegisterOffset,
177179 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.
178180 register_overflow: struct { reg: Register, eflags: Condition },
181 /// The value is a bool vector stored in a vector register with a different scalar type.
182 register_mask: struct { reg: Register, info: MaskInfo },
179183 /// The value is in memory at a hard-coded address.
180184 /// If the type is a pointer, it means the pointer address is stored at this memory location.
181185 memory: u64,
......@@ -211,7 +215,7 @@ pub const MCValue = union(enum) {
211215 /// Payload is a frame address.
212216 lea_frame: bits.FrameAddr,
213217 /// Supports integer_per_element abi
214 elementwise_regs_then_frame: packed struct { regs: u3 = 0, frame_off: i29 = 0, frame_index: FrameIndex },
218 elementwise_regs_then_frame: packed struct { regs: u3, frame_off: i29, frame_index: FrameIndex },
215219 /// This indicates that we have already allocated a frame index for this instruction,
216220 /// but it has not been spilled there yet in the current control flow.
217221 /// Payload is a frame index.
......@@ -226,6 +230,7 @@ pub const MCValue = union(enum) {
226230 .undef,
227231 .immediate,
228232 .register_offset,
233 .register_mask,
229234 .eflags,
230235 .register_overflow,
231236 .lea_symbol,
......@@ -239,6 +244,8 @@ pub const MCValue = union(enum) {
239244 => false,
240245 .register,
241246 .register_pair,
247 .register_triple,
248 .register_quadruple,
242249 .memory,
243250 .load_symbol,
244251 .load_got,
......@@ -250,13 +257,21 @@ pub const MCValue = union(enum) {
250257 };
251258 }
252259
253 fn isMemory(mcv: MCValue) bool {
260 // hack around linker relocation bugs
261 fn isBase(mcv: MCValue) bool {
254262 return switch (mcv) {
255263 .memory, .indirect, .load_frame => true,
256264 else => false,
257265 };
258266 }
259267
268 fn isMemory(mcv: MCValue) bool {
269 return switch (mcv) {
270 .memory, .indirect, .load_frame, .load_symbol => true,
271 else => false,
272 };
273 }
274
260275 fn isImmediate(mcv: MCValue) bool {
261276 return switch (mcv) {
262277 .immediate => true,
......@@ -284,16 +299,23 @@ pub const MCValue = union(enum) {
284299 .register => |reg| reg,
285300 .register_offset, .indirect => |ro| ro.reg,
286301 .register_overflow => |ro| ro.reg,
302 .register_mask => |rm| rm.reg,
287303 else => null,
288304 };
289305 }
290306
291307 fn getRegs(mcv: *const MCValue) []const Register {
292308 return switch (mcv.*) {
293 .register => |*reg| @as(*const [1]Register, reg),
294 .register_pair => |*regs| regs,
295 .register_offset, .indirect => |*ro| @as(*const [1]Register, &ro.reg),
296 .register_overflow => |*ro| @as(*const [1]Register, &ro.reg),
309 .register => |*reg| reg[0..1],
310 inline .register_pair,
311 .register_triple,
312 .register_quadruple,
313 => |*regs| regs,
314 inline .register_offset,
315 .indirect,
316 .register_overflow,
317 .register_mask,
318 => |*pl| (&pl.reg)[0..1],
297319 else => &.{},
298320 };
299321 }
......@@ -306,6 +328,13 @@ pub const MCValue = union(enum) {
306328 };
307329 }
308330
331 fn isAddress(mcv: MCValue) bool {
332 return switch (mcv) {
333 .immediate, .register, .register_offset, .lea_frame => true,
334 else => false,
335 };
336 }
337
309338 fn address(mcv: MCValue) MCValue {
310339 return switch (mcv) {
311340 .none,
......@@ -316,8 +345,11 @@ pub const MCValue = union(enum) {
316345 .eflags,
317346 .register,
318347 .register_pair,
348 .register_triple,
349 .register_quadruple,
319350 .register_offset,
320351 .register_overflow,
352 .register_mask,
321353 .lea_symbol,
322354 .lea_direct,
323355 .lea_got,
......@@ -348,7 +380,10 @@ pub const MCValue = union(enum) {
348380 .undef,
349381 .eflags,
350382 .register_pair,
383 .register_triple,
384 .register_quadruple,
351385 .register_overflow,
386 .register_mask,
352387 .memory,
353388 .indirect,
354389 .load_direct,
......@@ -383,7 +418,10 @@ pub const MCValue = union(enum) {
383418 => unreachable, // not valid
384419 .eflags,
385420 .register_pair,
421 .register_triple,
422 .register_quadruple,
386423 .register_overflow,
424 .register_mask,
387425 .memory,
388426 .indirect,
389427 .load_direct,
......@@ -410,7 +448,7 @@ pub const MCValue = union(enum) {
410448 };
411449 }
412450
413 fn mem(mcv: MCValue, function: *Self, size: Memory.Size) !Memory {
451 fn mem(mcv: MCValue, function: *CodeGen, mod_rm: Memory.Mod.Rm) !Memory {
414452 return switch (mcv) {
415453 .none,
416454 .unreach,
......@@ -420,8 +458,11 @@ pub const MCValue = union(enum) {
420458 .eflags,
421459 .register,
422460 .register_pair,
461 .register_triple,
462 .register_quadruple,
423463 .register_offset,
424464 .register_overflow,
465 .register_mask,
425466 .load_direct,
426467 .lea_direct,
427468 .load_got,
......@@ -433,25 +474,31 @@ pub const MCValue = union(enum) {
433474 .reserved_frame,
434475 .lea_symbol,
435476 => unreachable,
436 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{
477 .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{
437478 .base = .{ .reg = .ds },
438479 .mod = .{ .rm = .{
439 .size = size,
440 .disp = small_addr,
480 .size = mod_rm.size,
481 .index = mod_rm.index,
482 .scale = mod_rm.scale,
483 .disp = small_addr + mod_rm.disp,
441484 } },
442485 } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } },
443486 .indirect => |reg_off| .{
444 .base = .{ .reg = reg_off.reg },
487 .base = .{ .reg = registerAlias(reg_off.reg, @divExact(function.target.ptrBitWidth(), 8)) },
445488 .mod = .{ .rm = .{
446 .size = size,
447 .disp = reg_off.off,
489 .size = mod_rm.size,
490 .index = mod_rm.index,
491 .scale = mod_rm.scale,
492 .disp = reg_off.off + mod_rm.disp,
448493 } },
449494 },
450495 .load_frame => |frame_addr| .{
451496 .base = .{ .frame = frame_addr.index },
452497 .mod = .{ .rm = .{
453 .size = size,
454 .disp = frame_addr.off,
498 .size = mod_rm.size,
499 .index = mod_rm.index,
500 .scale = mod_rm.scale,
501 .disp = frame_addr.off + mod_rm.disp,
455502 } },
456503 },
457504 .load_symbol => |sym_off| {
......@@ -459,12 +506,14 @@ pub const MCValue = union(enum) {
459506 return .{
460507 .base = .{ .reloc = sym_off.sym_index },
461508 .mod = .{ .rm = .{
462 .size = size,
463 .disp = sym_off.off,
509 .size = mod_rm.size,
510 .index = mod_rm.index,
511 .scale = mod_rm.scale,
512 .disp = sym_off.off + mod_rm.disp,
464513 } },
465514 };
466515 },
467 .air_ref => |ref| (try function.resolveInst(ref)).mem(function, size),
516 .air_ref => |ref| (try function.resolveInst(ref)).mem(function, mod_rm),
468517 };
469518 }
470519
......@@ -480,12 +529,25 @@ pub const MCValue = union(enum) {
480529 .memory => |pl| try writer.print("[ds:0x{x}]", .{pl}),
481530 inline .eflags, .register => |pl| try writer.print("{s}", .{@tagName(pl)}),
482531 .register_pair => |pl| try writer.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }),
532 .register_triple => |pl| try writer.print("{s}:{s}:{s}", .{
533 @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]),
534 }),
535 .register_quadruple => |pl| try writer.print("{s}:{s}:{s}:{s}", .{
536 @tagName(pl[3]), @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]),
537 }),
483538 .register_offset => |pl| try writer.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }),
484539 .register_overflow => |pl| try writer.print("{s}:{s}", .{
485 @tagName(pl.eflags), @tagName(pl.reg),
540 @tagName(pl.eflags),
541 @tagName(pl.reg),
486542 }),
487 .load_symbol => |pl| try writer.print("[{} + 0x{x}]", .{ pl.sym_index, pl.off }),
488 .lea_symbol => |pl| try writer.print("{} + 0x{x}", .{ pl.sym_index, pl.off }),
543 .register_mask => |pl| try writer.print("mask({s},{}):{c}{s}", .{
544 @tagName(pl.info.kind),
545 pl.info.scalar,
546 @as(u8, if (pl.info.inverted) '!' else ' '),
547 @tagName(pl.reg),
548 }),
549 .load_symbol => |pl| try writer.print("[sym:{} + 0x{x}]", .{ pl.sym_index, pl.off }),
550 .lea_symbol => |pl| try writer.print("sym:{} + 0x{x}", .{ pl.sym_index, pl.off }),
489551 .indirect => |pl| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
490552 .load_direct => |pl| try writer.print("[direct:{d}]", .{pl}),
491553 .lea_direct => |pl| try writer.print("direct:{d}", .{pl}),
......@@ -494,7 +556,9 @@ pub const MCValue = union(enum) {
494556 .load_tlv => |pl| try writer.print("[tlv:{d}]", .{pl}),
495557 .lea_tlv => |pl| try writer.print("tlv:{d}", .{pl}),
496558 .load_frame => |pl| try writer.print("[{} + 0x{x}]", .{ pl.index, pl.off }),
497 .elementwise_regs_then_frame => |pl| try writer.print("elementwise:{d}:[{} + 0x{x}]", .{ pl.regs, pl.frame_index, pl.frame_off }),
559 .elementwise_regs_then_frame => |pl| try writer.print("elementwise:{d}:[{} + 0x{x}]", .{
560 pl.regs, pl.frame_index, pl.frame_off,
561 }),
498562 .lea_frame => |pl| try writer.print("{} + 0x{x}", .{ pl.index, pl.off }),
499563 .reserved_frame => |pl| try writer.print("(dead:{})", .{pl}),
500564 .air_ref => |pl| try writer.print("(air:0x{x})", .{@intFromEnum(pl)}),
......@@ -534,8 +598,11 @@ const InstTracking = struct {
534598 .eflags,
535599 .register,
536600 .register_pair,
601 .register_triple,
602 .register_quadruple,
537603 .register_offset,
538604 .register_overflow,
605 .register_mask,
539606 .indirect,
540607 => .none,
541608 }, .short = result };
......@@ -553,64 +620,48 @@ const InstTracking = struct {
553620 return self.short.getCondition();
554621 }
555622
556 fn spill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {
623 fn spill(self: *InstTracking, cg: *CodeGen, inst: Air.Inst.Index) !void {
557624 if (std.meta.eql(self.long, self.short)) return; // Already spilled
558625 // Allocate or reuse frame index
559626 switch (self.long) {
560 .none => self.long = try function.allocRegOrMem(inst, false),
627 .none => self.long = try cg.allocRegOrMem(inst, false),
561628 .load_frame => {},
562629 .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } },
563630 else => unreachable,
564631 }
565 tracking_log.debug("spill %{d} from {} to {}", .{ inst, self.short, self.long });
566 try function.genCopy(function.typeOfIndex(inst), self.long, self.short, .{});
632 tracking_log.debug("spill {} from {} to {}", .{ inst, self.short, self.long });
633 try cg.genCopy(cg.typeOfIndex(inst), self.long, self.short, .{});
567634 }
568635
569636 fn reuseFrame(self: *InstTracking) void {
570 switch (self.long) {
571 .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } },
572 else => {},
573 }
574 self.short = switch (self.long) {
575 .none,
576 .unreach,
577 .undef,
578 .immediate,
579 .memory,
580 .load_direct,
581 .lea_direct,
582 .load_got,
583 .lea_got,
584 .load_tlv,
585 .lea_tlv,
586 .load_frame,
587 .lea_frame,
588 .load_symbol,
589 .lea_symbol,
590 => self.long,
591 .dead,
592 .eflags,
593 .register,
594 .register_pair,
595 .register_offset,
596 .register_overflow,
597 .indirect,
598 .elementwise_regs_then_frame,
599 .reserved_frame,
600 .air_ref,
601 => unreachable,
602 };
637 self.* = .init(switch (self.long) {
638 .none => switch (self.short) {
639 .dead => .none,
640 else => |short| short,
641 },
642 .reserved_frame => |index| .{ .load_frame = .{ .index = index } },
643 else => |long| long,
644 });
603645 }
604646
605 fn trackSpill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {
647 fn trackSpill(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void {
606648 try function.freeValue(self.short);
607649 self.reuseFrame();
608 tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* });
650 tracking_log.debug("{} => {} (spilled)", .{ inst, self.* });
609651 }
610652
611653 fn verifyMaterialize(self: InstTracking, target: InstTracking) void {
612654 switch (self.long) {
613655 .none,
656 .load_frame,
657 .reserved_frame,
658 => switch (target.long) {
659 .none,
660 .load_frame,
661 .reserved_frame,
662 => {},
663 else => unreachable,
664 },
614665 .unreach,
615666 .undef,
616667 .immediate,
......@@ -625,21 +676,15 @@ const InstTracking = struct {
625676 .load_symbol,
626677 .lea_symbol,
627678 => assert(std.meta.eql(self.long, target.long)),
628 .load_frame,
629 .reserved_frame,
630 => switch (target.long) {
631 .none,
632 .load_frame,
633 .reserved_frame,
634 => {},
635 else => unreachable,
636 },
637679 .dead,
638680 .eflags,
639681 .register,
640682 .register_pair,
683 .register_triple,
684 .register_quadruple,
641685 .register_offset,
642686 .register_overflow,
687 .register_mask,
643688 .indirect,
644689 .elementwise_regs_then_frame,
645690 .air_ref,
......@@ -649,7 +694,7 @@ const InstTracking = struct {
649694
650695 fn materialize(
651696 self: *InstTracking,
652 function: *Self,
697 function: *CodeGen,
653698 inst: Air.Inst.Index,
654699 target: InstTracking,
655700 ) !void {
......@@ -659,7 +704,7 @@ const InstTracking = struct {
659704
660705 fn materializeUnsafe(
661706 self: InstTracking,
662 function: *Self,
707 function: *CodeGen,
663708 inst: Air.Inst.Index,
664709 target: InstTracking,
665710 ) !void {
......@@ -678,43 +723,42 @@ const InstTracking = struct {
678723 else => target.long,
679724 } else target.long;
680725 self.short = target.short;
681 tracking_log.debug("%{d} => {} (materialize)", .{ inst, self.* });
726 tracking_log.debug("{} => {} (materialize)", .{ inst, self.* });
682727 }
683728
684 fn resurrect(self: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {
729 fn resurrect(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index, scope_generation: u32) !void {
685730 switch (self.short) {
686731 .dead => |die_generation| if (die_generation >= scope_generation) {
687732 self.reuseFrame();
688 tracking_log.debug("%{d} => {} (resurrect)", .{ inst, self.* });
733 try function.getValue(self.short, inst);
734 tracking_log.debug("{} => {} (resurrect)", .{ inst, self.* });
689735 },
690736 else => {},
691737 }
692738 }
693739
694 fn die(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {
740 fn die(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void {
695741 if (self.short == .dead) return;
696742 try function.freeValue(self.short);
743 if (self.long == .none) self.long = self.short;
697744 self.short = .{ .dead = function.scope_generation };
698 tracking_log.debug("%{d} => {} (death)", .{ inst, self.* });
745 tracking_log.debug("{} => {} (death)", .{ inst, self.* });
699746 }
700747
701748 fn reuse(
702749 self: *InstTracking,
703 function: *Self,
750 function: *CodeGen,
704751 new_inst: ?Air.Inst.Index,
705752 old_inst: Air.Inst.Index,
706753 ) void {
707754 self.short = .{ .dead = function.scope_generation };
708 if (new_inst) |inst|
709 tracking_log.debug("%{d} => {} (reuse %{d})", .{ inst, self.*, old_inst })
710 else
711 tracking_log.debug("tmp => {} (reuse %{d})", .{ self.*, old_inst });
755 tracking_log.debug("{?} => {} (reuse {})", .{ new_inst, self.*, old_inst });
712756 }
713757
714 fn liveOut(self: *InstTracking, function: *Self, inst: Air.Inst.Index) void {
758 fn liveOut(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) void {
715759 for (self.getRegs()) |reg| {
716760 if (function.register_manager.isRegFree(reg)) {
717 tracking_log.debug("%{d} => {} (live-out)", .{ inst, self.* });
761 tracking_log.debug("{} => {} (live-out)", .{ inst, self.* });
718762 continue;
719763 }
720764
......@@ -741,28 +785,28 @@ const InstTracking = struct {
741785 // Perform side-effects of freeValue manually.
742786 function.register_manager.freeReg(reg);
743787
744 tracking_log.debug("%{d} => {} (live-out %{d})", .{ inst, self.*, tracked_inst });
788 tracking_log.debug("{} => {} (live-out {})", .{ inst, self.*, tracked_inst });
745789 }
746790 }
747791
748792 pub fn format(
749 self: InstTracking,
793 tracking: InstTracking,
750794 comptime _: []const u8,
751795 _: std.fmt.FormatOptions,
752796 writer: anytype,
753797 ) @TypeOf(writer).Error!void {
754 if (!std.meta.eql(self.long, self.short)) try writer.print("|{}| ", .{self.long});
755 try writer.print("{}", .{self.short});
798 if (!std.meta.eql(tracking.long, tracking.short)) try writer.print("|{}| ", .{tracking.long});
799 try writer.print("{}", .{tracking.short});
756800 }
757801};
758802
759803const FrameAlloc = struct {
760804 abi_size: u31,
761805 spill_pad: u3,
762 abi_align: Alignment,
806 abi_align: InternPool.Alignment,
763807 ref_count: u16,
764808
765 fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc {
809 fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: InternPool.Alignment }) FrameAlloc {
766810 return .{
767811 .abi_size = @intCast(alloc_abi.size),
768812 .spill_pad = alloc_abi.pad,
......@@ -779,14 +823,14 @@ const FrameAlloc = struct {
779823 fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc {
780824 const abi_size = ty.abiSize(zcu);
781825 const spill_size = if (abi_size < 8)
782 math.ceilPowerOfTwoAssert(u64, abi_size)
826 std.math.ceilPowerOfTwoAssert(u64, abi_size)
783827 else
784828 std.mem.alignForward(u64, abi_size, 8);
785829 return init(.{
786830 .size = spill_size,
787831 .pad = @intCast(spill_size - abi_size),
788832 .alignment = ty.abiAlignment(zcu).maxStrict(
789 Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
833 .fromNonzeroByteUnits(@min(spill_size, 8)),
790834 ),
791835 });
792836 }
......@@ -808,7 +852,7 @@ const BlockData = struct {
808852 }
809853};
810854
811const Self = @This();
855const CodeGen = @This();
812856
813857pub fn generate(
814858 bin_file: *link.File,
......@@ -819,16 +863,16 @@ pub fn generate(
819863 liveness: Liveness,
820864 code: *std.ArrayListUnmanaged(u8),
821865 debug_output: link.File.DebugInfoOutput,
822) CodeGenError!void {
866) codegen.CodeGenError!void {
823867 const zcu = pt.zcu;
824868 const comp = zcu.comp;
825869 const gpa = zcu.gpa;
826870 const ip = &zcu.intern_pool;
827871 const func = zcu.funcInfo(func_index);
828 const fn_type = Type.fromInterned(func.ty);
872 const fn_type: Type = .fromInterned(func.ty);
829873 const mod = zcu.navFileScope(func.owner_nav).mod;
830874
831 var function: Self = .{
875 var function: CodeGen = .{
832876 .gpa = gpa,
833877 .pt = pt,
834878 .air = air,
......@@ -853,14 +897,21 @@ pub fn generate(
853897 function.free_frame_indices.deinit(gpa);
854898 function.frame_locs.deinit(gpa);
855899 function.loops.deinit(gpa);
900 function.loop_switches.deinit(gpa);
856901 var block_it = function.blocks.valueIterator();
857902 while (block_it.next()) |block| block.deinit(gpa);
858903 function.blocks.deinit(gpa);
859904 function.inst_tracking.deinit(gpa);
860905 function.const_tracking.deinit(gpa);
861 function.exitlude_jump_relocs.deinit(gpa);
906 function.epilogue_relocs.deinit(gpa);
862907 function.mir_instructions.deinit(gpa);
863908 function.mir_extra.deinit(gpa);
909 function.mir_table.deinit(gpa);
910 }
911 try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max);
912 for (0..Temp.Index.max) |temp_index| {
913 const temp: Temp.Index = @enumFromInt(temp_index);
914 function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), .init(.none));
864915 }
865916
866917 wip_mir_log.debug("{}:", .{fmtNav(func.owner_nav, ip)});
......@@ -868,15 +919,14 @@ pub fn generate(
868919 try function.frame_allocs.resize(gpa, FrameIndex.named_count);
869920 function.frame_allocs.set(
870921 @intFromEnum(FrameIndex.stack_frame),
871 FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),
922 .init(.{ .size = 0, .alignment = .@"1" }),
872923 );
873924 function.frame_allocs.set(
874925 @intFromEnum(FrameIndex.call_frame),
875 FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),
926 .init(.{ .size = 0, .alignment = .@"1" }),
876927 );
877928
878929 const fn_info = zcu.typeToFunc(fn_type).?;
879 const cc = abi.resolveCallingConvention(fn_info.cc, function.target.*);
880930 var call_info = function.resolveCallingConventionValues(fn_info, &.{}, .args_frame) catch |err| switch (err) {
881931 error.CodegenFail => return error.CodegenFail,
882932 else => |e| return e,
......@@ -885,25 +935,24 @@ pub fn generate(
885935
886936 function.args = call_info.args;
887937 function.ret_mcv = call_info.return_value;
888 function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), FrameAlloc.init(.{
938 function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), .init(.{
889939 .size = Type.usize.abiSize(zcu),
890940 .alignment = Type.usize.abiAlignment(zcu).min(call_info.stack_align),
891941 }));
892 function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{
942 function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), .init(.{
893943 .size = Type.usize.abiSize(zcu),
894 .alignment = Alignment.min(
895 call_info.stack_align,
896 Alignment.fromNonzeroByteUnits(function.target.stackAlignment()),
944 .alignment = call_info.stack_align.min(
945 .fromNonzeroByteUnits(function.target.stackAlignment()),
897946 ),
898947 }));
899948 function.frame_allocs.set(
900949 @intFromEnum(FrameIndex.args_frame),
901 FrameAlloc.init(.{
950 .init(.{
902951 .size = call_info.stack_byte_count,
903952 .alignment = call_info.stack_align,
904953 }),
905954 );
906 function.va_info = switch (cc) {
955 function.va_info = switch (fn_info.cc) {
907956 else => undefined,
908957 .x86_64_sysv => .{ .sysv = .{
909958 .gp_count = call_info.gp_count,
......@@ -923,6 +972,7 @@ pub fn generate(
923972 var mir: Mir = .{
924973 .instructions = function.mir_instructions.toOwnedSlice(),
925974 .extra = try function.mir_extra.toOwnedSlice(gpa),
975 .table = try function.mir_table.toOwnedSlice(gpa),
926976 .frame_locs = function.frame_locs.toOwnedSlice(),
927977 };
928978 defer mir.deinit(gpa);
......@@ -931,9 +981,10 @@ pub fn generate(
931981 .air = function.air,
932982 .lower = .{
933983 .bin_file = bin_file,
984 .target = function.target,
934985 .allocator = gpa,
935986 .mir = mir,
936 .cc = cc,
987 .cc = fn_info.cc,
937988 .src_loc = src_loc,
938989 .output_mode = comp.config.output_mode,
939990 .link_mode = comp.config.link_mode,
......@@ -956,7 +1007,6 @@ pub fn generate(
9561007 },
9571008 .prev_di_pc = 0,
9581009 };
959 defer emit.deinit();
9601010 emit.emitMir() catch |err| switch (err) {
9611011 error.LowerFail, error.EmitFail => return function.failMsg(emit.lower.err_msg.?),
9621012
......@@ -972,12 +1022,12 @@ pub fn generateLazy(
9721022 lazy_sym: link.File.LazySymbol,
9731023 code: *std.ArrayListUnmanaged(u8),
9741024 debug_output: link.File.DebugInfoOutput,
975) CodeGenError!void {
1025) codegen.CodeGenError!void {
9761026 const comp = bin_file.comp;
9771027 const gpa = comp.gpa;
9781028 // This function is for generating global code, so we use the root module.
9791029 const mod = comp.root_mod;
980 var function: Self = .{
1030 var function: CodeGen = .{
9811031 .gpa = gpa,
9821032 .pt = pt,
9831033 .air = undefined,
......@@ -1000,6 +1050,7 @@ pub fn generateLazy(
10001050 defer {
10011051 function.mir_instructions.deinit(gpa);
10021052 function.mir_extra.deinit(gpa);
1053 function.mir_table.deinit(gpa);
10031054 }
10041055
10051056 function.genLazy(lazy_sym) catch |err| switch (err) {
......@@ -1011,6 +1062,7 @@ pub fn generateLazy(
10111062 var mir: Mir = .{
10121063 .instructions = function.mir_instructions.toOwnedSlice(),
10131064 .extra = try function.mir_extra.toOwnedSlice(gpa),
1065 .table = try function.mir_table.toOwnedSlice(gpa),
10141066 .frame_locs = function.frame_locs.toOwnedSlice(),
10151067 };
10161068 defer mir.deinit(gpa);
......@@ -1019,9 +1071,10 @@ pub fn generateLazy(
10191071 .air = function.air,
10201072 .lower = .{
10211073 .bin_file = bin_file,
1074 .target = function.target,
10221075 .allocator = gpa,
10231076 .mir = mir,
1024 .cc = abi.resolveCallingConvention(.auto, function.target.*),
1077 .cc = .auto,
10251078 .src_loc = src_loc,
10261079 .output_mode = comp.config.output_mode,
10271080 .link_mode = comp.config.link_mode,
......@@ -1036,7 +1089,6 @@ pub fn generateLazy(
10361089 .prev_di_loc = undefined, // no debug info yet
10371090 .prev_di_pc = undefined, // no debug info yet
10381091 };
1039 defer emit.deinit();
10401092 emit.emitMir() catch |err| switch (err) {
10411093 error.LowerFail, error.EmitFail => return function.failMsg(emit.lower.err_msg.?),
10421094 error.InvalidInstruction => return function.fail("failed to find a viable x86 instruction (Zig compiler bug)", .{}),
......@@ -1065,7 +1117,7 @@ fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Format
10651117}
10661118
10671119const FormatAirData = struct {
1068 self: *Self,
1120 self: *CodeGen,
10691121 inst: Air.Inst.Index,
10701122};
10711123fn formatAir(
......@@ -1081,12 +1133,12 @@ fn formatAir(
10811133 data.self.liveness,
10821134 );
10831135}
1084fn fmtAir(self: *Self, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) {
1136fn fmtAir(self: *CodeGen, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) {
10851137 return .{ .data = .{ .self = self, .inst = inst } };
10861138}
10871139
10881140const FormatWipMirData = struct {
1089 self: *Self,
1141 self: *CodeGen,
10901142 inst: Mir.Inst.Index,
10911143};
10921144fn formatWipMir(
......@@ -1099,10 +1151,12 @@ fn formatWipMir(
10991151 const mod = comp.root_mod;
11001152 var lower: Lower = .{
11011153 .bin_file = data.self.bin_file,
1154 .target = data.self.target,
11021155 .allocator = data.self.gpa,
11031156 .mir = .{
11041157 .instructions = data.self.mir_instructions.slice(),
11051158 .extra = data.self.mir_extra.items,
1159 .table = data.self.mir_table.items,
11061160 .frame_locs = (std.MultiArrayList(Mir.FrameLoc){}).slice(),
11071161 },
11081162 .cc = .auto,
......@@ -1169,14 +1223,14 @@ fn formatWipMir(
11691223 lower.mir.extraData(Mir.Imm64, mir_inst.data.ai.i).data.decode(),
11701224 }),
11711225 .pseudo_dbg_local_as => {
1172 const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{
1226 const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
11731227 .base = .{ .reloc = mir_inst.data.as.sym_index },
11741228 }) };
11751229 try writer.print(" {}, {}", .{ mir_inst.data.as.air_inst, mem_op.fmt(.m) });
11761230 },
11771231 .pseudo_dbg_local_aso => {
11781232 const sym_off = lower.mir.extraData(bits.SymbolOffset, mir_inst.data.ax.payload).data;
1179 const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{
1233 const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
11801234 .base = .{ .reloc = sym_off.sym_index },
11811235 .disp = sym_off.off,
11821236 }) };
......@@ -1184,7 +1238,7 @@ fn formatWipMir(
11841238 },
11851239 .pseudo_dbg_local_aro => {
11861240 const air_off = lower.mir.extraData(Mir.AirOffset, mir_inst.data.rx.payload).data;
1187 const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{
1241 const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
11881242 .base = .{ .reg = mir_inst.data.rx.r1 },
11891243 .disp = air_off.off,
11901244 }) };
......@@ -1192,14 +1246,14 @@ fn formatWipMir(
11921246 },
11931247 .pseudo_dbg_local_af => {
11941248 const frame_addr = lower.mir.extraData(bits.FrameAddr, mir_inst.data.ax.payload).data;
1195 const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{
1249 const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
11961250 .base = .{ .frame = frame_addr.index },
11971251 .disp = frame_addr.off,
11981252 }) };
1199 try writer.print(" {}, {d}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });
1253 try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });
12001254 },
12011255 .pseudo_dbg_local_am => {
1202 const mem_op: Instruction.Operand = .{
1256 const mem_op: encoder.Instruction.Operand = .{
12031257 .mem = lower.mir.extraData(Mir.Memory, mir_inst.data.ax.payload).data.decode(),
12041258 };
12051259 try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });
......@@ -1207,12 +1261,12 @@ fn formatWipMir(
12071261 }
12081262 }
12091263}
1210fn fmtWipMir(self: *Self, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {
1264fn fmtWipMir(self: *CodeGen, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {
12111265 return .{ .data = .{ .self = self, .inst = inst } };
12121266}
12131267
12141268const FormatTrackingData = struct {
1215 self: *Self,
1269 self: *CodeGen,
12161270};
12171271fn formatTracking(
12181272 data: FormatTrackingData,
......@@ -1221,28 +1275,28 @@ fn formatTracking(
12211275 writer: anytype,
12221276) @TypeOf(writer).Error!void {
12231277 var it = data.self.inst_tracking.iterator();
1224 while (it.next()) |entry| try writer.print("\n%{d} = {}", .{ entry.key_ptr.*, entry.value_ptr.* });
1278 while (it.next()) |entry| try writer.print("\n{} = {}", .{ entry.key_ptr.*, entry.value_ptr.* });
12251279}
1226fn fmtTracking(self: *Self) std.fmt.Formatter(formatTracking) {
1280fn fmtTracking(self: *CodeGen) std.fmt.Formatter(formatTracking) {
12271281 return .{ .data = .{ .self = self } };
12281282}
12291283
1230fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
1284fn addInst(self: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
12311285 const gpa = self.gpa;
12321286 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
12331287 const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);
12341288 self.mir_instructions.appendAssumeCapacity(inst);
1235 wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});
1289 if (inst.ops != .pseudo_dead_none) wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});
12361290 return result_index;
12371291}
12381292
1239fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {
1293fn addExtra(self: *CodeGen, extra: anytype) Allocator.Error!u32 {
12401294 const fields = std.meta.fields(@TypeOf(extra));
12411295 try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);
12421296 return self.addExtraAssumeCapacity(extra);
12431297}
12441298
1245fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1299fn addExtraAssumeCapacity(self: *CodeGen, extra: anytype) u32 {
12461300 const fields = std.meta.fields(@TypeOf(extra));
12471301 const result: u32 = @intCast(self.mir_extra.items.len);
12481302 inline for (fields) |field| {
......@@ -1256,9 +1310,83 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
12561310 return result;
12571311}
12581312
1313fn asmOps(self: *CodeGen, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {
1314 return switch (ops[0]) {
1315 .none => self.asmOpOnly(tag),
1316 .reg => |reg0| switch (ops[1]) {
1317 .none => self.asmRegister(tag, reg0),
1318 .reg => |reg1| switch (ops[2]) {
1319 .none => self.asmRegisterRegister(tag, reg0, reg1),
1320 .reg => |reg2| switch (ops[3]) {
1321 .none => self.asmRegisterRegisterRegister(tag, reg0, reg1, reg2),
1322 .reg => |reg3| self.asmRegisterRegisterRegisterRegister(tag, reg0, reg1, reg2, reg3),
1323 .imm => |imm3| self.asmRegisterRegisterRegisterImmediate(tag, reg0, reg1, reg2, imm3),
1324 else => error.InvalidInstruction,
1325 },
1326 .mem => |mem2| switch (ops[3]) {
1327 .none => self.asmRegisterRegisterMemory(tag, reg0, reg1, mem2),
1328 .reg => |reg3| self.asmRegisterRegisterMemoryRegister(tag, reg0, reg1, mem2, reg3),
1329 .imm => |imm3| self.asmRegisterRegisterMemoryImmediate(tag, reg0, reg1, mem2, imm3),
1330 else => error.InvalidInstruction,
1331 },
1332 .imm => |imm2| switch (ops[3]) {
1333 .none => self.asmRegisterRegisterImmediate(tag, reg0, reg1, imm2),
1334 else => error.InvalidInstruction,
1335 },
1336 else => error.InvalidInstruction,
1337 },
1338 .mem => |mem1| switch (ops[2]) {
1339 .none => self.asmRegisterMemory(tag, reg0, mem1),
1340 .reg => |reg2| switch (ops[3]) {
1341 .none => self.asmRegisterMemoryRegister(tag, reg0, mem1, reg2),
1342 else => error.InvalidInstruction,
1343 },
1344 .imm => |imm2| switch (ops[3]) {
1345 .none => self.asmRegisterMemoryImmediate(tag, reg0, mem1, imm2),
1346 else => error.InvalidInstruction,
1347 },
1348 else => error.InvalidInstruction,
1349 },
1350 .imm => |imm1| switch (ops[2]) {
1351 .none => self.asmRegisterImmediate(tag, reg0, imm1),
1352 else => error.InvalidInstruction,
1353 },
1354 else => error.InvalidInstruction,
1355 },
1356 .mem => |mem0| switch (ops[1]) {
1357 .none => self.asmMemory(tag, mem0),
1358 .reg => |reg1| switch (ops[2]) {
1359 .none => self.asmMemoryRegister(tag, mem0, reg1),
1360 .reg => |reg2| switch (ops[3]) {
1361 .none => self.asmMemoryRegisterRegister(tag, mem0, reg1, reg2),
1362 else => error.InvalidInstruction,
1363 },
1364 .imm => |imm2| switch (ops[3]) {
1365 .none => self.asmMemoryRegisterImmediate(tag, mem0, reg1, imm2),
1366 else => error.InvalidInstruction,
1367 },
1368 else => error.InvalidInstruction,
1369 },
1370 .imm => |imm1| switch (ops[2]) {
1371 .none => self.asmMemoryImmediate(tag, mem0, imm1),
1372 else => error.InvalidInstruction,
1373 },
1374 else => error.InvalidInstruction,
1375 },
1376 .imm => |imm0| switch (ops[1]) {
1377 .none => self.asmImmediate(tag, imm0),
1378 else => error.InvalidInstruction,
1379 },
1380 .inst => |inst0| switch (ops[1]) {
1381 .none => self.asmReloc(tag, inst0),
1382 else => error.InvalidInstruction,
1383 },
1384 };
1385}
1386
12591387/// A `cc` of `.z_and_np` clobbers `reg2`!
1260fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: Register) !void {
1261 _ = try self.addInst(.{
1388fn asmCmovccRegisterRegister(self: *CodeGen, cc: Condition, reg1: Register, reg2: Register) !void {
1389 if (self.hasFeature(.cmov)) _ = try self.addInst(.{
12621390 .tag = switch (cc) {
12631391 else => .cmov,
12641392 .z_and_np, .nz_or_p => .pseudo,
......@@ -1270,18 +1398,22 @@ fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: R
12701398 },
12711399 .data = .{ .rr = .{
12721400 .fixes = switch (cc) {
1273 else => Mir.Inst.Fixes.fromCondition(cc),
1401 else => .fromCondition(cc),
12741402 .z_and_np, .nz_or_p => ._,
12751403 },
12761404 .r1 = reg1,
12771405 .r2 = reg2,
12781406 } },
1279 });
1407 }) else {
1408 const reloc = try self.asmJccReloc(cc.negate(), undefined);
1409 try self.asmRegisterRegister(.{ ._, .mov }, reg1, reg2);
1410 self.performReloc(reloc);
1411 }
12801412}
12811413
12821414/// A `cc` of `.z_and_np` is not supported by this encoding!
1283fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory) !void {
1284 _ = try self.addInst(.{
1415fn asmCmovccRegisterMemory(self: *CodeGen, cc: Condition, reg: Register, m: Memory) !void {
1416 if (self.hasFeature(.cmov)) _ = try self.addInst(.{
12851417 .tag = switch (cc) {
12861418 else => .cmov,
12871419 .z_and_np => unreachable,
......@@ -1294,17 +1426,21 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)
12941426 },
12951427 .data = .{ .rx = .{
12961428 .fixes = switch (cc) {
1297 else => Mir.Inst.Fixes.fromCondition(cc),
1429 else => .fromCondition(cc),
12981430 .z_and_np => unreachable,
12991431 .nz_or_p => ._,
13001432 },
13011433 .r1 = reg,
13021434 .payload = try self.addExtra(Mir.Memory.encode(m)),
13031435 } },
1304 });
1436 }) else {
1437 const reloc = try self.asmJccReloc(cc.negate(), undefined);
1438 try self.asmRegisterMemory(.{ ._, .mov }, reg, m);
1439 self.performReloc(reloc);
1440 }
13051441}
13061442
1307fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
1443fn asmSetccRegister(self: *CodeGen, cc: Condition, reg: Register) !void {
13081444 _ = try self.addInst(.{
13091445 .tag = switch (cc) {
13101446 else => .set,
......@@ -1317,7 +1453,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
13171453 },
13181454 .data = switch (cc) {
13191455 else => .{ .r = .{
1320 .fixes = Mir.Inst.Fixes.fromCondition(cc),
1456 .fixes = .fromCondition(cc),
13211457 .r1 = reg,
13221458 } },
13231459 .z_and_np, .nz_or_p => .{ .rr = .{
......@@ -1328,7 +1464,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
13281464 });
13291465}
13301466
1331fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
1467fn asmSetccMemory(self: *CodeGen, cc: Condition, m: Memory) !void {
13321468 const payload = try self.addExtra(Mir.Memory.encode(m));
13331469 _ = try self.addInst(.{
13341470 .tag = switch (cc) {
......@@ -1342,7 +1478,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
13421478 },
13431479 .data = switch (cc) {
13441480 else => .{ .x = .{
1345 .fixes = Mir.Inst.Fixes.fromCondition(cc),
1481 .fixes = .fromCondition(cc),
13461482 .payload = payload,
13471483 } },
13481484 .z_and_np, .nz_or_p => .{ .rx = .{
......@@ -1353,7 +1489,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
13531489 });
13541490}
13551491
1356fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
1492fn asmJmpReloc(self: *CodeGen, target: Mir.Inst.Index) !Mir.Inst.Index {
13571493 return self.addInst(.{
13581494 .tag = .jmp,
13591495 .ops = .inst,
......@@ -1363,7 +1499,7 @@ fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
13631499 });
13641500}
13651501
1366fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Index {
1502fn asmJccReloc(self: *CodeGen, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Index {
13671503 return self.addInst(.{
13681504 .tag = switch (cc) {
13691505 else => .j,
......@@ -1376,7 +1512,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind
13761512 },
13771513 .data = .{ .inst = .{
13781514 .fixes = switch (cc) {
1379 else => Mir.Inst.Fixes.fromCondition(cc),
1515 else => .fromCondition(cc),
13801516 .z_and_np, .nz_or_p => ._,
13811517 },
13821518 .inst = target,
......@@ -1384,7 +1520,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind
13841520 });
13851521}
13861522
1387fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
1523fn asmReloc(self: *CodeGen, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
13881524 _ = try self.addInst(.{
13891525 .tag = tag[1],
13901526 .ops = .inst,
......@@ -1395,7 +1531,7 @@ fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
13951531 });
13961532}
13971533
1398fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
1534fn asmPlaceholder(self: *CodeGen) !Mir.Inst.Index {
13991535 return self.addInst(.{
14001536 .tag = .pseudo,
14011537 .ops = .pseudo_dead_none,
......@@ -1405,7 +1541,7 @@ fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
14051541
14061542const MirTagAir = enum { dbg_local };
14071543
1408fn asmAir(self: *Self, tag: MirTagAir, inst: Air.Inst.Index) !void {
1544fn asmAir(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index) !void {
14091545 _ = try self.addInst(.{
14101546 .tag = .pseudo,
14111547 .ops = switch (tag) {
......@@ -1415,7 +1551,7 @@ fn asmAir(self: *Self, tag: MirTagAir, inst: Air.Inst.Index) !void {
14151551 });
14161552}
14171553
1418fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immediate) !void {
1554fn asmAirImmediate(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index, imm: Immediate) !void {
14191555 switch (imm) {
14201556 .signed => |s| _ = try self.addInst(.{
14211557 .tag = .pseudo,
......@@ -1427,7 +1563,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed
14271563 .i = @bitCast(s),
14281564 } },
14291565 }),
1430 .unsigned => |u| _ = if (math.cast(u32, u)) |small| try self.addInst(.{
1566 .unsigned => |u| _ = if (std.math.cast(u32, u)) |small| try self.addInst(.{
14311567 .tag = .pseudo,
14321568 .ops = switch (tag) {
14331569 .dbg_local => .pseudo_dbg_local_ai_u,
......@@ -1469,7 +1605,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed
14691605}
14701606
14711607fn asmAirRegisterImmediate(
1472 self: *Self,
1608 self: *CodeGen,
14731609 tag: MirTagAir,
14741610 inst: Air.Inst.Index,
14751611 reg: Register,
......@@ -1491,7 +1627,7 @@ fn asmAirRegisterImmediate(
14911627}
14921628
14931629fn asmAirFrameAddress(
1494 self: *Self,
1630 self: *CodeGen,
14951631 tag: MirTagAir,
14961632 inst: Air.Inst.Index,
14971633 frame_addr: bits.FrameAddr,
......@@ -1508,7 +1644,7 @@ fn asmAirFrameAddress(
15081644 });
15091645}
15101646
1511fn asmAirMemory(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !void {
1647fn asmAirMemory(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !void {
15121648 _ = try self.addInst(.{
15131649 .tag = .pseudo,
15141650 .ops = switch (tag) {
......@@ -1521,7 +1657,7 @@ fn asmAirMemory(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !v
15211657 });
15221658}
15231659
1524fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {
1660fn asmOpOnly(self: *CodeGen, tag: Mir.Inst.FixedTag) !void {
15251661 _ = try self.addInst(.{
15261662 .tag = tag[1],
15271663 .ops = .none,
......@@ -1531,7 +1667,7 @@ fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {
15311667 });
15321668}
15331669
1534fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
1670fn asmPseudo(self: *CodeGen, ops: Mir.Inst.Ops) !void {
15351671 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
15361672 std.mem.endsWith(u8, @tagName(ops), "_none"));
15371673 _ = try self.addInst(.{
......@@ -1541,7 +1677,7 @@ fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
15411677 });
15421678}
15431679
1544fn asmPseudoRegister(self: *Self, ops: Mir.Inst.Ops, reg: Register) !void {
1680fn asmPseudoRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register) !void {
15451681 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
15461682 std.mem.endsWith(u8, @tagName(ops), "_r"));
15471683 _ = try self.addInst(.{
......@@ -1551,7 +1687,7 @@ fn asmPseudoRegister(self: *Self, ops: Mir.Inst.Ops, reg: Register) !void {
15511687 });
15521688}
15531689
1554fn asmPseudoImmediate(self: *Self, ops: Mir.Inst.Ops, imm: Immediate) !void {
1690fn asmPseudoImmediate(self: *CodeGen, ops: Mir.Inst.Ops, imm: Immediate) !void {
15551691 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
15561692 std.mem.endsWith(u8, @tagName(ops), "_i_s"));
15571693 _ = try self.addInst(.{
......@@ -1561,7 +1697,7 @@ fn asmPseudoImmediate(self: *Self, ops: Mir.Inst.Ops, imm: Immediate) !void {
15611697 });
15621698}
15631699
1564fn asmPseudoRegisterRegister(self: *Self, ops: Mir.Inst.Ops, reg1: Register, reg2: Register) !void {
1700fn asmPseudoRegisterRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg1: Register, reg2: Register) !void {
15651701 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
15661702 std.mem.endsWith(u8, @tagName(ops), "_rr"));
15671703 _ = try self.addInst(.{
......@@ -1571,7 +1707,7 @@ fn asmPseudoRegisterRegister(self: *Self, ops: Mir.Inst.Ops, reg1: Register, reg
15711707 });
15721708}
15731709
1574fn asmPseudoRegisterImmediate(self: *Self, ops: Mir.Inst.Ops, reg: Register, imm: Immediate) !void {
1710fn asmPseudoRegisterImmediate(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register, imm: Immediate) !void {
15751711 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
15761712 std.mem.endsWith(u8, @tagName(ops), "_ri_s"));
15771713 _ = try self.addInst(.{
......@@ -1581,7 +1717,7 @@ fn asmPseudoRegisterImmediate(self: *Self, ops: Mir.Inst.Ops, reg: Register, imm
15811717 });
15821718}
15831719
1584fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {
1720fn asmRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register) !void {
15851721 _ = try self.addInst(.{
15861722 .tag = tag[1],
15871723 .ops = .r,
......@@ -1592,7 +1728,7 @@ fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {
15921728 });
15931729}
15941730
1595fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
1731fn asmImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
15961732 _ = try self.addInst(.{
15971733 .tag = tag[1],
15981734 .ops = switch (imm) {
......@@ -1617,7 +1753,7 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
16171753 });
16181754}
16191755
1620fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {
1756fn asmRegisterRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {
16211757 _ = try self.addInst(.{
16221758 .tag = tag[1],
16231759 .ops = .rr,
......@@ -1629,10 +1765,10 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2
16291765 });
16301766}
16311767
1632fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
1768fn asmRegisterImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
16331769 const ops: Mir.Inst.Ops, const i: u32 = switch (imm) {
16341770 .signed => |s| .{ .ri_s, @bitCast(s) },
1635 .unsigned => |u| if (math.cast(u32, u)) |small|
1771 .unsigned => |u| if (std.math.cast(u32, u)) |small|
16361772 .{ .ri_u, small }
16371773 else
16381774 .{ .ri_64, try self.addExtra(Mir.Imm64.encode(imm.unsigned)) },
......@@ -1650,7 +1786,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm:
16501786}
16511787
16521788fn asmRegisterRegisterRegister(
1653 self: *Self,
1789 self: *CodeGen,
16541790 tag: Mir.Inst.FixedTag,
16551791 reg1: Register,
16561792 reg2: Register,
......@@ -1669,7 +1805,7 @@ fn asmRegisterRegisterRegister(
16691805}
16701806
16711807fn asmRegisterRegisterRegisterRegister(
1672 self: *Self,
1808 self: *CodeGen,
16731809 tag: Mir.Inst.FixedTag,
16741810 reg1: Register,
16751811 reg2: Register,
......@@ -1690,7 +1826,7 @@ fn asmRegisterRegisterRegisterRegister(
16901826}
16911827
16921828fn asmRegisterRegisterRegisterImmediate(
1693 self: *Self,
1829 self: *CodeGen,
16941830 tag: Mir.Inst.FixedTag,
16951831 reg1: Register,
16961832 reg2: Register,
......@@ -1715,7 +1851,7 @@ fn asmRegisterRegisterRegisterImmediate(
17151851}
17161852
17171853fn asmRegisterRegisterImmediate(
1718 self: *Self,
1854 self: *CodeGen,
17191855 tag: Mir.Inst.FixedTag,
17201856 reg1: Register,
17211857 reg2: Register,
......@@ -1742,7 +1878,7 @@ fn asmRegisterRegisterImmediate(
17421878}
17431879
17441880fn asmRegisterRegisterMemory(
1745 self: *Self,
1881 self: *CodeGen,
17461882 tag: Mir.Inst.FixedTag,
17471883 reg1: Register,
17481884 reg2: Register,
......@@ -1761,7 +1897,7 @@ fn asmRegisterRegisterMemory(
17611897}
17621898
17631899fn asmRegisterRegisterMemoryRegister(
1764 self: *Self,
1900 self: *CodeGen,
17651901 tag: Mir.Inst.FixedTag,
17661902 reg1: Register,
17671903 reg2: Register,
......@@ -1781,7 +1917,7 @@ fn asmRegisterRegisterMemoryRegister(
17811917 });
17821918}
17831919
1784fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1920fn asmMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory) !void {
17851921 _ = try self.addInst(.{
17861922 .tag = tag[1],
17871923 .ops = .m,
......@@ -1792,7 +1928,7 @@ fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
17921928 });
17931929}
17941930
1795fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
1931fn asmRegisterMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
17961932 _ = try self.addInst(.{
17971933 .tag = tag[1],
17981934 .ops = .rm,
......@@ -1805,7 +1941,7 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo
18051941}
18061942
18071943fn asmRegisterMemoryRegister(
1808 self: *Self,
1944 self: *CodeGen,
18091945 tag: Mir.Inst.FixedTag,
18101946 reg1: Register,
18111947 m: Memory,
......@@ -1824,15 +1960,15 @@ fn asmRegisterMemoryRegister(
18241960}
18251961
18261962fn asmRegisterMemoryImmediate(
1827 self: *Self,
1963 self: *CodeGen,
18281964 tag: Mir.Inst.FixedTag,
18291965 reg: Register,
18301966 m: Memory,
18311967 imm: Immediate,
18321968) !void {
18331969 if (switch (imm) {
1834 .signed => |s| if (math.cast(i16, s)) |x| @as(u16, @bitCast(x)) else null,
1835 .unsigned => |u| math.cast(u16, u),
1970 .signed => |s| if (std.math.cast(i16, s)) |x| @as(u16, @bitCast(x)) else null,
1971 .unsigned => |u| std.math.cast(u16, u),
18361972 .reloc => unreachable,
18371973 }) |small_imm| {
18381974 _ = try self.addInst(.{
......@@ -1869,7 +2005,7 @@ fn asmRegisterMemoryImmediate(
18692005}
18702006
18712007fn asmRegisterRegisterMemoryImmediate(
1872 self: *Self,
2008 self: *CodeGen,
18732009 tag: Mir.Inst.FixedTag,
18742010 reg1: Register,
18752011 reg2: Register,
......@@ -1889,7 +2025,7 @@ fn asmRegisterRegisterMemoryImmediate(
18892025 });
18902026}
18912027
1892fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
2028fn asmMemoryRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
18932029 _ = try self.addInst(.{
18942030 .tag = tag[1],
18952031 .ops = .mr,
......@@ -1901,7 +2037,7 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Regist
19012037 });
19022038}
19032039
1904fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
2040fn asmMemoryImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
19052041 const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
19062042 .signed => |s| @bitCast(s),
19072043 .unsigned => |u| @intCast(u),
......@@ -1923,7 +2059,7 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed
19232059}
19242060
19252061fn asmMemoryRegisterRegister(
1926 self: *Self,
2062 self: *CodeGen,
19272063 tag: Mir.Inst.FixedTag,
19282064 m: Memory,
19292065 reg1: Register,
......@@ -1942,7 +2078,7 @@ fn asmMemoryRegisterRegister(
19422078}
19432079
19442080fn asmMemoryRegisterImmediate(
1945 self: *Self,
2081 self: *CodeGen,
19462082 tag: Mir.Inst.FixedTag,
19472083 m: Memory,
19482084 reg: Register,
......@@ -1960,15 +2096,14 @@ fn asmMemoryRegisterImmediate(
19602096 });
19612097}
19622098
1963fn gen(self: *Self) InnerError!void {
2099fn gen(self: *CodeGen) InnerError!void {
19642100 const pt = self.pt;
19652101 const zcu = pt.zcu;
19662102 const fn_info = zcu.typeToFunc(self.fn_type).?;
1967 const cc = abi.resolveCallingConvention(fn_info.cc, self.target.*);
1968 if (cc != .naked) {
2103 if (fn_info.cc != .naked) {
19692104 try self.asmRegister(.{ ._, .push }, .rbp);
1970 try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, Immediate.s(8));
1971 try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, Immediate.s(0));
2105 try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, .s(8));
2106 try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, .s(0));
19722107 try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
19732108 try self.asmPseudoRegister(.pseudo_cfi_def_cfa_register_r, .rbp);
19742109 const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
......@@ -1983,11 +2118,11 @@ fn gen(self: *Self) InnerError!void {
19832118 // The address where to store the return value for the caller is in a
19842119 // register which the callee is free to clobber. Therefore, we purposely
19852120 // spill it to stack immediately.
1986 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(Type.usize, zcu));
2121 const frame_index = try self.allocFrameIndex(.initSpill(.usize, zcu));
19872122 try self.genSetMem(
19882123 .{ .frame = frame_index },
19892124 0,
1990 Type.usize,
2125 .usize,
19912126 self.ret_mcv.long.address().offset(-self.ret_mcv.short.indirect.off),
19922127 .{},
19932128 );
......@@ -1997,10 +2132,10 @@ fn gen(self: *Self) InnerError!void {
19972132 else => unreachable,
19982133 }
19992134
2000 if (fn_info.is_var_args) switch (cc) {
2135 if (fn_info.is_var_args) switch (fn_info.cc) {
20012136 .x86_64_sysv => {
20022137 const info = &self.va_info.sysv;
2003 const reg_save_area_fi = try self.allocFrameIndex(FrameAlloc.init(.{
2138 const reg_save_area_fi = try self.allocFrameIndex(.init(.{
20042139 .size = abi.SysV.c_abi_int_param_regs.len * 8 +
20052140 abi.SysV.c_abi_sse_param_regs.len * 16,
20062141 .alignment = .@"16",
......@@ -2008,15 +2143,9 @@ fn gen(self: *Self) InnerError!void {
20082143 info.reg_save_area = .{ .index = reg_save_area_fi };
20092144
20102145 for (abi.SysV.c_abi_int_param_regs[info.gp_count..], info.gp_count..) |reg, reg_i|
2011 try self.genSetMem(
2012 .{ .frame = reg_save_area_fi },
2013 @intCast(reg_i * 8),
2014 Type.usize,
2015 .{ .register = reg },
2016 .{},
2017 );
2146 try self.genSetMem(.{ .frame = reg_save_area_fi }, @intCast(reg_i * 8), .usize, .{ .register = reg }, .{});
20182147
2019 try self.asmRegisterImmediate(.{ ._, .cmp }, .al, Immediate.u(info.fp_count));
2148 try self.asmRegisterImmediate(.{ ._, .cmp }, .al, .u(info.fp_count));
20202149 const skip_sse_reloc = try self.asmJccReloc(.na, undefined);
20212150
20222151 const vec_2_f64 = try pt.vectorType(.{ .len = 2, .child = .f64_type });
......@@ -2032,38 +2161,38 @@ fn gen(self: *Self) InnerError!void {
20322161 self.performReloc(skip_sse_reloc);
20332162 },
20342163 .x86_64_win => return self.fail("TODO implement gen var arg function for Win64", .{}),
2035 else => unreachable,
2164 else => |cc| return self.fail("{s} does not support var args", .{@tagName(cc)}),
20362165 };
20372166
20382167 try self.asmPseudo(.pseudo_dbg_prologue_end_none);
20392168
20402169 try self.genBody(self.air.getMainBody());
20412170
2042 // TODO can single exitlude jump reloc be elided? What if it is not at the end of the code?
2043 // Example:
2044 // pub fn main() void {
2045 // maybeErr() catch return;
2046 // unreachable;
2047 // }
2048 // Eliding the reloc will cause a miscompilation in this case.
2049 for (self.exitlude_jump_relocs.items) |jmp_reloc| {
2050 self.mir_instructions.items(.data)[jmp_reloc].inst.inst =
2051 @intCast(self.mir_instructions.len);
2052 }
2171 const epilogue = if (self.epilogue_relocs.items.len > 0) epilogue: {
2172 const epilogue_relocs_last_index = self.epilogue_relocs.items.len - 1;
2173 for (if (self.epilogue_relocs.items[epilogue_relocs_last_index] == self.mir_instructions.len - 1) epilogue_relocs: {
2174 _ = self.mir_instructions.pop();
2175 break :epilogue_relocs self.epilogue_relocs.items[0..epilogue_relocs_last_index];
2176 } else self.epilogue_relocs.items) |epilogue_reloc| self.performReloc(epilogue_reloc);
2177
2178 try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
2179 const backpatch_stack_dealloc = try self.asmPlaceholder();
2180 const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
2181 try self.asmRegister(.{ ._, .pop }, .rbp);
2182 try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, .s(8));
2183 try self.asmOpOnly(.{ ._, .ret });
2184 break :epilogue .{
2185 .backpatch_stack_dealloc = backpatch_stack_dealloc,
2186 .backpatch_pop_callee_preserved_regs = backpatch_pop_callee_preserved_regs,
2187 };
2188 } else null;
20532189
2054 try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
2055 const backpatch_stack_dealloc = try self.asmPlaceholder();
2056 const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
2057 try self.asmRegister(.{ ._, .pop }, .rbp);
2058 try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, Immediate.s(8));
2059 try self.asmOpOnly(.{ ._, .ret });
2060
2061 const frame_layout = try self.computeFrameLayout(cc);
2062 const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
2190 const frame_layout = try self.computeFrameLayout(fn_info.cc);
2191 const need_frame_align = frame_layout.stack_mask != std.math.maxInt(u32);
20632192 const need_stack_adjust = frame_layout.stack_adjust > 0;
20642193 const need_save_reg = frame_layout.save_reg_list.count() > 0;
20652194 if (need_frame_align) {
2066 const page_align = @as(u32, math.maxInt(u32)) << 12;
2195 const page_align = @as(u32, std.math.maxInt(u32)) << 12;
20672196 self.mir_instructions.set(backpatch_frame_align, .{
20682197 .tag = .@"and",
20692198 .ops = .ri_s,
......@@ -2106,12 +2235,13 @@ fn gen(self: *Self) InnerError!void {
21062235 } },
21072236 });
21082237 } else {
2238 const scratch_reg = abi.getCAbiLinkerScratchReg(fn_info.cc);
21092239 self.mir_instructions.set(backpatch_stack_alloc, .{
21102240 .tag = .pseudo,
21112241 .ops = .pseudo_probe_adjust_setup_rri_s,
21122242 .data = .{ .rri = .{
21132243 .r1 = .rsp,
2114 .r2 = .rax,
2244 .r2 = scratch_reg,
21152245 .i = frame_layout.stack_adjust,
21162246 } },
21172247 });
......@@ -2120,34 +2250,44 @@ fn gen(self: *Self) InnerError!void {
21202250 .ops = .pseudo_probe_adjust_loop_rr,
21212251 .data = .{ .rr = .{
21222252 .r1 = .rsp,
2123 .r2 = .rax,
2253 .r2 = scratch_reg,
21242254 } },
21252255 });
21262256 }
21272257 }
2128 if (need_frame_align or need_stack_adjust) {
2129 self.mir_instructions.set(backpatch_stack_dealloc, .{
2130 .tag = .lea,
2131 .ops = .rm,
2132 .data = .{ .rx = .{
2133 .r1 = .rsp,
2134 .payload = try self.addExtra(Mir.Memory.encode(.{
2135 .base = .{ .reg = .rbp },
2136 .mod = .{ .rm = .{
2137 .size = .qword,
2138 .disp = -frame_layout.save_reg_list.size(),
2139 } },
2140 })),
2141 } },
2258 if (epilogue) |e| if (need_frame_align or need_stack_adjust) {
2259 self.mir_instructions.set(e.backpatch_stack_dealloc, switch (-frame_layout.save_reg_list.size(self.target)) {
2260 0 => .{
2261 .tag = .mov,
2262 .ops = .rr,
2263 .data = .{ .rr = .{
2264 .r1 = .rsp,
2265 .r2 = .rbp,
2266 } },
2267 },
2268 else => |disp| .{
2269 .tag = .lea,
2270 .ops = .rm,
2271 .data = .{ .rx = .{
2272 .r1 = .rsp,
2273 .payload = try self.addExtra(Mir.Memory.encode(.{
2274 .base = .{ .reg = .rbp },
2275 .mod = .{ .rm = .{
2276 .size = .qword,
2277 .disp = disp,
2278 } },
2279 })),
2280 } },
2281 },
21422282 });
2143 }
2283 };
21442284 if (need_save_reg) {
21452285 self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
21462286 .tag = .pseudo,
21472287 .ops = .pseudo_push_reg_list,
21482288 .data = .{ .reg_list = frame_layout.save_reg_list },
21492289 });
2150 self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{
2290 if (epilogue) |e| self.mir_instructions.set(e.backpatch_pop_callee_preserved_regs, .{
21512291 .tag = .pseudo,
21522292 .ops = .pseudo_pop_reg_list,
21532293 .data = .{ .reg_list = frame_layout.save_reg_list },
......@@ -2170,97 +2310,85 @@ fn gen(self: *Self) InnerError!void {
21702310 });
21712311}
21722312
2173fn checkInvariantsAfterAirInst(self: *Self, inst: Air.Inst.Index, old_air_bookkeeping: @TypeOf(air_bookkeeping_init)) void {
2313fn checkInvariantsAfterAirInst(self: *CodeGen) void {
21742314 assert(!self.register_manager.lockedRegsExist());
21752315
21762316 if (std.debug.runtime_safety) {
2177 if (self.air_bookkeeping < old_air_bookkeeping + 1) {
2178 std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, self.air.instructions.items(.tag)[@intFromEnum(inst)] });
2179 }
2180
2181 { // check consistency of tracked registers
2182 var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });
2183 while (it.next()) |index| {
2184 const tracked_inst = self.register_manager.registers[index];
2185 const tracking = self.getResolvedInstValue(tracked_inst);
2186 for (tracking.getRegs()) |reg| {
2187 if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break;
2188 } else unreachable; // tracked register not in use
2189 }
2317 // check consistency of tracked registers
2318 var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });
2319 while (it.next()) |index| {
2320 const tracked_inst = self.register_manager.registers[index];
2321 const tracking = self.getResolvedInstValue(tracked_inst);
2322 for (tracking.getRegs()) |reg| {
2323 if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break;
2324 } else unreachable; // tracked register not in use
21902325 }
21912326 }
21922327}
21932328
2194fn genBodyBlock(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2329fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
21952330 try self.asmPseudo(.pseudo_dbg_enter_block_none);
21962331 try self.genBody(body);
21972332 try self.asmPseudo(.pseudo_dbg_leave_block_none);
21982333}
21992334
2200fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2201 const pt = self.pt;
2335fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2336 @setEvalBranchQuota(1_700);
2337 const pt = cg.pt;
22022338 const zcu = pt.zcu;
22032339 const ip = &zcu.intern_pool;
2204 const air_tags = self.air.instructions.items(.tag);
2340 const air_tags = cg.air.instructions.items(.tag);
2341 const air_datas = cg.air.instructions.items(.data);
2342 const use_old = cg.target.ofmt == .coff;
22052343
2206 self.arg_index = 0;
2344 cg.arg_index = 0;
22072345 for (body) |inst| switch (air_tags[@intFromEnum(inst)]) {
22082346 .arg => {
2209 wip_mir_log.debug("{}", .{self.fmtAir(inst)});
2210 verbose_tracking_log.debug("{}", .{self.fmtTracking()});
2347 wip_mir_log.debug("{}", .{cg.fmtAir(inst)});
2348 verbose_tracking_log.debug("{}", .{cg.fmtTracking()});
22112349
2212 const old_air_bookkeeping = self.air_bookkeeping;
2213 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
2350 cg.reused_operands = .initEmpty();
2351 try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1);
22142352
2215 try self.airArg(inst);
2353 try cg.airArg(inst);
22162354
2217 self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping);
2355 cg.resetTemps();
2356 cg.checkInvariantsAfterAirInst();
22182357 },
22192358 else => break,
22202359 };
22212360
2222 if (self.arg_index == 0) try self.airDbgVarArgs();
2223 self.arg_index = 0;
2361 if (cg.arg_index == 0) try cg.airDbgVarArgs();
2362 cg.arg_index = 0;
22242363 for (body) |inst| {
2225 if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
2226 wip_mir_log.debug("{}", .{self.fmtAir(inst)});
2227 verbose_tracking_log.debug("{}", .{self.fmtTracking()});
2364 if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) continue;
2365 wip_mir_log.debug("{}", .{cg.fmtAir(inst)});
2366 verbose_tracking_log.debug("{}", .{cg.fmtTracking()});
22282367
2229 const old_air_bookkeeping = self.air_bookkeeping;
2230 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
2368 cg.reused_operands = .initEmpty();
2369 try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1);
22312370 switch (air_tags[@intFromEnum(inst)]) {
22322371 // zig fmt: off
2233 .not,
2234 => |tag| try self.airUnOp(inst, tag),
2235
22362372 .add,
22372373 .add_wrap,
22382374 .sub,
22392375 .sub_wrap,
2240 .bool_and,
2241 .bool_or,
2242 .bit_and,
2243 .bit_or,
2244 .xor,
22452376 .min,
22462377 .max,
2247 => |tag| try self.airBinOp(inst, tag),
2378 => |air_tag| try cg.airBinOp(inst, air_tag),
22482379
2249 .ptr_add, .ptr_sub => |tag| try self.airPtrArithmetic(inst, tag),
2380 .shr, .shr_exact => try cg.airShlShrBinOp(inst),
2381 .shl, .shl_exact => try cg.airShlShrBinOp(inst),
22502382
2251 .shr, .shr_exact => try self.airShlShrBinOp(inst),
2252 .shl, .shl_exact => try self.airShlShrBinOp(inst),
2383 .mul => try cg.airMulDivBinOp(inst),
2384 .mul_wrap => try cg.airMulDivBinOp(inst),
2385 .rem => try cg.airMulDivBinOp(inst),
2386 .mod => try cg.airMulDivBinOp(inst),
22532387
2254 .mul => try self.airMulDivBinOp(inst),
2255 .mul_wrap => try self.airMulDivBinOp(inst),
2256 .rem => try self.airMulDivBinOp(inst),
2257 .mod => try self.airMulDivBinOp(inst),
2258
2259 .add_sat => try self.airAddSat(inst),
2260 .sub_sat => try self.airSubSat(inst),
2261 .mul_sat => try self.airMulSat(inst),
2262 .shl_sat => try self.airShlSat(inst),
2263 .slice => try self.airSlice(inst),
2388 .add_sat => try cg.airAddSat(inst),
2389 .sub_sat => try cg.airSubSat(inst),
2390 .mul_sat => try cg.airMulSat(inst),
2391 .shl_sat => try cg.airShlSat(inst),
22642392
22652393 .sin,
22662394 .cos,
......@@ -2271,162 +2399,80 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
22712399 .log2,
22722400 .log10,
22732401 .round,
2274 => |tag| try self.airUnaryMath(inst, tag),
2275
2276 .floor => try self.airRound(inst, .{ .mode = .down, .precision = .inexact }),
2277 .ceil => try self.airRound(inst, .{ .mode = .up, .precision = .inexact }),
2278 .trunc_float => try self.airRound(inst, .{ .mode = .zero, .precision = .inexact }),
2279 .sqrt => try self.airSqrt(inst),
2280 .neg => try self.airFloatSign(inst),
2281
2282 .abs => try self.airAbs(inst),
2283
2284 .add_with_overflow => try self.airAddSubWithOverflow(inst),
2285 .sub_with_overflow => try self.airAddSubWithOverflow(inst),
2286 .mul_with_overflow => try self.airMulWithOverflow(inst),
2287 .shl_with_overflow => try self.airShlWithOverflow(inst),
2288
2289 .div_float, .div_trunc, .div_floor, .div_exact => try self.airMulDivBinOp(inst),
2290
2291 .cmp_lt => try self.airCmp(inst, .lt),
2292 .cmp_lte => try self.airCmp(inst, .lte),
2293 .cmp_eq => try self.airCmp(inst, .eq),
2294 .cmp_gte => try self.airCmp(inst, .gte),
2295 .cmp_gt => try self.airCmp(inst, .gt),
2296 .cmp_neq => try self.airCmp(inst, .neq),
2297
2298 .cmp_vector => try self.airCmpVector(inst),
2299 .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),
2300
2301 .alloc => try self.airAlloc(inst),
2302 .ret_ptr => try self.airRetPtr(inst),
2303 .arg => try self.airDbgArg(inst),
2304 .assembly => try self.airAsm(inst),
2305 .bitcast => try self.airBitCast(inst),
2306 .block => try self.airBlock(inst),
2307 .br => try self.airBr(inst),
2308 .repeat => try self.airRepeat(inst),
2309 .switch_dispatch => try self.airSwitchDispatch(inst),
2310 .trap => try self.airTrap(),
2311 .breakpoint => try self.airBreakpoint(),
2312 .ret_addr => try self.airRetAddr(inst),
2313 .frame_addr => try self.airFrameAddress(inst),
2314 .cond_br => try self.airCondBr(inst),
2315 .fptrunc => try self.airFptrunc(inst),
2316 .fpext => try self.airFpext(inst),
2317 .intcast => try self.airIntCast(inst),
2318 .trunc => try self.airTrunc(inst),
2319 .int_from_bool => try self.airIntFromBool(inst),
2320 .is_non_null => try self.airIsNonNull(inst),
2321 .is_non_null_ptr => try self.airIsNonNullPtr(inst),
2322 .is_null => try self.airIsNull(inst),
2323 .is_null_ptr => try self.airIsNullPtr(inst),
2324 .is_non_err => try self.airIsNonErr(inst),
2325 .is_non_err_ptr => try self.airIsNonErrPtr(inst),
2326 .is_err => try self.airIsErr(inst),
2327 .is_err_ptr => try self.airIsErrPtr(inst),
2328 .load => try self.airLoad(inst),
2329 .loop => try self.airLoop(inst),
2330 .int_from_ptr => try self.airIntFromPtr(inst),
2331 .ret => try self.airRet(inst, false),
2332 .ret_safe => try self.airRet(inst, true),
2333 .ret_load => try self.airRetLoad(inst),
2334 .store => try self.airStore(inst, false),
2335 .store_safe => try self.airStore(inst, true),
2336 .struct_field_ptr=> try self.airStructFieldPtr(inst),
2337 .struct_field_val=> try self.airStructFieldVal(inst),
2338 .array_to_slice => try self.airArrayToSlice(inst),
2339 .float_from_int => try self.airFloatFromInt(inst),
2340 .int_from_float => try self.airIntFromFloat(inst),
2341 .cmpxchg_strong => try self.airCmpxchg(inst),
2342 .cmpxchg_weak => try self.airCmpxchg(inst),
2343 .atomic_rmw => try self.airAtomicRmw(inst),
2344 .atomic_load => try self.airAtomicLoad(inst),
2345 .memcpy => try self.airMemcpy(inst),
2346 .memset => try self.airMemset(inst, false),
2347 .memset_safe => try self.airMemset(inst, true),
2348 .set_union_tag => try self.airSetUnionTag(inst),
2349 .get_union_tag => try self.airGetUnionTag(inst),
2350 .clz => try self.airClz(inst),
2351 .ctz => try self.airCtz(inst),
2352 .popcount => try self.airPopCount(inst),
2353 .byte_swap => try self.airByteSwap(inst),
2354 .bit_reverse => try self.airBitReverse(inst),
2355 .tag_name => try self.airTagName(inst),
2356 .error_name => try self.airErrorName(inst),
2357 .splat => try self.airSplat(inst),
2358 .select => try self.airSelect(inst),
2359 .shuffle => try self.airShuffle(inst),
2360 .reduce => try self.airReduce(inst),
2361 .aggregate_init => try self.airAggregateInit(inst),
2362 .union_init => try self.airUnionInit(inst),
2363 .prefetch => try self.airPrefetch(inst),
2364 .mul_add => try self.airMulAdd(inst),
2365 .addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
2366
2367 .@"try" => try self.airTry(inst),
2368 .try_cold => try self.airTry(inst), // TODO
2369 .try_ptr => try self.airTryPtr(inst),
2370 .try_ptr_cold => try self.airTryPtr(inst), // TODO
2371
2372 .dbg_stmt => try self.airDbgStmt(inst),
2373 .dbg_empty_stmt => try self.airDbgEmptyStmt(),
2374 .dbg_inline_block => try self.airDbgInlineBlock(inst),
2375 .dbg_var_ptr,
2376 .dbg_var_val,
2377 .dbg_arg_inline,
2378 => try self.airDbgVar(inst),
2379
2380 .call => try self.airCall(inst, .auto),
2381 .call_always_tail => try self.airCall(inst, .always_tail),
2382 .call_never_tail => try self.airCall(inst, .never_tail),
2383 .call_never_inline => try self.airCall(inst, .never_inline),
2384
2385 .atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
2386 .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
2387 .atomic_store_release => try self.airAtomicStore(inst, .release),
2388 .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
2389
2390 .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
2391 .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
2392 .struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
2393 .struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
2394
2395 .field_parent_ptr => try self.airFieldParentPtr(inst),
2396
2397 .switch_br => try self.airSwitchBr(inst),
2398 .loop_switch_br => try self.airLoopSwitchBr(inst),
2399 .slice_ptr => try self.airSlicePtr(inst),
2400 .slice_len => try self.airSliceLen(inst),
2401
2402 .ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
2403 .ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
2404
2405 .array_elem_val => try self.airArrayElemVal(inst),
2406 .slice_elem_val => try self.airSliceElemVal(inst),
2407 .slice_elem_ptr => try self.airSliceElemPtr(inst),
2408 .ptr_elem_val => try self.airPtrElemVal(inst),
2409 .ptr_elem_ptr => try self.airPtrElemPtr(inst),
2410
2411 .inferred_alloc, .inferred_alloc_comptime => unreachable,
2412 .unreach => self.finishAirBookkeeping(),
2413
2414 .optional_payload => try self.airOptionalPayload(inst),
2415 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
2416 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
2417 .unwrap_errunion_err => try self.airUnwrapErrUnionErr(inst),
2418 .unwrap_errunion_payload => try self.airUnwrapErrUnionPayload(inst),
2419 .unwrap_errunion_err_ptr => try self.airUnwrapErrUnionErrPtr(inst),
2420 .unwrap_errunion_payload_ptr=> try self.airUnwrapErrUnionPayloadPtr(inst),
2421 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
2422 .err_return_trace => try self.airErrReturnTrace(inst),
2423 .set_err_return_trace => try self.airSetErrReturnTrace(inst),
2424 .save_err_return_trace_index=> try self.airSaveErrReturnTraceIndex(inst),
2425
2426 .wrap_optional => try self.airWrapOptional(inst),
2427 .wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
2428 .wrap_errunion_err => try self.airWrapErrUnionErr(inst),
2402 => |air_tag| try cg.airUnaryMath(inst, air_tag),
2403
2404 .floor => try cg.airRound(inst, .{ .mode = .down, .precision = .inexact }),
2405 .ceil => try cg.airRound(inst, .{ .mode = .up, .precision = .inexact }),
2406 .trunc_float => try cg.airRound(inst, .{ .mode = .zero, .precision = .inexact }),
2407 .sqrt => try cg.airSqrt(inst),
2408 .neg => try cg.airFloatSign(inst),
2409
2410 .abs => try cg.airAbs(inst),
2411
2412 .add_with_overflow => try cg.airAddSubWithOverflow(inst),
2413 .sub_with_overflow => try cg.airAddSubWithOverflow(inst),
2414 .mul_with_overflow => try cg.airMulWithOverflow(inst),
2415 .shl_with_overflow => try cg.airShlWithOverflow(inst),
2416
2417 .div_float, .div_trunc, .div_floor, .div_exact => try cg.airMulDivBinOp(inst),
2418
2419 .cmp_lt_errors_len => try cg.airCmpLtErrorsLen(inst),
2420
2421 .bitcast => try cg.airBitCast(inst),
2422 .fptrunc => try cg.airFptrunc(inst),
2423 .fpext => try cg.airFpext(inst),
2424 .intcast => try cg.airIntCast(inst),
2425 .trunc => try cg.airTrunc(inst),
2426 .is_non_null => try cg.airIsNonNull(inst),
2427 .is_null => try cg.airIsNull(inst),
2428 .is_non_err => try cg.airIsNonErr(inst),
2429 .is_err => try cg.airIsErr(inst),
2430 .float_from_int => try cg.airFloatFromInt(inst),
2431 .int_from_float => try cg.airIntFromFloat(inst),
2432 .cmpxchg_strong => try cg.airCmpxchg(inst),
2433 .cmpxchg_weak => try cg.airCmpxchg(inst),
2434 .atomic_rmw => try cg.airAtomicRmw(inst),
2435 .atomic_load => try cg.airAtomicLoad(inst),
2436 .memcpy => try cg.airMemcpy(inst),
2437 .memset => try cg.airMemset(inst, false),
2438 .memset_safe => try cg.airMemset(inst, true),
2439 .ctz => try cg.airCtz(inst),
2440 .popcount => try cg.airPopCount(inst),
2441 .byte_swap => try cg.airByteSwap(inst),
2442 .bit_reverse => try cg.airBitReverse(inst),
2443 .tag_name => try cg.airTagName(inst),
2444 .error_name => try cg.airErrorName(inst),
2445 .splat => try cg.airSplat(inst),
2446 .select => try cg.airSelect(inst),
2447 .shuffle => try cg.airShuffle(inst),
2448 .reduce => try cg.airReduce(inst),
2449 .aggregate_init => try cg.airAggregateInit(inst),
2450 .prefetch => try cg.airPrefetch(inst),
2451 .mul_add => try cg.airMulAdd(inst),
2452
2453 .atomic_store_unordered => try cg.airAtomicStore(inst, .unordered),
2454 .atomic_store_monotonic => try cg.airAtomicStore(inst, .monotonic),
2455 .atomic_store_release => try cg.airAtomicStore(inst, .release),
2456 .atomic_store_seq_cst => try cg.airAtomicStore(inst, .seq_cst),
2457
2458 .array_elem_val => try cg.airArrayElemVal(inst),
2459
2460 .optional_payload => try cg.airOptionalPayload(inst),
2461 .unwrap_errunion_err => try cg.airUnwrapErrUnionErr(inst),
2462 .unwrap_errunion_payload => try cg.airUnwrapErrUnionPayload(inst),
2463 .err_return_trace => try cg.airErrReturnTrace(inst),
2464 .set_err_return_trace => try cg.airSetErrReturnTrace(inst),
2465 .save_err_return_trace_index=> try cg.airSaveErrReturnTraceIndex(inst),
2466
2467 .wrap_optional => try cg.airWrapOptional(inst),
2468 .wrap_errunion_payload => try cg.airWrapErrUnionPayload(inst),
2469 .wrap_errunion_err => try cg.airWrapErrUnionErr(inst),
2470 // zig fmt: on
24292471
2472 .add_safe,
2473 .sub_safe,
2474 .mul_safe,
2475 => return cg.fail("TODO implement safety_checked_instructions", .{}),
24302476 .add_optimized,
24312477 .sub_optimized,
24322478 .mul_optimized,
......@@ -2437,179 +2483,7698 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
24372483 .rem_optimized,
24382484 .mod_optimized,
24392485 .neg_optimized,
2440 .cmp_lt_optimized,
2441 .cmp_lte_optimized,
2442 .cmp_eq_optimized,
2443 .cmp_gte_optimized,
2444 .cmp_gt_optimized,
2445 .cmp_neq_optimized,
2446 .cmp_vector_optimized,
24472486 .reduce_optimized,
24482487 .int_from_float_optimized,
2449 => return self.fail("TODO implement optimized float mode", .{}),
2450
2451 .add_safe,
2452 .sub_safe,
2453 .mul_safe,
2454 => return self.fail("TODO implement safety_checked_instructions", .{}),
2455
2456 .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),
2457 .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),
2458 .vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),
2459
2460 .c_va_arg => try self.airVaArg(inst),
2461 .c_va_copy => try self.airVaCopy(inst),
2462 .c_va_end => try self.airVaEnd(inst),
2463 .c_va_start => try self.airVaStart(inst),
2464
2465 .wasm_memory_size => unreachable,
2466 .wasm_memory_grow => unreachable,
2467
2468 .work_item_id => unreachable,
2469 .work_group_size => unreachable,
2470 .work_group_id => unreachable,
2471 // zig fmt: on
2472 }
2473 self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping);
2474 }
2475 verbose_tracking_log.debug("{}", .{self.fmtTracking()});
2476}
2477
2478fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2479 const pt = self.pt;
2480 const zcu = pt.zcu;
2481 const ip = &zcu.intern_pool;
2482 switch (Type.fromInterned(lazy_sym.ty).zigTypeTag(zcu)) {
2483 .@"enum" => {
2484 const enum_ty = Type.fromInterned(lazy_sym.ty);
2485 wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)});
2486
2487 const resolved_cc = abi.resolveCallingConvention(.auto, self.target.*);
2488 const param_regs = abi.getCAbiIntParamRegs(resolved_cc);
2489 const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
2490 defer for (param_locks) |lock| self.register_manager.unlockReg(lock);
2491
2492 const ret_reg = param_regs[0];
2493 const enum_mcv = MCValue{ .register = param_regs[1] };
2494
2495 const exitlude_jump_relocs = try self.gpa.alloc(Mir.Inst.Index, enum_ty.enumFieldCount(zcu));
2496 defer self.gpa.free(exitlude_jump_relocs);
2497
2498 const data_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
2499 const data_lock = self.register_manager.lockRegAssumeUnused(data_reg);
2500 defer self.register_manager.unlockReg(data_lock);
2501 try self.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = enum_ty.toIntern() });
2502
2503 var data_off: i32 = 0;
2504 const tag_names = enum_ty.enumFields(zcu);
2505 for (exitlude_jump_relocs, 0..) |*exitlude_jump_reloc, tag_index| {
2506 const tag_name_len = tag_names.get(ip)[tag_index].length(ip);
2507 const tag_val = try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index));
2508 const tag_mcv = try self.genTypedValue(tag_val);
2509 try self.genBinOpMir(.{ ._, .cmp }, enum_ty, enum_mcv, tag_mcv);
2510 const skip_reloc = try self.asmJccReloc(.ne, undefined);
2511
2512 try self.genSetMem(
2513 .{ .reg = ret_reg },
2514 0,
2515 Type.usize,
2516 .{ .register_offset = .{ .reg = data_reg, .off = data_off } },
2517 .{},
2518 );
2519 try self.genSetMem(
2520 .{ .reg = ret_reg },
2521 8,
2522 Type.usize,
2523 .{ .immediate = tag_name_len },
2524 .{},
2525 );
2526
2527 exitlude_jump_reloc.* = try self.asmJmpReloc(undefined);
2528 self.performReloc(skip_reloc);
2529
2530 data_off += @intCast(tag_name_len + 1);
2531 }
2532
2533 try self.airTrap();
2534
2535 for (exitlude_jump_relocs) |reloc| self.performReloc(reloc);
2536 try self.asmOpOnly(.{ ._, .ret });
2537 },
2538 else => return self.fail(
2539 "TODO implement {s} for {}",
2540 .{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) },
2541 ),
2542 }
2543}
2544
2545fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void {
2546 for (value.getRegs()) |reg| try self.register_manager.getReg(reg, inst);
2547}
2548
2549fn getValueIfFree(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
2550 for (value.getRegs()) |reg| if (self.register_manager.isRegFree(reg))
2551 self.register_manager.getRegAssumeFree(reg, inst);
2552}
2553
2554fn freeValue(self: *Self, value: MCValue) !void {
2555 switch (value) {
2556 .register => |reg| {
2557 self.register_manager.freeReg(reg);
2558 if (reg.class() == .x87) try self.asmRegister(.{ .f_, .free }, reg);
2559 },
2560 .register_pair => |regs| for (regs) |reg| self.register_manager.freeReg(reg),
2561 .register_offset => |reg_off| self.register_manager.freeReg(reg_off.reg),
2562 .register_overflow => |reg_ov| {
2563 self.register_manager.freeReg(reg_ov.reg);
2564 self.eflags_inst = null;
2565 },
2566 .eflags => self.eflags_inst = null,
2567 else => {}, // TODO process stack allocation death
2568 }
2569}
2570
2571fn feed(self: *Self, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) !void {
2572 if (bt.feed()) if (operand.toIndex()) |inst| try self.processDeath(inst);
2488 => return cg.fail("TODO implement optimized float mode", .{}),
2489
2490 .arg => try cg.airDbgArg(inst),
2491 .ptr_add => |air_tag| if (use_old) try cg.airPtrArithmetic(inst, air_tag) else {
2492 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
2493 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
2494 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
2495 try ops[0].toSlicePtr(cg);
2496 var res: [1]Temp = undefined;
2497 if (ty_pl.ty.toType().elemType2(zcu).hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{ty_pl.ty.toType()}, &ops, comptime &.{ .{
2498 .patterns = &.{
2499 .{ .src = .{ .to_gpr, .simm32 } },
2500 },
2501 .dst_temps = .{.{ .rc = .general_purpose }},
2502 .each = .{ .once = &.{
2503 .{ ._, ._, .lea, .dst0p, .leaa(.none, .src0, .add_src0_elem_size_times_src1), ._, ._ },
2504 } },
2505 }, .{
2506 .dst_constraints = .{.{ .elem_size_is = 1 }},
2507 .patterns = &.{
2508 .{ .src = .{ .to_gpr, .to_gpr } },
2509 },
2510 .dst_temps = .{.{ .rc = .general_purpose }},
2511 .each = .{ .once = &.{
2512 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .src1), ._, ._ },
2513 } },
2514 }, .{
2515 .dst_constraints = .{.{ .elem_size_is = 2 }},
2516 .patterns = &.{
2517 .{ .src = .{ .to_gpr, .to_gpr } },
2518 },
2519 .dst_temps = .{.{ .rc = .general_purpose }},
2520 .each = .{ .once = &.{
2521 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"2", .src1), ._, ._ },
2522 } },
2523 }, .{
2524 .dst_constraints = .{.{ .elem_size_is = 2 + 1 }},
2525 .patterns = &.{
2526 .{ .src = .{ .to_gpr, .to_gpr } },
2527 },
2528 .dst_temps = .{.{ .rc = .general_purpose }},
2529 .each = .{ .once = &.{
2530 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"2", .src1), ._, ._ },
2531 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2532 } },
2533 }, .{
2534 .dst_constraints = .{.{ .elem_size_is = 4 }},
2535 .patterns = &.{
2536 .{ .src = .{ .to_gpr, .to_gpr } },
2537 },
2538 .dst_temps = .{.{ .rc = .general_purpose }},
2539 .each = .{ .once = &.{
2540 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"4", .src1), ._, ._ },
2541 } },
2542 }, .{
2543 .dst_constraints = .{.{ .elem_size_is = 4 + 1 }},
2544 .patterns = &.{
2545 .{ .src = .{ .to_gpr, .to_gpr } },
2546 },
2547 .dst_temps = .{.{ .ref = .src1 }},
2548 .each = .{ .once = &.{
2549 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"4", .src1), ._, ._ },
2550 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2551 } },
2552 }, .{
2553 .required_features = .{ .@"64bit", null, null, null },
2554 .dst_constraints = .{.{ .elem_size_is = 8 }},
2555 .patterns = &.{
2556 .{ .src = .{ .to_gpr, .to_gpr } },
2557 },
2558 .dst_temps = .{.{ .rc = .general_purpose }},
2559 .each = .{ .once = &.{
2560 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"8", .src1), ._, ._ },
2561 } },
2562 }, .{
2563 .required_features = .{ .@"64bit", null, null, null },
2564 .dst_constraints = .{.{ .elem_size_is = 8 + 1 }},
2565 .patterns = &.{
2566 .{ .src = .{ .to_gpr, .to_gpr } },
2567 },
2568 .dst_temps = .{.{ .ref = .src1 }},
2569 .each = .{ .once = &.{
2570 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"8", .src1), ._, ._ },
2571 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2572 } },
2573 }, .{
2574 .dst_constraints = .{.po2_elem_size},
2575 .patterns = &.{
2576 .{ .src = .{ .to_gpr, .to_mut_gpr } },
2577 },
2578 .dst_temps = .{.{ .ref = .src1 }},
2579 .clobbers = .{ .eflags = true },
2580 .each = .{ .once = &.{
2581 .{ ._, ._l, .sh, .src1p, .sa(.none, .add_log2_src0_elem_size), ._, ._ },
2582 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .src1), ._, ._ },
2583 } },
2584 }, .{
2585 .patterns = &.{
2586 .{ .src = .{ .to_gpr, .to_gpr } },
2587 },
2588 .dst_temps = .{.{ .rc = .general_purpose }},
2589 .clobbers = .{ .eflags = true },
2590 .each = .{ .once = &.{
2591 .{ ._, .i_, .mul, .dst0p, .src1p, .sa(.none, .add_src0_elem_size), ._ },
2592 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2593 } },
2594 } }) catch |err| switch (err) {
2595 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
2596 @tagName(air_tag),
2597 cg.typeOf(bin_op.lhs).fmt(pt),
2598 ops[0].tracking(cg),
2599 ops[1].tracking(cg),
2600 }),
2601 else => |e| return e,
2602 } else { // hack around Sema OPV bugs
2603 res[0] = ops[0];
2604 }
2605 for (ops) |op| for (res) |r| {
2606 if (op.index == r.index) break;
2607 } else try op.die(cg);
2608 try res[0].moveTo(inst, cg);
2609 },
2610 .ptr_sub => |air_tag| if (use_old) try cg.airPtrArithmetic(inst, air_tag) else {
2611 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
2612 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
2613 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
2614 try ops[0].toSlicePtr(cg);
2615 var res: [1]Temp = undefined;
2616 if (ty_pl.ty.toType().elemType2(zcu).hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{ty_pl.ty.toType()}, &ops, comptime &.{ .{
2617 .patterns = &.{
2618 .{ .src = .{ .to_gpr, .simm32 } },
2619 },
2620 .dst_temps = .{.{ .rc = .general_purpose }},
2621 .each = .{ .once = &.{
2622 .{ ._, ._, .lea, .dst0p, .leaa(.none, .src0, .sub_src0_elem_size_times_src1), ._, ._ },
2623 } },
2624 }, .{
2625 .dst_constraints = .{.{ .elem_size_is = 1 }},
2626 .patterns = &.{
2627 .{ .src = .{ .to_gpr, .to_mut_gpr } },
2628 },
2629 .dst_temps = .{.{ .ref = .src1 }},
2630 .clobbers = .{ .eflags = true },
2631 .each = .{ .once = &.{
2632 .{ ._, ._, .neg, .src1p, ._, ._, ._ },
2633 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .src1), ._, ._ },
2634 } },
2635 }, .{
2636 .dst_constraints = .{.{ .elem_size_is = 2 }},
2637 .patterns = &.{
2638 .{ .src = .{ .to_gpr, .to_mut_gpr } },
2639 },
2640 .dst_temps = .{.{ .ref = .src1 }},
2641 .clobbers = .{ .eflags = true },
2642 .each = .{ .once = &.{
2643 .{ ._, ._, .neg, .src1p, ._, ._, ._ },
2644 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"2", .src1), ._, ._ },
2645 } },
2646 }, .{
2647 .dst_constraints = .{.{ .elem_size_is = 2 + 1 }},
2648 .patterns = &.{
2649 .{ .src = .{ .to_gpr, .to_gpr } },
2650 },
2651 .dst_temps = .{.{ .rc = .general_purpose }},
2652 .clobbers = .{ .eflags = true },
2653 .each = .{ .once = &.{
2654 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"2", .src1), ._, ._ },
2655 .{ ._, ._, .neg, .dst0p, ._, ._, ._ },
2656 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2657 } },
2658 }, .{
2659 .dst_constraints = .{.{ .elem_size_is = 4 }},
2660 .patterns = &.{
2661 .{ .src = .{ .to_gpr, .to_mut_gpr } },
2662 },
2663 .dst_temps = .{.{ .ref = .src1 }},
2664 .clobbers = .{ .eflags = true },
2665 .each = .{ .once = &.{
2666 .{ ._, ._, .neg, .src1p, ._, ._, ._ },
2667 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"4", .src1), ._, ._ },
2668 } },
2669 }, .{
2670 .dst_constraints = .{.{ .elem_size_is = 4 + 1 }},
2671 .patterns = &.{
2672 .{ .src = .{ .to_gpr, .to_gpr } },
2673 },
2674 .dst_temps = .{.{ .rc = .general_purpose }},
2675 .clobbers = .{ .eflags = true },
2676 .each = .{ .once = &.{
2677 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"4", .src1), ._, ._ },
2678 .{ ._, ._, .neg, .dst0p, ._, ._, ._ },
2679 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2680 } },
2681 }, .{
2682 .required_features = .{ .@"64bit", null, null, null },
2683 .dst_constraints = .{.{ .elem_size_is = 8 }},
2684 .patterns = &.{
2685 .{ .src = .{ .to_gpr, .to_mut_gpr } },
2686 },
2687 .dst_temps = .{.{ .ref = .src1 }},
2688 .clobbers = .{ .eflags = true },
2689 .each = .{ .once = &.{
2690 .{ ._, ._, .neg, .src1p, ._, ._, ._ },
2691 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"8", .src1), ._, ._ },
2692 } },
2693 }, .{
2694 .required_features = .{ .@"64bit", null, null, null },
2695 .dst_constraints = .{.{ .elem_size_is = 8 + 1 }},
2696 .patterns = &.{
2697 .{ .src = .{ .to_gpr, .to_gpr } },
2698 },
2699 .dst_temps = .{.{ .rc = .general_purpose }},
2700 .clobbers = .{ .eflags = true },
2701 .each = .{ .once = &.{
2702 .{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"8", .src1), ._, ._ },
2703 .{ ._, ._, .neg, .dst0p, ._, ._, ._ },
2704 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2705 } },
2706 }, .{
2707 .dst_constraints = .{.po2_elem_size},
2708 .patterns = &.{
2709 .{ .src = .{ .to_gpr, .to_mut_gpr } },
2710 },
2711 .dst_temps = .{.{ .ref = .src1 }},
2712 .clobbers = .{ .eflags = true },
2713 .each = .{ .once = &.{
2714 .{ ._, ._l, .sa, .src1p, .sa(.none, .add_log2_src0_elem_size), ._, ._ },
2715 .{ ._, ._, .neg, .src1p, ._, ._, ._ },
2716 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .src1), ._, ._ },
2717 } },
2718 }, .{
2719 .patterns = &.{
2720 .{ .src = .{ .to_gpr, .to_gpr } },
2721 },
2722 .dst_temps = .{.{ .rc = .general_purpose }},
2723 .clobbers = .{ .eflags = true },
2724 .each = .{ .once = &.{
2725 .{ ._, .i_, .mul, .dst0p, .src1p, .sa(.none, .sub_src0_elem_size), ._ },
2726 .{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2727 } },
2728 } }) catch |err| switch (err) {
2729 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
2730 @tagName(air_tag),
2731 cg.typeOf(bin_op.lhs).fmt(pt),
2732 ops[0].tracking(cg),
2733 ops[1].tracking(cg),
2734 }),
2735 else => |e| return e,
2736 } else {
2737 // hack around Sema OPV bugs
2738 res[0] = ops[0];
2739 }
2740 for (ops) |op| for (res) |r| {
2741 if (op.index == r.index) break;
2742 } else try op.die(cg);
2743 try res[0].moveTo(inst, cg);
2744 },
2745 .alloc => if (use_old) try cg.airAlloc(inst) else {
2746 const ty = air_datas[@intFromEnum(inst)].ty;
2747 var slot = try cg.tempInit(ty, .{ .lea_frame = .{
2748 .index = try cg.allocMemPtr(inst),
2749 } });
2750 try slot.moveTo(inst, cg);
2751 },
2752 .inferred_alloc, .inferred_alloc_comptime => unreachable,
2753 .ret_ptr => if (use_old) try cg.airRetPtr(inst) else {
2754 const ty = air_datas[@intFromEnum(inst)].ty;
2755 var slot = switch (cg.ret_mcv.long) {
2756 else => unreachable,
2757 .none => try cg.tempInit(ty, .{ .lea_frame = .{
2758 .index = try cg.allocMemPtr(inst),
2759 } }),
2760 .load_frame => slot: {
2761 var slot = try cg.tempInit(ty, cg.ret_mcv.long);
2762 try slot.toOffset(cg.ret_mcv.short.indirect.off, cg);
2763 break :slot slot;
2764 },
2765 };
2766 try slot.moveTo(inst, cg);
2767 },
2768 .assembly => try cg.airAsm(inst),
2769 .bit_and, .bit_or, .xor, .bool_and, .bool_or => |air_tag| if (use_old) try cg.airBinOp(inst, air_tag) else {
2770 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
2771 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
2772 var res: [1]Temp = undefined;
2773 cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, switch (@as(Mir.Inst.Tag, switch (air_tag) {
2774 else => unreachable,
2775 .bit_and, .bool_and => .@"and",
2776 .bit_or, .bool_or => .@"or",
2777 .xor => .xor,
2778 })) {
2779 else => unreachable,
2780 inline .@"and", .@"or", .xor => |mir_tag| comptime &.{ .{
2781 .src_constraints = .{ .{ .size = .byte }, .{ .size = .byte } },
2782 .patterns = &.{
2783 .{ .src = .{ .mut_mem, .imm8 } },
2784 .{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } },
2785 .{ .src = .{ .to_mut_gpr, .imm8 } },
2786 .{ .src = .{ .imm8, .to_mut_gpr }, .commute = .{ 0, 1 } },
2787 .{ .src = .{ .mut_mem, .to_gpr } },
2788 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
2789 .{ .src = .{ .to_mut_gpr, .mem } },
2790 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
2791 .{ .src = .{ .to_mut_gpr, .to_gpr } },
2792 },
2793 .dst_temps = .{.{ .ref = .src0 }},
2794 .clobbers = .{ .eflags = true },
2795 .each = .{ .once = &.{
2796 .{ ._, ._, mir_tag, .dst0b, .src1b, ._, ._ },
2797 } },
2798 }, .{
2799 .src_constraints = .{ .{ .size = .word }, .{ .size = .word } },
2800 .patterns = &.{
2801 .{ .src = .{ .mut_mem, .imm16 } },
2802 .{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } },
2803 .{ .src = .{ .to_mut_gpr, .imm16 } },
2804 .{ .src = .{ .imm16, .to_mut_gpr }, .commute = .{ 0, 1 } },
2805 .{ .src = .{ .mut_mem, .to_gpr } },
2806 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
2807 .{ .src = .{ .to_mut_gpr, .mem } },
2808 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
2809 .{ .src = .{ .to_mut_gpr, .to_gpr } },
2810 },
2811 .dst_temps = .{.{ .ref = .src0 }},
2812 .clobbers = .{ .eflags = true },
2813 .each = .{ .once = &.{
2814 .{ ._, ._, mir_tag, .dst0w, .src1w, ._, ._ },
2815 } },
2816 }, .{
2817 .src_constraints = .{ .{ .size = .dword }, .{ .size = .dword } },
2818 .patterns = &.{
2819 .{ .src = .{ .mut_mem, .imm32 } },
2820 .{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } },
2821 .{ .src = .{ .to_mut_gpr, .imm32 } },
2822 .{ .src = .{ .imm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
2823 .{ .src = .{ .mut_mem, .to_gpr } },
2824 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
2825 .{ .src = .{ .to_mut_gpr, .mem } },
2826 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
2827 .{ .src = .{ .to_mut_gpr, .to_gpr } },
2828 },
2829 .dst_temps = .{.{ .ref = .src0 }},
2830 .clobbers = .{ .eflags = true },
2831 .each = .{ .once = &.{
2832 .{ ._, ._, mir_tag, .dst0d, .src1d, ._, ._ },
2833 } },
2834 }, .{
2835 .required_features = .{ .@"64bit", null, null, null },
2836 .src_constraints = .{ .{ .size = .qword }, .{ .size = .qword } },
2837 .patterns = &.{
2838 .{ .src = .{ .mut_mem, .simm32 } },
2839 .{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } },
2840 .{ .src = .{ .to_mut_gpr, .simm32 } },
2841 .{ .src = .{ .simm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
2842 .{ .src = .{ .mut_mem, .to_gpr } },
2843 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
2844 .{ .src = .{ .to_mut_gpr, .mem } },
2845 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
2846 .{ .src = .{ .to_mut_gpr, .to_gpr } },
2847 },
2848 .dst_temps = .{.{ .ref = .src0 }},
2849 .clobbers = .{ .eflags = true },
2850 .each = .{ .once = &.{
2851 .{ ._, ._, mir_tag, .dst0q, .src1q, ._, ._ },
2852 } },
2853 }, .{
2854 .required_features = .{ .mmx, null, null, null },
2855 .src_constraints = .{ .{ .size = .qword }, .{ .size = .qword } },
2856 .patterns = &.{
2857 .{ .src = .{ .to_mut_mm, .mem } },
2858 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
2859 .{ .src = .{ .to_mut_mm, .to_mm } },
2860 },
2861 .dst_temps = .{.{ .ref = .src0 }},
2862 .each = .{ .once = &.{
2863 .{ ._, .p_, mir_tag, .dst0q, .src1q, ._, ._ },
2864 } },
2865 }, .{
2866 .required_features = .{ .avx, null, null, null },
2867 .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword } },
2868 .patterns = &.{
2869 .{ .src = .{ .to_xmm, .mem } },
2870 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
2871 .{ .src = .{ .to_xmm, .to_xmm } },
2872 },
2873 .dst_temps = .{.{ .rc = .sse }},
2874 .each = .{ .once = &.{
2875 .{ ._, .vp_, mir_tag, .dst0x, .src0x, .src1x, ._ },
2876 } },
2877 }, .{
2878 .required_features = .{ .sse2, null, null, null },
2879 .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword } },
2880 .patterns = &.{
2881 .{ .src = .{ .to_mut_xmm, .mem } },
2882 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
2883 .{ .src = .{ .to_mut_xmm, .to_xmm } },
2884 },
2885 .dst_temps = .{.{ .ref = .src0 }},
2886 .each = .{ .once = &.{
2887 .{ ._, .p_, mir_tag, .dst0x, .src1x, ._, ._ },
2888 } },
2889 }, .{
2890 .required_features = .{ .sse, null, null, null },
2891 .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword } },
2892 .patterns = &.{
2893 .{ .src = .{ .to_mut_xmm, .mem } },
2894 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
2895 .{ .src = .{ .to_mut_xmm, .to_xmm } },
2896 },
2897 .dst_temps = .{.{ .ref = .src0 }},
2898 .each = .{ .once = &.{
2899 .{ ._, ._ps, mir_tag, .dst0x, .src1x, ._, ._ },
2900 } },
2901 }, .{
2902 .required_features = .{ .avx2, null, null, null },
2903 .src_constraints = .{ .{ .size = .yword }, .{ .size = .yword } },
2904 .patterns = &.{
2905 .{ .src = .{ .to_ymm, .mem } },
2906 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
2907 .{ .src = .{ .to_ymm, .to_ymm } },
2908 },
2909 .dst_temps = .{.{ .rc = .sse }},
2910 .each = .{ .once = &.{
2911 .{ ._, .vp_, mir_tag, .dst0y, .src0y, .src1y, ._ },
2912 } },
2913 }, .{
2914 .required_features = .{ .avx, null, null, null },
2915 .src_constraints = .{ .{ .size = .yword }, .{ .size = .yword } },
2916 .patterns = &.{
2917 .{ .src = .{ .to_ymm, .mem } },
2918 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
2919 .{ .src = .{ .to_ymm, .to_ymm } },
2920 },
2921 .dst_temps = .{.{ .rc = .sse }},
2922 .each = .{ .once = &.{
2923 .{ ._, .v_pd, mir_tag, .dst0y, .src0y, .src1y, ._ },
2924 } },
2925 }, .{
2926 .required_features = .{ .avx2, null, null, null },
2927 .src_constraints = .{ .{ .multiple_size = .yword }, .{ .multiple_size = .yword } },
2928 .patterns = &.{
2929 .{ .src = .{ .to_mem, .to_mem } },
2930 },
2931 .extra_temps = .{
2932 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
2933 .{ .kind = .{ .rc = .sse } },
2934 .unused,
2935 .unused,
2936 .unused,
2937 .unused,
2938 },
2939 .dst_temps = .{.mem},
2940 .clobbers = .{ .eflags = true },
2941 .each = .{ .once = &.{
2942 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
2943 .{ .@"0:", .v_dqu, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
2944 .{ ._, .vp_, mir_tag, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ },
2945 .{ ._, .v_dqu, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ },
2946 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
2947 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2948 } },
2949 }, .{
2950 .required_features = .{ .avx, null, null, null },
2951 .src_constraints = .{ .{ .multiple_size = .yword }, .{ .multiple_size = .yword } },
2952 .patterns = &.{
2953 .{ .src = .{ .to_mem, .to_mem } },
2954 },
2955 .extra_temps = .{
2956 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
2957 .{ .kind = .{ .rc = .sse } },
2958 .unused,
2959 .unused,
2960 .unused,
2961 .unused,
2962 },
2963 .dst_temps = .{.mem},
2964 .clobbers = .{ .eflags = true },
2965 .each = .{ .once = &.{
2966 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
2967 .{ .@"0:", .v_pd, .movu, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
2968 .{ ._, .v_pd, mir_tag, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ },
2969 .{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ },
2970 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
2971 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2972 } },
2973 }, .{
2974 .required_features = .{ .avx, null, null, null },
2975 .src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword } },
2976 .patterns = &.{
2977 .{ .src = .{ .to_mem, .to_mem } },
2978 },
2979 .extra_temps = .{
2980 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
2981 .{ .kind = .{ .rc = .sse } },
2982 .unused,
2983 .unused,
2984 .unused,
2985 .unused,
2986 },
2987 .dst_temps = .{.mem},
2988 .clobbers = .{ .eflags = true },
2989 .each = .{ .once = &.{
2990 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
2991 .{ .@"0:", .v_dqu, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
2992 .{ ._, .vp_, mir_tag, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ },
2993 .{ ._, .v_dqu, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
2994 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
2995 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2996 } },
2997 }, .{
2998 .required_features = .{ .sse2, null, null, null },
2999 .src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword } },
3000 .patterns = &.{
3001 .{ .src = .{ .to_mem, .to_mem } },
3002 },
3003 .extra_temps = .{
3004 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3005 .{ .kind = .{ .rc = .sse } },
3006 .unused,
3007 .unused,
3008 .unused,
3009 .unused,
3010 },
3011 .dst_temps = .{.mem},
3012 .clobbers = .{ .eflags = true },
3013 .each = .{ .once = &.{
3014 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3015 .{ .@"0:", ._dqu, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3016 .{ ._, .p_, mir_tag, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
3017 .{ ._, ._dqu, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
3018 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3019 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3020 } },
3021 }, .{
3022 .required_features = .{ .sse, null, null, null },
3023 .src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword } },
3024 .patterns = &.{
3025 .{ .src = .{ .to_mem, .to_mem } },
3026 },
3027 .extra_temps = .{
3028 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3029 .{ .kind = .{ .rc = .sse } },
3030 .unused,
3031 .unused,
3032 .unused,
3033 .unused,
3034 },
3035 .dst_temps = .{.mem},
3036 .clobbers = .{ .eflags = true },
3037 .each = .{ .once = &.{
3038 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3039 .{ .@"0:", ._ps, .movu, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3040 .{ ._, ._ps, mir_tag, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
3041 .{ ._, ._ps, .movu, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
3042 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3043 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3044 } },
3045 }, .{
3046 .required_features = .{ .mmx, null, null, null },
3047 .src_constraints = .{ .{ .multiple_size = .qword }, .{ .multiple_size = .qword } },
3048 .patterns = &.{
3049 .{ .src = .{ .to_mem, .to_mem } },
3050 },
3051 .extra_temps = .{
3052 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3053 .{ .kind = .{ .rc = .mmx } },
3054 .unused,
3055 .unused,
3056 .unused,
3057 .unused,
3058 },
3059 .dst_temps = .{.mem},
3060 .clobbers = .{ .eflags = true },
3061 .each = .{ .once = &.{
3062 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3063 .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
3064 .{ ._, .p_, mir_tag, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
3065 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ },
3066 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3067 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3068 } },
3069 }, .{
3070 .src_constraints = .{ .{ .multiple_size = .qword }, .{ .multiple_size = .qword } },
3071 .patterns = &.{
3072 .{ .src = .{ .to_mem, .to_mem } },
3073 },
3074 .extra_temps = .{
3075 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3076 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3077 .unused,
3078 .unused,
3079 .unused,
3080 .unused,
3081 },
3082 .dst_temps = .{.mem},
3083 .clobbers = .{ .eflags = true },
3084 .each = .{ .once = &.{
3085 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3086 .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
3087 .{ ._, ._, mir_tag, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
3088 .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
3089 .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
3090 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3091 } },
3092 } },
3093 }) catch |err| switch (err) {
3094 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
3095 @tagName(air_tag),
3096 cg.typeOf(bin_op.lhs).fmt(pt),
3097 ops[0].tracking(cg),
3098 ops[1].tracking(cg),
3099 }),
3100 else => |e| return e,
3101 };
3102 for (ops) |op| for (res) |r| {
3103 if (op.index == r.index) break;
3104 } else try op.die(cg);
3105 try res[0].moveTo(inst, cg);
3106 },
3107 .not => |air_tag| if (use_old) try cg.airUnOp(inst, air_tag) else {
3108 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
3109 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
3110 var res: [1]Temp = undefined;
3111 cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{
3112 .src_constraints = .{ .{ .signed_or_exact_int = .byte }, .any },
3113 .patterns = &.{
3114 .{ .src = .{ .mut_mem, .none } },
3115 .{ .src = .{ .to_mut_gpr, .none } },
3116 },
3117 .dst_temps = .{.{ .ref = .src0 }},
3118 .each = .{ .once = &.{
3119 .{ ._, ._, .not, .dst0b, ._, ._, ._ },
3120 } },
3121 }, .{
3122 .src_constraints = .{ .{ .unsigned_int = .byte }, .any },
3123 .patterns = &.{
3124 .{ .src = .{ .mut_mem, .none } },
3125 .{ .src = .{ .to_mut_gpr, .none } },
3126 },
3127 .dst_temps = .{.{ .ref = .src0 }},
3128 .clobbers = .{ .eflags = true },
3129 .each = .{ .once = &.{
3130 .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_umax), ._, ._ },
3131 } },
3132 }, .{
3133 .src_constraints = .{ .{ .signed_or_exact_int = .word }, .any },
3134 .patterns = &.{
3135 .{ .src = .{ .mut_mem, .none } },
3136 .{ .src = .{ .to_mut_gpr, .none } },
3137 },
3138 .dst_temps = .{.{ .ref = .src0 }},
3139 .each = .{ .once = &.{
3140 .{ ._, ._, .not, .dst0w, ._, ._, ._ },
3141 } },
3142 }, .{
3143 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
3144 .patterns = &.{
3145 .{ .src = .{ .mut_mem, .none } },
3146 .{ .src = .{ .to_mut_gpr, .none } },
3147 },
3148 .dst_temps = .{.{ .ref = .src0 }},
3149 .clobbers = .{ .eflags = true },
3150 .each = .{ .once = &.{
3151 .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_umax), ._, ._ },
3152 } },
3153 }, .{
3154 .src_constraints = .{ .{ .signed_or_exact_int = .dword }, .any },
3155 .patterns = &.{
3156 .{ .src = .{ .mut_mem, .none } },
3157 .{ .src = .{ .to_mut_gpr, .none } },
3158 },
3159 .dst_temps = .{.{ .ref = .src0 }},
3160 .each = .{ .once = &.{
3161 .{ ._, ._, .not, .dst0d, ._, ._, ._ },
3162 } },
3163 }, .{
3164 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
3165 .patterns = &.{
3166 .{ .src = .{ .mut_mem, .none } },
3167 .{ .src = .{ .to_mut_gpr, .none } },
3168 },
3169 .dst_temps = .{.{ .ref = .src0 }},
3170 .clobbers = .{ .eflags = true },
3171 .each = .{ .once = &.{
3172 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_umax), ._, ._ },
3173 } },
3174 }, .{
3175 .required_features = .{ .@"64bit", null, null, null },
3176 .src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any },
3177 .patterns = &.{
3178 .{ .src = .{ .mut_mem, .none } },
3179 .{ .src = .{ .to_mut_gpr, .none } },
3180 },
3181 .dst_temps = .{.{ .ref = .src0 }},
3182 .each = .{ .once = &.{
3183 .{ ._, ._, .not, .dst0q, ._, ._, ._ },
3184 } },
3185 }, .{
3186 .required_features = .{ .@"64bit", null, null, null },
3187 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
3188 .patterns = &.{
3189 .{ .src = .{ .mem, .none } },
3190 .{ .src = .{ .to_gpr, .none } },
3191 },
3192 .dst_temps = .{.{ .rc = .general_purpose }},
3193 .each = .{ .once = &.{
3194 .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
3195 .{ ._, ._, .xor, .dst0q, .src0q, ._, ._ },
3196 } },
3197 }, .{
3198 .required_features = .{ .mmx, null, null, null },
3199 .src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any },
3200 .patterns = &.{
3201 .{ .src = .{ .mem, .none } },
3202 .{ .src = .{ .to_mm, .none } },
3203 },
3204 .dst_temps = .{.{ .rc = .mmx }},
3205 .each = .{ .once = &.{
3206 .{ ._, .p_d, .cmpeq, .dst0q, .dst0q, ._, ._ },
3207 .{ ._, .p_, .xor, .dst0q, .src0q, ._, ._ },
3208 } },
3209 }, .{
3210 .required_features = .{ .mmx, null, null, null },
3211 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
3212 .patterns = &.{
3213 .{ .src = .{ .to_mut_mm, .none } },
3214 },
3215 .extra_temps = .{
3216 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3217 .{ .kind = .{ .umax_mem = .src0 } },
3218 .unused,
3219 .unused,
3220 .unused,
3221 .unused,
3222 },
3223 .dst_temps = .{.{ .ref = .src0 }},
3224 .each = .{ .once = &.{
3225 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3226 .{ ._, .p_, .xor, .dst0q, .lea(.qword, .tmp0), ._, ._ },
3227 } },
3228 }, .{
3229 .required_features = .{ .avx, null, null, null },
3230 .src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any },
3231 .patterns = &.{
3232 .{ .src = .{ .mem, .none } },
3233 .{ .src = .{ .to_xmm, .none } },
3234 },
3235 .dst_temps = .{.{ .rc = .sse }},
3236 .each = .{ .once = &.{
3237 .{ ._, .vp_q, .cmpeq, .dst0x, .dst0x, .dst0x, ._ },
3238 .{ ._, .vp_, .xor, .dst0x, .dst0x, .src0x, ._ },
3239 } },
3240 }, .{
3241 .required_features = .{ .avx, null, null, null },
3242 .src_constraints = .{ .{ .unsigned_int = .xword }, .any },
3243 .patterns = &.{
3244 .{ .src = .{ .to_xmm, .none } },
3245 },
3246 .extra_temps = .{
3247 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3248 .{ .kind = .{ .umax_mem = .src0 } },
3249 .unused,
3250 .unused,
3251 .unused,
3252 .unused,
3253 },
3254 .dst_temps = .{.{ .rc = .sse }},
3255 .each = .{ .once = &.{
3256 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3257 .{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.xword, .tmp0), ._ },
3258 } },
3259 }, .{
3260 .required_features = .{ .sse2, null, null, null },
3261 .src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any },
3262 .patterns = &.{
3263 .{ .src = .{ .mem, .none } },
3264 .{ .src = .{ .to_xmm, .none } },
3265 },
3266 .dst_temps = .{.{ .rc = .sse }},
3267 .each = .{ .once = &.{
3268 .{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ },
3269 .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ },
3270 } },
3271 }, .{
3272 .required_features = .{ .sse2, null, null, null },
3273 .src_constraints = .{ .{ .unsigned_int = .xword }, .any },
3274 .patterns = &.{
3275 .{ .src = .{ .to_mut_xmm, .none } },
3276 },
3277 .extra_temps = .{
3278 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3279 .{ .kind = .{ .umax_mem = .src0 } },
3280 .unused,
3281 .unused,
3282 .unused,
3283 .unused,
3284 },
3285 .dst_temps = .{.{ .ref = .src0 }},
3286 .each = .{ .once = &.{
3287 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3288 .{ ._, .p_, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
3289 } },
3290 }, .{
3291 .required_features = .{ .sse, null, null, null },
3292 .src_constraints = .{ .{ .int = .xword }, .any },
3293 .patterns = &.{
3294 .{ .src = .{ .to_mut_xmm, .none } },
3295 },
3296 .extra_temps = .{
3297 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3298 .{ .kind = .{ .umax_mem = .src0 } },
3299 .unused,
3300 .unused,
3301 .unused,
3302 .unused,
3303 },
3304 .dst_temps = .{.{ .ref = .src0 }},
3305 .each = .{ .once = &.{
3306 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3307 .{ ._, ._ps, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
3308 } },
3309 }, .{
3310 .required_features = .{ .avx2, null, null, null },
3311 .src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any },
3312 .patterns = &.{
3313 .{ .src = .{ .mem, .none } },
3314 .{ .src = .{ .to_ymm, .none } },
3315 },
3316 .dst_temps = .{.{ .rc = .sse }},
3317 .each = .{ .once = &.{
3318 .{ ._, .vp_q, .cmpeq, .dst0y, .dst0y, .dst0y, ._ },
3319 .{ ._, .vp_, .xor, .dst0y, .dst0y, .src0y, ._ },
3320 } },
3321 }, .{
3322 .required_features = .{ .avx2, null, null, null },
3323 .src_constraints = .{ .{ .unsigned_int = .yword }, .any },
3324 .patterns = &.{
3325 .{ .src = .{ .to_ymm, .none } },
3326 },
3327 .extra_temps = .{
3328 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3329 .{ .kind = .{ .umax_mem = .src0 } },
3330 .unused,
3331 .unused,
3332 .unused,
3333 .unused,
3334 },
3335 .dst_temps = .{.{ .rc = .sse }},
3336 .each = .{ .once = &.{
3337 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3338 .{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
3339 } },
3340 }, .{
3341 .required_features = .{ .avx, null, null, null },
3342 .src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any },
3343 .patterns = &.{
3344 .{ .src = .{ .mem, .none } },
3345 .{ .src = .{ .to_ymm, .none } },
3346 },
3347 .dst_temps = .{.{ .rc = .sse }},
3348 .each = .{ .once = &.{
3349 .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .si(0b01111) },
3350 .{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ },
3351 } },
3352 }, .{
3353 .required_features = .{ .avx, null, null, null },
3354 .src_constraints = .{ .{ .unsigned_int = .yword }, .any },
3355 .patterns = &.{
3356 .{ .src = .{ .to_ymm, .none } },
3357 },
3358 .extra_temps = .{
3359 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3360 .{ .kind = .{ .umax_mem = .src0 } },
3361 .unused,
3362 .unused,
3363 .unused,
3364 .unused,
3365 },
3366 .dst_temps = .{.{ .rc = .sse }},
3367 .each = .{ .once = &.{
3368 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3369 .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
3370 } },
3371 }, .{
3372 .required_features = .{ .avx2, null, null, null },
3373 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any },
3374 .patterns = &.{
3375 .{ .src = .{ .to_mem, .none } },
3376 },
3377 .extra_temps = .{
3378 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3379 .{ .kind = .{ .rc = .sse } },
3380 .{ .kind = .{ .rc = .sse } },
3381 .unused,
3382 .unused,
3383 .unused,
3384 },
3385 .dst_temps = .{.mem},
3386 .clobbers = .{ .eflags = true },
3387 .each = .{ .once = &.{
3388 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3389 .{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ },
3390 .{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ },
3391 .{ ._, .v_dqu, .mov, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ },
3392 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
3393 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3394 .{ .@"0:", .vp_, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ },
3395 .{ ._, .v_dqa, .mov, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ },
3396 } },
3397 }, .{
3398 .required_features = .{ .avx2, null, null, null },
3399 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any },
3400 .patterns = &.{
3401 .{ .src = .{ .to_mem, .none } },
3402 },
3403 .extra_temps = .{
3404 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3405 .{ .kind = .{ .rc = .sse } },
3406 .{ .kind = .{ .rc = .sse } },
3407 .unused,
3408 .unused,
3409 .unused,
3410 },
3411 .dst_temps = .{.mem},
3412 .clobbers = .{ .eflags = true },
3413 .each = .{ .once = &.{
3414 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3415 .{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ },
3416 .{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ },
3417 .{ ._, .v_dqu, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ },
3418 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
3419 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3420 } },
3421 }, .{
3422 .required_features = .{ .avx, null, null, null },
3423 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any },
3424 .patterns = &.{
3425 .{ .src = .{ .to_mem, .none } },
3426 },
3427 .extra_temps = .{
3428 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3429 .{ .kind = .{ .rc = .sse } },
3430 .{ .kind = .{ .rc = .sse } },
3431 .unused,
3432 .unused,
3433 .unused,
3434 },
3435 .dst_temps = .{.mem},
3436 .clobbers = .{ .eflags = true },
3437 .each = .{ .once = &.{
3438 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3439 .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .si(0b01111) },
3440 .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ },
3441 .{ ._, .v_pd, .movu, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ },
3442 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
3443 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3444 .{ .@"0:", .v_pd, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ },
3445 .{ ._, .v_pd, .mova, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ },
3446 } },
3447 }, .{
3448 .required_features = .{ .avx, null, null, null },
3449 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any },
3450 .patterns = &.{
3451 .{ .src = .{ .to_mem, .none } },
3452 },
3453 .extra_temps = .{
3454 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3455 .{ .kind = .{ .rc = .sse } },
3456 .{ .kind = .{ .rc = .sse } },
3457 .unused,
3458 .unused,
3459 .unused,
3460 },
3461 .dst_temps = .{.mem},
3462 .clobbers = .{ .eflags = true },
3463 .each = .{ .once = &.{
3464 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3465 .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .si(0b01111) },
3466 .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ },
3467 .{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ },
3468 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
3469 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3470 } },
3471 }, .{
3472 .required_features = .{ .avx, null, null, null },
3473 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3474 .patterns = &.{
3475 .{ .src = .{ .to_mem, .none } },
3476 },
3477 .extra_temps = .{
3478 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3479 .{ .kind = .{ .rc = .sse } },
3480 .{ .kind = .{ .rc = .sse } },
3481 .unused,
3482 .unused,
3483 .unused,
3484 },
3485 .dst_temps = .{.mem},
3486 .clobbers = .{ .eflags = true },
3487 .each = .{ .once = &.{
3488 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3489 .{ ._, .vp_q, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ },
3490 .{ .@"0:", .v_, .xor, .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._ },
3491 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ },
3492 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3493 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3494 } },
3495 }, .{
3496 .required_features = .{ .sse2, null, null, null },
3497 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3498 .patterns = &.{
3499 .{ .src = .{ .to_mem, .none } },
3500 },
3501 .extra_temps = .{
3502 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3503 .{ .kind = .{ .rc = .sse } },
3504 .{ .kind = .{ .rc = .sse } },
3505 .unused,
3506 .unused,
3507 .unused,
3508 },
3509 .dst_temps = .{.mem},
3510 .clobbers = .{ .eflags = true },
3511 .each = .{ .once = &.{
3512 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3513 .{ ._, .p_d, .cmpeq, .tmp1x, .tmp1x, ._, ._ },
3514 .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3515 .{ ._, .p_, .xor, .tmp2x, .tmp1x, ._, ._ },
3516 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ },
3517 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3518 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3519 } },
3520 }, .{
3521 .required_features = .{ .@"64bit", null, null, null },
3522 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3523 .patterns = &.{
3524 .{ .src = .{ .mut_mem, .none } },
3525 },
3526 .extra_temps = .{
3527 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3528 .unused,
3529 .unused,
3530 .unused,
3531 .unused,
3532 .unused,
3533 },
3534 .dst_temps = .{.{ .ref = .src0 }},
3535 .clobbers = .{ .eflags = true },
3536 .each = .{ .once = &.{
3537 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3538 .{ .@"0:", ._, .not, .memia(.dst0q, .tmp0, .add_size), ._, ._, ._ },
3539 .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, 8), ._, ._, ._ },
3540 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3541 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3542 } },
3543 }, .{
3544 .required_features = .{ .@"64bit", null, null, null },
3545 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3546 .patterns = &.{
3547 .{ .src = .{ .to_mem, .none } },
3548 },
3549 .extra_temps = .{
3550 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3551 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3552 .unused,
3553 .unused,
3554 .unused,
3555 .unused,
3556 },
3557 .dst_temps = .{.mem},
3558 .clobbers = .{ .eflags = true },
3559 .each = .{ .once = &.{
3560 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3561 .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
3562 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3563 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ },
3564 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3565 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3566 } },
3567 }, .{
3568 .required_features = .{ .@"64bit", null, null, null },
3569 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = .dword } }, .any },
3570 .patterns = &.{
3571 .{ .src = .{ .mut_mem, .none } },
3572 },
3573 .extra_temps = .{
3574 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3575 .unused,
3576 .unused,
3577 .unused,
3578 .unused,
3579 .unused,
3580 },
3581 .dst_temps = .{.{ .ref = .src0 }},
3582 .clobbers = .{ .eflags = true },
3583 .each = .{ .once = &.{
3584 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3585 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
3586 .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
3587 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3588 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3589 .{ ._, ._, .not, .memad(.dst0d, .add_size, -16), ._, ._, ._ },
3590 } },
3591 }, .{
3592 .required_features = .{ .@"64bit", null, null, null },
3593 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = .dword } }, .any },
3594 .patterns = &.{
3595 .{ .src = .{ .to_mem, .none } },
3596 },
3597 .extra_temps = .{
3598 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3599 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3600 .unused,
3601 .unused,
3602 .unused,
3603 .unused,
3604 },
3605 .dst_temps = .{.mem},
3606 .clobbers = .{ .eflags = true },
3607 .each = .{ .once = &.{
3608 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3609 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ },
3610 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3611 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ },
3612 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3613 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3614 .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -16), ._, ._ },
3615 .{ ._, ._, .not, .tmp0d, ._, ._, ._ },
3616 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16), .tmp0d, ._, ._ },
3617 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16 + 4), .si(0), ._, ._ },
3618 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16 + 8), .si(0), ._, ._ },
3619 } },
3620 }, .{
3621 .required_features = .{ .@"64bit", null, null, null },
3622 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .qword, .is = .qword } }, .any },
3623 .patterns = &.{
3624 .{ .src = .{ .mut_mem, .none } },
3625 },
3626 .extra_temps = .{
3627 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3628 .unused,
3629 .unused,
3630 .unused,
3631 .unused,
3632 .unused,
3633 },
3634 .dst_temps = .{.{ .ref = .src0 }},
3635 .clobbers = .{ .eflags = true },
3636 .each = .{ .once = &.{
3637 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3638 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
3639 .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
3640 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3641 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3642 .{ ._, ._, .not, .memad(.dst0q, .add_size, -16), ._, ._, ._ },
3643 } },
3644 }, .{
3645 .required_features = .{ .@"64bit", null, null, null },
3646 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .qword, .is = .qword } }, .any },
3647 .patterns = &.{
3648 .{ .src = .{ .to_mem, .none } },
3649 },
3650 .extra_temps = .{
3651 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3652 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3653 .unused,
3654 .unused,
3655 .unused,
3656 .unused,
3657 },
3658 .dst_temps = .{.mem},
3659 .clobbers = .{ .eflags = true },
3660 .each = .{ .once = &.{
3661 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3662 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
3663 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3664 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
3665 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3666 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3667 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ },
3668 } },
3669 }, .{
3670 .required_features = .{ .@"64bit", null, null, null },
3671 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .dword, .is = .dword } }, .any },
3672 .patterns = &.{
3673 .{ .src = .{ .mut_mem, .none } },
3674 },
3675 .extra_temps = .{
3676 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3677 .unused,
3678 .unused,
3679 .unused,
3680 .unused,
3681 .unused,
3682 },
3683 .dst_temps = .{.{ .ref = .src0 }},
3684 .clobbers = .{ .eflags = true },
3685 .each = .{ .once = &.{
3686 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3687 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ },
3688 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3689 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3690 .{ ._, ._, .not, .memad(.dst0d, .add_size, -8), ._, ._, ._ },
3691 } },
3692 }, .{
3693 .required_features = .{ .@"64bit", null, null, null },
3694 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .dword, .is = .dword } }, .any },
3695 .patterns = &.{
3696 .{ .src = .{ .to_mem, .none } },
3697 },
3698 .extra_temps = .{
3699 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3700 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3701 .unused,
3702 .unused,
3703 .unused,
3704 .unused,
3705 },
3706 .dst_temps = .{.mem},
3707 .clobbers = .{ .eflags = true },
3708 .each = .{ .once = &.{
3709 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3710 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
3711 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3712 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
3713 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3714 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3715 .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -8), ._, ._ },
3716 .{ ._, ._, .not, .tmp0d, ._, ._, ._ },
3717 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8), .tmp0d, ._, ._ },
3718 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8 + 4), .si(0), ._, ._ },
3719 } },
3720 }, .{
3721 .required_features = .{ .@"64bit", null, null, null },
3722 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .dword } }, .any },
3723 .patterns = &.{
3724 .{ .src = .{ .mut_mem, .none } },
3725 },
3726 .extra_temps = .{
3727 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3728 .unused,
3729 .unused,
3730 .unused,
3731 .unused,
3732 .unused,
3733 },
3734 .dst_temps = .{.{ .ref = .src0 }},
3735 .clobbers = .{ .eflags = true },
3736 .each = .{ .once = &.{
3737 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3738 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
3739 .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
3740 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3741 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3742 .{ ._, ._, .xor, .memad(.dst0d, .add_size, -16), .sa(.src0, .add_umax), ._, ._ },
3743 } },
3744 }, .{
3745 .required_features = .{ .@"64bit", null, null, null },
3746 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .dword } }, .any },
3747 .patterns = &.{
3748 .{ .src = .{ .to_mem, .none } },
3749 },
3750 .extra_temps = .{
3751 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3752 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3753 .unused,
3754 .unused,
3755 .unused,
3756 .unused,
3757 },
3758 .dst_temps = .{.mem},
3759 .clobbers = .{ .eflags = true },
3760 .each = .{ .once = &.{
3761 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3762 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ },
3763 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3764 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ },
3765 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3766 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3767 .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -16), ._, ._ },
3768 .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_umax), ._, ._ },
3769 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16), .tmp0d, ._, ._ },
3770 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16 + 4), .si(0), ._, ._ },
3771 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16 + 8), .si(0), ._, ._ },
3772 } },
3773 }, .{
3774 .required_features = .{ .@"64bit", null, null, null },
3775 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .dword } }, .any },
3776 .patterns = &.{
3777 .{ .src = .{ .mut_mem, .none } },
3778 },
3779 .extra_temps = .{
3780 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3781 .unused,
3782 .unused,
3783 .unused,
3784 .unused,
3785 .unused,
3786 },
3787 .dst_temps = .{.{ .ref = .src0 }},
3788 .clobbers = .{ .eflags = true },
3789 .each = .{ .once = &.{
3790 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3791 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ },
3792 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3793 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3794 .{ ._, ._, .xor, .memad(.dst0d, .add_size, -8), .sa(.src0, .add_umax), ._, ._ },
3795 } },
3796 }, .{
3797 .required_features = .{ .@"64bit", null, null, null },
3798 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .dword } }, .any },
3799 .patterns = &.{
3800 .{ .src = .{ .to_mem, .none } },
3801 },
3802 .extra_temps = .{
3803 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3804 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3805 .unused,
3806 .unused,
3807 .unused,
3808 .unused,
3809 },
3810 .dst_temps = .{.mem},
3811 .clobbers = .{ .eflags = true },
3812 .each = .{ .once = &.{
3813 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3814 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
3815 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3816 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
3817 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3818 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3819 .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -8), ._, ._ },
3820 .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_umax), ._, ._ },
3821 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8), .tmp0d, ._, ._ },
3822 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8 + 4), .si(0), ._, ._ },
3823 } },
3824 }, .{
3825 .required_features = .{ .@"64bit", null, null, null },
3826 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
3827 .patterns = &.{
3828 .{ .src = .{ .mut_mem, .none } },
3829 },
3830 .extra_temps = .{
3831 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3832 .unused,
3833 .unused,
3834 .unused,
3835 .unused,
3836 .unused,
3837 },
3838 .dst_temps = .{.{ .ref = .src0 }},
3839 .clobbers = .{ .eflags = true },
3840 .each = .{ .once = &.{
3841 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3842 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
3843 .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
3844 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3845 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3846 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
3847 .{ ._, ._, .xor, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ },
3848 } },
3849 }, .{
3850 .required_features = .{ .@"64bit", null, null, null },
3851 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
3852 .patterns = &.{
3853 .{ .src = .{ .to_mem, .none } },
3854 },
3855 .extra_temps = .{
3856 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3857 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3858 .unused,
3859 .unused,
3860 .unused,
3861 .unused,
3862 },
3863 .dst_temps = .{.mem},
3864 .clobbers = .{ .eflags = true },
3865 .each = .{ .once = &.{
3866 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3867 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ },
3868 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3869 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ },
3870 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3871 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3872 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
3873 .{ ._, ._, .xor, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ },
3874 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ },
3875 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ },
3876 } },
3877 }, .{
3878 .required_features = .{ .@"64bit", null, null, null },
3879 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3880 .patterns = &.{
3881 .{ .src = .{ .mut_mem, .none } },
3882 },
3883 .extra_temps = .{
3884 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3885 .unused,
3886 .unused,
3887 .unused,
3888 .unused,
3889 .unused,
3890 },
3891 .dst_temps = .{.{ .ref = .src0 }},
3892 .clobbers = .{ .eflags = true },
3893 .each = .{ .once = &.{
3894 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3895 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ },
3896 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3897 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3898 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
3899 .{ ._, ._, .xor, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ },
3900 } },
3901 }, .{
3902 .required_features = .{ .@"64bit", null, null, null },
3903 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3904 .patterns = &.{
3905 .{ .src = .{ .to_mem, .none } },
3906 },
3907 .extra_temps = .{
3908 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3909 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3910 .unused,
3911 .unused,
3912 .unused,
3913 .unused,
3914 },
3915 .dst_temps = .{.mem},
3916 .clobbers = .{ .eflags = true },
3917 .each = .{ .once = &.{
3918 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3919 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
3920 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3921 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
3922 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3923 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3924 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
3925 .{ ._, ._, .xor, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ },
3926 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ },
3927 } },
3928 }, .{
3929 .required_features = .{ .mmx, null, null, null },
3930 .src_constraints = .{ .{ .signed_int_or_full_vec = .qword }, .any },
3931 .patterns = &.{
3932 .{ .src = .{ .mem, .none } },
3933 .{ .src = .{ .to_mm, .none } },
3934 },
3935 .dst_temps = .{.{ .rc = .mmx }},
3936 .each = .{ .once = &.{
3937 .{ ._, .p_d, .cmpeq, .dst0q, .dst0q, ._, ._ },
3938 .{ ._, .p_, .xor, .dst0q, .src0q, ._, ._ },
3939 } },
3940 }, .{
3941 .required_features = .{ .mmx, null, null, null },
3942 .src_constraints = .{ .{ .unsigned_int_vec = .qword }, .any },
3943 .patterns = &.{
3944 .{ .src = .{ .to_mut_mm, .none } },
3945 },
3946 .extra_temps = .{
3947 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3948 .{ .kind = .{ .umax_mem = .src0 } },
3949 .unused,
3950 .unused,
3951 .unused,
3952 .unused,
3953 },
3954 .dst_temps = .{.{ .ref = .src0 }},
3955 .each = .{ .once = &.{
3956 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3957 .{ ._, .p_, .xor, .dst0q, .lea(.qword, .tmp0), ._, ._ },
3958 } },
3959 }, .{
3960 .required_features = .{ .avx, null, null, null },
3961 .src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any },
3962 .patterns = &.{
3963 .{ .src = .{ .mem, .none } },
3964 .{ .src = .{ .to_xmm, .none } },
3965 },
3966 .dst_temps = .{.{ .rc = .sse }},
3967 .each = .{ .once = &.{
3968 .{ ._, .vp_q, .cmpeq, .dst0x, .dst0x, .dst0x, ._ },
3969 .{ ._, .vp_, .xor, .dst0x, .dst0x, .src0x, ._ },
3970 } },
3971 }, .{
3972 .required_features = .{ .avx, null, null, null },
3973 .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any },
3974 .patterns = &.{
3975 .{ .src = .{ .to_xmm, .none } },
3976 },
3977 .extra_temps = .{
3978 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3979 .{ .kind = .{ .umax_mem = .src0 } },
3980 .unused,
3981 .unused,
3982 .unused,
3983 .unused,
3984 },
3985 .dst_temps = .{.{ .rc = .sse }},
3986 .each = .{ .once = &.{
3987 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3988 .{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.xword, .tmp0), ._ },
3989 } },
3990 }, .{
3991 .required_features = .{ .sse2, null, null, null },
3992 .src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any },
3993 .patterns = &.{
3994 .{ .src = .{ .mem, .none } },
3995 .{ .src = .{ .to_xmm, .none } },
3996 },
3997 .dst_temps = .{.{ .rc = .sse }},
3998 .each = .{ .once = &.{
3999 .{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ },
4000 .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ },
4001 } },
4002 }, .{
4003 .required_features = .{ .sse2, null, null, null },
4004 .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any },
4005 .patterns = &.{
4006 .{ .src = .{ .to_mut_xmm, .none } },
4007 },
4008 .extra_temps = .{
4009 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4010 .{ .kind = .{ .umax_mem = .src0 } },
4011 .unused,
4012 .unused,
4013 .unused,
4014 .unused,
4015 },
4016 .dst_temps = .{.{ .ref = .src0 }},
4017 .each = .{ .once = &.{
4018 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
4019 .{ ._, .p_, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
4020 } },
4021 }, .{
4022 .required_features = .{ .sse, null, null, null },
4023 .src_constraints = .{ .{ .vec = .xword }, .any },
4024 .patterns = &.{
4025 .{ .src = .{ .to_mut_xmm, .none } },
4026 },
4027 .extra_temps = .{
4028 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4029 .{ .kind = .{ .umax_mem = .src0 } },
4030 .unused,
4031 .unused,
4032 .unused,
4033 .unused,
4034 },
4035 .dst_temps = .{.{ .ref = .src0 }},
4036 .each = .{ .once = &.{
4037 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
4038 .{ ._, ._ps, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
4039 } },
4040 }, .{
4041 .required_features = .{ .avx2, null, null, null },
4042 .src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any },
4043 .patterns = &.{
4044 .{ .src = .{ .mem, .none } },
4045 .{ .src = .{ .to_ymm, .none } },
4046 },
4047 .dst_temps = .{.{ .rc = .sse }},
4048 .each = .{ .once = &.{
4049 .{ ._, .vp_q, .cmpeq, .dst0y, .dst0y, .dst0y, ._ },
4050 .{ ._, .vp_, .xor, .dst0y, .dst0y, .src0y, ._ },
4051 } },
4052 }, .{
4053 .required_features = .{ .avx2, null, null, null },
4054 .src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any },
4055 .patterns = &.{
4056 .{ .src = .{ .to_ymm, .none } },
4057 },
4058 .extra_temps = .{
4059 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4060 .{ .kind = .{ .umax_mem = .src0 } },
4061 .unused,
4062 .unused,
4063 .unused,
4064 .unused,
4065 },
4066 .dst_temps = .{.{ .rc = .sse }},
4067 .each = .{ .once = &.{
4068 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
4069 .{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
4070 } },
4071 }, .{
4072 .required_features = .{ .avx, null, null, null },
4073 .src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any },
4074 .patterns = &.{
4075 .{ .src = .{ .mem, .none } },
4076 .{ .src = .{ .to_ymm, .none } },
4077 },
4078 .dst_temps = .{.{ .rc = .sse }},
4079 .each = .{ .once = &.{
4080 .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .si(0b01111) },
4081 .{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ },
4082 } },
4083 }, .{
4084 .required_features = .{ .avx, null, null, null },
4085 .src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any },
4086 .patterns = &.{
4087 .{ .src = .{ .to_ymm, .none } },
4088 },
4089 .extra_temps = .{
4090 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4091 .{ .kind = .{ .umax_mem = .src0 } },
4092 .unused,
4093 .unused,
4094 .unused,
4095 .unused,
4096 },
4097 .dst_temps = .{.{ .rc = .sse }},
4098 .each = .{ .once = &.{
4099 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
4100 .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
4101 } },
4102 }, .{
4103 .required_features = .{ .@"64bit", null, null, null },
4104 .patterns = &.{
4105 .{ .src = .{ .to_mem, .none } },
4106 },
4107 .extra_temps = .{
4108 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4109 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4110 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4111 .{ .kind = .{ .umax_mem = .src0 } },
4112 .unused,
4113 .unused,
4114 },
4115 .dst_temps = .{.mem},
4116 .each = .{ .once = &.{
4117 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_src0_size), ._, ._ },
4118 .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ },
4119 .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_src0_size), ._, ._ },
4120 .{ ._, ._, .xor, .tmp2q, .leaia(.qword, .tmp1, .tmp0, .add_src0_size), ._, ._ },
4121 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_src0_size), .tmp2q, ._, ._ },
4122 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
4123 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4124 } },
4125 }, .{
4126 .patterns = &.{
4127 .{ .src = .{ .to_mem, .none } },
4128 },
4129 .extra_temps = .{
4130 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4131 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4132 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4133 .{ .kind = .{ .umax_mem = .src0 } },
4134 .unused,
4135 .unused,
4136 },
4137 .dst_temps = .{.mem},
4138 .each = .{ .once = &.{
4139 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_src0_size), ._, ._ },
4140 .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ },
4141 .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_src0_size), ._, ._ },
4142 .{ ._, ._, .xor, .tmp2d, .leaia(.dword, .tmp1, .tmp0, .add_src0_size), ._, ._ },
4143 .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_src0_size), .tmp2d, ._, ._ },
4144 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
4145 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4146 } },
4147 } }) catch |err| switch (err) {
4148 error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
4149 @tagName(air_tag),
4150 cg.typeOf(ty_op.operand).fmt(pt),
4151 ops[0].tracking(cg),
4152 }),
4153 else => |e| return e,
4154 };
4155 for (ops) |op| for (res) |r| {
4156 if (op.index == r.index) break;
4157 } else try op.die(cg);
4158 try res[0].moveTo(inst, cg);
4159 },
4160
4161 .block => if (use_old) try cg.airBlock(inst) else {
4162 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
4163 const extra = cg.air.extraData(Air.Block, ty_pl.payload);
4164 try cg.asmPseudo(.pseudo_dbg_enter_block_none);
4165 try cg.lowerBlock(inst, @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]));
4166 try cg.asmPseudo(.pseudo_dbg_leave_block_none);
4167 },
4168 .loop => if (use_old) try cg.airLoop(inst) else {
4169 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
4170 const extra = cg.air.extraData(Air.Block, ty_pl.payload);
4171 cg.scope_generation += 1;
4172 try cg.loops.putNoClobber(cg.gpa, inst, .{
4173 .state = try cg.saveState(),
4174 .target = @intCast(cg.mir_instructions.len),
4175 });
4176 defer assert(cg.loops.remove(inst));
4177 try cg.genBodyBlock(@ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]));
4178 },
4179 .repeat => if (use_old) try cg.airRepeat(inst) else {
4180 const repeat = air_datas[@intFromEnum(inst)].repeat;
4181 const loop = cg.loops.get(repeat.loop_inst).?;
4182 try cg.restoreState(loop.state, &.{}, .{
4183 .emit_instructions = true,
4184 .update_tracking = false,
4185 .resurrect = false,
4186 .close_scope = true,
4187 });
4188 _ = try cg.asmJmpReloc(loop.target);
4189 },
4190 .br => try cg.airBr(inst),
4191 .trap => try cg.asmOpOnly(.{ ._, .ud2 }),
4192 .breakpoint => try cg.asmOpOnly(.{ ._, .int3 }),
4193 .ret_addr => if (use_old) try cg.airRetAddr(inst) else {
4194 var slot = try cg.tempInit(.usize, .{ .load_frame = .{
4195 .index = .ret_addr,
4196 } });
4197 while (try slot.toRegClass(true, .general_purpose, cg)) {}
4198 try slot.moveTo(inst, cg);
4199 },
4200 .frame_addr => if (use_old) try cg.airFrameAddress(inst) else {
4201 var slot = try cg.tempInit(.usize, .{ .lea_frame = .{
4202 .index = .base_ptr,
4203 } });
4204 try slot.moveTo(inst, cg);
4205 },
4206 .call => try cg.airCall(inst, .auto, .{ .safety = true }),
4207 .call_always_tail => try cg.airCall(inst, .always_tail, .{ .safety = true }),
4208 .call_never_tail => try cg.airCall(inst, .never_tail, .{ .safety = true }),
4209 .call_never_inline => try cg.airCall(inst, .never_inline, .{ .safety = true }),
4210
4211 .clz => |air_tag| if (use_old) try cg.airClz(inst) else {
4212 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
4213 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
4214 var res: [1]Temp = undefined;
4215 cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{
4216 .required_features = .{ .slow_incdec, null, null, null },
4217 .src_constraints = .{ .{ .exact_signed_int = 1 }, .any },
4218 .patterns = &.{
4219 .{ .src = .{ .mut_mem, .none } },
4220 .{ .src = .{ .to_mut_gpr, .none } },
4221 },
4222 .dst_temps = .{.{ .ref = .src0 }},
4223 .clobbers = .{ .eflags = true },
4224 .each = .{ .once = &.{
4225 .{ ._, ._, .add, .dst0b, .si(1), ._, ._ },
4226 } },
4227 }, .{
4228 .src_constraints = .{ .{ .exact_signed_int = 1 }, .any },
4229 .patterns = &.{
4230 .{ .src = .{ .mut_mem, .none } },
4231 .{ .src = .{ .to_mut_gpr, .none } },
4232 },
4233 .dst_temps = .{.{ .ref = .src0 }},
4234 .clobbers = .{ .eflags = true },
4235 .each = .{ .once = &.{
4236 .{ ._, ._, .inc, .dst0b, ._, ._, ._ },
4237 } },
4238 }, .{
4239 .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any },
4240 .patterns = &.{
4241 .{ .src = .{ .mut_mem, .none } },
4242 .{ .src = .{ .to_mut_gpr, .none } },
4243 },
4244 .dst_temps = .{.{ .ref = .src0 }},
4245 .clobbers = .{ .eflags = true },
4246 .each = .{ .once = &.{
4247 .{ ._, ._, .xor, .dst0b, .si(1), ._, ._ },
4248 } },
4249 }, .{
4250 .required_features = .{ .lzcnt, null, null, null },
4251 .src_constraints = .{ .{ .unsigned_or_exact_int = .byte }, .any },
4252 .patterns = &.{
4253 .{ .src = .{ .mem, .none } },
4254 .{ .src = .{ .to_gpr, .none } },
4255 },
4256 .dst_temps = .{.{ .rc = .general_purpose }},
4257 .clobbers = .{ .eflags = true },
4258 .each = .{ .once = &.{
4259 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4260 .{ ._, ._, .lzcnt, .dst0d, .dst0d, ._, ._ },
4261 .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
4262 } },
4263 }, .{
4264 .required_features = .{ .lzcnt, null, null, null },
4265 .src_constraints = .{ .{ .signed_int = .byte }, .any },
4266 .patterns = &.{
4267 .{ .src = .{ .mem, .none } },
4268 .{ .src = .{ .to_gpr, .none } },
4269 },
4270 .dst_temps = .{.{ .rc = .general_purpose }},
4271 .clobbers = .{ .eflags = true },
4272 .each = .{ .once = &.{
4273 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4274 .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
4275 .{ ._, ._, .lzcnt, .dst0d, .dst0d, ._, ._ },
4276 .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
4277 } },
4278 }, .{
4279 .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
4280 .src_constraints = .{ .{ .exact_int = 16 }, .any },
4281 .patterns = &.{
4282 .{ .src = .{ .to_mut_gpr, .none } },
4283 },
4284 .dst_temps = .{.{ .ref = .src0 }},
4285 .clobbers = .{ .eflags = true },
4286 .each = .{ .once = &.{
4287 .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
4288 } },
4289 }, .{
4290 .required_features = .{ .lzcnt, null, null, null },
4291 .src_constraints = .{ .{ .exact_int = 16 }, .any },
4292 .patterns = &.{
4293 .{ .src = .{ .mem, .none } },
4294 .{ .src = .{ .to_gpr, .none } },
4295 },
4296 .dst_temps = .{.{ .rc = .general_purpose }},
4297 .clobbers = .{ .eflags = true },
4298 .each = .{ .once = &.{
4299 .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
4300 } },
4301 }, .{
4302 .required_features = .{ .lzcnt, null, null, null },
4303 .src_constraints = .{ .{ .signed_int = .word }, .any },
4304 .patterns = &.{
4305 .{ .src = .{ .to_mut_gpr, .none } },
4306 },
4307 .dst_temps = .{.{ .ref = .src0 }},
4308 .clobbers = .{ .eflags = true },
4309 .each = .{ .once = &.{
4310 .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
4311 .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
4312 .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ },
4313 } },
4314 }, .{
4315 .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
4316 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
4317 .patterns = &.{
4318 .{ .src = .{ .to_mut_gpr, .none } },
4319 },
4320 .dst_temps = .{.{ .ref = .src0 }},
4321 .clobbers = .{ .eflags = true },
4322 .each = .{ .once = &.{
4323 .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
4324 .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ },
4325 } },
4326 }, .{
4327 .required_features = .{ .lzcnt, null, null, null },
4328 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
4329 .patterns = &.{
4330 .{ .src = .{ .mem, .none } },
4331 .{ .src = .{ .to_gpr, .none } },
4332 },
4333 .dst_temps = .{.{ .rc = .general_purpose }},
4334 .clobbers = .{ .eflags = true },
4335 .each = .{ .once = &.{
4336 .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
4337 .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ },
4338 } },
4339 }, .{
4340 .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
4341 .src_constraints = .{ .{ .exact_int = 32 }, .any },
4342 .patterns = &.{
4343 .{ .src = .{ .to_mut_gpr, .none } },
4344 },
4345 .dst_temps = .{.{ .ref = .src0 }},
4346 .clobbers = .{ .eflags = true },
4347 .each = .{ .once = &.{
4348 .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
4349 } },
4350 }, .{
4351 .required_features = .{ .lzcnt, null, null, null },
4352 .src_constraints = .{ .{ .exact_int = 32 }, .any },
4353 .patterns = &.{
4354 .{ .src = .{ .mem, .none } },
4355 .{ .src = .{ .to_gpr, .none } },
4356 },
4357 .dst_temps = .{.{ .rc = .general_purpose }},
4358 .clobbers = .{ .eflags = true },
4359 .each = .{ .once = &.{
4360 .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
4361 } },
4362 }, .{
4363 .required_features = .{ .lzcnt, null, null, null },
4364 .src_constraints = .{ .{ .signed_int = .dword }, .any },
4365 .patterns = &.{
4366 .{ .src = .{ .to_mut_gpr, .none } },
4367 },
4368 .dst_temps = .{.{ .ref = .src0 }},
4369 .clobbers = .{ .eflags = true },
4370 .each = .{ .once = &.{
4371 .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
4372 .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
4373 .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
4374 } },
4375 }, .{
4376 .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
4377 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
4378 .patterns = &.{
4379 .{ .src = .{ .to_mut_gpr, .none } },
4380 },
4381 .dst_temps = .{.{ .ref = .src0 }},
4382 .clobbers = .{ .eflags = true },
4383 .each = .{ .once = &.{
4384 .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
4385 .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
4386 } },
4387 }, .{
4388 .required_features = .{ .lzcnt, null, null, null },
4389 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
4390 .patterns = &.{
4391 .{ .src = .{ .mem, .none } },
4392 .{ .src = .{ .to_gpr, .none } },
4393 },
4394 .dst_temps = .{.{ .rc = .general_purpose }},
4395 .clobbers = .{ .eflags = true },
4396 .each = .{ .once = &.{
4397 .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
4398 .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
4399 } },
4400 }, .{
4401 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
4402 .src_constraints = .{ .{ .exact_int = 64 }, .any },
4403 .patterns = &.{
4404 .{ .src = .{ .to_mut_gpr, .none } },
4405 },
4406 .dst_temps = .{.{ .ref = .src0 }},
4407 .clobbers = .{ .eflags = true },
4408 .each = .{ .once = &.{
4409 .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
4410 } },
4411 }, .{
4412 .required_features = .{ .@"64bit", .lzcnt, null, null },
4413 .src_constraints = .{ .{ .exact_int = 64 }, .any },
4414 .patterns = &.{
4415 .{ .src = .{ .mem, .none } },
4416 .{ .src = .{ .to_gpr, .none } },
4417 },
4418 .dst_temps = .{.{ .rc = .general_purpose }},
4419 .clobbers = .{ .eflags = true },
4420 .each = .{ .once = &.{
4421 .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
4422 } },
4423 }, .{
4424 .required_features = .{ .@"64bit", .lzcnt, null, null },
4425 .src_constraints = .{ .{ .signed_int = .qword }, .any },
4426 .patterns = &.{
4427 .{ .src = .{ .mem, .none } },
4428 .{ .src = .{ .to_gpr, .none } },
4429 },
4430 .dst_temps = .{.{ .rc = .general_purpose }},
4431 .clobbers = .{ .eflags = true },
4432 .each = .{ .once = &.{
4433 .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
4434 .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ },
4435 .{ ._, ._, .lzcnt, .dst0q, .dst0q, ._, ._ },
4436 .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ },
4437 } },
4438 }, .{
4439 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
4440 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
4441 .patterns = &.{
4442 .{ .src = .{ .to_mut_gpr, .none } },
4443 },
4444 .dst_temps = .{.{ .ref = .src0 }},
4445 .clobbers = .{ .eflags = true },
4446 .each = .{ .once = &.{
4447 .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
4448 .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ },
4449 } },
4450 }, .{
4451 .required_features = .{ .@"64bit", .lzcnt, null, null },
4452 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
4453 .patterns = &.{
4454 .{ .src = .{ .mem, .none } },
4455 .{ .src = .{ .to_gpr, .none } },
4456 },
4457 .dst_temps = .{.{ .rc = .general_purpose }},
4458 .clobbers = .{ .eflags = true },
4459 .each = .{ .once = &.{
4460 .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
4461 .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ },
4462 } },
4463 }, .{
4464 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4465 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any },
4466 .patterns = &.{
4467 .{ .src = .{ .mem, .none } },
4468 .{ .src = .{ .to_gpr, .none } },
4469 },
4470 .extra_temps = .{
4471 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4472 .unused,
4473 .unused,
4474 .unused,
4475 .unused,
4476 .unused,
4477 },
4478 .dst_temps = .{.{ .rc = .general_purpose }},
4479 .clobbers = .{ .eflags = true },
4480 .each = .{ .once = &.{
4481 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4482 .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
4483 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4484 .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ },
4485 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4486 } },
4487 }, .{
4488 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4489 .src_constraints = .{ .{ .signed_po2_int = .byte }, .any },
4490 .patterns = &.{
4491 .{ .src = .{ .mem, .none } },
4492 .{ .src = .{ .to_gpr, .none } },
4493 },
4494 .extra_temps = .{
4495 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4496 .unused,
4497 .unused,
4498 .unused,
4499 .unused,
4500 .unused,
4501 },
4502 .dst_temps = .{.{ .rc = .general_purpose }},
4503 .clobbers = .{ .eflags = true },
4504 .each = .{ .once = &.{
4505 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4506 .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
4507 .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
4508 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4509 .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ },
4510 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4511 } },
4512 }, .{
4513 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4514 .src_constraints = .{ .{ .signed_int = .byte }, .any },
4515 .patterns = &.{
4516 .{ .src = .{ .mem, .none } },
4517 .{ .src = .{ .to_gpr, .none } },
4518 },
4519 .extra_temps = .{
4520 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4521 .unused,
4522 .unused,
4523 .unused,
4524 .unused,
4525 .unused,
4526 },
4527 .dst_temps = .{.{ .rc = .general_purpose }},
4528 .clobbers = .{ .eflags = true },
4529 .each = .{ .once = &.{
4530 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4531 .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ },
4532 .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
4533 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
4534 .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ },
4535 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4536 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4537 } },
4538 }, .{
4539 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4540 .src_constraints = .{ .{ .unsigned_int = .byte }, .any },
4541 .patterns = &.{
4542 .{ .src = .{ .mem, .none } },
4543 .{ .src = .{ .to_gpr, .none } },
4544 },
4545 .extra_temps = .{
4546 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4547 .unused,
4548 .unused,
4549 .unused,
4550 .unused,
4551 .unused,
4552 },
4553 .dst_temps = .{.{ .rc = .general_purpose }},
4554 .clobbers = .{ .eflags = true },
4555 .each = .{ .once = &.{
4556 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4557 .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
4558 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
4559 .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ },
4560 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4561 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4562 } },
4563 }, .{
4564 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4565 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any },
4566 .patterns = &.{
4567 .{ .src = .{ .mem, .none } },
4568 .{ .src = .{ .to_gpr, .none } },
4569 },
4570 .dst_temps = .{.{ .rc = .general_purpose }},
4571 .clobbers = .{ .eflags = true },
4572 .each = .{ .once = &.{
4573 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4574 .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
4575 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
4576 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4577 .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4578 } },
4579 }, .{
4580 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4581 .src_constraints = .{ .{ .signed_po2_int = .byte }, .any },
4582 .patterns = &.{
4583 .{ .src = .{ .mem, .none } },
4584 .{ .src = .{ .to_gpr, .none } },
4585 },
4586 .dst_temps = .{.{ .rc = .general_purpose }},
4587 .clobbers = .{ .eflags = true },
4588 .each = .{ .once = &.{
4589 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4590 .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
4591 .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
4592 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
4593 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4594 .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4595 } },
4596 }, .{
4597 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4598 .src_constraints = .{ .{ .signed_int = .byte }, .any },
4599 .patterns = &.{
4600 .{ .src = .{ .mem, .none } },
4601 .{ .src = .{ .to_gpr, .none } },
4602 },
4603 .extra_temps = .{
4604 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4605 .unused,
4606 .unused,
4607 .unused,
4608 .unused,
4609 .unused,
4610 },
4611 .dst_temps = .{.{ .rc = .general_purpose }},
4612 .clobbers = .{ .eflags = true },
4613 .each = .{ .once = &.{
4614 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4615 .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ },
4616 .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
4617 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4618 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4619 .{ ._, ._c, .st, ._, ._, ._, ._ },
4620 .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ },
4621 } },
4622 }, .{
4623 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4624 .src_constraints = .{ .{ .unsigned_int = .byte }, .any },
4625 .patterns = &.{
4626 .{ .src = .{ .mem, .none } },
4627 .{ .src = .{ .to_gpr, .none } },
4628 },
4629 .extra_temps = .{
4630 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4631 .unused,
4632 .unused,
4633 .unused,
4634 .unused,
4635 .unused,
4636 },
4637 .dst_temps = .{.{ .rc = .general_purpose }},
4638 .clobbers = .{ .eflags = true },
4639 .each = .{ .once = &.{
4640 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4641 .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
4642 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4643 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4644 .{ ._, ._c, .st, ._, ._, ._, ._ },
4645 .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ },
4646 } },
4647 }, .{
4648 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any },
4649 .patterns = &.{
4650 .{ .src = .{ .mem, .none } },
4651 .{ .src = .{ .to_gpr, .none } },
4652 },
4653 .extra_temps = .{
4654 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4655 .unused,
4656 .unused,
4657 .unused,
4658 .unused,
4659 .unused,
4660 },
4661 .dst_temps = .{.{ .rc = .general_purpose }},
4662 .clobbers = .{ .eflags = true },
4663 .each = .{ .once = &.{
4664 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4665 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4666 .{ ._, ._r, .bs, .dst0d, .tmp0d, ._, ._ },
4667 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4668 } },
4669 }, .{
4670 .src_constraints = .{ .{ .signed_po2_int = .byte }, .any },
4671 .patterns = &.{
4672 .{ .src = .{ .mem, .none } },
4673 .{ .src = .{ .to_gpr, .none } },
4674 },
4675 .extra_temps = .{
4676 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4677 .unused,
4678 .unused,
4679 .unused,
4680 .unused,
4681 .unused,
4682 },
4683 .dst_temps = .{.{ .rc = .general_purpose }},
4684 .clobbers = .{ .eflags = true },
4685 .each = .{ .once = &.{
4686 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4687 .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ },
4688 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4689 .{ ._, ._r, .bs, .dst0d, .tmp0d, ._, ._ },
4690 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4691 } },
4692 }, .{
4693 .src_constraints = .{ .{ .signed_int = .byte }, .any },
4694 .patterns = &.{
4695 .{ .src = .{ .mem, .none } },
4696 .{ .src = .{ .to_gpr, .none } },
4697 },
4698 .extra_temps = .{
4699 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4700 .unused,
4701 .unused,
4702 .unused,
4703 .unused,
4704 .unused,
4705 },
4706 .dst_temps = .{.{ .rc = .general_purpose }},
4707 .clobbers = .{ .eflags = true },
4708 .each = .{ .once = &.{
4709 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4710 .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
4711 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
4712 .{ ._, ._r, .bs, .tmp0d, .dst0d, ._, ._ },
4713 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4714 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4715 } },
4716 }, .{
4717 .src_constraints = .{ .{ .unsigned_int = .byte }, .any },
4718 .patterns = &.{
4719 .{ .src = .{ .mem, .none } },
4720 .{ .src = .{ .to_gpr, .none } },
4721 },
4722 .extra_temps = .{
4723 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4724 .unused,
4725 .unused,
4726 .unused,
4727 .unused,
4728 .unused,
4729 },
4730 .dst_temps = .{.{ .rc = .general_purpose }},
4731 .clobbers = .{ .eflags = true },
4732 .each = .{ .once = &.{
4733 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4734 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
4735 .{ ._, ._r, .bs, .tmp0d, .dst0d, ._, ._ },
4736 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4737 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4738 } },
4739 }, .{
4740 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4741 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any },
4742 .patterns = &.{
4743 .{ .src = .{ .to_mut_gpr, .none } },
4744 },
4745 .dst_temps = .{.{ .rc = .general_purpose }},
4746 .clobbers = .{ .eflags = true },
4747 .each = .{ .once = &.{
4748 .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
4749 .{ ._, ._, .mov, .dst0w, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4750 .{ ._, ._nz, .cmov, .dst0w, .src0w, ._, ._ },
4751 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4752 } },
4753 }, .{
4754 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4755 .src_constraints = .{ .{ .signed_int = .word }, .any },
4756 .patterns = &.{
4757 .{ .src = .{ .to_mut_gpr, .none } },
4758 },
4759 .dst_temps = .{.{ .rc = .general_purpose }},
4760 .clobbers = .{ .eflags = true },
4761 .each = .{ .once = &.{
4762 .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
4763 .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
4764 .{ ._, ._, .mov, .dst0w, .si(0xff), ._, ._ },
4765 .{ ._, ._z, .cmov, .src0w, .dst0w, ._, ._ },
4766 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4767 .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
4768 } },
4769 }, .{
4770 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4771 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
4772 .patterns = &.{
4773 .{ .src = .{ .to_mut_gpr, .none } },
4774 },
4775 .dst_temps = .{.{ .rc = .general_purpose }},
4776 .clobbers = .{ .eflags = true },
4777 .each = .{ .once = &.{
4778 .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
4779 .{ ._, ._, .mov, .dst0w, .si(0xff), ._, ._ },
4780 .{ ._, ._z, .cmov, .src0w, .dst0w, ._, ._ },
4781 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4782 .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
4783 } },
4784 }, .{
4785 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4786 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any },
4787 .patterns = &.{
4788 .{ .src = .{ .to_mut_gpr, .none } },
4789 },
4790 .dst_temps = .{.{ .ref = .src0 }},
4791 .clobbers = .{ .eflags = true },
4792 .each = .{ .once = &.{
4793 .{ ._, ._r, .bs, .dst0w, .src0w, ._, ._ },
4794 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
4795 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4796 .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4797 } },
4798 }, .{
4799 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4800 .src_constraints = .{ .{ .signed_int = .word }, .any },
4801 .patterns = &.{
4802 .{ .src = .{ .to_mut_gpr, .none } },
4803 },
4804 .dst_temps = .{.{ .rc = .general_purpose }},
4805 .clobbers = .{ .eflags = true },
4806 .each = .{ .once = &.{
4807 .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
4808 .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
4809 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4810 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4811 .{ ._, ._c, .st, ._, ._, ._, ._ },
4812 .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
4813 } },
4814 }, .{
4815 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4816 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
4817 .patterns = &.{
4818 .{ .src = .{ .to_mut_gpr, .none } },
4819 },
4820 .dst_temps = .{.{ .rc = .general_purpose }},
4821 .clobbers = .{ .eflags = true },
4822 .each = .{ .once = &.{
4823 .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
4824 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4825 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4826 .{ ._, ._c, .st, ._, ._, ._, ._ },
4827 .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
4828 } },
4829 }, .{
4830 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any },
4831 .patterns = &.{
4832 .{ .src = .{ .mem, .none } },
4833 .{ .src = .{ .to_gpr, .none } },
4834 },
4835 .dst_temps = .{.{ .rc = .general_purpose }},
4836 .clobbers = .{ .eflags = true },
4837 .each = .{ .once = &.{
4838 .{ ._, ._, .mov, .dst0w, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4839 .{ ._, ._r, .bs, .dst0w, .src0w, ._, ._ },
4840 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4841 } },
4842 }, .{
4843 .src_constraints = .{ .{ .signed_int = .word }, .any },
4844 .patterns = &.{
4845 .{ .src = .{ .to_mut_gpr, .none } },
4846 },
4847 .extra_temps = .{
4848 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4849 .unused,
4850 .unused,
4851 .unused,
4852 .unused,
4853 .unused,
4854 },
4855 .dst_temps = .{.{ .rc = .general_purpose }},
4856 .clobbers = .{ .eflags = true },
4857 .each = .{ .once = &.{
4858 .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
4859 .{ ._, ._, .mov, .tmp0w, .si(0xff), ._, ._ },
4860 .{ ._, ._r, .bs, .tmp0w, .src0w, ._, ._ },
4861 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4862 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4863 } },
4864 }, .{
4865 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
4866 .patterns = &.{
4867 .{ .src = .{ .mem, .none } },
4868 .{ .src = .{ .to_gpr, .none } },
4869 },
4870 .extra_temps = .{
4871 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4872 .unused,
4873 .unused,
4874 .unused,
4875 .unused,
4876 .unused,
4877 },
4878 .dst_temps = .{.{ .rc = .general_purpose }},
4879 .clobbers = .{ .eflags = true },
4880 .each = .{ .once = &.{
4881 .{ ._, ._, .mov, .tmp0w, .si(0xff), ._, ._ },
4882 .{ ._, ._r, .bs, .tmp0w, .src0w, ._, ._ },
4883 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4884 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4885 } },
4886 }, .{
4887 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4888 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any },
4889 .patterns = &.{
4890 .{ .src = .{ .to_mut_gpr, .none } },
4891 },
4892 .dst_temps = .{.{ .rc = .general_purpose }},
4893 .clobbers = .{ .eflags = true },
4894 .each = .{ .once = &.{
4895 .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
4896 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4897 .{ ._, ._nz, .cmov, .dst0d, .src0d, ._, ._ },
4898 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4899 } },
4900 }, .{
4901 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4902 .src_constraints = .{ .{ .signed_int = .dword }, .any },
4903 .patterns = &.{
4904 .{ .src = .{ .to_mut_gpr, .none } },
4905 },
4906 .dst_temps = .{.{ .rc = .general_purpose }},
4907 .clobbers = .{ .eflags = true },
4908 .each = .{ .once = &.{
4909 .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
4910 .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
4911 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
4912 .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ },
4913 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4914 .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
4915 } },
4916 }, .{
4917 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4918 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
4919 .patterns = &.{
4920 .{ .src = .{ .to_mut_gpr, .none } },
4921 },
4922 .dst_temps = .{.{ .rc = .general_purpose }},
4923 .clobbers = .{ .eflags = true },
4924 .each = .{ .once = &.{
4925 .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
4926 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
4927 .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ },
4928 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4929 .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
4930 } },
4931 }, .{
4932 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4933 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any },
4934 .patterns = &.{
4935 .{ .src = .{ .to_mut_gpr, .none } },
4936 },
4937 .dst_temps = .{.{ .ref = .src0 }},
4938 .clobbers = .{ .eflags = true },
4939 .each = .{ .once = &.{
4940 .{ ._, ._r, .bs, .dst0d, .src0d, ._, ._ },
4941 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
4942 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4943 .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4944 } },
4945 }, .{
4946 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4947 .src_constraints = .{ .{ .signed_int = .dword }, .any },
4948 .patterns = &.{
4949 .{ .src = .{ .to_mut_gpr, .none } },
4950 },
4951 .dst_temps = .{.{ .rc = .general_purpose }},
4952 .clobbers = .{ .eflags = true },
4953 .each = .{ .once = &.{
4954 .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
4955 .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
4956 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4957 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4958 .{ ._, ._c, .st, ._, ._, ._, ._ },
4959 .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
4960 } },
4961 }, .{
4962 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4963 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
4964 .patterns = &.{
4965 .{ .src = .{ .to_mut_gpr, .none } },
4966 },
4967 .dst_temps = .{.{ .rc = .general_purpose }},
4968 .clobbers = .{ .eflags = true },
4969 .each = .{ .once = &.{
4970 .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
4971 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4972 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4973 .{ ._, ._c, .st, ._, ._, ._, ._ },
4974 .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
4975 } },
4976 }, .{
4977 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any },
4978 .patterns = &.{
4979 .{ .src = .{ .mem, .none } },
4980 .{ .src = .{ .to_gpr, .none } },
4981 },
4982 .dst_temps = .{.{ .rc = .general_purpose }},
4983 .clobbers = .{ .eflags = true },
4984 .each = .{ .once = &.{
4985 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4986 .{ ._, ._r, .bs, .dst0d, .src0d, ._, ._ },
4987 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4988 } },
4989 }, .{
4990 .src_constraints = .{ .{ .signed_int = .dword }, .any },
4991 .patterns = &.{
4992 .{ .src = .{ .to_mut_gpr, .none } },
4993 },
4994 .extra_temps = .{
4995 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4996 .unused,
4997 .unused,
4998 .unused,
4999 .unused,
5000 .unused,
5001 },
5002 .dst_temps = .{.{ .rc = .general_purpose }},
5003 .clobbers = .{ .eflags = true },
5004 .each = .{ .once = &.{
5005 .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
5006 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
5007 .{ ._, ._r, .bs, .tmp0d, .src0d, ._, ._ },
5008 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5009 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
5010 } },
5011 }, .{
5012 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
5013 .patterns = &.{
5014 .{ .src = .{ .mem, .none } },
5015 .{ .src = .{ .to_gpr, .none } },
5016 },
5017 .extra_temps = .{
5018 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5019 .unused,
5020 .unused,
5021 .unused,
5022 .unused,
5023 .unused,
5024 },
5025 .dst_temps = .{.{ .rc = .general_purpose }},
5026 .clobbers = .{ .eflags = true },
5027 .each = .{ .once = &.{
5028 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
5029 .{ ._, ._r, .bs, .tmp0d, .src0d, ._, ._ },
5030 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5031 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
5032 } },
5033 }, .{
5034 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
5035 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any },
5036 .patterns = &.{
5037 .{ .src = .{ .to_mut_gpr, .none } },
5038 },
5039 .dst_temps = .{.{ .rc = .general_purpose }},
5040 .clobbers = .{ .eflags = true },
5041 .each = .{ .once = &.{
5042 .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ },
5043 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
5044 .{ ._, ._nz, .cmov, .dst0d, .src0d, ._, ._ },
5045 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5046 } },
5047 }, .{
5048 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
5049 .src_constraints = .{ .{ .signed_int = .qword }, .any },
5050 .patterns = &.{
5051 .{ .src = .{ .mem, .none } },
5052 .{ .src = .{ .to_gpr, .none } },
5053 },
5054 .extra_temps = .{
5055 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5056 .unused,
5057 .unused,
5058 .unused,
5059 .unused,
5060 .unused,
5061 },
5062 .dst_temps = .{.{ .rc = .general_purpose }},
5063 .clobbers = .{ .eflags = true },
5064 .each = .{ .once = &.{
5065 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
5066 .{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ },
5067 .{ ._, ._r, .bs, .tmp0q, .tmp0q, ._, ._ },
5068 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
5069 .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ },
5070 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5071 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
5072 } },
5073 }, .{
5074 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
5075 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
5076 .patterns = &.{
5077 .{ .src = .{ .to_mut_gpr, .none } },
5078 },
5079 .dst_temps = .{.{ .rc = .general_purpose }},
5080 .clobbers = .{ .eflags = true },
5081 .each = .{ .once = &.{
5082 .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ },
5083 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
5084 .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ },
5085 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5086 .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
5087 } },
5088 }, .{
5089 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
5090 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any },
5091 .patterns = &.{
5092 .{ .src = .{ .to_mut_gpr, .none } },
5093 },
5094 .dst_temps = .{.{ .ref = .src0 }},
5095 .clobbers = .{ .eflags = true },
5096 .each = .{ .once = &.{
5097 .{ ._, ._r, .bs, .dst0q, .src0q, ._, ._ },
5098 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5099 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
5100 .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5101 } },
5102 }, .{
5103 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
5104 .src_constraints = .{ .{ .signed_int = .qword }, .any },
5105 .patterns = &.{
5106 .{ .src = .{ .mem, .none } },
5107 .{ .src = .{ .to_gpr, .none } },
5108 },
5109 .extra_temps = .{
5110 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5111 .unused,
5112 .unused,
5113 .unused,
5114 .unused,
5115 .unused,
5116 },
5117 .dst_temps = .{.{ .rc = .general_purpose }},
5118 .clobbers = .{ .eflags = true },
5119 .each = .{ .once = &.{
5120 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
5121 .{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ },
5122 .{ ._, ._r, .bs, .tmp0q, .tmp0q, ._, ._ },
5123 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
5124 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
5125 .{ ._, ._c, .st, ._, ._, ._, ._ },
5126 .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ },
5127 } },
5128 }, .{
5129 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
5130 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
5131 .patterns = &.{
5132 .{ .src = .{ .to_mut_gpr, .none } },
5133 },
5134 .dst_temps = .{.{ .rc = .general_purpose }},
5135 .clobbers = .{ .eflags = true },
5136 .each = .{ .once = &.{
5137 .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ },
5138 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
5139 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
5140 .{ ._, ._c, .st, ._, ._, ._, ._ },
5141 .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
5142 } },
5143 }, .{
5144 .required_features = .{ .@"64bit", null, null, null },
5145 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any },
5146 .patterns = &.{
5147 .{ .src = .{ .mem, .none } },
5148 .{ .src = .{ .to_gpr, .none } },
5149 },
5150 .dst_temps = .{.{ .rc = .general_purpose }},
5151 .clobbers = .{ .eflags = true },
5152 .each = .{ .once = &.{
5153 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
5154 .{ ._, ._r, .bs, .dst0q, .src0q, ._, ._ },
5155 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5156 } },
5157 }, .{
5158 .required_features = .{ .@"64bit", null, null, null },
5159 .src_constraints = .{ .{ .signed_int = .qword }, .any },
5160 .patterns = &.{
5161 .{ .src = .{ .mem, .none } },
5162 .{ .src = .{ .to_gpr, .none } },
5163 },
5164 .extra_temps = .{
5165 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5166 .unused,
5167 .unused,
5168 .unused,
5169 .unused,
5170 .unused,
5171 },
5172 .dst_temps = .{.{ .rc = .general_purpose }},
5173 .clobbers = .{ .eflags = true },
5174 .each = .{ .once = &.{
5175 .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
5176 .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ },
5177 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
5178 .{ ._, ._r, .bs, .tmp0q, .dst0q, ._, ._ },
5179 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5180 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
5181 } },
5182 }, .{
5183 .required_features = .{ .@"64bit", null, null, null },
5184 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
5185 .patterns = &.{
5186 .{ .src = .{ .mem, .none } },
5187 .{ .src = .{ .to_gpr, .none } },
5188 },
5189 .extra_temps = .{
5190 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5191 .unused,
5192 .unused,
5193 .unused,
5194 .unused,
5195 .unused,
5196 },
5197 .dst_temps = .{.{ .rc = .general_purpose }},
5198 .clobbers = .{ .eflags = true },
5199 .each = .{ .once = &.{
5200 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
5201 .{ ._, ._r, .bs, .tmp0q, .src0q, ._, ._ },
5202 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5203 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
5204 } },
5205 }, .{
5206 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
5207 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5208 .patterns = &.{
5209 .{ .src = .{ .to_mem, .none } },
5210 },
5211 .extra_temps = .{
5212 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5213 .unused,
5214 .unused,
5215 .unused,
5216 .unused,
5217 .unused,
5218 },
5219 .dst_temps = .{.{ .rc = .general_purpose }},
5220 .clobbers = .{ .eflags = true },
5221 .each = .{ .once = &.{
5222 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5223 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
5224 .{ ._, ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5225 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5226 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5227 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5228 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5229 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5230 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5231 } },
5232 }, .{
5233 .required_features = .{ .@"64bit", .lzcnt, null, null },
5234 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5235 .patterns = &.{
5236 .{ .src = .{ .to_mem, .none } },
5237 },
5238 .extra_temps = .{
5239 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5240 .unused,
5241 .unused,
5242 .unused,
5243 .unused,
5244 .unused,
5245 },
5246 .dst_temps = .{.{ .rc = .general_purpose }},
5247 .clobbers = .{ .eflags = true },
5248 .each = .{ .once = &.{
5249 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5250 .{ .@"0:", ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5251 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5252 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5253 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5254 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5255 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5256 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5257 } },
5258 }, .{
5259 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
5260 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5261 .patterns = &.{
5262 .{ .src = .{ .to_mem, .none } },
5263 },
5264 .extra_temps = .{
5265 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5266 .unused,
5267 .unused,
5268 .unused,
5269 .unused,
5270 .unused,
5271 },
5272 .dst_temps = .{.{ .rc = .general_purpose }},
5273 .clobbers = .{ .eflags = true },
5274 .each = .{ .once = &.{
5275 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5276 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
5277 .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5278 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5279 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5280 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5281 .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ },
5282 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5283 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5284 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5285 } },
5286 }, .{
5287 .required_features = .{ .@"64bit", null, null, null },
5288 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5289 .patterns = &.{
5290 .{ .src = .{ .to_mem, .none } },
5291 },
5292 .extra_temps = .{
5293 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5294 .unused,
5295 .unused,
5296 .unused,
5297 .unused,
5298 .unused,
5299 },
5300 .dst_temps = .{.{ .rc = .general_purpose }},
5301 .clobbers = .{ .eflags = true },
5302 .each = .{ .once = &.{
5303 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5304 .{ .@"0:", ._, .mov, .dst0d, .si(-1), ._, ._ },
5305 .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5306 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5307 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5308 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5309 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5310 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5311 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5312 } },
5313 }, .{
5314 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
5315 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5316 .patterns = &.{
5317 .{ .src = .{ .to_mem, .none } },
5318 },
5319 .extra_temps = .{
5320 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5321 .unused,
5322 .unused,
5323 .unused,
5324 .unused,
5325 .unused,
5326 },
5327 .dst_temps = .{.{ .rc = .general_purpose }},
5328 .clobbers = .{ .eflags = true },
5329 .each = .{ .once = &.{
5330 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5331 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
5332 .{ ._, ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5333 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5334 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5335 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5336 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5337 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5338 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5339 } },
5340 }, .{
5341 .required_features = .{ .@"64bit", .lzcnt, null, null },
5342 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5343 .patterns = &.{
5344 .{ .src = .{ .to_mem, .none } },
5345 },
5346 .extra_temps = .{
5347 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5348 .unused,
5349 .unused,
5350 .unused,
5351 .unused,
5352 .unused,
5353 },
5354 .dst_temps = .{.{ .rc = .general_purpose }},
5355 .clobbers = .{ .eflags = true },
5356 .each = .{ .once = &.{
5357 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5358 .{ .@"0:", ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5359 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5360 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5361 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5362 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5363 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5364 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5365 } },
5366 }, .{
5367 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
5368 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5369 .patterns = &.{
5370 .{ .src = .{ .to_mem, .none } },
5371 },
5372 .extra_temps = .{
5373 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5374 .unused,
5375 .unused,
5376 .unused,
5377 .unused,
5378 .unused,
5379 },
5380 .dst_temps = .{.{ .rc = .general_purpose }},
5381 .clobbers = .{ .eflags = true },
5382 .each = .{ .once = &.{
5383 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5384 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
5385 .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5386 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5387 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5388 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5389 .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ },
5390 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5391 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5392 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5393 } },
5394 }, .{
5395 .required_features = .{ .@"64bit", null, null, null },
5396 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5397 .patterns = &.{
5398 .{ .src = .{ .to_mem, .none } },
5399 },
5400 .extra_temps = .{
5401 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5402 .unused,
5403 .unused,
5404 .unused,
5405 .unused,
5406 .unused,
5407 },
5408 .dst_temps = .{.{ .rc = .general_purpose }},
5409 .clobbers = .{ .eflags = true },
5410 .each = .{ .once = &.{
5411 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5412 .{ .@"0:", ._, .mov, .dst0d, .si(-1), ._, ._ },
5413 .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5414 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5415 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5416 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5417 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5418 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5419 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5420 } },
5421 }, .{
5422 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
5423 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5424 .patterns = &.{
5425 .{ .src = .{ .to_mem, .none } },
5426 },
5427 .extra_temps = .{
5428 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5429 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5430 .unused,
5431 .unused,
5432 .unused,
5433 .unused,
5434 },
5435 .dst_temps = .{.{ .rc = .general_purpose }},
5436 .clobbers = .{ .eflags = true },
5437 .each = .{ .once = &.{
5438 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5439 .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5440 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
5441 .{ ._, ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
5442 .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
5443 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5444 .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
5445 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5446 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5447 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5448 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5449 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5450 } },
5451 }, .{
5452 .required_features = .{ .@"64bit", .lzcnt, null, null },
5453 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5454 .patterns = &.{
5455 .{ .src = .{ .to_mem, .none } },
5456 },
5457 .extra_temps = .{
5458 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5459 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5460 .unused,
5461 .unused,
5462 .unused,
5463 .unused,
5464 },
5465 .dst_temps = .{.{ .rc = .general_purpose }},
5466 .clobbers = .{ .eflags = true },
5467 .each = .{ .once = &.{
5468 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5469 .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5470 .{ .@"0:", ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
5471 .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
5472 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5473 .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
5474 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5475 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5476 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5477 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5478 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5479 } },
5480 }, .{
5481 .required_features = .{ .@"64bit", null, null, null },
5482 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5483 .patterns = &.{
5484 .{ .src = .{ .to_mem, .none } },
5485 },
5486 .extra_temps = .{
5487 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5488 .unused,
5489 .unused,
5490 .unused,
5491 .unused,
5492 .unused,
5493 },
5494 .dst_temps = .{.{ .rc = .general_purpose }},
5495 .clobbers = .{ .eflags = true },
5496 .each = .{ .once = &.{
5497 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5498 .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
5499 .{ .@"0:", ._, .@"and", .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5500 .{ ._, ._r, .bs, .dst0q, .dst0q, ._, ._ },
5501 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5502 .{ ._, ._, .mov, .dst0q, .si(-1), ._, ._ },
5503 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5504 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5505 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5506 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5507 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5508 } },
5509 }, .{
5510 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
5511 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5512 .patterns = &.{
5513 .{ .src = .{ .to_mem, .none } },
5514 },
5515 .extra_temps = .{
5516 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5517 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5518 .unused,
5519 .unused,
5520 .unused,
5521 .unused,
5522 },
5523 .dst_temps = .{.{ .rc = .general_purpose }},
5524 .clobbers = .{ .eflags = true },
5525 .each = .{ .once = &.{
5526 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5527 .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5528 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
5529 .{ ._, ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
5530 .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
5531 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5532 .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
5533 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5534 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5535 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5536 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5537 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5538 } },
5539 }, .{
5540 .required_features = .{ .@"64bit", .lzcnt, null, null },
5541 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5542 .patterns = &.{
5543 .{ .src = .{ .to_mem, .none } },
5544 },
5545 .extra_temps = .{
5546 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5547 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5548 .unused,
5549 .unused,
5550 .unused,
5551 .unused,
5552 },
5553 .dst_temps = .{.{ .rc = .general_purpose }},
5554 .clobbers = .{ .eflags = true },
5555 .each = .{ .once = &.{
5556 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5557 .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5558 .{ .@"0:", ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
5559 .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
5560 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5561 .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
5562 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5563 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5564 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5565 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5566 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5567 } },
5568 }, .{
5569 .required_features = .{ .@"64bit", null, null, null },
5570 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5571 .patterns = &.{
5572 .{ .src = .{ .to_mem, .none } },
5573 },
5574 .extra_temps = .{
5575 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5576 .unused,
5577 .unused,
5578 .unused,
5579 .unused,
5580 .unused,
5581 },
5582 .dst_temps = .{.{ .rc = .general_purpose }},
5583 .clobbers = .{ .eflags = true },
5584 .each = .{ .once = &.{
5585 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5586 .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
5587 .{ .@"0:", ._, .@"and", .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5588 .{ ._, ._r, .bs, .dst0q, .dst0q, ._, ._ },
5589 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5590 .{ ._, ._, .mov, .dst0q, .si(-1), ._, ._ },
5591 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5592 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5593 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5594 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5595 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5596 } },
5597 }, .{
5598 .required_features = .{ .lzcnt, .slow_incdec, null, null },
5599 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5600 .patterns = &.{
5601 .{ .src = .{ .to_mem, .none } },
5602 },
5603 .extra_temps = .{
5604 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5605 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5606 .unused,
5607 .unused,
5608 .unused,
5609 .unused,
5610 },
5611 .dst_temps = .{.mem},
5612 .clobbers = .{ .eflags = true },
5613 .each = .{ .once = &.{
5614 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5615 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5616 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5617 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5618 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5619 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5620 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5621 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5622 } },
5623 }, .{
5624 .required_features = .{ .lzcnt, null, null, null },
5625 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5626 .patterns = &.{
5627 .{ .src = .{ .to_mem, .none } },
5628 },
5629 .extra_temps = .{
5630 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5631 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5632 .unused,
5633 .unused,
5634 .unused,
5635 .unused,
5636 },
5637 .dst_temps = .{.mem},
5638 .clobbers = .{ .eflags = true },
5639 .each = .{ .once = &.{
5640 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5641 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5642 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5643 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5644 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5645 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5646 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5647 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5648 } },
5649 }, .{
5650 .required_features = .{ .lzcnt, .slow_incdec, null, null },
5651 .src_constraints = .{ .{ .scalar_int = .word }, .any },
5652 .patterns = &.{
5653 .{ .src = .{ .to_mem, .none } },
5654 },
5655 .extra_temps = .{
5656 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5657 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5658 .unused,
5659 .unused,
5660 .unused,
5661 .unused,
5662 },
5663 .dst_temps = .{.mem},
5664 .clobbers = .{ .eflags = true },
5665 .each = .{ .once = &.{
5666 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5667 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
5668 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5669 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5670 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5671 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5672 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5673 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5674 } },
5675 }, .{
5676 .required_features = .{ .lzcnt, null, null, null },
5677 .src_constraints = .{ .{ .scalar_int = .word }, .any },
5678 .patterns = &.{
5679 .{ .src = .{ .to_mem, .none } },
5680 },
5681 .extra_temps = .{
5682 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5683 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5684 .unused,
5685 .unused,
5686 .unused,
5687 .unused,
5688 },
5689 .dst_temps = .{.mem},
5690 .clobbers = .{ .eflags = true },
5691 .each = .{ .once = &.{
5692 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5693 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
5694 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5695 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5696 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5697 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5698 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5699 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5700 } },
5701 }, .{
5702 .required_features = .{ .lzcnt, .slow_incdec, null, null },
5703 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
5704 .patterns = &.{
5705 .{ .src = .{ .to_mem, .none } },
5706 },
5707 .extra_temps = .{
5708 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5709 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5710 .unused,
5711 .unused,
5712 .unused,
5713 .unused,
5714 },
5715 .dst_temps = .{.mem},
5716 .clobbers = .{ .eflags = true },
5717 .each = .{ .once = &.{
5718 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5719 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
5720 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5721 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5722 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5723 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5724 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5725 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5726 } },
5727 }, .{
5728 .required_features = .{ .lzcnt, null, null, null },
5729 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
5730 .patterns = &.{
5731 .{ .src = .{ .to_mem, .none } },
5732 },
5733 .extra_temps = .{
5734 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5735 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5736 .unused,
5737 .unused,
5738 .unused,
5739 .unused,
5740 },
5741 .dst_temps = .{.mem},
5742 .clobbers = .{ .eflags = true },
5743 .each = .{ .once = &.{
5744 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5745 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
5746 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5747 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5748 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5749 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5750 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5751 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5752 } },
5753 }, .{
5754 .required_features = .{ .@"64bit", .lzcnt, .slow_incdec, null },
5755 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
5756 .patterns = &.{
5757 .{ .src = .{ .to_mem, .none } },
5758 },
5759 .extra_temps = .{
5760 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5761 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5762 .unused,
5763 .unused,
5764 .unused,
5765 .unused,
5766 },
5767 .dst_temps = .{.mem},
5768 .clobbers = .{ .eflags = true },
5769 .each = .{ .once = &.{
5770 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5771 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5772 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
5773 .{ ._, ._, .lzcnt, .tmp1q, .tmp1q, ._, ._ },
5774 .{ ._, ._, .sub, .tmp1b, .sia(64, .src0, .sub_bit_size), ._, ._ },
5775 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5776 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5777 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5778 } },
5779 }, .{
5780 .required_features = .{ .@"64bit", .lzcnt, null, null },
5781 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
5782 .patterns = &.{
5783 .{ .src = .{ .to_mem, .none } },
5784 },
5785 .extra_temps = .{
5786 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5787 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5788 .unused,
5789 .unused,
5790 .unused,
5791 .unused,
5792 },
5793 .dst_temps = .{.mem},
5794 .clobbers = .{ .eflags = true },
5795 .each = .{ .once = &.{
5796 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5797 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5798 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
5799 .{ ._, ._, .lzcnt, .tmp1q, .tmp1q, ._, ._ },
5800 .{ ._, ._, .sub, .tmp1b, .sia(64, .src0, .sub_bit_size), ._, ._ },
5801 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5802 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5803 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5804 } },
5805 }, .{
5806 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null },
5807 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5808 .patterns = &.{
5809 .{ .src = .{ .to_mem, .none } },
5810 },
5811 .extra_temps = .{
5812 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5813 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5814 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5815 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
5816 .unused,
5817 .unused,
5818 },
5819 .dst_temps = .{.mem},
5820 .clobbers = .{ .eflags = true },
5821 .each = .{ .once = &.{
5822 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5823 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
5824 .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5825 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
5826 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
5827 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
5828 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5829 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
5830 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
5831 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5832 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5833 } },
5834 }, .{
5835 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
5836 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5837 .patterns = &.{
5838 .{ .src = .{ .to_mem, .none } },
5839 },
5840 .extra_temps = .{
5841 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5842 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5843 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5844 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
5845 .unused,
5846 .unused,
5847 },
5848 .dst_temps = .{.mem},
5849 .clobbers = .{ .eflags = true },
5850 .each = .{ .once = &.{
5851 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5852 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
5853 .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5854 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
5855 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
5856 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
5857 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5858 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
5859 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
5860 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5861 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5862 } },
5863 }, .{
5864 .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null },
5865 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5866 .patterns = &.{
5867 .{ .src = .{ .to_mem, .none } },
5868 },
5869 .extra_temps = .{
5870 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5871 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5872 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5873 .unused,
5874 .unused,
5875 .unused,
5876 },
5877 .dst_temps = .{.mem},
5878 .clobbers = .{ .eflags = true },
5879 .each = .{ .once = &.{
5880 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5881 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5882 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5883 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
5884 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
5885 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
5886 .{ ._, ._c, .st, ._, ._, ._, ._ },
5887 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
5888 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
5889 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5890 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5891 } },
5892 }, .{
5893 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
5894 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5895 .patterns = &.{
5896 .{ .src = .{ .to_mem, .none } },
5897 },
5898 .extra_temps = .{
5899 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5900 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5901 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5902 .unused,
5903 .unused,
5904 .unused,
5905 },
5906 .dst_temps = .{.mem},
5907 .clobbers = .{ .eflags = true },
5908 .each = .{ .once = &.{
5909 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5910 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5911 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5912 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
5913 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
5914 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
5915 .{ ._, ._c, .st, ._, ._, ._, ._ },
5916 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
5917 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
5918 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5919 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5920 } },
5921 }, .{
5922 .required_features = .{ .slow_incdec, null, null, null },
5923 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5924 .patterns = &.{
5925 .{ .src = .{ .to_mem, .none } },
5926 },
5927 .extra_temps = .{
5928 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5929 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5930 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5931 .unused,
5932 .unused,
5933 .unused,
5934 },
5935 .dst_temps = .{.mem},
5936 .clobbers = .{ .eflags = true },
5937 .each = .{ .once = &.{
5938 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5939 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5940 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5941 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
5942 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
5943 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5944 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
5945 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5946 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5947 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5948 } },
5949 }, .{
5950 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5951 .patterns = &.{
5952 .{ .src = .{ .to_mem, .none } },
5953 },
5954 .extra_temps = .{
5955 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5956 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5957 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5958 .unused,
5959 .unused,
5960 .unused,
5961 },
5962 .dst_temps = .{.mem},
5963 .clobbers = .{ .eflags = true },
5964 .each = .{ .once = &.{
5965 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5966 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5967 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5968 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
5969 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
5970 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5971 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
5972 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5973 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5974 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5975 } },
5976 }, .{
5977 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null },
5978 .src_constraints = .{ .{ .scalar_int = .word }, .any },
5979 .patterns = &.{
5980 .{ .src = .{ .to_mem, .none } },
5981 },
5982 .extra_temps = .{
5983 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5984 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5985 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5986 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
5987 .unused,
5988 .unused,
5989 },
5990 .dst_temps = .{.mem},
5991 .clobbers = .{ .eflags = true },
5992 .each = .{ .once = &.{
5993 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5994 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
5995 .{ .@"0:", ._, .movzx, .tmp2d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
5996 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
5997 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
5998 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
5999 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6000 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
6001 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6002 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6003 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6004 } },
6005 }, .{
6006 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
6007 .src_constraints = .{ .{ .scalar_int = .word }, .any },
6008 .patterns = &.{
6009 .{ .src = .{ .to_mem, .none } },
6010 },
6011 .extra_temps = .{
6012 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6013 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6014 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6015 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6016 .unused,
6017 .unused,
6018 },
6019 .dst_temps = .{.mem},
6020 .clobbers = .{ .eflags = true },
6021 .each = .{ .once = &.{
6022 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6023 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
6024 .{ .@"0:", ._, .movzx, .tmp2d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
6025 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
6026 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
6027 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
6028 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6029 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
6030 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6031 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6032 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6033 } },
6034 }, .{
6035 .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null },
6036 .src_constraints = .{ .{ .scalar_int = .word }, .any },
6037 .patterns = &.{
6038 .{ .src = .{ .to_mem, .none } },
6039 },
6040 .extra_temps = .{
6041 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6042 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6043 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6044 .unused,
6045 .unused,
6046 .unused,
6047 },
6048 .dst_temps = .{.mem},
6049 .clobbers = .{ .eflags = true },
6050 .each = .{ .once = &.{
6051 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6052 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
6053 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
6054 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
6055 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
6056 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
6057 .{ ._, ._c, .st, ._, ._, ._, ._ },
6058 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
6059 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
6060 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6061 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6062 } },
6063 }, .{
6064 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
6065 .src_constraints = .{ .{ .scalar_int = .word }, .any },
6066 .patterns = &.{
6067 .{ .src = .{ .to_mem, .none } },
6068 },
6069 .extra_temps = .{
6070 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6071 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6072 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6073 .unused,
6074 .unused,
6075 .unused,
6076 },
6077 .dst_temps = .{.mem},
6078 .clobbers = .{ .eflags = true },
6079 .each = .{ .once = &.{
6080 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6081 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
6082 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
6083 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
6084 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
6085 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
6086 .{ ._, ._c, .st, ._, ._, ._, ._ },
6087 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
6088 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
6089 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6090 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6091 } },
6092 }, .{
6093 .required_features = .{ .slow_incdec, null, null, null },
6094 .src_constraints = .{ .{ .scalar_int = .word }, .any },
6095 .patterns = &.{
6096 .{ .src = .{ .to_mem, .none } },
6097 },
6098 .extra_temps = .{
6099 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6100 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6101 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6102 .unused,
6103 .unused,
6104 .unused,
6105 },
6106 .dst_temps = .{.mem},
6107 .clobbers = .{ .eflags = true },
6108 .each = .{ .once = &.{
6109 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6110 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
6111 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
6112 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
6113 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
6114 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6115 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
6116 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
6117 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6118 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6119 } },
6120 }, .{
6121 .src_constraints = .{ .{ .scalar_int = .word }, .any },
6122 .patterns = &.{
6123 .{ .src = .{ .to_mem, .none } },
6124 },
6125 .extra_temps = .{
6126 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6127 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6128 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6129 .unused,
6130 .unused,
6131 .unused,
6132 },
6133 .dst_temps = .{.mem},
6134 .clobbers = .{ .eflags = true },
6135 .each = .{ .once = &.{
6136 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6137 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
6138 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
6139 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
6140 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
6141 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6142 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
6143 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
6144 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6145 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6146 } },
6147 }, .{
6148 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null },
6149 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
6150 .patterns = &.{
6151 .{ .src = .{ .to_mem, .none } },
6152 },
6153 .extra_temps = .{
6154 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6155 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6156 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6157 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6158 .unused,
6159 .unused,
6160 },
6161 .dst_temps = .{.mem},
6162 .clobbers = .{ .eflags = true },
6163 .each = .{ .once = &.{
6164 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6165 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
6166 .{ .@"0:", ._, .mov, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
6167 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
6168 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
6169 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
6170 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6171 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
6172 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6173 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6174 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6175 } },
6176 }, .{
6177 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
6178 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
6179 .patterns = &.{
6180 .{ .src = .{ .to_mem, .none } },
6181 },
6182 .extra_temps = .{
6183 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6184 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6185 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6186 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6187 .unused,
6188 .unused,
6189 },
6190 .dst_temps = .{.mem},
6191 .clobbers = .{ .eflags = true },
6192 .each = .{ .once = &.{
6193 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6194 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
6195 .{ .@"0:", ._, .mov, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
6196 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
6197 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
6198 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
6199 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6200 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
6201 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6202 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6203 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6204 } },
6205 }, .{
6206 .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null },
6207 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
6208 .patterns = &.{
6209 .{ .src = .{ .to_mem, .none } },
6210 },
6211 .extra_temps = .{
6212 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6213 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6214 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6215 .unused,
6216 .unused,
6217 .unused,
6218 },
6219 .dst_temps = .{.mem},
6220 .clobbers = .{ .eflags = true },
6221 .each = .{ .once = &.{
6222 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6223 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
6224 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
6225 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
6226 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
6227 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
6228 .{ ._, ._c, .st, ._, ._, ._, ._ },
6229 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
6230 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
6231 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6232 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6233 } },
6234 }, .{
6235 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
6236 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
6237 .patterns = &.{
6238 .{ .src = .{ .to_mem, .none } },
6239 },
6240 .extra_temps = .{
6241 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6242 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6243 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6244 .unused,
6245 .unused,
6246 .unused,
6247 },
6248 .dst_temps = .{.mem},
6249 .clobbers = .{ .eflags = true },
6250 .each = .{ .once = &.{
6251 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6252 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
6253 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
6254 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
6255 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
6256 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
6257 .{ ._, ._c, .st, ._, ._, ._, ._ },
6258 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
6259 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
6260 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6261 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6262 } },
6263 }, .{
6264 .required_features = .{ .slow_incdec, null, null, null },
6265 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
6266 .patterns = &.{
6267 .{ .src = .{ .to_mem, .none } },
6268 },
6269 .extra_temps = .{
6270 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6271 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6272 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6273 .unused,
6274 .unused,
6275 .unused,
6276 },
6277 .dst_temps = .{.mem},
6278 .clobbers = .{ .eflags = true },
6279 .each = .{ .once = &.{
6280 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6281 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
6282 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
6283 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
6284 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
6285 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6286 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
6287 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
6288 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6289 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6290 } },
6291 }, .{
6292 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
6293 .patterns = &.{
6294 .{ .src = .{ .to_mem, .none } },
6295 },
6296 .extra_temps = .{
6297 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6298 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6299 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6300 .unused,
6301 .unused,
6302 .unused,
6303 },
6304 .dst_temps = .{.mem},
6305 .clobbers = .{ .eflags = true },
6306 .each = .{ .once = &.{
6307 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6308 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
6309 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
6310 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
6311 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
6312 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6313 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
6314 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
6315 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6316 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6317 } },
6318 }, .{
6319 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec },
6320 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6321 .patterns = &.{
6322 .{ .src = .{ .to_mem, .none } },
6323 },
6324 .extra_temps = .{
6325 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6326 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6327 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6328 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6329 .unused,
6330 .unused,
6331 },
6332 .dst_temps = .{.mem},
6333 .clobbers = .{ .eflags = true },
6334 .each = .{ .once = &.{
6335 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6336 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
6337 .{ .@"0:", ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ },
6338 .{ ._, ._, .@"and", .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6339 .{ ._, ._r, .bs, .tmp2q, .tmp2q, ._, ._ },
6340 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
6341 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6342 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
6343 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6344 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6345 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6346 } },
6347 }, .{
6348 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
6349 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6350 .patterns = &.{
6351 .{ .src = .{ .to_mem, .none } },
6352 },
6353 .extra_temps = .{
6354 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6355 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6356 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6357 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6358 .unused,
6359 .unused,
6360 },
6361 .dst_temps = .{.mem},
6362 .clobbers = .{ .eflags = true },
6363 .each = .{ .once = &.{
6364 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6365 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
6366 .{ .@"0:", ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ },
6367 .{ ._, ._, .@"and", .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6368 .{ ._, ._r, .bs, .tmp2q, .tmp2q, ._, ._ },
6369 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
6370 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6371 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
6372 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6373 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6374 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6375 } },
6376 }, .{
6377 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, .slow_incdec, null },
6378 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6379 .patterns = &.{
6380 .{ .src = .{ .to_mem, .none } },
6381 },
6382 .extra_temps = .{
6383 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6384 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6385 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6386 .unused,
6387 .unused,
6388 .unused,
6389 },
6390 .dst_temps = .{.mem},
6391 .clobbers = .{ .eflags = true },
6392 .each = .{ .once = &.{
6393 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6394 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
6395 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6396 .{ ._, ._r, .bs, .tmp1q, .tmp1q, ._, ._ },
6397 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
6398 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
6399 .{ ._, ._c, .st, ._, ._, ._, ._ },
6400 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
6401 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
6402 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6403 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6404 } },
6405 }, .{
6406 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
6407 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6408 .patterns = &.{
6409 .{ .src = .{ .to_mem, .none } },
6410 },
6411 .extra_temps = .{
6412 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6413 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6414 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6415 .unused,
6416 .unused,
6417 .unused,
6418 },
6419 .dst_temps = .{.mem},
6420 .clobbers = .{ .eflags = true },
6421 .each = .{ .once = &.{
6422 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6423 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
6424 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6425 .{ ._, ._r, .bs, .tmp1q, .tmp1q, ._, ._ },
6426 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
6427 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
6428 .{ ._, ._c, .st, ._, ._, ._, ._ },
6429 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
6430 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
6431 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6432 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6433 } },
6434 }, .{
6435 .required_features = .{ .@"64bit", .slow_incdec, null, null },
6436 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6437 .patterns = &.{
6438 .{ .src = .{ .to_mem, .none } },
6439 },
6440 .extra_temps = .{
6441 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6442 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6443 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6444 .unused,
6445 .unused,
6446 .unused,
6447 },
6448 .dst_temps = .{.mem},
6449 .clobbers = .{ .eflags = true },
6450 .each = .{ .once = &.{
6451 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6452 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
6453 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6454 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
6455 .{ ._, ._r, .bs, .tmp2q, .tmp1q, ._, ._ },
6456 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6457 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
6458 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
6459 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6460 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6461 } },
6462 }, .{
6463 .required_features = .{ .@"64bit", null, null, null },
6464 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6465 .patterns = &.{
6466 .{ .src = .{ .to_mem, .none } },
6467 },
6468 .extra_temps = .{
6469 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6470 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6471 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6472 .unused,
6473 .unused,
6474 .unused,
6475 },
6476 .dst_temps = .{.mem},
6477 .clobbers = .{ .eflags = true },
6478 .each = .{ .once = &.{
6479 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6480 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
6481 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6482 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
6483 .{ ._, ._r, .bs, .tmp2q, .tmp1q, ._, ._ },
6484 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6485 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
6486 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
6487 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6488 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6489 } },
6490 }, .{
6491 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
6492 .dst_constraints = .{.{ .scalar_int = .byte }},
6493 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6494 .patterns = &.{
6495 .{ .src = .{ .to_mem, .none } },
6496 },
6497 .extra_temps = .{
6498 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6499 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6500 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6501 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6502 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6503 .unused,
6504 },
6505 .dst_temps = .{.mem},
6506 .clobbers = .{ .eflags = true },
6507 .each = .{ .once = &.{
6508 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6509 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6510 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6511 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6512 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6513 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6514 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6515 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6516 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6517 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6518 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6519 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6520 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6521 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6522 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6523 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6524 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6525 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6526 } },
6527 }, .{
6528 .required_features = .{ .@"64bit", .lzcnt, null, null },
6529 .dst_constraints = .{.{ .scalar_int = .byte }},
6530 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6531 .patterns = &.{
6532 .{ .src = .{ .to_mem, .none } },
6533 },
6534 .extra_temps = .{
6535 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6536 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6537 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6538 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6539 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6540 .unused,
6541 },
6542 .dst_temps = .{.mem},
6543 .clobbers = .{ .eflags = true },
6544 .each = .{ .once = &.{
6545 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6546 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6547 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6548 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6549 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6550 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6551 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6552 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6553 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6554 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6555 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6556 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6557 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6558 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6559 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6560 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6561 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6562 } },
6563 }, .{
6564 .required_features = .{ .@"64bit", null, null, null },
6565 .dst_constraints = .{.{ .scalar_int = .byte }},
6566 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6567 .patterns = &.{
6568 .{ .src = .{ .to_mem, .none } },
6569 },
6570 .extra_temps = .{
6571 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6572 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6573 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6574 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6575 .unused,
6576 .unused,
6577 },
6578 .dst_temps = .{.mem},
6579 .clobbers = .{ .eflags = true },
6580 .each = .{ .once = &.{
6581 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6582 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6583 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6584 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6585 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6586 .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
6587 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
6588 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6589 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6590 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6591 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6592 .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
6593 .{ ._, ._, .neg, .tmp3b, ._, ._, ._ },
6594 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6595 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6596 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6597 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6598 } },
6599 }, .{
6600 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
6601 .dst_constraints = .{.{ .scalar_int = .byte }},
6602 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6603 .patterns = &.{
6604 .{ .src = .{ .to_mem, .none } },
6605 },
6606 .extra_temps = .{
6607 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6608 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6609 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6610 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6611 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6612 .unused,
6613 },
6614 .dst_temps = .{.mem},
6615 .clobbers = .{ .eflags = true },
6616 .each = .{ .once = &.{
6617 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6618 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6619 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6620 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6621 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6622 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6623 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6624 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6625 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6626 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6627 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6628 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6629 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6630 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6631 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6632 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6633 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6634 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6635 } },
6636 }, .{
6637 .required_features = .{ .@"64bit", .lzcnt, null, null },
6638 .dst_constraints = .{.{ .scalar_int = .byte }},
6639 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6640 .patterns = &.{
6641 .{ .src = .{ .to_mem, .none } },
6642 },
6643 .extra_temps = .{
6644 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6645 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6646 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6647 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6648 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6649 .unused,
6650 },
6651 .dst_temps = .{.mem},
6652 .clobbers = .{ .eflags = true },
6653 .each = .{ .once = &.{
6654 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6655 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6656 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6657 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6658 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6659 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6660 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6661 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6662 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6663 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6664 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6665 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6666 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6667 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6668 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6669 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6670 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6671 } },
6672 }, .{
6673 .required_features = .{ .@"64bit", null, null, null },
6674 .dst_constraints = .{.{ .scalar_int = .byte }},
6675 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6676 .patterns = &.{
6677 .{ .src = .{ .to_mem, .none } },
6678 },
6679 .extra_temps = .{
6680 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6681 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6682 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6683 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6684 .unused,
6685 .unused,
6686 },
6687 .dst_temps = .{.mem},
6688 .clobbers = .{ .eflags = true },
6689 .each = .{ .once = &.{
6690 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6691 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6692 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6693 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6694 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6695 .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
6696 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
6697 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6698 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6699 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6700 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6701 .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
6702 .{ ._, ._, .neg, .tmp3b, ._, ._, ._ },
6703 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6704 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6705 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6706 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6707 } },
6708 }, .{
6709 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
6710 .dst_constraints = .{.{ .scalar_int = .word }},
6711 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6712 .patterns = &.{
6713 .{ .src = .{ .to_mem, .none } },
6714 },
6715 .extra_temps = .{
6716 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6717 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6718 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6719 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6720 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6721 .unused,
6722 },
6723 .dst_temps = .{.mem},
6724 .clobbers = .{ .eflags = true },
6725 .each = .{ .once = &.{
6726 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6727 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6728 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6729 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6730 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6731 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6732 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6733 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6734 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6735 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6736 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6737 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6738 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6739 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6740 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6741 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6742 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6743 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6744 } },
6745 }, .{
6746 .required_features = .{ .@"64bit", .lzcnt, null, null },
6747 .dst_constraints = .{.{ .scalar_int = .word }},
6748 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6749 .patterns = &.{
6750 .{ .src = .{ .to_mem, .none } },
6751 },
6752 .extra_temps = .{
6753 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6754 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6755 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6756 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6757 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6758 .unused,
6759 },
6760 .dst_temps = .{.mem},
6761 .clobbers = .{ .eflags = true },
6762 .each = .{ .once = &.{
6763 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6764 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6765 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6766 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6767 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6768 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6769 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6770 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6771 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6772 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6773 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6774 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6775 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6776 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6777 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6778 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6779 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6780 } },
6781 }, .{
6782 .required_features = .{ .@"64bit", null, null, null },
6783 .dst_constraints = .{.{ .scalar_int = .word }},
6784 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6785 .patterns = &.{
6786 .{ .src = .{ .to_mem, .none } },
6787 },
6788 .extra_temps = .{
6789 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6790 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6791 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6792 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6793 .unused,
6794 .unused,
6795 },
6796 .dst_temps = .{.mem},
6797 .clobbers = .{ .eflags = true },
6798 .each = .{ .once = &.{
6799 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6800 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6801 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6802 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6803 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6804 .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
6805 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
6806 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6807 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6808 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6809 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6810 .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
6811 .{ ._, ._, .neg, .tmp3d, ._, ._, ._ },
6812 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6813 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6814 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6815 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6816 } },
6817 }, .{
6818 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
6819 .dst_constraints = .{.{ .scalar_int = .word }},
6820 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6821 .patterns = &.{
6822 .{ .src = .{ .to_mem, .none } },
6823 },
6824 .extra_temps = .{
6825 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6826 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6827 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6828 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6829 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6830 .unused,
6831 },
6832 .dst_temps = .{.mem},
6833 .clobbers = .{ .eflags = true },
6834 .each = .{ .once = &.{
6835 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6836 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6837 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6838 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6839 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6840 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6841 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6842 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6843 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6844 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6845 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6846 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6847 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6848 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6849 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6850 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6851 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6852 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6853 } },
6854 }, .{
6855 .required_features = .{ .@"64bit", .lzcnt, null, null },
6856 .dst_constraints = .{.{ .scalar_int = .word }},
6857 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6858 .patterns = &.{
6859 .{ .src = .{ .to_mem, .none } },
6860 },
6861 .extra_temps = .{
6862 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6863 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6864 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6865 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6866 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6867 .unused,
6868 },
6869 .dst_temps = .{.mem},
6870 .clobbers = .{ .eflags = true },
6871 .each = .{ .once = &.{
6872 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6873 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6874 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6875 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6876 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6877 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6878 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6879 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6880 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6881 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6882 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6883 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6884 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6885 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6886 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6887 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6888 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6889 } },
6890 }, .{
6891 .required_features = .{ .@"64bit", null, null, null },
6892 .dst_constraints = .{.{ .scalar_int = .word }},
6893 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6894 .patterns = &.{
6895 .{ .src = .{ .to_mem, .none } },
6896 },
6897 .extra_temps = .{
6898 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6899 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6900 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6901 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6902 .unused,
6903 .unused,
6904 },
6905 .dst_temps = .{.mem},
6906 .clobbers = .{ .eflags = true },
6907 .each = .{ .once = &.{
6908 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6909 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6910 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6911 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6912 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6913 .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
6914 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
6915 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6916 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6917 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6918 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6919 .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
6920 .{ ._, ._, .neg, .tmp3d, ._, ._, ._ },
6921 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6922 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6923 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6924 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6925 } },
6926 } }) catch |err| switch (err) {
6927 error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
6928 @tagName(air_tag),
6929 cg.typeOf(ty_op.operand).fmt(pt),
6930 ops[0].tracking(cg),
6931 }),
6932 else => |e| return e,
6933 };
6934 for (ops) |op| for (res) |r| {
6935 if (op.index == r.index) break;
6936 } else try op.die(cg);
6937 try res[0].moveTo(inst, cg);
6938 },
6939
6940 .cmp_vector, .cmp_vector_optimized => |air_tag| if (use_old) try cg.airCmpVector(inst) else fallback: {
6941 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
6942 const extra = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data;
6943 switch (extra.compareOperator()) {
6944 .eq, .neq => if (cg.typeOf(extra.lhs).scalarType(zcu).isRuntimeFloat())
6945 break :fallback try cg.airCmpVector(inst),
6946 else => break :fallback try cg.airCmpVector(inst),
6947 }
6948 var ops = try cg.tempsFromOperands(inst, .{ extra.lhs, extra.rhs });
6949 var res: [1]Temp = undefined;
6950 switch (extra.compareOperator()) {
6951 .lt => unreachable,
6952 .lte => unreachable,
6953 .eq, .neq => |cmp_op| cg.select(&res, &.{ty_pl.ty.toType()}, &ops, switch (@as(Condition, switch (cmp_op) {
6954 else => unreachable,
6955 .eq => .e,
6956 .neq => .ne,
6957 })) {
6958 else => unreachable,
6959 inline .e, .ne => |cc| comptime &.{ .{
6960 .required_features = .{ .avx2, null, null, null },
6961 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
6962 .patterns = &.{
6963 .{ .src = .{ .to_ymm, .mem } },
6964 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
6965 .{ .src = .{ .to_ymm, .to_ymm } },
6966 },
6967 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
6968 .kind = .all,
6969 .inverted = switch (cc) {
6970 else => unreachable,
6971 .e => false,
6972 .ne => true,
6973 },
6974 .scalar = .byte,
6975 } } }},
6976 .each = .{ .once = &.{
6977 .{ ._, .vp_b, .cmpeq, .dst0y, .src0y, .src1y, ._ },
6978 } },
6979 }, .{
6980 .required_features = .{ .avx2, null, null, null },
6981 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
6982 .patterns = &.{
6983 .{ .src = .{ .to_ymm, .mem } },
6984 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
6985 .{ .src = .{ .to_ymm, .to_ymm } },
6986 },
6987 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
6988 .kind = .all,
6989 .inverted = switch (cc) {
6990 else => unreachable,
6991 .e => false,
6992 .ne => true,
6993 },
6994 .scalar = .word,
6995 } } }},
6996 .each = .{ .once = &.{
6997 .{ ._, .vp_w, .cmpeq, .dst0y, .src0y, .src1y, ._ },
6998 } },
6999 }, .{
7000 .required_features = .{ .avx2, null, null, null },
7001 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
7002 .patterns = &.{
7003 .{ .src = .{ .to_ymm, .mem } },
7004 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
7005 .{ .src = .{ .to_ymm, .to_ymm } },
7006 },
7007 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
7008 .kind = .all,
7009 .inverted = switch (cc) {
7010 else => unreachable,
7011 .e => false,
7012 .ne => true,
7013 },
7014 .scalar = .dword,
7015 } } }},
7016 .each = .{ .once = &.{
7017 .{ ._, .vp_d, .cmpeq, .dst0y, .src0y, .src1y, ._ },
7018 } },
7019 }, .{
7020 .required_features = .{ .avx2, null, null, null },
7021 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
7022 .patterns = &.{
7023 .{ .src = .{ .to_ymm, .mem } },
7024 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
7025 .{ .src = .{ .to_ymm, .to_ymm } },
7026 },
7027 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
7028 .kind = .all,
7029 .inverted = switch (cc) {
7030 else => unreachable,
7031 .e => false,
7032 .ne => true,
7033 },
7034 .scalar = .qword,
7035 } } }},
7036 .each = .{ .once = &.{
7037 .{ ._, .vp_q, .cmpeq, .dst0y, .src0y, .src1y, ._ },
7038 } },
7039 }, .{
7040 .required_features = .{ .avx, null, null, null },
7041 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
7042 .patterns = &.{
7043 .{ .src = .{ .to_xmm, .mem } },
7044 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
7045 .{ .src = .{ .to_xmm, .to_xmm } },
7046 },
7047 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
7048 .kind = .all,
7049 .inverted = switch (cc) {
7050 else => unreachable,
7051 .e => false,
7052 .ne => true,
7053 },
7054 .scalar = .byte,
7055 } } }},
7056 .each = .{ .once = &.{
7057 .{ ._, .vp_b, .cmpeq, .dst0x, .src0x, .src1x, ._ },
7058 } },
7059 }, .{
7060 .required_features = .{ .avx, null, null, null },
7061 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
7062 .patterns = &.{
7063 .{ .src = .{ .to_xmm, .mem } },
7064 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
7065 .{ .src = .{ .to_xmm, .to_xmm } },
7066 },
7067 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
7068 .kind = .all,
7069 .inverted = switch (cc) {
7070 else => unreachable,
7071 .e => false,
7072 .ne => true,
7073 },
7074 .scalar = .word,
7075 } } }},
7076 .each = .{ .once = &.{
7077 .{ ._, .vp_w, .cmpeq, .dst0x, .src0x, .src1x, ._ },
7078 } },
7079 }, .{
7080 .required_features = .{ .avx, null, null, null },
7081 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
7082 .patterns = &.{
7083 .{ .src = .{ .to_xmm, .mem } },
7084 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
7085 .{ .src = .{ .to_xmm, .to_xmm } },
7086 },
7087 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
7088 .kind = .all,
7089 .inverted = switch (cc) {
7090 else => unreachable,
7091 .e => false,
7092 .ne => true,
7093 },
7094 .scalar = .dword,
7095 } } }},
7096 .each = .{ .once = &.{
7097 .{ ._, .vp_d, .cmpeq, .dst0x, .src0x, .src1x, ._ },
7098 } },
7099 }, .{
7100 .required_features = .{ .avx, null, null, null },
7101 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
7102 .patterns = &.{
7103 .{ .src = .{ .to_xmm, .mem } },
7104 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
7105 .{ .src = .{ .to_xmm, .to_xmm } },
7106 },
7107 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
7108 .kind = .all,
7109 .inverted = switch (cc) {
7110 else => unreachable,
7111 .e => false,
7112 .ne => true,
7113 },
7114 .scalar = .qword,
7115 } } }},
7116 .each = .{ .once = &.{
7117 .{ ._, .vp_q, .cmpeq, .dst0x, .src0x, .src1x, ._ },
7118 } },
7119 }, .{
7120 .required_features = .{ .sse2, null, null, null },
7121 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
7122 .patterns = &.{
7123 .{ .src = .{ .to_mut_xmm, .mem } },
7124 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
7125 .{ .src = .{ .to_mut_xmm, .to_xmm } },
7126 },
7127 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
7128 .kind = .all,
7129 .inverted = switch (cc) {
7130 else => unreachable,
7131 .e => false,
7132 .ne => true,
7133 },
7134 .scalar = .byte,
7135 } } }},
7136 .each = .{ .once = &.{
7137 .{ ._, .p_b, .cmpeq, .dst0x, .src1x, ._, ._ },
7138 } },
7139 }, .{
7140 .required_features = .{ .sse2, null, null, null },
7141 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
7142 .patterns = &.{
7143 .{ .src = .{ .to_mut_xmm, .mem } },
7144 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
7145 .{ .src = .{ .to_mut_xmm, .to_xmm } },
7146 },
7147 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
7148 .kind = .all,
7149 .inverted = switch (cc) {
7150 else => unreachable,
7151 .e => false,
7152 .ne => true,
7153 },
7154 .scalar = .word,
7155 } } }},
7156 .each = .{ .once = &.{
7157 .{ ._, .p_w, .cmpeq, .dst0x, .src1x, ._, ._ },
7158 } },
7159 }, .{
7160 .required_features = .{ .sse2, null, null, null },
7161 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
7162 .patterns = &.{
7163 .{ .src = .{ .to_mut_xmm, .mem } },
7164 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
7165 .{ .src = .{ .to_mut_xmm, .to_xmm } },
7166 },
7167 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
7168 .kind = .all,
7169 .inverted = switch (cc) {
7170 else => unreachable,
7171 .e => false,
7172 .ne => true,
7173 },
7174 .scalar = .dword,
7175 } } }},
7176 .each = .{ .once = &.{
7177 .{ ._, .p_d, .cmpeq, .dst0x, .src1x, ._, ._ },
7178 } },
7179 }, .{
7180 .required_features = .{ .sse4_1, null, null, null },
7181 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
7182 .patterns = &.{
7183 .{ .src = .{ .to_mut_xmm, .mem } },
7184 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
7185 .{ .src = .{ .to_mut_xmm, .to_xmm } },
7186 },
7187 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
7188 .kind = .all,
7189 .inverted = switch (cc) {
7190 else => unreachable,
7191 .e => false,
7192 .ne => true,
7193 },
7194 .scalar = .qword,
7195 } } }},
7196 .each = .{ .once = &.{
7197 .{ ._, .p_q, .cmpeq, .dst0x, .src1x, ._, ._ },
7198 } },
7199 }, .{
7200 .required_features = .{ .mmx, null, null, null },
7201 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
7202 .patterns = &.{
7203 .{ .src = .{ .to_mut_mm, .mem } },
7204 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
7205 .{ .src = .{ .to_mut_mm, .to_mm } },
7206 },
7207 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
7208 .kind = .all,
7209 .inverted = switch (cc) {
7210 else => unreachable,
7211 .e => false,
7212 .ne => true,
7213 },
7214 .scalar = .byte,
7215 } } }},
7216 .each = .{ .once = &.{
7217 .{ ._, .p_b, .cmpeq, .dst0q, .src1q, ._, ._ },
7218 } },
7219 }, .{
7220 .required_features = .{ .mmx, null, null, null },
7221 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
7222 .patterns = &.{
7223 .{ .src = .{ .to_mut_mm, .mem } },
7224 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
7225 .{ .src = .{ .to_mut_mm, .to_mm } },
7226 },
7227 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
7228 .kind = .all,
7229 .inverted = switch (cc) {
7230 else => unreachable,
7231 .e => false,
7232 .ne => true,
7233 },
7234 .scalar = .word,
7235 } } }},
7236 .each = .{ .once = &.{
7237 .{ ._, .p_w, .cmpeq, .dst0q, .src1q, ._, ._ },
7238 } },
7239 }, .{
7240 .required_features = .{ .mmx, null, null, null },
7241 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
7242 .patterns = &.{
7243 .{ .src = .{ .to_mut_mm, .mem } },
7244 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
7245 .{ .src = .{ .to_mut_mm, .to_mm } },
7246 },
7247 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
7248 .kind = .all,
7249 .inverted = switch (cc) {
7250 else => unreachable,
7251 .e => false,
7252 .ne => true,
7253 },
7254 .scalar = .dword,
7255 } } }},
7256 .each = .{ .once = &.{
7257 .{ ._, .p_d, .cmpeq, .dst0q, .src1q, ._, ._ },
7258 } },
7259 }, .{
7260 .src_constraints = .{ .{ .bool_vec = .byte }, .{ .bool_vec = .byte } },
7261 .patterns = &.{
7262 .{ .src = .{ .mut_mem, .imm8 } },
7263 .{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } },
7264 .{ .src = .{ .to_mut_gpr, .imm8 } },
7265 .{ .src = .{ .imm8, .to_mut_gpr }, .commute = .{ 0, 1 } },
7266 .{ .src = .{ .mut_mem, .to_gpr } },
7267 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
7268 .{ .src = .{ .to_mut_gpr, .mem } },
7269 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
7270 .{ .src = .{ .to_mut_gpr, .to_gpr } },
7271 },
7272 .dst_temps = .{.{ .ref = .src0 }},
7273 .clobbers = .{ .eflags = true },
7274 .each = .{ .once = switch (cc) {
7275 else => unreachable,
7276 .e => &.{
7277 .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ },
7278 .{ ._, ._, .not, .dst0b, ._, ._, ._ },
7279 },
7280 .ne => &.{
7281 .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ },
7282 },
7283 } },
7284 }, .{
7285 .src_constraints = .{ .{ .bool_vec = .word }, .{ .bool_vec = .word } },
7286 .patterns = &.{
7287 .{ .src = .{ .mut_mem, .imm16 } },
7288 .{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } },
7289 .{ .src = .{ .to_mut_gpr, .imm16 } },
7290 .{ .src = .{ .imm16, .to_mut_gpr }, .commute = .{ 0, 1 } },
7291 .{ .src = .{ .mut_mem, .to_gpr } },
7292 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
7293 .{ .src = .{ .to_mut_gpr, .mem } },
7294 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
7295 .{ .src = .{ .to_mut_gpr, .to_gpr } },
7296 },
7297 .dst_temps = .{.{ .ref = .src0 }},
7298 .clobbers = .{ .eflags = true },
7299 .each = .{ .once = switch (cc) {
7300 else => unreachable,
7301 .e => &.{
7302 .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ },
7303 .{ ._, ._, .not, .dst0w, ._, ._, ._ },
7304 },
7305 .ne => &.{
7306 .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ },
7307 },
7308 } },
7309 }, .{
7310 .src_constraints = .{ .{ .bool_vec = .dword }, .{ .bool_vec = .dword } },
7311 .patterns = &.{
7312 .{ .src = .{ .mut_mem, .imm32 } },
7313 .{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } },
7314 .{ .src = .{ .to_mut_gpr, .imm32 } },
7315 .{ .src = .{ .imm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
7316 .{ .src = .{ .mut_mem, .to_gpr } },
7317 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
7318 .{ .src = .{ .to_mut_gpr, .mem } },
7319 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
7320 .{ .src = .{ .to_mut_gpr, .to_gpr } },
7321 },
7322 .dst_temps = .{.{ .ref = .src0 }},
7323 .clobbers = .{ .eflags = true },
7324 .each = .{ .once = switch (cc) {
7325 else => unreachable,
7326 .e => &.{
7327 .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ },
7328 .{ ._, ._, .not, .dst0d, ._, ._, ._ },
7329 },
7330 .ne => &.{
7331 .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ },
7332 },
7333 } },
7334 }, .{
7335 .required_features = .{ .@"64bit", null, null, null },
7336 .src_constraints = .{ .{ .bool_vec = .qword }, .{ .bool_vec = .qword } },
7337 .patterns = &.{
7338 .{ .src = .{ .mut_mem, .simm32 } },
7339 .{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } },
7340 .{ .src = .{ .to_mut_gpr, .simm32 } },
7341 .{ .src = .{ .simm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
7342 .{ .src = .{ .mut_mem, .to_gpr } },
7343 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
7344 .{ .src = .{ .to_mut_gpr, .mem } },
7345 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
7346 .{ .src = .{ .to_mut_gpr, .to_gpr } },
7347 },
7348 .dst_temps = .{.{ .ref = .src0 }},
7349 .clobbers = .{ .eflags = true },
7350 .each = .{ .once = switch (cc) {
7351 else => unreachable,
7352 .e => &.{
7353 .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ },
7354 .{ ._, ._, .not, .dst0q, ._, ._, ._ },
7355 },
7356 .ne => &.{
7357 .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ },
7358 },
7359 } },
7360 }, .{
7361 .src_constraints = .{ .any_bool_vec, .any_bool_vec },
7362 .patterns = &.{
7363 .{ .src = .{ .to_mem, .to_mem } },
7364 },
7365 .extra_temps = .{
7366 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7367 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7368 .unused,
7369 .unused,
7370 .unused,
7371 .unused,
7372 },
7373 .dst_temps = .{.mem},
7374 .clobbers = .{ .eflags = true },
7375 .each = .{ .once = switch (cc) {
7376 else => unreachable,
7377 .e => &.{
7378 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7379 .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
7380 .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
7381 .{ ._, ._, .not, .tmp1p, ._, ._, ._ },
7382 .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
7383 .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
7384 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7385 },
7386 .ne => &.{
7387 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7388 .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
7389 .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
7390 .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
7391 .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
7392 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7393 },
7394 } },
7395 }, .{
7396 .required_features = .{ .avx2, null, null, null },
7397 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
7398 .patterns = &.{
7399 .{ .src = .{ .to_mem, .to_mem } },
7400 },
7401 .extra_temps = .{
7402 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7403 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7404 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7405 .{ .kind = .{ .rc = .sse } },
7406 .unused,
7407 .unused,
7408 },
7409 .dst_temps = .{.mem},
7410 .clobbers = .{ .eflags = true },
7411 .each = .{ .once = switch (cc) {
7412 else => unreachable,
7413 .e => &.{
7414 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7415 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7416 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7417 .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7418 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
7419 .{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ },
7420 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7421 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
7422 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7423 },
7424 .ne => &.{
7425 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7426 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7427 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7428 .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7429 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
7430 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
7431 .{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ },
7432 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7433 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
7434 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7435 },
7436 } },
7437 }, .{
7438 .required_features = .{ .avx2, null, null, null },
7439 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
7440 .patterns = &.{
7441 .{ .src = .{ .to_mem, .to_mem } },
7442 },
7443 .extra_temps = .{
7444 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7445 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7446 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
7447 .{ .kind = .{ .rc = .sse } },
7448 .unused,
7449 .unused,
7450 },
7451 .dst_temps = .{.mem},
7452 .clobbers = .{ .eflags = true },
7453 .each = .{ .once = switch (cc) {
7454 else => unreachable,
7455 .e => &.{
7456 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7457 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7458 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7459 .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7460 .{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ },
7461 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
7462 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
7463 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7464 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
7465 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7466 },
7467 .ne => &.{
7468 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7469 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7470 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7471 .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7472 .{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ },
7473 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
7474 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
7475 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
7476 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7477 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
7478 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7479 },
7480 } },
7481 }, .{
7482 .required_features = .{ .avx2, null, null, null },
7483 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
7484 .patterns = &.{
7485 .{ .src = .{ .to_mem, .to_mem } },
7486 },
7487 .extra_temps = .{
7488 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7489 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7490 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7491 .{ .kind = .{ .rc = .sse } },
7492 .unused,
7493 .unused,
7494 },
7495 .dst_temps = .{.mem},
7496 .clobbers = .{ .eflags = true },
7497 .each = .{ .once = switch (cc) {
7498 else => unreachable,
7499 .e => &.{
7500 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7501 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7502 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7503 .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7504 .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ },
7505 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
7506 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
7507 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
7508 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7509 },
7510 .ne => &.{
7511 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7512 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7513 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7514 .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7515 .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ },
7516 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
7517 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
7518 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
7519 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
7520 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7521 },
7522 } },
7523 }, .{
7524 .required_features = .{ .avx2, null, null, null },
7525 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
7526 .patterns = &.{
7527 .{ .src = .{ .to_mem, .to_mem } },
7528 },
7529 .extra_temps = .{
7530 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7531 .{ .type = .u32, .kind = .{ .reg = .rcx } },
7532 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7533 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7534 .{ .kind = .{ .rc = .sse } },
7535 .unused,
7536 },
7537 .dst_temps = .{.mem},
7538 .clobbers = .{ .eflags = true },
7539 .each = .{ .once = switch (cc) {
7540 else => unreachable,
7541 .e => &.{
7542 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7543 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7544 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7545 .{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7546 .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ },
7547 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ },
7548 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7549 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7550 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7551 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7552 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7553 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7554 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7555 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7556 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7557 .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ },
7558 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7559 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7560 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7561 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7562 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7563 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7564 },
7565 .ne => &.{
7566 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7567 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7568 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7569 .{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7570 .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ },
7571 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ },
7572 .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ },
7573 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7574 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7575 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7576 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7577 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7578 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7579 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7580 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7581 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7582 .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ },
7583 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7584 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7585 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7586 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7587 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7588 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7589 },
7590 } },
7591 }, .{
7592 .required_features = .{ .avx, null, null, null },
7593 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
7594 .patterns = &.{
7595 .{ .src = .{ .to_mem, .to_mem } },
7596 },
7597 .extra_temps = .{
7598 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7599 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7600 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
7601 .{ .kind = .{ .rc = .sse } },
7602 .unused,
7603 .unused,
7604 },
7605 .dst_temps = .{.mem},
7606 .clobbers = .{ .eflags = true },
7607 .each = .{ .once = switch (cc) {
7608 else => unreachable,
7609 .e => &.{
7610 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7611 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7612 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7613 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
7614 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7615 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
7616 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7617 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7618 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7619 },
7620 .ne => &.{
7621 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7622 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7623 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7624 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
7625 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7626 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
7627 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
7628 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7629 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7630 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7631 },
7632 } },
7633 }, .{
7634 .required_features = .{ .avx, null, null, null },
7635 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
7636 .patterns = &.{
7637 .{ .src = .{ .to_mem, .to_mem } },
7638 },
7639 .extra_temps = .{
7640 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7641 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7642 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7643 .{ .kind = .{ .rc = .sse } },
7644 .unused,
7645 .unused,
7646 },
7647 .dst_temps = .{.mem},
7648 .clobbers = .{ .eflags = true },
7649 .each = .{ .once = switch (cc) {
7650 else => unreachable,
7651 .e => &.{
7652 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7653 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7654 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7655 .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
7656 .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ },
7657 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7658 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
7659 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
7660 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7661 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7662 },
7663 .ne => &.{
7664 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7665 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7666 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7667 .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
7668 .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ },
7669 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7670 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
7671 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
7672 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
7673 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7674 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7675 },
7676 } },
7677 }, .{
7678 .required_features = .{ .avx, null, null, null },
7679 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
7680 .patterns = &.{
7681 .{ .src = .{ .to_mem, .to_mem } },
7682 },
7683 .extra_temps = .{
7684 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7685 .{ .type = .u32, .kind = .{ .reg = .rcx } },
7686 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7687 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7688 .{ .kind = .{ .rc = .sse } },
7689 .unused,
7690 },
7691 .dst_temps = .{.mem},
7692 .clobbers = .{ .eflags = true },
7693 .each = .{ .once = switch (cc) {
7694 else => unreachable,
7695 .e => &.{
7696 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7697 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7698 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7699 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7700 .{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
7701 .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7702 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7703 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7704 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7705 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7706 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7707 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7708 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7709 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7710 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7711 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
7712 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7713 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7714 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7715 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7716 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7717 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7718 },
7719 .ne => &.{
7720 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7721 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7722 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7723 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7724 .{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
7725 .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7726 .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ },
7727 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7728 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7729 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7730 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7731 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7732 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7733 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7734 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7735 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7736 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
7737 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7738 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7739 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7740 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7741 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7742 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7743 },
7744 } },
7745 }, .{
7746 .required_features = .{ .avx, null, null, null },
7747 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
7748 .patterns = &.{
7749 .{ .src = .{ .to_mem, .to_mem } },
7750 },
7751 .extra_temps = .{
7752 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7753 .{ .type = .u32, .kind = .{ .reg = .rcx } },
7754 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7755 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7756 .{ .kind = .{ .rc = .sse } },
7757 .unused,
7758 },
7759 .dst_temps = .{.mem},
7760 .clobbers = .{ .eflags = true },
7761 .each = .{ .once = switch (cc) {
7762 else => unreachable,
7763 .e => &.{
7764 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7765 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7766 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7767 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7768 .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
7769 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7770 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7771 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7772 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7773 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7774 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7775 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7776 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7777 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7778 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7779 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
7780 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7781 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7782 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7783 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7784 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7785 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7786 },
7787 .ne => &.{
7788 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7789 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7790 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7791 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7792 .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
7793 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7794 .{ ._, ._, .xor, .tmp3b, .si(0b11), ._, ._ },
7795 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7796 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7797 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7798 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7799 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7800 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7801 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7802 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7803 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7804 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
7805 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7806 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7807 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7808 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7809 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7810 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7811 },
7812 } },
7813 }, .{
7814 .required_features = .{ .sse2, null, null, null },
7815 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
7816 .patterns = &.{
7817 .{ .src = .{ .to_mem, .to_mem } },
7818 },
7819 .extra_temps = .{
7820 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7821 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7822 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
7823 .{ .kind = .{ .rc = .sse } },
7824 .unused,
7825 .unused,
7826 },
7827 .dst_temps = .{.mem},
7828 .clobbers = .{ .eflags = true },
7829 .each = .{ .once = switch (cc) {
7830 else => unreachable,
7831 .e => &.{
7832 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7833 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7834 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7835 .{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7836 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7837 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
7838 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7839 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7840 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7841 },
7842 .ne => &.{
7843 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7844 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7845 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7846 .{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7847 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7848 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
7849 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
7850 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7851 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7852 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7853 },
7854 } },
7855 }, .{
7856 .required_features = .{ .sse2, null, null, null },
7857 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
7858 .patterns = &.{
7859 .{ .src = .{ .to_mem, .to_mem } },
7860 },
7861 .extra_temps = .{
7862 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7863 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7864 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7865 .{ .kind = .{ .rc = .sse } },
7866 .unused,
7867 .unused,
7868 },
7869 .dst_temps = .{.mem},
7870 .clobbers = .{ .eflags = true },
7871 .each = .{ .once = switch (cc) {
7872 else => unreachable,
7873 .e => &.{
7874 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7875 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7876 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7877 .{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7878 .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ },
7879 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7880 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
7881 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
7882 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7883 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7884 },
7885 .ne => &.{
7886 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7887 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7888 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7889 .{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7890 .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ },
7891 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7892 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
7893 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
7894 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
7895 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7896 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7897 },
7898 } },
7899 }, .{
7900 .required_features = .{ .sse2, null, null, null },
7901 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
7902 .patterns = &.{
7903 .{ .src = .{ .to_mem, .to_mem } },
7904 },
7905 .extra_temps = .{
7906 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7907 .{ .type = .u32, .kind = .{ .reg = .rcx } },
7908 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7909 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7910 .{ .kind = .{ .rc = .sse } },
7911 .unused,
7912 },
7913 .dst_temps = .{.mem},
7914 .clobbers = .{ .eflags = true },
7915 .each = .{ .once = switch (cc) {
7916 else => unreachable,
7917 .e => &.{
7918 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7919 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7920 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7921 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7922 .{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7923 .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7924 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7925 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7926 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7927 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7928 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7929 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7930 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7931 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7932 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7933 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
7934 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7935 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7936 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7937 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7938 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7939 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7940 },
7941 .ne => &.{
7942 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7943 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7944 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7945 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7946 .{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7947 .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7948 .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ },
7949 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7950 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7951 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7952 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7953 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7954 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7955 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7956 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7957 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7958 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
7959 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7960 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7961 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7962 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7963 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7964 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7965 },
7966 } },
7967 }, .{
7968 .required_features = .{ .sse4_1, null, null, null },
7969 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
7970 .patterns = &.{
7971 .{ .src = .{ .to_mem, .to_mem } },
7972 },
7973 .extra_temps = .{
7974 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7975 .{ .type = .u32, .kind = .{ .reg = .rcx } },
7976 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7977 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7978 .{ .kind = .{ .rc = .sse } },
7979 .unused,
7980 },
7981 .dst_temps = .{.mem},
7982 .clobbers = .{ .eflags = true },
7983 .each = .{ .once = switch (cc) {
7984 else => unreachable,
7985 .e => &.{
7986 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7987 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7988 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7989 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7990 .{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7991 .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7992 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7993 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7994 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7995 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7996 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7997 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7998 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7999 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
8000 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
8001 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
8002 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8003 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
8004 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
8005 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
8006 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
8007 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
8008 },
8009 .ne => &.{
8010 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8011 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
8012 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
8013 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
8014 .{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
8015 .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
8016 .{ ._, ._, .xor, .tmp3b, .si(0b11), ._, ._ },
8017 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
8018 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
8019 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
8020 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
8021 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
8022 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
8023 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
8024 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
8025 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
8026 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
8027 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8028 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
8029 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
8030 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
8031 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
8032 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
8033 },
8034 } },
8035 }, .{
8036 .required_features = .{ .sse, .mmx, null, null },
8037 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
8038 .patterns = &.{
8039 .{ .src = .{ .to_mem, .to_mem } },
8040 },
8041 .extra_temps = .{
8042 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8043 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8044 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
8045 .{ .kind = .{ .rc = .mmx } },
8046 .unused,
8047 .unused,
8048 },
8049 .dst_temps = .{.mem},
8050 .clobbers = .{ .eflags = true },
8051 .each = .{ .once = switch (cc) {
8052 else => unreachable,
8053 .e => &.{
8054 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8055 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
8056 .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8057 .{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8058 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
8059 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
8060 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
8061 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
8062 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8063 },
8064 .ne => &.{
8065 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8066 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
8067 .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8068 .{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8069 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
8070 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
8071 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
8072 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
8073 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
8074 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8075 },
8076 } },
8077 }, .{
8078 .required_features = .{ .sse, .mmx, null, null },
8079 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
8080 .patterns = &.{
8081 .{ .src = .{ .to_mem, .to_mem } },
8082 },
8083 .extra_temps = .{
8084 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8085 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8086 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
8087 .{ .kind = .{ .rc = .mmx } },
8088 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
8089 .{ .kind = .{ .rc = .mmx } },
8090 },
8091 .dst_temps = .{.mem},
8092 .clobbers = .{ .eflags = true },
8093 .each = .{ .once = switch (cc) {
8094 else => unreachable,
8095 .e => &.{
8096 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8097 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
8098 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
8099 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
8100 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8101 .{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8102 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
8103 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
8104 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
8105 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
8106 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
8107 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
8108 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
8109 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
8110 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
8111 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
8112 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
8113 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
8114 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8115 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
8116 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
8117 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
8118 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
8119 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
8120 },
8121 .ne => &.{
8122 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8123 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
8124 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
8125 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
8126 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8127 .{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8128 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
8129 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
8130 .{ ._, ._, .xor, .tmp4b, .si(0b1111), ._, ._ },
8131 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
8132 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
8133 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
8134 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
8135 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
8136 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
8137 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
8138 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
8139 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
8140 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
8141 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8142 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
8143 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
8144 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
8145 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
8146 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
8147 },
8148 } },
8149 }, .{
8150 .required_features = .{ .sse, .mmx, null, null },
8151 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
8152 .patterns = &.{
8153 .{ .src = .{ .to_mem, .to_mem } },
8154 },
8155 .extra_temps = .{
8156 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8157 .{ .type = .u32, .kind = .{ .reg = .rcx } },
8158 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
8159 .{ .kind = .{ .rc = .mmx } },
8160 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
8161 .{ .kind = .{ .rc = .mmx } },
8162 },
8163 .dst_temps = .{.mem},
8164 .clobbers = .{ .eflags = true },
8165 .each = .{ .once = switch (cc) {
8166 else => unreachable,
8167 .e => &.{
8168 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8169 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
8170 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
8171 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
8172 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8173 .{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8174 .{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ },
8175 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
8176 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
8177 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
8178 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
8179 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
8180 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
8181 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
8182 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
8183 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
8184 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
8185 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
8186 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
8187 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8188 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
8189 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
8190 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
8191 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
8192 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
8193 },
8194 .ne => &.{
8195 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8196 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
8197 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
8198 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
8199 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8200 .{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8201 .{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ },
8202 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
8203 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
8204 .{ ._, ._, .xor, .tmp4b, .si(0b11), ._, ._ },
8205 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
8206 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
8207 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
8208 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
8209 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
8210 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
8211 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
8212 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
8213 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
8214 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
8215 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8216 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
8217 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
8218 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
8219 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
8220 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
8221 },
8222 } },
8223 }, .{
8224 .dst_constraints = .{.{ .bool_vec = .byte }},
8225 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
8226 .patterns = &.{
8227 .{ .src = .{ .to_mem, .to_mem } },
8228 },
8229 .extra_temps = .{
8230 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8231 .{ .type = .u8, .kind = .{ .reg = .cl } },
8232 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
8233 .unused,
8234 .unused,
8235 .unused,
8236 },
8237 .dst_temps = .{.{ .rc = .general_purpose }},
8238 .clobbers = .{ .eflags = true },
8239 .each = .{ .once = &.{
8240 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
8241 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8242 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8243 .{ .@"0:", ._, .mov, .tmp2b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
8244 .{ ._, ._, .cmp, .tmp2b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
8245 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8246 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
8247 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
8248 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8249 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
8250 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8251 } },
8252 }, .{
8253 .dst_constraints = .{.{ .bool_vec = .byte }},
8254 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
8255 .patterns = &.{
8256 .{ .src = .{ .to_mem, .to_mem } },
8257 },
8258 .extra_temps = .{
8259 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8260 .{ .type = .u8, .kind = .{ .reg = .cl } },
8261 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
8262 .unused,
8263 .unused,
8264 .unused,
8265 },
8266 .dst_temps = .{.{ .rc = .general_purpose }},
8267 .clobbers = .{ .eflags = true },
8268 .each = .{ .once = &.{
8269 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
8270 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8271 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8272 .{ .@"0:", ._, .mov, .tmp2w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
8273 .{ ._, ._, .cmp, .tmp2w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
8274 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8275 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
8276 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
8277 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8278 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
8279 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8280 } },
8281 }, .{
8282 .dst_constraints = .{.{ .bool_vec = .byte }},
8283 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
8284 .patterns = &.{
8285 .{ .src = .{ .to_mem, .to_mem } },
8286 },
8287 .extra_temps = .{
8288 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8289 .{ .type = .u8, .kind = .{ .reg = .cl } },
8290 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8291 .unused,
8292 .unused,
8293 .unused,
8294 },
8295 .dst_temps = .{.{ .rc = .general_purpose }},
8296 .clobbers = .{ .eflags = true },
8297 .each = .{ .once = &.{
8298 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
8299 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8300 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8301 .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
8302 .{ ._, ._, .cmp, .tmp2d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
8303 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8304 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
8305 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
8306 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8307 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
8308 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8309 } },
8310 }, .{
8311 .required_features = .{ .@"64bit", null, null, null },
8312 .dst_constraints = .{.{ .bool_vec = .byte }},
8313 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
8314 .patterns = &.{
8315 .{ .src = .{ .to_mem, .to_mem } },
8316 },
8317 .extra_temps = .{
8318 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8319 .{ .type = .u8, .kind = .{ .reg = .cl } },
8320 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8321 .unused,
8322 .unused,
8323 .unused,
8324 },
8325 .dst_temps = .{.{ .rc = .general_purpose }},
8326 .clobbers = .{ .eflags = true },
8327 .each = .{ .once = &.{
8328 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
8329 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8330 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8331 .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8332 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8333 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8334 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
8335 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
8336 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8337 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
8338 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8339 } },
8340 }, .{
8341 .dst_constraints = .{.{ .bool_vec = .byte }},
8342 .patterns = &.{
8343 .{ .src = .{ .to_mem, .to_mem } },
8344 },
8345 .extra_temps = .{
8346 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8347 .{ .type = .u8, .kind = .{ .reg = .cl } },
8348 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8349 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8350 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8351 .unused,
8352 },
8353 .dst_temps = .{.{ .rc = .general_purpose }},
8354 .clobbers = .{ .eflags = true },
8355 .each = .{ .once = &.{
8356 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
8357 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
8358 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8359 .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ },
8360 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
8361 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
8362 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
8363 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
8364 .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ },
8365 .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ },
8366 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
8367 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
8368 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8369 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
8370 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
8371 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8372 .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ },
8373 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
8374 } },
8375 }, .{
8376 .dst_constraints = .{.{ .bool_vec = .dword }},
8377 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
8378 .patterns = &.{
8379 .{ .src = .{ .to_mem, .to_mem } },
8380 },
8381 .extra_temps = .{
8382 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8383 .{ .type = .u8, .kind = .{ .reg = .cl } },
8384 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8385 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
8386 .unused,
8387 .unused,
8388 },
8389 .dst_temps = .{.{ .rc = .general_purpose }},
8390 .clobbers = .{ .eflags = true },
8391 .each = .{ .once = &.{
8392 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
8393 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8394 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8395 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8396 .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
8397 .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
8398 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8399 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
8400 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
8401 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8402 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
8403 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8404 } },
8405 }, .{
8406 .dst_constraints = .{.{ .bool_vec = .dword }},
8407 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
8408 .patterns = &.{
8409 .{ .src = .{ .to_mem, .to_mem } },
8410 },
8411 .extra_temps = .{
8412 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8413 .{ .type = .u8, .kind = .{ .reg = .cl } },
8414 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8415 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
8416 .unused,
8417 .unused,
8418 },
8419 .dst_temps = .{.{ .rc = .general_purpose }},
8420 .clobbers = .{ .eflags = true },
8421 .each = .{ .once = &.{
8422 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
8423 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8424 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8425 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8426 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
8427 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
8428 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8429 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
8430 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
8431 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8432 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
8433 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8434 } },
8435 }, .{
8436 .dst_constraints = .{.{ .bool_vec = .dword }},
8437 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
8438 .patterns = &.{
8439 .{ .src = .{ .to_mem, .to_mem } },
8440 },
8441 .extra_temps = .{
8442 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8443 .{ .type = .u8, .kind = .{ .reg = .cl } },
8444 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8445 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8446 .unused,
8447 .unused,
8448 },
8449 .dst_temps = .{.{ .rc = .general_purpose }},
8450 .clobbers = .{ .eflags = true },
8451 .each = .{ .once = &.{
8452 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
8453 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8454 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8455 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8456 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
8457 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
8458 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8459 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
8460 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
8461 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8462 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
8463 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8464 } },
8465 }, .{
8466 .required_features = .{ .@"64bit", null, null, null },
8467 .dst_constraints = .{.{ .bool_vec = .dword }},
8468 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
8469 .patterns = &.{
8470 .{ .src = .{ .to_mem, .to_mem } },
8471 },
8472 .extra_temps = .{
8473 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8474 .{ .type = .u8, .kind = .{ .reg = .cl } },
8475 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8476 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8477 .unused,
8478 .unused,
8479 },
8480 .dst_temps = .{.{ .rc = .general_purpose }},
8481 .clobbers = .{ .eflags = true },
8482 .each = .{ .once = &.{
8483 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
8484 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8485 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8486 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8487 .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8488 .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8489 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8490 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
8491 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
8492 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8493 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
8494 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8495 } },
8496 }, .{
8497 .dst_constraints = .{.{ .bool_vec = .dword }},
8498 .patterns = &.{
8499 .{ .src = .{ .to_mem, .to_mem } },
8500 },
8501 .extra_temps = .{
8502 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8503 .{ .type = .u8, .kind = .{ .reg = .cl } },
8504 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8505 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8506 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8507 .unused,
8508 },
8509 .dst_temps = .{.{ .rc = .general_purpose }},
8510 .clobbers = .{ .eflags = true },
8511 .each = .{ .once = &.{
8512 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
8513 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
8514 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8515 .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ },
8516 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
8517 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
8518 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
8519 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
8520 .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ },
8521 .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ },
8522 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
8523 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8524 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
8525 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8526 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
8527 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
8528 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8529 .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ },
8530 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
8531 } },
8532 }, .{
8533 .required_features = .{ .@"64bit", null, null, null },
8534 .dst_constraints = .{.{ .bool_vec = .qword }},
8535 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
8536 .patterns = &.{
8537 .{ .src = .{ .to_mem, .to_mem } },
8538 },
8539 .extra_temps = .{
8540 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8541 .{ .type = .u8, .kind = .{ .reg = .cl } },
8542 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8543 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
8544 .unused,
8545 .unused,
8546 },
8547 .dst_temps = .{.{ .rc = .general_purpose }},
8548 .clobbers = .{ .eflags = true },
8549 .each = .{ .once = &.{
8550 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
8551 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8552 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8553 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8554 .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
8555 .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
8556 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8557 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
8558 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
8559 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8560 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
8561 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8562 } },
8563 }, .{
8564 .required_features = .{ .@"64bit", null, null, null },
8565 .dst_constraints = .{.{ .bool_vec = .qword }},
8566 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
8567 .patterns = &.{
8568 .{ .src = .{ .to_mem, .to_mem } },
8569 },
8570 .extra_temps = .{
8571 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8572 .{ .type = .u8, .kind = .{ .reg = .cl } },
8573 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8574 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
8575 .unused,
8576 .unused,
8577 },
8578 .dst_temps = .{.{ .rc = .general_purpose }},
8579 .clobbers = .{ .eflags = true },
8580 .each = .{ .once = &.{
8581 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
8582 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8583 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8584 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
8585 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
8586 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8587 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
8588 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
8589 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8590 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
8591 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8592 } },
8593 }, .{
8594 .required_features = .{ .@"64bit", null, null, null },
8595 .dst_constraints = .{.{ .bool_vec = .qword }},
8596 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
8597 .patterns = &.{
8598 .{ .src = .{ .to_mem, .to_mem } },
8599 },
8600 .extra_temps = .{
8601 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8602 .{ .type = .u8, .kind = .{ .reg = .cl } },
8603 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8604 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8605 .unused,
8606 .unused,
8607 },
8608 .dst_temps = .{.{ .rc = .general_purpose }},
8609 .clobbers = .{ .eflags = true },
8610 .each = .{ .once = &.{
8611 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
8612 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8613 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8614 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8615 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
8616 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
8617 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8618 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
8619 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
8620 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8621 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
8622 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8623 } },
8624 }, .{
8625 .required_features = .{ .@"64bit", null, null, null },
8626 .dst_constraints = .{.{ .bool_vec = .qword }},
8627 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
8628 .patterns = &.{
8629 .{ .src = .{ .to_mem, .to_mem } },
8630 },
8631 .extra_temps = .{
8632 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8633 .{ .type = .u8, .kind = .{ .reg = .cl } },
8634 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8635 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8636 .unused,
8637 .unused,
8638 },
8639 .dst_temps = .{.{ .rc = .general_purpose }},
8640 .clobbers = .{ .eflags = true },
8641 .each = .{ .once = &.{
8642 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
8643 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8644 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8645 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8646 .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8647 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8648 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8649 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
8650 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
8651 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8652 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
8653 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8654 } },
8655 }, .{
8656 .required_features = .{ .@"64bit", null, null, null },
8657 .dst_constraints = .{.{ .bool_vec = .qword }},
8658 .patterns = &.{
8659 .{ .src = .{ .to_mem, .to_mem } },
8660 },
8661 .extra_temps = .{
8662 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8663 .{ .type = .u8, .kind = .{ .reg = .cl } },
8664 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8665 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8666 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8667 .unused,
8668 },
8669 .dst_temps = .{.{ .rc = .general_purpose }},
8670 .clobbers = .{ .eflags = true },
8671 .each = .{ .once = &.{
8672 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
8673 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
8674 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8675 .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ },
8676 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
8677 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
8678 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
8679 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
8680 .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ },
8681 .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ },
8682 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
8683 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8684 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
8685 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
8686 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
8687 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
8688 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8689 .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ },
8690 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
8691 } },
8692 }, .{
8693 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
8694 .patterns = &.{
8695 .{ .src = .{ .to_mem, .to_mem } },
8696 },
8697 .extra_temps = .{
8698 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8699 .{ .type = .u32, .kind = .{ .reg = .ecx } },
8700 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8701 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8702 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
8703 .unused,
8704 },
8705 .dst_temps = .{.mem},
8706 .clobbers = .{ .eflags = true },
8707 .each = .{ .once = &.{
8708 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8709 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
8710 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8711 .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ },
8712 .{ ._, ._, .mov, .tmp4b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
8713 .{ ._, ._, .cmp, .tmp4b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
8714 .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ },
8715 .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ },
8716 .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ },
8717 .{ ._, ._, .add, .tmp1d, .si(1), ._, ._ },
8718 .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ },
8719 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
8720 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
8721 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
8722 .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ },
8723 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8724 .{ .@"1:", ._, .add, .tmp0p, .si(1), ._, ._ },
8725 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8726 .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ },
8727 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
8728 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
8729 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
8730 .{ ._, ._, .mov, .memi(.dst0p, .tmp3), .tmp2p, ._, ._ },
8731 } },
8732 } },
8733 }) catch |err| switch (err) {
8734 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
8735 @tagName(air_tag),
8736 cg.typeOf(extra.lhs).fmt(pt),
8737 ops[0].tracking(cg),
8738 ops[1].tracking(cg),
8739 }),
8740 else => |e| return e,
8741 },
8742 .gte => unreachable,
8743 .gt => unreachable,
8744 }
8745 for (ops) |op| for (res) |r| {
8746 if (op.index == r.index) break;
8747 } else try op.die(cg);
8748 try res[0].moveTo(inst, cg);
8749 },
8750
8751 .cmp_lt,
8752 .cmp_lt_optimized,
8753 .cmp_lte,
8754 .cmp_lte_optimized,
8755 .cmp_gte,
8756 .cmp_gte_optimized,
8757 .cmp_gt,
8758 .cmp_gt_optimized,
8759 => |air_tag| if (use_old) try cg.airCmp(inst, switch (air_tag) {
8760 else => unreachable,
8761 .cmp_lt, .cmp_lt_optimized => .lt,
8762 .cmp_lte, .cmp_lte_optimized => .lte,
8763 .cmp_gte, .cmp_gte_optimized => .gte,
8764 .cmp_gt, .cmp_gt_optimized => .gt,
8765 }) else fallback: {
8766 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
8767 const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu);
8768 if (scalar_ty.isRuntimeFloat()) break :fallback try cg.airCmp(inst, switch (air_tag) {
8769 else => unreachable,
8770 .cmp_lt, .cmp_lt_optimized => .lt,
8771 .cmp_lte, .cmp_lte_optimized => .lte,
8772 .cmp_gte, .cmp_gte_optimized => .gte,
8773 .cmp_gt, .cmp_gt_optimized => .gt,
8774 });
8775 const signedness = if (scalar_ty.isAbiInt(zcu))
8776 scalar_ty.intInfo(zcu).signedness
8777 else
8778 .unsigned;
8779 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
8780 var res: [1]Temp = undefined;
8781 cg.select(&res, &.{.bool}, &ops, switch (@as(Condition, switch (signedness) {
8782 .signed => switch (air_tag) {
8783 else => unreachable,
8784 .cmp_lt, .cmp_lt_optimized => .l,
8785 .cmp_lte, .cmp_lte_optimized => .le,
8786 .cmp_gte, .cmp_gte_optimized => .ge,
8787 .cmp_gt, .cmp_gt_optimized => .g,
8788 },
8789 .unsigned => switch (air_tag) {
8790 else => unreachable,
8791 .cmp_lt, .cmp_lt_optimized => .b,
8792 .cmp_lte, .cmp_lte_optimized => .be,
8793 .cmp_gte, .cmp_gte_optimized => .ae,
8794 .cmp_gt, .cmp_gt_optimized => .a,
8795 },
8796 })) {
8797 else => unreachable,
8798 inline .l, .le, .ge, .g, .b, .be, .ae, .a => |cc| comptime &.{ .{
8799 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
8800 .patterns = &.{
8801 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
8802 .{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
8803 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
8804 },
8805 .dst_temps = .{.{ .cc = cc.commute() }},
8806 .clobbers = .{ .eflags = true },
8807 .each = .{ .once = &.{
8808 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
8809 } },
8810 }, .{
8811 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
8812 .patterns = &.{
8813 .{ .src = .{ .mem, .imm8 } },
8814 .{ .src = .{ .to_gpr, .imm8 } },
8815 .{ .src = .{ .to_gpr, .mem } },
8816 .{ .src = .{ .to_gpr, .to_gpr } },
8817 },
8818 .dst_temps = .{.{ .cc = cc }},
8819 .clobbers = .{ .eflags = true },
8820 .each = .{ .once = &.{
8821 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
8822 } },
8823 }, .{
8824 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
8825 .patterns = &.{
8826 .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
8827 .{ .src = .{ .imm16, .to_gpr }, .commute = .{ 0, 1 } },
8828 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
8829 },
8830 .dst_temps = .{.{ .cc = cc.commute() }},
8831 .clobbers = .{ .eflags = true },
8832 .each = .{ .once = &.{
8833 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
8834 } },
8835 }, .{
8836 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
8837 .patterns = &.{
8838 .{ .src = .{ .mem, .imm16 } },
8839 .{ .src = .{ .to_gpr, .imm16 } },
8840 .{ .src = .{ .to_gpr, .mem } },
8841 .{ .src = .{ .to_gpr, .to_gpr } },
8842 },
8843 .dst_temps = .{.{ .cc = cc }},
8844 .clobbers = .{ .eflags = true },
8845 .each = .{ .once = &.{
8846 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
8847 } },
8848 }, .{
8849 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
8850 .patterns = &.{
8851 .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
8852 .{ .src = .{ .imm32, .to_gpr }, .commute = .{ 0, 1 } },
8853 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
8854 },
8855 .dst_temps = .{.{ .cc = cc.commute() }},
8856 .clobbers = .{ .eflags = true },
8857 .each = .{ .once = &.{
8858 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
8859 } },
8860 }, .{
8861 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
8862 .patterns = &.{
8863 .{ .src = .{ .mem, .imm32 } },
8864 .{ .src = .{ .to_gpr, .imm32 } },
8865 .{ .src = .{ .to_gpr, .mem } },
8866 .{ .src = .{ .to_gpr, .to_gpr } },
8867 },
8868 .dst_temps = .{.{ .cc = cc }},
8869 .clobbers = .{ .eflags = true },
8870 .each = .{ .once = &.{
8871 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
8872 } },
8873 }, .{
8874 .required_features = .{ .@"64bit", null, null, null },
8875 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
8876 .patterns = &.{
8877 .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
8878 .{ .src = .{ .simm32, .to_gpr }, .commute = .{ 0, 1 } },
8879 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
8880 },
8881 .dst_temps = .{.{ .cc = cc.commute() }},
8882 .clobbers = .{ .eflags = true },
8883 .each = .{ .once = &.{
8884 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
8885 } },
8886 }, .{
8887 .required_features = .{ .@"64bit", null, null, null },
8888 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
8889 .patterns = &.{
8890 .{ .src = .{ .mem, .simm32 } },
8891 .{ .src = .{ .to_gpr, .simm32 } },
8892 .{ .src = .{ .to_gpr, .mem } },
8893 .{ .src = .{ .to_gpr, .to_gpr } },
8894 },
8895 .dst_temps = .{.{ .cc = cc }},
8896 .clobbers = .{ .eflags = true },
8897 .each = .{ .once = &.{
8898 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
8899 } },
8900 }, .{
8901 .src_constraints = .{ .any_int, .any_int },
8902 .patterns = &.{
8903 .{ .src = .{ .to_mem, .to_mem }, .commute = switch (cc) {
8904 else => unreachable,
8905 .l, .ge, .b, .ae => .{ 0, 0 },
8906 .le, .g, .be, .a => .{ 0, 1 },
8907 } },
8908 },
8909 .extra_temps = .{
8910 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8911 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8912 .unused,
8913 .unused,
8914 .unused,
8915 .unused,
8916 },
8917 .dst_temps = .{.{ .rc = .general_purpose }},
8918 .clobbers = .{ .eflags = true },
8919 .each = .{ .once = &.{
8920 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8921 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8922 .{ .@"0:", ._r, .sh, .tmp1b, .si(1), ._, ._ },
8923 .{ ._, ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
8924 .{ ._, ._, .sbb, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
8925 .{ ._, ._c, .set, .tmp1b, ._, ._, ._ },
8926 .{ ._, .fromCondition(switch (cc) {
8927 else => unreachable,
8928 .l, .ge, .b, .ae => cc,
8929 .le, .g, .be, .a => cc.commute(),
8930 }), .set, .dst0b, ._, ._, ._ },
8931 .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
8932 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8933 } },
8934 } },
8935 }) catch |err| switch (err) {
8936 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
8937 @tagName(air_tag),
8938 cg.typeOf(bin_op.lhs).fmt(pt),
8939 ops[0].tracking(cg),
8940 ops[1].tracking(cg),
8941 }),
8942 else => |e| return e,
8943 };
8944 for (ops) |op| for (res) |r| {
8945 if (op.index == r.index) break;
8946 } else try op.die(cg);
8947 try res[0].moveTo(inst, cg);
8948 },
8949 .cmp_eq,
8950 .cmp_eq_optimized,
8951 .cmp_neq,
8952 .cmp_neq_optimized,
8953 => |air_tag| if (use_old) try cg.airCmp(inst, switch (air_tag) {
8954 else => unreachable,
8955 .cmp_eq, .cmp_eq_optimized => .eq,
8956 .cmp_neq, .cmp_neq_optimized => .neq,
8957 }) else fallback: {
8958 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
8959 const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu);
8960 if (intInfo(scalar_ty, cg) == null) break :fallback try cg.airCmp(inst, switch (air_tag) {
8961 else => unreachable,
8962 .cmp_eq, .cmp_eq_optimized => .eq,
8963 .cmp_neq, .cmp_neq_optimized => .neq,
8964 });
8965 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
8966 var res: [1]Temp = undefined;
8967 cg.select(&res, &.{.bool}, &ops, switch (@as(Condition, switch (air_tag) {
8968 else => unreachable,
8969 .cmp_eq, .cmp_eq_optimized => .e,
8970 .cmp_neq, .cmp_neq_optimized => .ne,
8971 })) {
8972 else => unreachable,
8973 inline .e, .ne => |cc| comptime &.{ .{
8974 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
8975 .patterns = &.{
8976 .{ .src = .{ .mem, .imm8 } },
8977 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
8978 .{ .src = .{ .to_gpr, .imm8 } },
8979 .{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
8980 .{ .src = .{ .to_gpr, .mem } },
8981 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
8982 .{ .src = .{ .to_gpr, .to_gpr } },
8983 },
8984 .dst_temps = .{.{ .cc = cc }},
8985 .clobbers = .{ .eflags = true },
8986 .each = .{ .once = &.{
8987 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
8988 } },
8989 }, .{
8990 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
8991 .patterns = &.{
8992 .{ .src = .{ .mem, .imm16 } },
8993 .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
8994 .{ .src = .{ .to_gpr, .imm16 } },
8995 .{ .src = .{ .imm16, .to_gpr }, .commute = .{ 0, 1 } },
8996 .{ .src = .{ .to_gpr, .mem } },
8997 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
8998 .{ .src = .{ .to_gpr, .to_gpr } },
8999 },
9000 .dst_temps = .{.{ .cc = cc }},
9001 .clobbers = .{ .eflags = true },
9002 .each = .{ .once = &.{
9003 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
9004 } },
9005 }, .{
9006 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
9007 .patterns = &.{
9008 .{ .src = .{ .mem, .imm32 } },
9009 .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
9010 .{ .src = .{ .to_gpr, .imm32 } },
9011 .{ .src = .{ .imm32, .to_gpr }, .commute = .{ 0, 1 } },
9012 .{ .src = .{ .to_gpr, .mem } },
9013 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
9014 .{ .src = .{ .to_gpr, .to_gpr } },
9015 },
9016 .dst_temps = .{.{ .cc = cc }},
9017 .clobbers = .{ .eflags = true },
9018 .each = .{ .once = &.{
9019 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
9020 } },
9021 }, .{
9022 .required_features = .{ .@"64bit", null, null, null },
9023 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
9024 .patterns = &.{
9025 .{ .src = .{ .mem, .simm32 } },
9026 .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
9027 .{ .src = .{ .to_gpr, .simm32 } },
9028 .{ .src = .{ .simm32, .to_gpr }, .commute = .{ 0, 1 } },
9029 .{ .src = .{ .to_gpr, .mem } },
9030 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
9031 .{ .src = .{ .to_gpr, .to_gpr } },
9032 },
9033 .dst_temps = .{.{ .cc = cc }},
9034 .clobbers = .{ .eflags = true },
9035 .each = .{ .once = &.{
9036 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
9037 } },
9038 }, .{
9039 .required_features = .{ .sse, .mmx, null, null },
9040 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
9041 .patterns = &.{
9042 .{ .src = .{ .to_mut_mm, .mem } },
9043 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
9044 .{ .src = .{ .to_mut_mm, .to_mm } },
9045 },
9046 .extra_temps = .{
9047 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
9048 .{ .kind = .{ .rc = .mmx } },
9049 .unused,
9050 .unused,
9051 .unused,
9052 .unused,
9053 },
9054 .dst_temps = .{.{ .cc = cc }},
9055 .clobbers = .{ .eflags = true },
9056 .each = .{ .once = &.{
9057 .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ },
9058 .{ ._, .p_, .xor, .src0q, .src1q, ._, ._ },
9059 .{ ._, .p_b, .cmpeq, .tmp1q, .src0q, ._, ._ },
9060 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ },
9061 .{ ._, ._, .xor, .tmp0d, .si(0xff), ._, ._ },
9062 } },
9063 }, .{
9064 .required_features = .{ .avx, null, null, null },
9065 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
9066 .patterns = &.{
9067 .{ .src = .{ .to_xmm, .mem } },
9068 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
9069 .{ .src = .{ .to_xmm, .to_xmm } },
9070 },
9071 .extra_temps = .{
9072 .{ .kind = .{ .rc = .sse } },
9073 .unused,
9074 .unused,
9075 .unused,
9076 .unused,
9077 .unused,
9078 },
9079 .dst_temps = .{.{ .cc = cc }},
9080 .clobbers = .{ .eflags = true },
9081 .each = .{ .once = &.{
9082 .{ ._, .vp_, .xor, .tmp0x, .src0x, .src1x, ._ },
9083 .{ ._, .vp_, .@"test", .tmp0x, .tmp0x, ._, ._ },
9084 } },
9085 }, .{
9086 .required_features = .{ .sse4_1, null, null, null },
9087 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
9088 .patterns = &.{
9089 .{ .src = .{ .to_mut_xmm, .mem } },
9090 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
9091 .{ .src = .{ .to_mut_xmm, .to_xmm } },
9092 },
9093 .dst_temps = .{.{ .cc = cc }},
9094 .clobbers = .{ .eflags = true },
9095 .each = .{ .once = &.{
9096 .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
9097 .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ },
9098 } },
9099 }, .{
9100 .required_features = .{ .sse2, null, null, null },
9101 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
9102 .patterns = &.{
9103 .{ .src = .{ .to_mut_xmm, .mem } },
9104 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
9105 .{ .src = .{ .to_mut_xmm, .to_xmm } },
9106 },
9107 .extra_temps = .{
9108 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
9109 .{ .kind = .{ .rc = .sse } },
9110 .unused,
9111 .unused,
9112 .unused,
9113 .unused,
9114 },
9115 .dst_temps = .{.{ .cc = cc }},
9116 .clobbers = .{ .eflags = true },
9117 .each = .{ .once = &.{
9118 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
9119 .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
9120 .{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ },
9121 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ },
9122 .{ ._, ._, .xor, .tmp0d, .si(0xffff), ._, ._ },
9123 } },
9124 }, .{
9125 .required_features = .{ .avx2, null, null, null },
9126 .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword } },
9127 .patterns = &.{
9128 .{ .src = .{ .to_ymm, .mem } },
9129 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
9130 .{ .src = .{ .to_ymm, .to_ymm } },
9131 },
9132 .extra_temps = .{
9133 .{ .kind = .{ .rc = .sse } },
9134 .unused,
9135 .unused,
9136 .unused,
9137 .unused,
9138 .unused,
9139 },
9140 .dst_temps = .{.{ .cc = cc }},
9141 .clobbers = .{ .eflags = true },
9142 .each = .{ .once = &.{
9143 .{ ._, .vp_, .xor, .tmp0y, .src0y, .src1y, ._ },
9144 .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
9145 } },
9146 }, .{
9147 .required_features = .{ .avx, null, null, null },
9148 .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword } },
9149 .patterns = &.{
9150 .{ .src = .{ .to_ymm, .mem } },
9151 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
9152 .{ .src = .{ .to_ymm, .to_ymm } },
9153 },
9154 .extra_temps = .{
9155 .{ .kind = .{ .rc = .sse } },
9156 .unused,
9157 .unused,
9158 .unused,
9159 .unused,
9160 .unused,
9161 },
9162 .dst_temps = .{.{ .cc = cc }},
9163 .clobbers = .{ .eflags = true },
9164 .each = .{ .once = &.{
9165 .{ ._, .v_pd, .xor, .tmp0y, .src0y, .src1y, ._ },
9166 .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
9167 } },
9168 }, .{
9169 .required_features = .{ .avx2, null, null, null },
9170 .src_constraints = .{
9171 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
9172 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
9173 },
9174 .patterns = &.{
9175 .{ .src = .{ .to_mem, .to_mem } },
9176 },
9177 .extra_temps = .{
9178 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9179 .{ .kind = .{ .rc = .sse } },
9180 .{ .kind = .{ .rc = .sse } },
9181 .unused,
9182 .unused,
9183 .unused,
9184 },
9185 .dst_temps = .{.{ .cc = cc }},
9186 .clobbers = .{ .eflags = true },
9187 .each = .{ .once = &.{
9188 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
9189 .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
9190 .{ .@"0:", .v_dqu, .mov, .tmp2y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ },
9191 .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .memiad(.src1y, .tmp0, .add_size, -16), ._ },
9192 .{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
9193 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
9194 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9195 .{ ._, .v_dqa, .mov, .tmp2x, .memad(.src0x, .add_size, -16), ._, ._ },
9196 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .memad(.src1x, .add_size, -16), ._ },
9197 .{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
9198 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
9199 } },
9200 }, .{
9201 .required_features = .{ .avx2, null, null, null },
9202 .patterns = &.{
9203 .{ .src = .{ .to_mem, .to_mem } },
9204 },
9205 .extra_temps = .{
9206 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9207 .{ .kind = .{ .rc = .sse } },
9208 .{ .kind = .{ .rc = .sse } },
9209 .unused,
9210 .unused,
9211 .unused,
9212 },
9213 .dst_temps = .{.{ .cc = cc }},
9214 .clobbers = .{ .eflags = true },
9215 .each = .{ .once = &.{
9216 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
9217 .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
9218 .{ .@"0:", .v_dqu, .mov, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
9219 .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._ },
9220 .{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
9221 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
9222 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9223 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
9224 } },
9225 }, .{
9226 .required_features = .{ .avx, null, null, null },
9227 .src_constraints = .{
9228 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
9229 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
9230 },
9231 .patterns = &.{
9232 .{ .src = .{ .to_mem, .to_mem } },
9233 },
9234 .extra_temps = .{
9235 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9236 .{ .kind = .{ .rc = .sse } },
9237 .{ .kind = .{ .rc = .sse } },
9238 .unused,
9239 .unused,
9240 .unused,
9241 },
9242 .dst_temps = .{.{ .cc = cc }},
9243 .clobbers = .{ .eflags = true },
9244 .each = .{ .once = &.{
9245 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
9246 .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
9247 .{ .@"0:", .v_pd, .movu, .tmp2y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ },
9248 .{ ._, .v_pd, .xor, .tmp2y, .tmp2y, .memiad(.src1y, .tmp0, .add_size, -16), ._ },
9249 .{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
9250 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
9251 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9252 .{ ._, .v_pd, .mova, .tmp2x, .memad(.src0x, .add_size, -16), ._, ._ },
9253 .{ ._, .v_pd, .xor, .tmp2x, .tmp2x, .memad(.src1x, .add_size, -16), ._ },
9254 .{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
9255 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
9256 } },
9257 }, .{
9258 .required_features = .{ .avx, null, null, null },
9259 .patterns = &.{
9260 .{ .src = .{ .to_mem, .to_mem } },
9261 },
9262 .extra_temps = .{
9263 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9264 .{ .kind = .{ .rc = .sse } },
9265 .{ .kind = .{ .rc = .sse } },
9266 .unused,
9267 .unused,
9268 .unused,
9269 },
9270 .dst_temps = .{.{ .cc = cc }},
9271 .clobbers = .{ .eflags = true },
9272 .each = .{ .once = &.{
9273 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
9274 .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
9275 .{ .@"0:", .v_pd, .movu, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
9276 .{ ._, .v_pd, .xor, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._ },
9277 .{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
9278 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
9279 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9280 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
9281 } },
9282 }, .{
9283 .required_features = .{ .avx, null, null, null },
9284 .patterns = &.{
9285 .{ .src = .{ .to_mem, .to_mem } },
9286 },
9287 .extra_temps = .{
9288 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9289 .{ .kind = .{ .rc = .sse } },
9290 .{ .kind = .{ .rc = .sse } },
9291 .unused,
9292 .unused,
9293 .unused,
9294 },
9295 .dst_temps = .{.{ .cc = cc }},
9296 .clobbers = .{ .eflags = true },
9297 .each = .{ .once = &.{
9298 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
9299 .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ },
9300 .{ .@"0:", .v_dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
9301 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._ },
9302 .{ ._, .vp_, .@"or", .tmp1x, .tmp1x, .tmp2x, ._ },
9303 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
9304 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9305 .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ },
9306 } },
9307 }, .{
9308 .required_features = .{ .sse4_1, null, null, null },
9309 .patterns = &.{
9310 .{ .src = .{ .to_mem, .to_mem } },
9311 },
9312 .extra_temps = .{
9313 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9314 .{ .kind = .{ .rc = .sse } },
9315 .{ .kind = .{ .rc = .sse } },
9316 .unused,
9317 .unused,
9318 .unused,
9319 },
9320 .dst_temps = .{.{ .cc = cc }},
9321 .clobbers = .{ .eflags = true },
9322 .each = .{ .once = &.{
9323 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
9324 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
9325 .{ .@"0:", ._dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
9326 .{ ._, .p_, .xor, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
9327 .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ },
9328 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
9329 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9330 .{ ._, .p_, .@"test", .tmp1x, .tmp1x, ._, ._ },
9331 } },
9332 }, .{
9333 .required_features = .{ .sse2, null, null, null },
9334 .patterns = &.{
9335 .{ .src = .{ .to_mem, .to_mem } },
9336 },
9337 .extra_temps = .{
9338 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9339 .{ .kind = .{ .rc = .sse } },
9340 .{ .kind = .{ .rc = .sse } },
9341 .unused,
9342 .unused,
9343 .unused,
9344 },
9345 .dst_temps = .{.{ .cc = cc }},
9346 .clobbers = .{ .eflags = true },
9347 .each = .{ .once = &.{
9348 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
9349 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
9350 .{ .@"0:", ._dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
9351 .{ ._, .p_, .xor, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
9352 .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ },
9353 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
9354 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9355 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
9356 .{ ._, .p_b, .cmpeq, .tmp1x, .tmp2x, ._, ._ },
9357 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ },
9358 .{ ._, ._, .cmp, .tmp0d, .si(0xffff), ._, ._ },
9359 } },
9360 }, .{
9361 .required_features = .{ .sse, .mmx, null, null },
9362 .patterns = &.{
9363 .{ .src = .{ .to_mem, .to_mem } },
9364 },
9365 .extra_temps = .{
9366 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9367 .{ .kind = .{ .rc = .mmx } },
9368 .{ .kind = .{ .rc = .mmx } },
9369 .unused,
9370 .unused,
9371 .unused,
9372 },
9373 .dst_temps = .{.{ .cc = cc }},
9374 .clobbers = .{ .eflags = true },
9375 .each = .{ .once = &.{
9376 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
9377 .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ },
9378 .{ .@"0:", ._q, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
9379 .{ ._, .p_, .xor, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
9380 .{ ._, .p_, .@"or", .tmp1q, .tmp2q, ._, ._ },
9381 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
9382 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9383 .{ ._, .p_, .xor, .tmp2q, .tmp2q, ._, ._ },
9384 .{ ._, .p_b, .cmpeq, .tmp1q, .tmp2q, ._, ._ },
9385 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ },
9386 .{ ._, ._, .cmp, .tmp0d, .si(0xff), ._, ._ },
9387 } },
9388 }, .{
9389 .patterns = &.{
9390 .{ .src = .{ .to_mem, .to_mem } },
9391 },
9392 .extra_temps = .{
9393 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9394 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9395 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9396 .unused,
9397 .unused,
9398 .unused,
9399 },
9400 .dst_temps = .{.{ .cc = cc }},
9401 .clobbers = .{ .eflags = true },
9402 .each = .{ .once = &.{
9403 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
9404 .{ ._, ._, .xor, .tmp1p, .tmp1p, ._, ._ },
9405 .{ .@"0:", ._, .mov, .tmp2p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
9406 .{ ._, ._, .xor, .tmp2p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
9407 .{ ._, ._, .@"or", .tmp1p, .tmp2p, ._, ._ },
9408 .{ ._, ._, .add, .tmp0p, .sa(.tmp2, .add_size), ._, ._ },
9409 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9410 .{ ._, ._, .@"test", .tmp1p, .tmp1p, ._, ._ },
9411 } },
9412 } },
9413 }) catch |err| switch (err) {
9414 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
9415 @tagName(air_tag),
9416 cg.typeOf(bin_op.lhs).fmt(pt),
9417 ops[0].tracking(cg),
9418 ops[1].tracking(cg),
9419 }),
9420 else => |e| return e,
9421 };
9422 for (ops) |op| for (res) |r| {
9423 if (op.index == r.index) break;
9424 } else try op.die(cg);
9425 try res[0].moveTo(inst, cg);
9426 },
9427
9428 .cond_br => try cg.airCondBr(inst),
9429 .switch_br => try cg.airSwitchBr(inst),
9430 .loop_switch_br => try cg.airLoopSwitchBr(inst),
9431 .switch_dispatch => try cg.airSwitchDispatch(inst),
9432 .@"try", .try_cold => try cg.airTry(inst),
9433 .try_ptr, .try_ptr_cold => try cg.airTryPtr(inst),
9434 .dbg_stmt => if (use_old) try cg.airDbgStmt(inst) else {
9435 const dbg_stmt = air_datas[@intFromEnum(inst)].dbg_stmt;
9436 _ = try cg.addInst(.{
9437 .tag = .pseudo,
9438 .ops = .pseudo_dbg_line_line_column,
9439 .data = .{ .line_column = .{
9440 .line = dbg_stmt.line,
9441 .column = dbg_stmt.column,
9442 } },
9443 });
9444 },
9445 .dbg_empty_stmt => if (use_old) try cg.airDbgEmptyStmt() else {
9446 if (cg.mir_instructions.len > 0) {
9447 const prev_mir_op = &cg.mir_instructions.items(.ops)[cg.mir_instructions.len - 1];
9448 if (prev_mir_op.* == .pseudo_dbg_line_stmt_line_column)
9449 prev_mir_op.* = .pseudo_dbg_line_line_column;
9450 }
9451 try cg.asmOpOnly(.{ ._, .nop });
9452 },
9453 .dbg_inline_block => if (use_old) try cg.airDbgInlineBlock(inst) else {
9454 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
9455 const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
9456 const old_inline_func = cg.inline_func;
9457 defer cg.inline_func = old_inline_func;
9458 cg.inline_func = extra.data.func;
9459 _ = try cg.addInst(.{
9460 .tag = .pseudo,
9461 .ops = .pseudo_dbg_enter_inline_func,
9462 .data = .{ .func = extra.data.func },
9463 });
9464 try cg.lowerBlock(inst, @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]));
9465 _ = try cg.addInst(.{
9466 .tag = .pseudo,
9467 .ops = .pseudo_dbg_leave_inline_func,
9468 .data = .{ .func = old_inline_func },
9469 });
9470 },
9471 .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => if (use_old) try cg.airDbgVar(inst) else {
9472 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
9473 var ops = try cg.tempsFromOperands(inst, .{pl_op.operand});
9474 try cg.genLocalDebugInfo(inst, ops[0].tracking(cg).short);
9475 try ops[0].die(cg);
9476 },
9477 .is_null_ptr => if (use_old) try cg.airIsNullPtr(inst) else {
9478 const un_op = air_datas[@intFromEnum(inst)].un_op;
9479 const opt_ty = cg.typeOf(un_op).childType(zcu);
9480 const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu);
9481 const opt_child_ty = opt_ty.optionalChild(zcu);
9482 const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu));
9483 var ops = try cg.tempsFromOperands(inst, .{un_op});
9484 if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg);
9485 while (try ops[0].toLea(cg)) {}
9486 try cg.asmMemoryImmediate(
9487 .{ ._, .cmp },
9488 try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl)
9489 .byte
9490 else if (opt_child_ty.isSlice(zcu))
9491 .qword
9492 else
9493 .fromSize(opt_child_abi_size) }),
9494 .u(0),
9495 );
9496 var is_null = try cg.tempInit(.bool, .{ .eflags = .e });
9497 try ops[0].die(cg);
9498 try is_null.moveTo(inst, cg);
9499 },
9500 .is_non_null_ptr => if (use_old) try cg.airIsNonNullPtr(inst) else {
9501 const un_op = air_datas[@intFromEnum(inst)].un_op;
9502 const opt_ty = cg.typeOf(un_op).childType(zcu);
9503 const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu);
9504 const opt_child_ty = opt_ty.optionalChild(zcu);
9505 const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu));
9506 var ops = try cg.tempsFromOperands(inst, .{un_op});
9507 if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg);
9508 while (try ops[0].toLea(cg)) {}
9509 try cg.asmMemoryImmediate(
9510 .{ ._, .cmp },
9511 try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl)
9512 .byte
9513 else if (opt_child_ty.isSlice(zcu))
9514 .qword
9515 else
9516 .fromSize(opt_child_abi_size) }),
9517 .u(0),
9518 );
9519 var is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne });
9520 try ops[0].die(cg);
9521 try is_non_null.moveTo(inst, cg);
9522 },
9523 .is_err_ptr => if (use_old) try cg.airIsErrPtr(inst) else {
9524 const un_op = air_datas[@intFromEnum(inst)].un_op;
9525 const eu_ty = cg.typeOf(un_op).childType(zcu);
9526 const eu_err_ty = eu_ty.errorUnionSet(zcu);
9527 const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
9528 const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
9529 var ops = try cg.tempsFromOperands(inst, .{un_op});
9530 try ops[0].toOffset(eu_err_off, cg);
9531 while (try ops[0].toLea(cg)) {}
9532 try cg.asmMemoryImmediate(
9533 .{ ._, .cmp },
9534 try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }),
9535 .u(0),
9536 );
9537 var is_err = try cg.tempInit(.bool, .{ .eflags = .ne });
9538 try ops[0].die(cg);
9539 try is_err.moveTo(inst, cg);
9540 },
9541 .is_non_err_ptr => if (use_old) try cg.airIsNonErrPtr(inst) else {
9542 const un_op = air_datas[@intFromEnum(inst)].un_op;
9543 const eu_ty = cg.typeOf(un_op).childType(zcu);
9544 const eu_err_ty = eu_ty.errorUnionSet(zcu);
9545 const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
9546 const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
9547 var ops = try cg.tempsFromOperands(inst, .{un_op});
9548 try ops[0].toOffset(eu_err_off, cg);
9549 while (try ops[0].toLea(cg)) {}
9550 try cg.asmMemoryImmediate(
9551 .{ ._, .cmp },
9552 try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }),
9553 .u(0),
9554 );
9555 var is_non_err = try cg.tempInit(.bool, .{ .eflags = .e });
9556 try ops[0].die(cg);
9557 try is_non_err.moveTo(inst, cg);
9558 },
9559 .load => if (use_old) try cg.airLoad(inst) else fallback: {
9560 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9561 const val_ty = ty_op.ty.toType();
9562 const ptr_ty = cg.typeOf(ty_op.operand);
9563 const ptr_info = ptr_ty.ptrInfo(zcu);
9564 if (ptr_info.packed_offset.host_size > 0 and
9565 (ptr_info.flags.vector_index == .none or val_ty.toIntern() == .bool_type))
9566 break :fallback try cg.airLoad(inst);
9567 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9568 var res = try ops[0].load(val_ty, .{
9569 .disp = switch (ptr_info.flags.vector_index) {
9570 .none => 0,
9571 .runtime => unreachable,
9572 else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)),
9573 },
9574 }, cg);
9575 for (ops) |op| if (op.index != res.index) try op.die(cg);
9576 try res.moveTo(inst, cg);
9577 },
9578 .int_from_ptr => if (use_old) try cg.airIntFromPtr(inst) else {
9579 const un_op = air_datas[@intFromEnum(inst)].un_op;
9580 var ops = try cg.tempsFromOperands(inst, .{un_op});
9581 try ops[0].toSlicePtr(cg);
9582 try ops[0].moveTo(inst, cg);
9583 },
9584 .int_from_bool => if (use_old) try cg.airIntFromBool(inst) else {
9585 const un_op = air_datas[@intFromEnum(inst)].un_op;
9586 var ops = try cg.tempsFromOperands(inst, .{un_op});
9587 try ops[0].moveTo(inst, cg);
9588 },
9589 .ret => try cg.airRet(inst, false),
9590 .ret_safe => try cg.airRet(inst, true),
9591 .ret_load => try cg.airRetLoad(inst),
9592 .store, .store_safe => |air_tag| if (use_old) try cg.airStore(inst, switch (air_tag) {
9593 else => unreachable,
9594 .store => false,
9595 .store_safe => true,
9596 }) else fallback: {
9597 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
9598 const ptr_ty = cg.typeOf(bin_op.lhs);
9599 const ptr_info = ptr_ty.ptrInfo(zcu);
9600 const val_ty = cg.typeOf(bin_op.rhs);
9601 if (ptr_info.packed_offset.host_size > 0 and
9602 (ptr_info.flags.vector_index == .none or val_ty.toIntern() == .bool_type))
9603 break :fallback try cg.airStore(inst, switch (air_tag) {
9604 else => unreachable,
9605 .store => false,
9606 .store_safe => true,
9607 });
9608 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
9609 try ops[0].store(&ops[1], .{
9610 .disp = switch (ptr_info.flags.vector_index) {
9611 .none => 0,
9612 .runtime => unreachable,
9613 else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)),
9614 },
9615 .safe = switch (air_tag) {
9616 else => unreachable,
9617 .store => false,
9618 .store_safe => true,
9619 },
9620 }, cg);
9621 for (ops) |op| try op.die(cg);
9622 },
9623 .unreach => {},
9624 .optional_payload_ptr => if (use_old) try cg.airOptionalPayloadPtr(inst) else {
9625 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9626 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9627 try ops[0].moveTo(inst, cg);
9628 },
9629 .optional_payload_ptr_set => if (use_old) try cg.airOptionalPayloadPtrSet(inst) else {
9630 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9631 const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);
9632 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9633 if (!opt_ty.optionalReprIsPayload(zcu)) {
9634 const opt_child_ty = opt_ty.optionalChild(zcu);
9635 const opt_child_abi_size: i32 = @intCast(opt_child_ty.abiSize(zcu));
9636 try ops[0].toOffset(opt_child_abi_size, cg);
9637 var has_value = try cg.tempInit(.bool, .{ .immediate = 1 });
9638 try ops[0].store(&has_value, .{}, cg);
9639 try has_value.die(cg);
9640 try ops[0].toOffset(-opt_child_abi_size, cg);
9641 }
9642 try ops[0].moveTo(inst, cg);
9643 },
9644 .unwrap_errunion_payload_ptr => if (use_old) try cg.airUnwrapErrUnionPayloadPtr(inst) else {
9645 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9646 const eu_ty = cg.typeOf(ty_op.operand).childType(zcu);
9647 const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
9648 const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu));
9649 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9650 try ops[0].toOffset(eu_pl_off, cg);
9651 try ops[0].moveTo(inst, cg);
9652 },
9653 .unwrap_errunion_err_ptr => if (use_old) try cg.airUnwrapErrUnionErrPtr(inst) else {
9654 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9655 const eu_ty = cg.typeOf(ty_op.operand).childType(zcu);
9656 const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
9657 const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
9658 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9659 try ops[0].toOffset(eu_err_off, cg);
9660 var err = try ops[0].load(eu_ty.errorUnionSet(zcu), .{}, cg);
9661 try ops[0].die(cg);
9662 try err.moveTo(inst, cg);
9663 },
9664 .errunion_payload_ptr_set => if (use_old) try cg.airErrUnionPayloadPtrSet(inst) else {
9665 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9666 const eu_ty = cg.typeOf(ty_op.operand).childType(zcu);
9667 const eu_err_ty = eu_ty.errorUnionSet(zcu);
9668 const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
9669 const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
9670 const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu));
9671 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9672 try ops[0].toOffset(eu_err_off, cg);
9673 var no_err = try cg.tempInit(eu_err_ty, .{ .immediate = 0 });
9674 try ops[0].store(&no_err, .{}, cg);
9675 try no_err.die(cg);
9676 try ops[0].toOffset(eu_pl_off - eu_err_off, cg);
9677 try ops[0].moveTo(inst, cg);
9678 },
9679 .struct_field_ptr => if (use_old) try cg.airStructFieldPtr(inst) else {
9680 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
9681 const extra = cg.air.extraData(Air.StructField, ty_pl.payload).data;
9682 var ops = try cg.tempsFromOperands(inst, .{extra.struct_operand});
9683 try ops[0].toOffset(cg.fieldOffset(
9684 cg.typeOf(extra.struct_operand),
9685 ty_pl.ty.toType(),
9686 extra.field_index,
9687 ), cg);
9688 try ops[0].moveTo(inst, cg);
9689 },
9690 .struct_field_ptr_index_0,
9691 .struct_field_ptr_index_1,
9692 .struct_field_ptr_index_2,
9693 .struct_field_ptr_index_3,
9694 => |air_tag| if (use_old) try cg.airStructFieldPtrIndex(inst, switch (air_tag) {
9695 else => unreachable,
9696 .struct_field_ptr_index_0 => 0,
9697 .struct_field_ptr_index_1 => 1,
9698 .struct_field_ptr_index_2 => 2,
9699 .struct_field_ptr_index_3 => 3,
9700 }) else {
9701 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9702 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9703 try ops[0].toOffset(cg.fieldOffset(
9704 cg.typeOf(ty_op.operand),
9705 ty_op.ty.toType(),
9706 switch (air_tag) {
9707 else => unreachable,
9708 .struct_field_ptr_index_0 => 0,
9709 .struct_field_ptr_index_1 => 1,
9710 .struct_field_ptr_index_2 => 2,
9711 .struct_field_ptr_index_3 => 3,
9712 },
9713 ), cg);
9714 try ops[0].moveTo(inst, cg);
9715 },
9716 .struct_field_val => if (use_old) try cg.airStructFieldVal(inst) else fallback: {
9717 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
9718 const extra = cg.air.extraData(Air.StructField, ty_pl.payload).data;
9719 const agg_ty = cg.typeOf(extra.struct_operand);
9720 const field_ty = ty_pl.ty.toType();
9721 const field_off: u31 = switch (agg_ty.containerLayout(zcu)) {
9722 .auto, .@"extern" => @intCast(agg_ty.structFieldOffset(extra.field_index, zcu)),
9723 .@"packed" => break :fallback try cg.airStructFieldVal(inst),
9724 };
9725 var ops = try cg.tempsFromOperands(inst, .{extra.struct_operand});
9726 // hack around Sema OPV bugs
9727 var res = if (field_ty.hasRuntimeBitsIgnoreComptime(zcu))
9728 try ops[0].read(field_ty, .{ .disp = field_off }, cg)
9729 else
9730 try cg.tempInit(field_ty, .none);
9731 for (ops) |op| if (op.index != res.index) try op.die(cg);
9732 try res.moveTo(inst, cg);
9733 },
9734 .set_union_tag => if (use_old) try cg.airSetUnionTag(inst) else {
9735 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
9736 const union_ty = cg.typeOf(bin_op.lhs).childType(zcu);
9737 const union_layout = union_ty.unionGetLayout(zcu);
9738 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
9739 // hack around Sema OPV bugs
9740 if (union_layout.tag_size > 0) try ops[0].store(&ops[1], .{
9741 .disp = @intCast(union_layout.tagOffset()),
9742 }, cg);
9743 for (ops) |op| try op.die(cg);
9744 },
9745 .get_union_tag => if (use_old) try cg.airGetUnionTag(inst) else {
9746 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9747 const union_ty = cg.typeOf(ty_op.operand);
9748 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9749 const union_layout = union_ty.unionGetLayout(zcu);
9750 assert(union_layout.tag_size > 0);
9751 var res = try ops[0].read(ty_op.ty.toType(), .{
9752 .disp = @intCast(union_layout.tagOffset()),
9753 }, cg);
9754 for (ops) |op| if (op.index != res.index) try op.die(cg);
9755 try res.moveTo(inst, cg);
9756 },
9757 .slice => if (use_old) try cg.airSlice(inst) else {
9758 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
9759 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
9760 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
9761 try ops[0].toPair(&ops[1], cg);
9762 try ops[0].moveTo(inst, cg);
9763 },
9764 .slice_len => if (use_old) try cg.airSliceLen(inst) else {
9765 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9766 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9767 try ops[0].toSliceLen(cg);
9768 try ops[0].moveTo(inst, cg);
9769 },
9770 .slice_ptr => if (use_old) try cg.airSlicePtr(inst) else {
9771 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9772 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9773 try ops[0].toSlicePtr(cg);
9774 try ops[0].moveTo(inst, cg);
9775 },
9776 .ptr_slice_len_ptr => if (use_old) try cg.airPtrSliceLenPtr(inst) else {
9777 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9778 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9779 try ops[0].toOffset(8, cg);
9780 try ops[0].moveTo(inst, cg);
9781 },
9782 .ptr_slice_ptr_ptr => if (use_old) try cg.airPtrSlicePtrPtr(inst) else {
9783 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9784 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9785 try ops[0].toOffset(0, cg);
9786 try ops[0].moveTo(inst, cg);
9787 },
9788 .slice_elem_val, .ptr_elem_val => |air_tag| if (use_old) switch (air_tag) {
9789 else => unreachable,
9790 .slice_elem_val => try cg.airSliceElemVal(inst),
9791 .ptr_elem_val => try cg.airPtrElemVal(inst),
9792 } else {
9793 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
9794 const res_ty = cg.typeOf(bin_op.lhs).elemType2(zcu);
9795 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
9796 try ops[0].toSlicePtr(cg);
9797 var res: [1]Temp = undefined;
9798 if (res_ty.hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{res_ty}, &ops, comptime &.{ .{
9799 .dst_constraints = .{.{ .int = .byte }},
9800 .patterns = &.{
9801 .{ .src = .{ .to_gpr, .simm32 } },
9802 },
9803 .dst_temps = .{.{ .rc = .general_purpose }},
9804 .each = .{ .once = &.{
9805 .{ ._, ._, .movzx, .dst0d, .leaa(.byte, .src0, .add_src0_elem_size_times_src1), ._, ._ },
9806 } },
9807 }, .{
9808 .dst_constraints = .{.{ .int = .byte }},
9809 .patterns = &.{
9810 .{ .src = .{ .to_gpr, .to_gpr } },
9811 },
9812 .dst_temps = .{.{ .rc = .general_purpose }},
9813 .each = .{ .once = &.{
9814 .{ ._, ._, .movzx, .dst0d, .leai(.byte, .src0, .src1), ._, ._ },
9815 } },
9816 }, .{
9817 .dst_constraints = .{.{ .int = .word }},
9818 .patterns = &.{
9819 .{ .src = .{ .to_gpr, .simm32 } },
9820 },
9821 .dst_temps = .{.{ .rc = .general_purpose }},
9822 .each = .{ .once = &.{
9823 .{ ._, ._, .movzx, .dst0d, .leaa(.word, .src0, .add_src0_elem_size_times_src1), ._, ._ },
9824 } },
9825 }, .{
9826 .dst_constraints = .{.{ .int = .word }},
9827 .patterns = &.{
9828 .{ .src = .{ .to_gpr, .to_gpr } },
9829 },
9830 .dst_temps = .{.{ .rc = .general_purpose }},
9831 .each = .{ .once = &.{
9832 .{ ._, ._, .movzx, .dst0d, .leasi(.word, .src0, .@"2", .src1), ._, ._ },
9833 } },
9834 }, .{
9835 .dst_constraints = .{.{ .int = .dword }},
9836 .patterns = &.{
9837 .{ .src = .{ .to_gpr, .simm32 } },
9838 },
9839 .dst_temps = .{.{ .rc = .general_purpose }},
9840 .each = .{ .once = &.{
9841 .{ ._, ._, .mov, .dst0d, .leaa(.dword, .src0, .add_src0_elem_size_times_src1), ._, ._ },
9842 } },
9843 }, .{
9844 .dst_constraints = .{.{ .int = .dword }},
9845 .patterns = &.{
9846 .{ .src = .{ .to_gpr, .to_gpr } },
9847 },
9848 .dst_temps = .{.{ .rc = .general_purpose }},
9849 .each = .{ .once = &.{
9850 .{ ._, ._, .mov, .dst0d, .leasi(.dword, .src0, .@"4", .src1), ._, ._ },
9851 } },
9852 }, .{
9853 .dst_constraints = .{.{ .int = .qword }},
9854 .patterns = &.{
9855 .{ .src = .{ .to_gpr, .simm32 } },
9856 },
9857 .dst_temps = .{.{ .rc = .general_purpose }},
9858 .each = .{ .once = &.{
9859 .{ ._, ._, .mov, .dst0q, .leaa(.qword, .src0, .add_src0_elem_size_times_src1), ._, ._ },
9860 } },
9861 }, .{
9862 .required_features = .{ .@"64bit", null, null, null },
9863 .dst_constraints = .{.{ .int = .qword }},
9864 .patterns = &.{
9865 .{ .src = .{ .to_gpr, .to_gpr } },
9866 },
9867 .dst_temps = .{.{ .rc = .general_purpose }},
9868 .each = .{ .once = &.{
9869 .{ ._, ._, .mov, .dst0q, .leasi(.qword, .src0, .@"8", .src1), ._, ._ },
9870 } },
9871 } }) catch |err| switch (err) {
9872 error.SelectFailed => {
9873 const elem_size = res_ty.abiSize(zcu);
9874 while (true) for (&ops) |*op| {
9875 if (try op.toRegClass(true, .general_purpose, cg)) break;
9876 } else break;
9877 const lhs_reg = ops[0].unwrap(cg).temp.tracking(cg).short.register.to64();
9878 const rhs_reg = ops[1].unwrap(cg).temp.tracking(cg).short.register.to64();
9879 if (!std.math.isPowerOfTwo(elem_size)) {
9880 try cg.spillEflagsIfOccupied();
9881 try cg.asmRegisterRegisterImmediate(
9882 .{ .i_, .mul },
9883 rhs_reg,
9884 rhs_reg,
9885 .u(elem_size),
9886 );
9887 try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
9888 .base = .{ .reg = lhs_reg },
9889 .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
9890 });
9891 } else if (elem_size > 8) {
9892 try cg.spillEflagsIfOccupied();
9893 try cg.asmRegisterImmediate(
9894 .{ ._l, .sh },
9895 rhs_reg,
9896 .u(std.math.log2_int(u64, elem_size)),
9897 );
9898 try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
9899 .base = .{ .reg = lhs_reg },
9900 .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
9901 });
9902 } else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
9903 .base = .{ .reg = lhs_reg },
9904 .mod = .{ .rm = .{
9905 .size = .qword,
9906 .index = rhs_reg,
9907 .scale = .fromFactor(@intCast(elem_size)),
9908 } },
9909 });
9910 res[0] = try ops[0].load(res_ty, .{}, cg);
9911 },
9912 else => |e| return e,
9913 } else {
9914 // hack around Sema OPV bugs
9915 res[0] = try cg.tempInit(res_ty, .none);
9916 }
9917 for (ops) |op| for (res) |r| {
9918 if (op.index == r.index) break;
9919 } else try op.die(cg);
9920 try res[0].moveTo(inst, cg);
9921 },
9922 .slice_elem_ptr, .ptr_elem_ptr => |air_tag| if (use_old) switch (air_tag) {
9923 else => unreachable,
9924 .slice_elem_ptr => try cg.airSliceElemPtr(inst),
9925 .ptr_elem_ptr => try cg.airPtrElemPtr(inst),
9926 } else {
9927 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
9928 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
9929 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
9930 try ops[0].toSlicePtr(cg);
9931 const dst_ty = ty_pl.ty.toType();
9932 if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: {
9933 const elem_size = dst_ty.childType(zcu).abiSize(zcu);
9934 // hack around Sema OPV bugs
9935 if (elem_size == 0) break :zero_offset;
9936 while (true) for (&ops) |*op| {
9937 if (try op.toRegClass(true, .general_purpose, cg)) break;
9938 } else break;
9939 const lhs_reg = ops[0].unwrap(cg).temp.tracking(cg).short.register.to64();
9940 const rhs_reg = ops[1].unwrap(cg).temp.tracking(cg).short.register.to64();
9941 if (!std.math.isPowerOfTwo(elem_size)) {
9942 try cg.spillEflagsIfOccupied();
9943 try cg.asmRegisterRegisterImmediate(
9944 .{ .i_, .mul },
9945 rhs_reg,
9946 rhs_reg,
9947 .u(elem_size),
9948 );
9949 try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
9950 .base = .{ .reg = lhs_reg },
9951 .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
9952 });
9953 } else if (elem_size > 8) {
9954 try cg.spillEflagsIfOccupied();
9955 try cg.asmRegisterImmediate(
9956 .{ ._l, .sh },
9957 rhs_reg,
9958 .u(std.math.log2_int(u64, elem_size)),
9959 );
9960 try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
9961 .base = .{ .reg = lhs_reg },
9962 .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
9963 });
9964 } else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
9965 .base = .{ .reg = lhs_reg },
9966 .mod = .{ .rm = .{
9967 .size = .qword,
9968 .index = rhs_reg,
9969 .scale = .fromFactor(@intCast(elem_size)),
9970 } },
9971 });
9972 }
9973 try ops[1].die(cg);
9974 try ops[0].moveTo(inst, cg);
9975 },
9976 .array_to_slice => if (use_old) try cg.airArrayToSlice(inst) else {
9977 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9978 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9979 var len = try cg.tempInit(.usize, .{
9980 .immediate = cg.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu),
9981 });
9982 try ops[0].toPair(&len, cg);
9983 try ops[0].moveTo(inst, cg);
9984 },
9985 .error_set_has_value => return cg.fail("TODO implement error_set_has_value", .{}),
9986 .union_init => if (use_old) try cg.airUnionInit(inst) else {
9987 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
9988 const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data;
9989 const union_ty = ty_pl.ty.toType();
9990 var ops = try cg.tempsFromOperands(inst, .{extra.init});
9991 var res = try cg.tempAllocMem(union_ty);
9992 const union_layout = union_ty.unionGetLayout(zcu);
9993 if (union_layout.tag_size > 0) {
9994 var tag_temp = try cg.tempFromValue(try pt.enumValueFieldIndex(
9995 union_ty.unionTagTypeSafety(zcu).?,
9996 extra.field_index,
9997 ));
9998 try res.write(&tag_temp, .{
9999 .disp = @intCast(union_layout.tagOffset()),
10000 }, cg);
10001 try tag_temp.die(cg);
10002 }
10003 try res.write(&ops[0], .{
10004 .disp = @intCast(union_layout.payloadOffset()),
10005 }, cg);
10006 try ops[0].die(cg);
10007 try res.moveTo(inst, cg);
10008 },
10009 .field_parent_ptr => if (use_old) try cg.airFieldParentPtr(inst) else {
10010 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
10011 const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
10012 var ops = try cg.tempsFromOperands(inst, .{extra.field_ptr});
10013 try ops[0].toOffset(-cg.fieldOffset(
10014 ty_pl.ty.toType(),
10015 cg.typeOf(extra.field_ptr),
10016 extra.field_index,
10017 ), cg);
10018 try ops[0].moveTo(inst, cg);
10019 },
10020
10021 .is_named_enum_value => return cg.fail("TODO implement is_named_enum_value", .{}),
10022
10023 .wasm_memory_size => unreachable,
10024 .wasm_memory_grow => unreachable,
10025
10026 .addrspace_cast => {
10027 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
10028 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
10029 try ops[0].moveTo(inst, cg);
10030 },
10031
10032 .vector_store_elem => return cg.fail("TODO implement vector_store_elem", .{}),
10033
10034 .c_va_arg => try cg.airVaArg(inst),
10035 .c_va_copy => try cg.airVaCopy(inst),
10036 .c_va_end => try cg.airVaEnd(inst),
10037 .c_va_start => try cg.airVaStart(inst),
10038
10039 .work_item_id => unreachable,
10040 .work_group_size => unreachable,
10041 .work_group_id => unreachable,
10042 }
10043 cg.resetTemps();
10044 cg.checkInvariantsAfterAirInst();
10045 }
10046 verbose_tracking_log.debug("{}", .{cg.fmtTracking()});
257310047}
257410048
2575/// Asserts there is already capacity to insert into top branch inst_table.
2576fn processDeath(self: *Self, inst: Air.Inst.Index) !void {
2577 try self.inst_tracking.getPtr(inst).?.die(self, inst);
10049fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
10050 const pt = self.pt;
10051 const zcu = pt.zcu;
10052 const ip = &zcu.intern_pool;
10053 switch (Type.fromInterned(lazy_sym.ty).zigTypeTag(zcu)) {
10054 .@"enum" => {
10055 const enum_ty: Type = .fromInterned(lazy_sym.ty);
10056 wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)});
10057
10058 const param_regs = abi.getCAbiIntParamRegs(.auto);
10059 const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
10060 defer for (param_locks) |lock| self.register_manager.unlockReg(lock);
10061
10062 const ret_reg = param_regs[0];
10063 const enum_mcv = MCValue{ .register = param_regs[1] };
10064
10065 const epilogue_relocs = try self.gpa.alloc(Mir.Inst.Index, enum_ty.enumFieldCount(zcu));
10066 defer self.gpa.free(epilogue_relocs);
10067
10068 const data_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
10069 const data_lock = self.register_manager.lockRegAssumeUnused(data_reg);
10070 defer self.register_manager.unlockReg(data_lock);
10071 try self.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = enum_ty.toIntern() });
10072
10073 var data_off: i32 = 0;
10074 const tag_names = enum_ty.enumFields(zcu);
10075 for (epilogue_relocs, 0..) |*epilogue_reloc, tag_index| {
10076 const tag_name_len = tag_names.get(ip)[tag_index].length(ip);
10077 const tag_val = try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index));
10078 const tag_mcv = try self.genTypedValue(tag_val);
10079 try self.genBinOpMir(.{ ._, .cmp }, enum_ty, enum_mcv, tag_mcv);
10080 const skip_reloc = try self.asmJccReloc(.ne, undefined);
10081
10082 try self.genSetMem(
10083 .{ .reg = ret_reg },
10084 0,
10085 .usize,
10086 .{ .register_offset = .{ .reg = data_reg, .off = data_off } },
10087 .{},
10088 );
10089 try self.genSetMem(.{ .reg = ret_reg }, 8, .usize, .{ .immediate = tag_name_len }, .{});
10090
10091 epilogue_reloc.* = try self.asmJmpReloc(undefined);
10092 self.performReloc(skip_reloc);
10093
10094 data_off += @intCast(tag_name_len + 1);
10095 }
10096
10097 try self.asmOpOnly(.{ ._, .ud2 });
10098
10099 for (epilogue_relocs) |reloc| self.performReloc(reloc);
10100 try self.asmOpOnly(.{ ._, .ret });
10101 },
10102 else => return self.fail(
10103 "TODO implement {s} for {}",
10104 .{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) },
10105 ),
10106 }
257810107}
257910108
2580/// Called when there are no operands, and the instruction is always unreferenced.
2581fn finishAirBookkeeping(self: *Self) void {
2582 if (std.debug.runtime_safety) {
2583 self.air_bookkeeping += 1;
10109fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void {
10110 for (value.getRegs()) |reg| try self.register_manager.getReg(reg, inst);
10111 switch (value) {
10112 else => {},
10113 .eflags, .register_overflow => self.eflags_inst = inst,
10114 }
10115}
10116
10117fn getValueIfFree(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) void {
10118 for (value.getRegs()) |reg| if (self.register_manager.isRegFree(reg))
10119 self.register_manager.getRegAssumeFree(reg, inst);
10120}
10121
10122fn freeReg(self: *CodeGen, reg: Register) !void {
10123 self.register_manager.freeReg(reg);
10124 if (reg.class() == .x87) try self.asmRegister(.{ .f_, .free }, reg);
10125}
10126
10127fn freeValue(self: *CodeGen, value: MCValue) !void {
10128 switch (value) {
10129 .register => |reg| try self.freeReg(reg),
10130 inline .register_pair,
10131 .register_triple,
10132 .register_quadruple,
10133 => |regs| for (regs) |reg| try self.freeReg(reg),
10134 .register_offset, .indirect => |reg_off| try self.freeReg(reg_off.reg),
10135 .register_overflow => |reg_ov| {
10136 try self.freeReg(reg_ov.reg);
10137 self.eflags_inst = null;
10138 },
10139 .register_mask => |reg_mask| try self.freeReg(reg_mask.reg),
10140 .eflags => self.eflags_inst = null,
10141 else => {}, // TODO process stack allocation death
258410142 }
258510143}
258610144
2587fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
10145fn feed(self: *CodeGen, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) !void {
10146 if (bt.feed()) if (operand.toIndex()) |inst| try self.processDeath(inst);
10147}
10148
10149/// Asserts there is already capacity to insert into top branch inst_table.
10150fn processDeath(self: *CodeGen, inst: Air.Inst.Index) !void {
10151 try self.inst_tracking.getPtr(inst).?.die(self, inst);
10152}
10153
10154fn finishAirResult(self: *CodeGen, inst: Air.Inst.Index, result: MCValue) void {
258810155 if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) {
258910156 .none, .dead, .unreach => {},
259010157 else => unreachable, // Why didn't the result die?
259110158 } else {
2592 tracking_log.debug("%{d} => {} (birth)", .{ inst, result });
2593 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
10159 tracking_log.debug("{} => {} (birth)", .{ inst, result });
10160 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));
259410161 // In some cases, an operand may be reused as the result.
259510162 // If that operand died and was a register, it was freed by
259610163 // processDeath, so we have to "re-allocate" the register.
259710164 self.getValueIfFree(result, inst);
259810165 }
2599 self.finishAirBookkeeping();
260010166}
260110167
260210168fn finishAir(
2603 self: *Self,
10169 self: *CodeGen,
260410170 inst: Air.Inst.Index,
260510171 result: MCValue,
260610172 operands: [Liveness.bpi - 1]Air.Inst.Ref,
260710173) !void {
2608 var tomb_bits = self.liveness.getTombBits(inst);
2609 for (operands) |op| {
2610 const dies = @as(u1, @truncate(tomb_bits)) != 0;
2611 tomb_bits >>= 1;
2612 if (!dies) continue;
10174 const tomb_bits = self.liveness.getTombBits(inst);
10175 for (0.., operands) |op_index, op| {
10176 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
10177 if (self.reused_operands.isSet(op_index)) continue;
261310178 try self.processDeath(op.toIndexAllowNone() orelse continue);
261410179 }
261510180 self.finishAirResult(inst, result);
......@@ -2622,7 +10187,7 @@ const FrameLayout = struct {
262210187};
262310188
262410189fn setFrameLoc(
2625 self: *Self,
10190 self: *CodeGen,
262610191 frame_index: FrameIndex,
262710192 base: Register,
262810193 offset: *i32,
......@@ -2637,7 +10202,7 @@ fn setFrameLoc(
263710202 offset.* += self.frame_allocs.items(.abi_size)[frame_i];
263810203}
263910204
2640fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayout {
10205fn computeFrameLayout(self: *CodeGen, cc: std.builtin.CallingConvention.Tag) !FrameLayout {
264110206 const frame_allocs_len = self.frame_allocs.len;
264210207 try self.frame_locs.resize(self.gpa, frame_allocs_len);
264310208 const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);
......@@ -2657,7 +10222,7 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
265710222 }
265810223 };
265910224 const sort_context = SortContext{ .frame_align = frame_align };
2660 mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
10225 std.mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
266110226 }
266210227
266310228 const call_frame_align = frame_align[@intFromEnum(FrameIndex.call_frame)];
......@@ -2668,9 +10233,8 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
266810233
266910234 // Create list of registers to save in the prologue.
267010235 // TODO handle register classes
2671 var save_reg_list = Mir.RegisterList{};
2672 const callee_preserved_regs =
2673 abi.getCalleePreservedRegs(abi.resolveCallingConvention(cc, self.target.*));
10236 var save_reg_list: Mir.RegisterList = .empty;
10237 const callee_preserved_regs = abi.getCalleePreservedRegs(cc);
267410238 for (callee_preserved_regs) |reg| {
267510239 if (self.register_manager.isRegAllocated(reg)) {
267610240 save_reg_list.push(callee_preserved_regs, reg);
......@@ -2684,7 +10248,7 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
268410248 const stack_frame_align_offset = if (need_align_stack)
268510249 0
268610250 else
2687 save_reg_list.size() + frame_offset[@intFromEnum(FrameIndex.args_frame)];
10251 save_reg_list.size(self.target) + frame_offset[@intFromEnum(FrameIndex.args_frame)];
268810252
268910253 var rsp_offset: i32 = 0;
269010254 self.setFrameLoc(.call_frame, .rsp, &rsp_offset, true);
......@@ -2697,22 +10261,22 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
269710261 @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.stack_frame)]);
269810262
269910263 return .{
2700 .stack_mask = @as(u32, math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0),
10264 .stack_mask = @as(u32, std.math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0),
270110265 .stack_adjust = @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)]),
270210266 .save_reg_list = save_reg_list,
270310267 };
270410268}
270510269
2706fn getFrameAddrAlignment(self: *Self, frame_addr: bits.FrameAddr) Alignment {
10270fn getFrameAddrAlignment(self: *CodeGen, frame_addr: bits.FrameAddr) InternPool.Alignment {
270710271 const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align;
270810272 return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off)));
270910273}
271010274
2711fn getFrameAddrSize(self: *Self, frame_addr: bits.FrameAddr) u32 {
10275fn getFrameAddrSize(self: *CodeGen, frame_addr: bits.FrameAddr) u32 {
271210276 return self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_size - @as(u31, @intCast(frame_addr.off));
271310277}
271410278
2715fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
10279fn allocFrameIndex(self: *CodeGen, alloc: FrameAlloc) !FrameIndex {
271610280 const frame_allocs_slice = self.frame_allocs.slice();
271710281 const frame_size = frame_allocs_slice.items(.abi_size);
271810282 const frame_align = frame_allocs_slice.items(.abi_align);
......@@ -2735,36 +10299,36 @@ fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
273510299}
273610300
273710301/// Use a pointer instruction as the basis for allocating stack memory.
2738fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
10302fn allocMemPtr(self: *CodeGen, inst: Air.Inst.Index) !FrameIndex {
273910303 const pt = self.pt;
274010304 const zcu = pt.zcu;
274110305 const ptr_ty = self.typeOfIndex(inst);
274210306 const val_ty = ptr_ty.childType(zcu);
2743 return self.allocFrameIndex(FrameAlloc.init(.{
2744 .size = math.cast(u32, val_ty.abiSize(zcu)) orelse {
10307 return self.allocFrameIndex(.init(.{
10308 .size = std.math.cast(u32, val_ty.abiSize(zcu)) orelse {
274510309 return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(pt)});
274610310 },
274710311 .alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"),
274810312 }));
274910313}
275010314
2751fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
10315fn allocRegOrMem(self: *CodeGen, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
275210316 return self.allocRegOrMemAdvanced(self.typeOfIndex(inst), inst, reg_ok);
275310317}
275410318
2755fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue {
10319fn allocTempRegOrMem(self: *CodeGen, elem_ty: Type, reg_ok: bool) !MCValue {
275610320 return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok);
275710321}
275810322
2759fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
10323fn allocRegOrMemAdvanced(self: *CodeGen, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
276010324 const pt = self.pt;
276110325 const zcu = pt.zcu;
2762 const abi_size = math.cast(u32, ty.abiSize(zcu)) orelse {
10326 const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse {
276310327 return self.fail("type '{}' too big to fit into stack frame", .{ty.fmt(pt)});
276410328 };
276510329
276610330 if (reg_ok) need_mem: {
2767 if (abi_size <= @as(u32, switch (ty.zigTypeTag(zcu)) {
10331 if (std.math.isPowerOfTwo(abi_size) and abi_size <= @as(u32, switch (ty.zigTypeTag(zcu)) {
276810332 .float => switch (ty.floatBits(self.target.*)) {
276910333 16, 32, 64, 128 => 16,
277010334 80 => break :need_mem,
......@@ -2772,43 +10336,76 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
277210336 },
277310337 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
277410338 .float => switch (ty.childType(zcu).floatBits(self.target.*)) {
2775 16, 32, 64, 128 => if (self.hasFeature(.avx)) 32 else 16,
10339 16, 32, 64, 128 => self.vectorSize(.float),
277610340 80 => break :need_mem,
277710341 else => unreachable,
277810342 },
2779 else => if (self.hasFeature(.avx)) 32 else 16,
10343 else => self.vectorSize(.int),
278010344 },
278110345 else => 8,
278210346 })) {
2783 if (self.register_manager.tryAllocReg(inst, self.regClassForType(ty))) |reg| {
10347 if (self.register_manager.tryAllocReg(inst, self.regSetForType(ty))) |reg| {
278410348 return MCValue{ .register = registerAlias(reg, abi_size) };
278510349 }
278610350 }
278710351 }
278810352
2789 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, zcu));
10353 const frame_index = try self.allocFrameIndex(.initSpill(ty, zcu));
279010354 return .{ .load_frame = .{ .index = frame_index } };
279110355}
279210356
2793fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {
10357fn regClassForType(self: *CodeGen, ty: Type) Register.Class {
279410358 const pt = self.pt;
279510359 const zcu = pt.zcu;
279610360 return switch (ty.zigTypeTag(zcu)) {
279710361 .float => switch (ty.floatBits(self.target.*)) {
2798 80 => abi.RegisterClass.x87,
2799 else => abi.RegisterClass.sse,
10362 80 => .x87,
10363 else => .sse,
280010364 },
280110365 .vector => switch (ty.childType(zcu).toIntern()) {
2802 .bool_type, .u1_type => abi.RegisterClass.gp,
2803 else => if (ty.isAbiInt(zcu) and ty.intInfo(zcu).bits == 1)
2804 abi.RegisterClass.gp
2805 else
2806 abi.RegisterClass.sse,
10366 .bool_type => .general_purpose,
10367 else => .sse,
280710368 },
2808 else => abi.RegisterClass.gp,
10369 else => .general_purpose,
280910370 };
281010371}
281110372
10373fn regSetForRegClass(rc: Register.Class) RegisterManager.RegisterBitSet {
10374 return switch (rc) {
10375 .general_purpose => abi.RegisterClass.gp,
10376 .segment, .ip => unreachable,
10377 .x87 => abi.RegisterClass.x87,
10378 .mmx => @panic("TODO"),
10379 .sse => abi.RegisterClass.sse,
10380 };
10381}
10382
10383fn regSetForType(self: *CodeGen, ty: Type) RegisterManager.RegisterBitSet {
10384 return regSetForRegClass(self.regClassForType(ty));
10385}
10386
10387fn vectorSize(cg: *CodeGen, kind: enum { int, float }) u6 {
10388 return if (cg.hasFeature(switch (kind) {
10389 .int => .avx2,
10390 .float => .avx,
10391 })) 32 else if (cg.hasFeature(.sse)) 16 else 8;
10392}
10393
10394fn limbType(cg: *CodeGen, ty: Type) Type {
10395 const pt = cg.pt;
10396 const zcu = pt.zcu;
10397 const vector_size = cg.vectorSize(if (ty.isRuntimeFloat()) .float else .int);
10398 const scalar_ty, const scalar_size = scalar: {
10399 const scalar_ty = ty.scalarType(zcu);
10400 const scalar_size = scalar_ty.abiSize(zcu);
10401 if (scalar_size <= vector_size) break :scalar .{ scalar_ty, scalar_size };
10402 };
10403 pt.vectorType(.{
10404 .len = @divExact(vector_size, scalar_size),
10405 .child = scalar_ty.toIntern(),
10406 });
10407}
10408
281210409const State = struct {
281310410 registers: RegisterManager.TrackedRegisters,
281410411 reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
......@@ -2817,14 +10414,14 @@ const State = struct {
281710414 scope_generation: u32,
281810415};
281910416
2820fn initRetroactiveState(self: *Self) State {
10417fn initRetroactiveState(self: *CodeGen) State {
282110418 var state: State = undefined;
282210419 state.inst_tracking_len = @intCast(self.inst_tracking.count());
282310420 state.scope_generation = self.scope_generation;
282410421 return state;
282510422}
282610423
2827fn saveRetroactiveState(self: *Self, state: *State) !void {
10424fn saveRetroactiveState(self: *CodeGen, state: *State) !void {
282810425 try self.spillEflagsIfOccupied();
282910426 const free_registers = self.register_manager.free_registers;
283010427 var it = free_registers.iterator(.{ .kind = .unset });
......@@ -2836,13 +10433,13 @@ fn saveRetroactiveState(self: *Self, state: *State) !void {
283610433 state.free_registers = free_registers;
283710434}
283810435
2839fn saveState(self: *Self) !State {
10436fn saveState(self: *CodeGen) !State {
284010437 var state = self.initRetroactiveState();
284110438 try self.saveRetroactiveState(&state);
284210439 return state;
284310440}
284410441
2845fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, comptime opts: struct {
10442fn restoreState(self: *CodeGen, state: State, deaths: []const Air.Inst.Index, comptime opts: struct {
284610443 emit_instructions: bool,
284710444 update_tracking: bool,
284810445 resurrect: bool,
......@@ -2857,9 +10454,9 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
285710454 }
285810455
285910456 if (opts.resurrect) for (
2860 self.inst_tracking.keys()[0..state.inst_tracking_len],
2861 self.inst_tracking.values()[0..state.inst_tracking_len],
2862 ) |inst, *tracking| tracking.resurrect(inst, state.scope_generation);
10457 self.inst_tracking.keys()[Temp.Index.max..state.inst_tracking_len],
10458 self.inst_tracking.values()[Temp.Index.max..state.inst_tracking_len],
10459 ) |inst, *tracking| try tracking.resurrect(self, inst, state.scope_generation);
286310460 for (deaths) |death| try self.processDeath(death);
286410461
286510462 const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterLock;
......@@ -2876,43 +10473,34 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
287610473 reg_locks.deinit();
287710474 };
287810475
2879 for (0..state.registers.len) |index| {
2880 const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
2881 null
2882 else
2883 self.register_manager.registers[index];
2884 const target_maybe_inst = if (state.free_registers.isSet(index))
2885 null
2886 else
2887 state.registers[index];
10476 for (
10477 0..,
10478 self.register_manager.registers,
10479 state.registers,
10480 state.reg_tracking,
10481 ) |reg_i, current_slot, target_slot, reg_tracking| {
10482 const reg_index: RegisterManager.TrackedIndex = @intCast(reg_i);
10483 const current_maybe_inst = if (self.register_manager.isRegIndexFree(reg_index)) null else current_slot;
10484 const target_maybe_inst = if (state.free_registers.isSet(reg_index)) null else target_slot;
288810485 if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst|
288910486 assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);
289010487 if (opts.emit_instructions) {
2891 if (current_maybe_inst) |current_inst| {
10488 if (current_maybe_inst) |current_inst|
289210489 try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);
2893 }
2894 if (target_maybe_inst) |target_inst| {
2895 const target_tracking = self.inst_tracking.getPtr(target_inst).?;
2896 try target_tracking.materialize(self, target_inst, state.reg_tracking[index]);
2897 }
10490 if (target_maybe_inst) |target_inst|
10491 try self.inst_tracking.getPtr(target_inst).?.materialize(self, target_inst, reg_tracking);
289810492 }
289910493 if (opts.update_tracking) {
290010494 if (current_maybe_inst) |current_inst| {
290110495 try self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
2902 }
2903 {
2904 const reg = RegisterManager.regAtTrackedIndex(@intCast(index));
2905 self.register_manager.freeReg(reg);
2906 self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
10496 self.register_manager.freeRegIndex(reg_index);
290710497 }
290810498 if (target_maybe_inst) |target_inst| {
2909 self.inst_tracking.getPtr(target_inst).?.trackMaterialize(
2910 target_inst,
2911 state.reg_tracking[index],
2912 );
10499 self.register_manager.getRegIndexAssumeFree(reg_index, target_maybe_inst);
10500 self.inst_tracking.getPtr(target_inst).?.trackMaterialize(target_inst, reg_tracking);
291310501 }
291410502 } else if (target_maybe_inst) |_|
2915 try reg_locks.append(self.register_manager.lockRegIndexAssumeUnused(@intCast(index)));
10503 try reg_locks.append(self.register_manager.lockRegIndexAssumeUnused(reg_index));
291610504 }
291710505 if (opts.emit_instructions) if (self.eflags_inst) |inst|
291810506 try self.inst_tracking.getPtr(inst).?.spill(self, inst);
......@@ -2930,7 +10518,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
293010518 }
293110519}
293210520
2933pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
10521pub fn spillInstruction(self: *CodeGen, reg: Register, inst: Air.Inst.Index) !void {
293410522 const tracking = self.inst_tracking.getPtr(inst) orelse return;
293510523 for (tracking.getRegs()) |tracked_reg| {
293610524 if (tracked_reg.id() == reg.id()) break;
......@@ -2939,7 +10527,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
293910527 try tracking.trackSpill(self, inst);
294010528}
294110529
2942pub fn spillEflagsIfOccupied(self: *Self) !void {
10530pub fn spillEflagsIfOccupied(self: *CodeGen) !void {
294310531 if (self.eflags_inst) |inst| {
294410532 self.eflags_inst = null;
294510533 const tracking = self.inst_tracking.getPtr(inst).?;
......@@ -2949,23 +10537,22 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
294910537 }
295010538}
295110539
2952pub fn spillCallerPreservedRegs(self: *Self, cc: std.builtin.CallingConvention) !void {
10540pub fn spillCallerPreservedRegs(self: *CodeGen, cc: std.builtin.CallingConvention.Tag) !void {
295310541 switch (cc) {
2954 .x86_64_sysv => try self.spillRegisters(abi.getCallerPreservedRegs(.{ .x86_64_sysv = .{} })),
2955 .x86_64_win => try self.spillRegisters(abi.getCallerPreservedRegs(.{ .x86_64_win = .{} })),
10542 inline .auto, .x86_64_sysv, .x86_64_win => |tag| try self.spillRegisters(abi.getCallerPreservedRegs(tag)),
295610543 else => unreachable,
295710544 }
295810545}
295910546
2960pub fn spillRegisters(self: *Self, comptime registers: []const Register) !void {
10547pub fn spillRegisters(self: *CodeGen, comptime registers: []const Register) !void {
296110548 inline for (registers) |reg| try self.register_manager.getKnownReg(reg, null);
296210549}
296310550
296410551/// Copies a value to a register without tracking the register. The register is not considered
296510552/// allocated. A second call to `copyToTmpRegister` may return the same register.
296610553/// This can have a side effect of spilling instructions to the stack to free up a register.
2967fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
2968 const reg = try self.register_manager.allocReg(null, self.regClassForType(ty));
10554fn copyToTmpRegister(self: *CodeGen, ty: Type, mcv: MCValue) !Register {
10555 const reg = try self.register_manager.allocReg(null, self.regSetForType(ty));
296910556 try self.genSetReg(reg, ty, mcv, .{});
297010557 return reg;
297110558}
......@@ -2975,22 +10562,22 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
297510562/// This can have a side effect of spilling instructions to the stack to free up a register.
297610563/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
297710564fn copyToRegisterWithInstTracking(
2978 self: *Self,
10565 self: *CodeGen,
297910566 reg_owner: Air.Inst.Index,
298010567 ty: Type,
298110568 mcv: MCValue,
298210569) !MCValue {
2983 const reg: Register = try self.register_manager.allocReg(reg_owner, self.regClassForType(ty));
10570 const reg: Register = try self.register_manager.allocReg(reg_owner, self.regSetForType(ty));
298410571 try self.genSetReg(reg, ty, mcv, .{});
298510572 return MCValue{ .register = reg };
298610573}
298710574
2988fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
10575fn airAlloc(self: *CodeGen, inst: Air.Inst.Index) !void {
298910576 const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };
299010577 return self.finishAir(inst, result, .{ .none, .none, .none });
299110578}
299210579
2993fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
10580fn airRetPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
299410581 const result: MCValue = switch (self.ret_mcv.long) {
299510582 else => unreachable,
299610583 .none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },
......@@ -3006,7 +10593,7 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
300610593 return self.finishAir(inst, result, .{ .none, .none, .none });
300710594}
300810595
3009fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
10596fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void {
301010597 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
301110598 const dst_ty = self.typeOfIndex(inst);
301210599 const dst_bits = dst_ty.floatBits(self.target.*);
......@@ -3043,7 +10630,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
304310630 floatCompilerRtAbiName(src_bits),
304410631 floatCompilerRtAbiName(dst_bits),
304510632 }) catch unreachable,
3046 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
10633 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
304710634 }
304810635
304910636 const src_mcv = try self.resolveInst(ty_op.operand);
......@@ -3067,18 +10654,18 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
306710654 .{ .v_, .cvtps2ph },
306810655 dst_reg,
306910656 mat_src_reg.to128(),
3070 Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10657 .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
307110658 );
307210659 },
307310660 else => unreachable,
307410661 }
307510662 } else {
307610663 assert(src_bits == 64 and dst_bits == 32);
3077 if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
10664 if (self.hasFeature(.avx)) if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
307810665 .{ .v_ss, .cvtsd2 },
307910666 dst_reg,
308010667 dst_reg,
3081 try src_mcv.mem(self, .qword),
10668 try src_mcv.mem(self, .{ .size = .qword }),
308210669 ) else try self.asmRegisterRegisterRegister(
308310670 .{ .v_ss, .cvtsd2 },
308410671 dst_reg,
......@@ -3087,10 +10674,10 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
308710674 src_mcv.getReg().?
308810675 else
308910676 try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
3090 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
10677 ) else if (src_mcv.isBase()) try self.asmRegisterMemory(
309110678 .{ ._ss, .cvtsd2 },
309210679 dst_reg,
3093 try src_mcv.mem(self, .qword),
10680 try src_mcv.mem(self, .{ .size = .qword }),
309410681 ) else try self.asmRegisterRegister(
309510682 .{ ._ss, .cvtsd2 },
309610683 dst_reg,
......@@ -3105,7 +10692,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
310510692 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
310610693}
310710694
3108fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
10695fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void {
310910696 const pt = self.pt;
311010697 const zcu = pt.zcu;
311110698 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -3147,7 +10734,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
314710734 floatCompilerRtAbiName(src_bits),
314810735 floatCompilerRtAbiName(dst_bits),
314910736 }) catch unreachable,
3150 } }, &.{src_scalar_ty}, &.{.{ .air_ref = ty_op.operand }});
10737 } }, &.{src_scalar_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
315110738 }
315210739
315310740 const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu));
......@@ -3186,11 +10773,11 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
318610773 } else {
318710774 assert(src_bits == 32 and dst_bits == 64);
318810775 if (self.hasFeature(.avx)) switch (vec_len) {
3189 1 => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
10776 1 => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
319010777 .{ .v_sd, .cvtss2 },
319110778 dst_alias,
319210779 dst_alias,
3193 try src_mcv.mem(self, self.memSize(src_ty)),
10780 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
319410781 ) else try self.asmRegisterRegisterRegister(
319510782 .{ .v_sd, .cvtss2 },
319610783 dst_alias,
......@@ -3200,10 +10787,10 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
320010787 else
320110788 try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
320210789 ),
3203 2...4 => if (src_mcv.isMemory()) try self.asmRegisterMemory(
10790 2...4 => if (src_mcv.isBase()) try self.asmRegisterMemory(
320410791 .{ .v_pd, .cvtps2 },
320510792 dst_alias,
3206 try src_mcv.mem(self, self.memSize(src_ty)),
10793 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
320710794 ) else try self.asmRegisterRegister(
320810795 .{ .v_pd, .cvtps2 },
320910796 dst_alias,
......@@ -3213,14 +10800,14 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
321310800 try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
321410801 ),
321510802 else => break :result null,
3216 } else if (src_mcv.isMemory()) try self.asmRegisterMemory(
10803 } else if (src_mcv.isBase()) try self.asmRegisterMemory(
321710804 switch (vec_len) {
321810805 1 => .{ ._sd, .cvtss2 },
321910806 2 => .{ ._pd, .cvtps2 },
322010807 else => break :result null,
322110808 },
322210809 dst_alias,
3223 try src_mcv.mem(self, self.memSize(src_ty)),
10810 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
322410811 ) else try self.asmRegisterRegister(
322510812 switch (vec_len) {
322610813 1 => .{ ._sd, .cvtss2 },
......@@ -3241,7 +10828,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
324110828 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
324210829}
324310830
3244fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
10831fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void {
324510832 const pt = self.pt;
324610833 const zcu = pt.zcu;
324710834 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -3249,7 +10836,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
324910836 const dst_ty = self.typeOfIndex(inst);
325010837
325110838 const result = @as(?MCValue, result: {
3252 const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
10839 const src_abi_size: u31 = @intCast(src_ty.abiSize(zcu));
10840 const dst_abi_size: u31 = @intCast(dst_ty.abiSize(zcu));
325310841
325410842 const src_int_info = src_ty.intInfo(zcu);
325510843 const dst_int_info = dst_ty.intInfo(zcu);
......@@ -3260,14 +10848,13 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
326010848
326110849 const src_mcv = try self.resolveInst(ty_op.operand);
326210850 if (dst_ty.isVector(zcu)) {
3263 const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu));
326410851 const max_abi_size = @max(dst_abi_size, src_abi_size);
3265 if (max_abi_size > @as(u32, if (self.hasFeature(.avx2)) 32 else 16)) break :result null;
10852 if (max_abi_size > self.vectorSize(.int)) break :result null;
326610853 const has_avx = self.hasFeature(.avx);
326710854
326810855 const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu);
326910856 const src_elem_abi_size = src_ty.childType(zcu).abiSize(zcu);
3270 switch (math.order(dst_elem_abi_size, src_elem_abi_size)) {
10857 switch (std.math.order(dst_elem_abi_size, src_elem_abi_size)) {
327110858 .lt => {
327210859 const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
327310860 else => break :result null,
......@@ -3354,10 +10941,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
335410941 const dst_reg = dst_mcv.getReg().?;
335510942 const dst_alias = registerAlias(dst_reg, dst_abi_size);
335610943
3357 if (src_mcv.isMemory()) try self.asmRegisterMemory(
10944 if (src_mcv.isBase()) try self.asmRegisterMemory(
335810945 mir_tag,
335910946 dst_alias,
3360 try src_mcv.mem(self, self.memSize(src_ty)),
10947 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
336110948 ) else try self.asmRegisterRegister(
336210949 mir_tag,
336310950 dst_alias,
......@@ -3430,9 +11017,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
343011017 else => src_int_info.bits,
343111018 };
343211019
3433 const dst_mcv = if (dst_int_info.bits <= src_storage_bits and
3434 math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
3435 math.divCeil(u32, src_storage_bits, 64) catch unreachable and
11020 const dst_mcv = if ((if (src_mcv.getReg()) |src_reg| src_reg.class() == .general_purpose else src_abi_size > 8) and
11021 dst_int_info.bits <= src_storage_bits and
11022 std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
11023 std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and
343611024 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
343711025 const dst_mcv = try self.allocRegOrMem(inst, true);
343811026 try self.genCopy(min_ty, dst_mcv, src_mcv, .{});
......@@ -3449,10 +11037,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
344911037 break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) };
345011038 }
345111039
3452 const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
3453 const dst_limbs_len = math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;
11040 const src_limbs_len = std.math.divCeil(u31, src_abi_size, 8) catch unreachable;
11041 const dst_limbs_len = @divExact(dst_abi_size, 8);
345411042
3455 const high_mcv: MCValue = if (dst_mcv.isMemory())
11043 const high_mcv: MCValue = if (dst_mcv.isBase())
345611044 dst_mcv.address().offset((src_limbs_len - 1) * 8).deref()
345711045 else
345811046 .{ .register = dst_mcv.register_pair[1] };
......@@ -3460,8 +11048,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
346011048 high_mcv.getReg().?
346111049 else
346211050 try self.copyToTmpRegister(switch (src_int_info.signedness) {
3463 .signed => Type.isize,
3464 .unsigned => Type.usize,
11051 .signed => .isize,
11052 .unsigned => .usize,
346511053 }, high_mcv);
346611054 const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);
346711055 defer self.register_manager.unlockReg(high_lock);
......@@ -3469,7 +11057,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
346911057 const high_bits = src_int_info.bits % 64;
347011058 if (high_bits > 0) {
347111059 try self.truncateRegister(src_ty, high_reg);
3472 const high_ty = if (dst_int_info.bits >= 64) Type.usize else dst_ty;
11060 const high_ty: Type = if (dst_int_info.bits >= 64) .usize else dst_ty;
347311061 try self.genCopy(high_ty, high_mcv, .{ .register = high_reg }, .{});
347411062 }
347511063
......@@ -3478,13 +11066,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
347811066 switch (extend) {
347911067 .signed => extend: {
348011068 const extend_mcv = MCValue{ .register = high_reg };
3481 try self.genShiftBinOpMir(
3482 .{ ._r, .sa },
3483 Type.isize,
3484 extend_mcv,
3485 Type.u8,
3486 .{ .immediate = 63 },
3487 );
11069 try self.genShiftBinOpMir(.{ ._r, .sa }, .isize, extend_mcv, .u8, .{ .immediate = 63 });
348811070 break :extend extend_mcv;
348911071 },
349011072 .unsigned => .{ .immediate = 0 },
......@@ -3500,7 +11082,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
350011082 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
350111083}
350211084
3503fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
11085fn airTrunc(self: *CodeGen, inst: Air.Inst.Index) !void {
350411086 const pt = self.pt;
350511087 const zcu = pt.zcu;
350611088 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -3516,7 +11098,8 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
351611098 if (src_mcv.getReg()) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;
351711099 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
351811100
3519 const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
11101 const dst_mcv = if (src_mcv.isRegister() and src_mcv.getReg().?.class() == self.regClassForType(dst_ty) and
11102 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
352011103 src_mcv
352111104 else if (dst_abi_size <= 8)
352211105 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv)
......@@ -3570,7 +11153,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
357011153 const dst_info = dst_elem_ty.intInfo(zcu);
357111154 const src_info = src_elem_ty.intInfo(zcu);
357211155
3573 const mask_val = try pt.intValue(src_elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
11156 const mask_val = try pt.intValue(src_elem_ty, @as(u64, std.math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
357411157
357511158 const splat_ty = try pt.vectorType(.{
357611159 .len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),
......@@ -3583,10 +11166,10 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
358311166 .storage = .{ .repeated_elem = mask_val.ip_index },
358411167 } });
358511168
3586 const splat_mcv = try self.genTypedValue(Value.fromInterned(splat_val));
11169 const splat_mcv = try self.genTypedValue(.fromInterned(splat_val));
358711170 const splat_addr_mcv: MCValue = switch (splat_mcv) {
358811171 .memory, .indirect, .load_frame => splat_mcv.address(),
3589 else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },
11172 else => .{ .register = try self.copyToTmpRegister(.usize, splat_mcv.address()) },
359011173 };
359111174
359211175 const dst_reg = dst_mcv.getReg().?;
......@@ -3596,7 +11179,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
359611179 .{ .vp_, .@"and" },
359711180 dst_alias,
359811181 dst_alias,
3599 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
11182 try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }),
360011183 );
360111184 if (src_abi_size > 16) {
360211185 const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
......@@ -3607,7 +11190,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
360711190 .{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },
360811191 registerAlias(temp_reg, dst_abi_size),
360911192 dst_alias,
3610 Immediate.u(1),
11193 .u(1),
361111194 );
361211195 try self.asmRegisterRegisterRegister(
361311196 mir_tag,
......@@ -3620,7 +11203,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
362011203 try self.asmRegisterMemory(
362111204 .{ .p_, .@"and" },
362211205 dst_alias,
3623 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
11206 try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }),
362411207 );
362511208 try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
362611209 }
......@@ -3646,7 +11229,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
364611229 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
364711230}
364811231
3649fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {
11232fn airIntFromBool(self: *CodeGen, inst: Air.Inst.Index) !void {
365011233 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
365111234 const ty = self.typeOfIndex(inst);
365211235
......@@ -3659,13 +11242,13 @@ fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {
365911242 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
366011243}
366111244
3662fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
11245fn airSlice(self: *CodeGen, inst: Air.Inst.Index) !void {
366311246 const zcu = self.pt.zcu;
366411247 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
366511248 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
366611249
366711250 const slice_ty = self.typeOfIndex(inst);
3668 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));
11251 const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu));
366911252
367011253 const ptr_ty = self.typeOf(bin_op.lhs);
367111254 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }, .{});
......@@ -3683,13 +11266,13 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
368311266 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
368411267}
368511268
3686fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
11269fn airUnOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
368711270 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
368811271 const dst_mcv = try self.genUnOp(inst, tag, ty_op.operand);
368911272 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
369011273}
369111274
3692fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
11275fn airBinOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
369311276 const pt = self.pt;
369411277 const zcu = pt.zcu;
369511278 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -3724,14 +11307,14 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
372411307 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
372511308}
372611309
3727fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
11310fn airPtrArithmetic(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
372811311 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
372911312 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
373011313 const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
373111314 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
373211315}
373311316
3734fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 {
11317fn activeIntBits(self: *CodeGen, dst_air: Air.Inst.Ref) u16 {
373511318 const pt = self.pt;
373611319 const zcu = pt.zcu;
373711320 const air_tag = self.air.instructions.items(.tag);
......@@ -3766,7 +11349,7 @@ fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 {
376611349 return dst_info.bits;
376711350}
376811351
3769fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
11352fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
377011353 const pt = self.pt;
377111354 const zcu = pt.zcu;
377211355 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -3802,11 +11385,11 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
380211385 state: State,
380311386 reloc: Mir.Inst.Index,
380411387 } = if (signed and tag == .div_floor) state: {
3805 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(Type.usize, zcu));
11388 const frame_index = try self.allocFrameIndex(.initType(.usize, zcu));
380611389 try self.asmMemoryImmediate(
380711390 .{ ._, .mov },
380811391 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
3809 Immediate.u(0),
11392 .u(0),
381011393 );
381111394
381211395 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
......@@ -3817,7 +11400,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
381711400 const mat_lhs_mcv = switch (lhs_mcv) {
381811401 .load_symbol => mat_lhs_mcv: {
381911402 // TODO clean this up!
3820 const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());
11403 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
382111404 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
382211405 },
382311406 else => lhs_mcv,
......@@ -3827,10 +11410,10 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
382711410 else => null,
382811411 };
382911412 defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock);
3830 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
11413 if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
383111414 .{ ._, .mov },
383211415 tmp_reg,
3833 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
11416 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
383411417 ) else try self.asmRegisterRegister(
383511418 .{ ._, .mov },
383611419 tmp_reg,
......@@ -3841,7 +11424,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
384111424 const mat_rhs_mcv = switch (rhs_mcv) {
384211425 .load_symbol => mat_rhs_mcv: {
384311426 // TODO clean this up!
3844 const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());
11427 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
384511428 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
384611429 },
384711430 else => rhs_mcv,
......@@ -3851,10 +11434,10 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
385111434 else => null,
385211435 };
385311436 defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
3854 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
11437 if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
385511438 .{ ._, .xor },
385611439 tmp_reg,
3857 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
11440 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
385811441 ) else try self.asmRegisterRegister(
385911442 .{ ._, .xor },
386011443 tmp_reg,
......@@ -3882,6 +11465,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
388211465 } },
388311466 &.{ src_ty, src_ty },
388411467 &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } },
11468 .{},
388511469 );
388611470 break :result if (signed) switch (tag) {
388711471 .div_floor => {
......@@ -3911,6 +11495,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
391111495 } },
391211496 &.{ src_ty, src_ty },
391311497 &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } },
11498 .{},
391411499 );
391511500 try self.asmRegisterMemory(
391611501 .{ ._, .sub },
......@@ -3920,11 +11505,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
392011505 .mod = .{ .rm = .{ .size = .qword } },
392111506 },
392211507 );
3923 try self.asmRegisterImmediate(
3924 .{ ._, .sbb },
3925 dst_mcv.register_pair[1],
3926 Immediate.u(0),
3927 );
11508 try self.asmRegisterImmediate(.{ ._, .sbb }, dst_mcv.register_pair[1], .u(0));
392811509 try self.freeValue(
392911510 .{ .load_frame = .{ .index = signed_div_floor_state.frame_index } },
393011511 );
......@@ -3936,7 +11517,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
393611517 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
393711518
393811519 const tmp_regs =
3939 try self.register_manager.allocRegs(2, .{null} ** 2, abi.RegisterClass.gp);
11520 try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
394011521 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
394111522 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
394211523
......@@ -3944,7 +11525,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
394411525 const mat_rhs_mcv = switch (rhs_mcv) {
394511526 .load_symbol => mat_rhs_mcv: {
394611527 // TODO clean this up!
3947 const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());
11528 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
394811529 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
394911530 },
395011531 else => rhs_mcv,
......@@ -3957,16 +11538,16 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
395711538
395811539 for (tmp_regs, dst_regs) |tmp_reg, dst_reg|
395911540 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_reg);
3960 if (mat_rhs_mcv.isMemory()) {
11541 if (mat_rhs_mcv.isBase()) {
396111542 try self.asmRegisterMemory(
396211543 .{ ._, .add },
396311544 tmp_regs[0],
3964 try mat_rhs_mcv.mem(self, .qword),
11545 try mat_rhs_mcv.mem(self, .{ .size = .qword }),
396511546 );
396611547 try self.asmRegisterMemory(
396711548 .{ ._, .adc },
396811549 tmp_regs[1],
3969 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
11550 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
397011551 );
397111552 } else for (
397211553 [_]Mir.Inst.Tag{ .add, .adc },
......@@ -3996,7 +11577,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
399611577 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
399711578}
399811579
3999fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
11580fn airAddSat(self: *CodeGen, inst: Air.Inst.Index) !void {
400011581 const pt = self.pt;
400111582 const zcu = pt.zcu;
400211583 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -4031,22 +11612,10 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
403111612 const reg_extra_bits = self.regExtraBits(ty);
403211613 const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
403311614 if (reg_extra_bits > 0) {
4034 try self.genShiftBinOpMir(
4035 .{ ._l, .sa },
4036 ty,
4037 dst_mcv,
4038 Type.u8,
4039 .{ .immediate = reg_extra_bits },
4040 );
11615 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
404111616 }
404211617 try self.genSetReg(limit_reg, ty, dst_mcv, .{});
4043 try self.genShiftBinOpMir(
4044 .{ ._r, .sa },
4045 ty,
4046 limit_mcv,
4047 Type.u8,
4048 .{ .immediate = reg_bits - 1 },
4049 );
11618 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
405011619 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
405111620 .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
405211621 });
......@@ -4056,19 +11625,13 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
405611625 const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);
405711626 defer self.register_manager.unlockReg(shifted_rhs_lock);
405811627
4059 try self.genShiftBinOpMir(
4060 .{ ._l, .sa },
4061 ty,
4062 shifted_rhs_mcv,
4063 Type.u8,
4064 .{ .immediate = reg_extra_bits },
4065 );
11628 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits });
406611629 try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, shifted_rhs_mcv);
406711630 } else try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
406811631 break :cc .o;
406911632 } else cc: {
407011633 try self.genSetReg(limit_reg, ty, .{
4071 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)),
11634 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)),
407211635 }, .{});
407311636
407411637 try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
......@@ -4086,18 +11649,13 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
408611649 registerAlias(limit_reg, cmov_abi_size),
408711650 );
408811651
4089 if (reg_extra_bits > 0 and ty.isSignedInt(zcu)) try self.genShiftBinOpMir(
4090 .{ ._r, .sa },
4091 ty,
4092 dst_mcv,
4093 Type.u8,
4094 .{ .immediate = reg_extra_bits },
4095 );
11652 if (reg_extra_bits > 0 and ty.isSignedInt(zcu))
11653 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
409611654
409711655 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
409811656}
409911657
4100fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
11658fn airSubSat(self: *CodeGen, inst: Air.Inst.Index) !void {
410111659 const pt = self.pt;
410211660 const zcu = pt.zcu;
410311661 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -4132,22 +11690,10 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
413211690 const reg_extra_bits = self.regExtraBits(ty);
413311691 const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
413411692 if (reg_extra_bits > 0) {
4135 try self.genShiftBinOpMir(
4136 .{ ._l, .sa },
4137 ty,
4138 dst_mcv,
4139 Type.u8,
4140 .{ .immediate = reg_extra_bits },
4141 );
11693 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
414211694 }
414311695 try self.genSetReg(limit_reg, ty, dst_mcv, .{});
4144 try self.genShiftBinOpMir(
4145 .{ ._r, .sa },
4146 ty,
4147 limit_mcv,
4148 Type.u8,
4149 .{ .immediate = reg_bits - 1 },
4150 );
11696 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
415111697 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
415211698 .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
415311699 });
......@@ -4157,13 +11703,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
415711703 const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);
415811704 defer self.register_manager.unlockReg(shifted_rhs_lock);
415911705
4160 try self.genShiftBinOpMir(
4161 .{ ._l, .sa },
4162 ty,
4163 shifted_rhs_mcv,
4164 Type.u8,
4165 .{ .immediate = reg_extra_bits },
4166 );
11706 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits });
416711707 try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, shifted_rhs_mcv);
416811708 } else try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
416911709 break :cc .o;
......@@ -4180,18 +11720,13 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
418011720 registerAlias(limit_reg, cmov_abi_size),
418111721 );
418211722
4183 if (reg_extra_bits > 0 and ty.isSignedInt(zcu)) try self.genShiftBinOpMir(
4184 .{ ._r, .sa },
4185 ty,
4186 dst_mcv,
4187 Type.u8,
4188 .{ .immediate = reg_extra_bits },
4189 );
11723 if (reg_extra_bits > 0 and ty.isSignedInt(zcu))
11724 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
419011725
419111726 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
419211727}
419311728
4194fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
11729fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void {
419511730 const pt = self.pt;
419611731 const zcu = pt.zcu;
419711732 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -4199,18 +11734,18 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
419911734
420011735 const result = result: {
420111736 if (ty.toIntern() == .i128_type) {
4202 const ptr_c_int = try pt.singleMutPtrType(Type.c_int);
4203 const overflow = try self.allocTempRegOrMem(Type.c_int, false);
11737 const ptr_c_int = try pt.singleMutPtrType(.c_int);
11738 const overflow = try self.allocTempRegOrMem(.c_int, false);
420411739
420511740 const dst_mcv = try self.genCall(.{ .lib = .{
420611741 .return_type = .i128_type,
420711742 .param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() },
420811743 .callee = "__muloti4",
4209 } }, &.{ Type.i128, Type.i128, ptr_c_int }, &.{
11744 } }, &.{ .i128, .i128, ptr_c_int }, &.{
421011745 .{ .air_ref = bin_op.lhs },
421111746 .{ .air_ref = bin_op.rhs },
421211747 overflow.address(),
4213 });
11748 }, .{});
421411749 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_mcv.register_pair);
421511750 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
421611751
......@@ -4222,7 +11757,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
422211757 const mat_lhs_mcv = switch (lhs_mcv) {
422311758 .load_symbol => mat_lhs_mcv: {
422411759 // TODO clean this up!
4225 const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());
11760 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
422611761 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
422711762 },
422811763 else => lhs_mcv,
......@@ -4232,10 +11767,10 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
423211767 else => null,
423311768 };
423411769 defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock);
4235 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
11770 if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
423611771 .{ ._, .mov },
423711772 tmp_reg,
4238 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
11773 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
423911774 ) else try self.asmRegisterRegister(
424011775 .{ ._, .mov },
424111776 tmp_reg,
......@@ -4246,7 +11781,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
424611781 const mat_rhs_mcv = switch (rhs_mcv) {
424711782 .load_symbol => mat_rhs_mcv: {
424811783 // TODO clean this up!
4249 const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());
11784 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
425011785 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
425111786 },
425211787 else => rhs_mcv,
......@@ -4256,22 +11791,22 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
425611791 else => null,
425711792 };
425811793 defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
4259 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
11794 if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
426011795 .{ ._, .xor },
426111796 tmp_reg,
4262 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
11797 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
426311798 ) else try self.asmRegisterRegister(
426411799 .{ ._, .xor },
426511800 tmp_reg,
426611801 mat_rhs_mcv.register_pair[1],
426711802 );
426811803
4269 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));
11804 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
427011805 try self.asmRegister(.{ ._, .not }, tmp_reg);
4271 try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), Immediate.s(0));
11806 try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .{ .size = .dword }), .s(0));
427211807 try self.freeValue(overflow);
427311808 try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
4274 try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63));
11809 try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, .u(63));
427511810 try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[1], tmp_reg);
427611811 break :result dst_mcv;
427711812 }
......@@ -4308,20 +11843,14 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
430811843 const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
430911844 try self.genSetReg(limit_reg, ty, lhs_mcv, .{});
431011845 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
4311 try self.genShiftBinOpMir(
4312 .{ ._r, .sa },
4313 ty,
4314 limit_mcv,
4315 Type.u8,
4316 .{ .immediate = reg_bits - 1 },
4317 );
11846 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
431811847 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
431911848 .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
432011849 });
432111850 break :cc .o;
432211851 } else cc: {
432311852 try self.genSetReg(limit_reg, ty, .{
4324 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - reg_bits),
11853 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - reg_bits),
432511854 }, .{});
432611855 break :cc .c;
432711856 };
......@@ -4338,7 +11867,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
433811867 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
433911868}
434011869
4341fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
11870fn airAddSubWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
434211871 const pt = self.pt;
434311872 const zcu = pt.zcu;
434411873 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -4366,7 +11895,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
436611895 };
436711896
436811897 const tuple_ty = self.typeOfIndex(inst);
4369 if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
11898 if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) {
437011899 switch (partial_mcv) {
437111900 .register => |reg| {
437211901 self.eflags_inst = inst;
......@@ -4375,12 +11904,11 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
437511904 else => {},
437611905 }
437711906
4378 const frame_index =
4379 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
11907 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
438011908 try self.genSetMem(
438111909 .{ .frame = frame_index },
438211910 @intCast(tuple_ty.structFieldOffset(1, zcu)),
4383 Type.u1,
11911 .u1,
438411912 .{ .eflags = cc },
438511913 .{},
438611914 );
......@@ -4394,8 +11922,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
439411922 break :result .{ .load_frame = .{ .index = frame_index } };
439511923 }
439611924
4397 const frame_index =
4398 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
11925 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
439911926 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
440011927 break :result .{ .load_frame = .{ .index = frame_index } };
440111928 },
......@@ -4405,7 +11932,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
440511932 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
440611933}
440711934
4408fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
11935fn airShlWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
440911936 const pt = self.pt;
441011937 const zcu = pt.zcu;
441111938 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -4444,7 +11971,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
444411971 const cc = Condition.ne;
444511972
444611973 const tuple_ty = self.typeOfIndex(inst);
4447 if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
11974 if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) {
444811975 switch (partial_mcv) {
444911976 .register => |reg| {
445011977 self.eflags_inst = inst;
......@@ -4453,8 +11980,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
445311980 else => {},
445411981 }
445511982
4456 const frame_index =
4457 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
11983 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
445811984 try self.genSetMem(
445911985 .{ .frame = frame_index },
446011986 @intCast(tuple_ty.structFieldOffset(1, zcu)),
......@@ -4473,7 +11999,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
447311999 }
447412000
447512001 const frame_index =
4476 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
12002 try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
447712003 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
447812004 break :result .{ .load_frame = .{ .index = frame_index } };
447912005 },
......@@ -4484,7 +12010,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
448412010}
448512011
448612012fn genSetFrameTruncatedOverflowCompare(
4487 self: *Self,
12013 self: *CodeGen,
448812014 tuple_ty: Type,
448912015 frame_index: FrameIndex,
449012016 src_mcv: MCValue,
......@@ -4499,6 +12025,7 @@ fn genSetFrameTruncatedOverflowCompare(
449912025 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
450012026
450112027 const ty = tuple_ty.fieldType(0, zcu);
12028 const ty_size = ty.abiSize(zcu);
450212029 const int_info = ty.intInfo(zcu);
450312030
450412031 const hi_bits = (int_info.bits - 1) % 64 + 1;
......@@ -4510,7 +12037,7 @@ fn genSetFrameTruncatedOverflowCompare(
451012037 const rest_ty = try pt.intType(.unsigned, int_info.bits - hi_bits);
451112038
451212039 const temp_regs =
4513 try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);
12040 try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp);
451412041 const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
451512042 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
451612043
......@@ -4530,13 +12057,15 @@ fn genSetFrameTruncatedOverflowCompare(
453012057 const eq_reg = temp_regs[2];
453112058 if (overflow_cc) |_| {
453212059 try self.asmSetccRegister(.ne, eq_reg.to8());
4533 try self.genBinOpMir(
4534 .{ ._, .@"or" },
4535 Type.u8,
4536 .{ .register = overflow_reg },
4537 .{ .register = eq_reg },
4538 );
12060 try self.genBinOpMir(.{ ._, .@"or" }, .u8, .{ .register = overflow_reg }, .{ .register = eq_reg });
453912061 }
12062 try self.genSetMem(
12063 .{ .frame = frame_index },
12064 @intCast(tuple_ty.structFieldOffset(1, zcu)),
12065 tuple_ty.fieldType(1, zcu),
12066 if (overflow_cc) |_| .{ .register = overflow_reg.to8() } else .{ .eflags = .ne },
12067 .{},
12068 );
454012069
454112070 const payload_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu));
454212071 if (hi_limb_off > 0) try self.genSetMem(
......@@ -4546,23 +12075,30 @@ fn genSetFrameTruncatedOverflowCompare(
454612075 src_mcv,
454712076 .{},
454812077 );
4549 try self.genSetMem(
4550 .{ .frame = frame_index },
4551 payload_off + hi_limb_off,
4552 limb_ty,
4553 .{ .register = scratch_reg },
4554 .{},
4555 );
4556 try self.genSetMem(
4557 .{ .frame = frame_index },
4558 @intCast(tuple_ty.structFieldOffset(1, zcu)),
4559 tuple_ty.fieldType(1, zcu),
4560 if (overflow_cc) |_| .{ .register = overflow_reg.to8() } else .{ .eflags = .ne },
12078 try self.genSetMem(
12079 .{ .frame = frame_index },
12080 payload_off + hi_limb_off,
12081 limb_ty,
12082 .{ .register = scratch_reg },
456112083 .{},
456212084 );
12085 var ext_off: i32 = hi_limb_off + 8;
12086 if (ext_off < ty_size) {
12087 switch (int_info.signedness) {
12088 .signed => try self.asmRegisterImmediate(.{ ._r, .sa }, scratch_reg.to64(), .s(63)),
12089 .unsigned => try self.asmRegisterRegister(.{ ._, .xor }, scratch_reg.to32(), scratch_reg.to32()),
12090 }
12091 while (ext_off < ty_size) : (ext_off += 8) try self.genSetMem(
12092 .{ .frame = frame_index },
12093 payload_off + ext_off,
12094 limb_ty,
12095 .{ .register = scratch_reg },
12096 .{},
12097 );
12098 }
456312099}
456412100
4565fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
12101fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
456612102 const pt = self.pt;
456712103 const zcu = pt.zcu;
456812104 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -4576,7 +12112,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
457612112 if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {
457712113 const slow_inc = self.hasFeature(.slow_incdec);
457812114 const abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
4579 const limb_len = math.divCeil(u32, abi_size, 8) catch unreachable;
12115 const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
458012116
458112117 try self.spillRegisters(&.{ .rax, .rcx, .rdx });
458212118 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });
......@@ -4593,7 +12129,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
459312129 const rhs_mcv = try self.resolveInst(bin_op.rhs);
459412130
459512131 const temp_regs =
4596 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
12132 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
459712133 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
459812134 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
459912135
......@@ -4618,7 +12154,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
461812154 try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);
461912155
462012156 const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
4621 try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, Immediate.u(1));
12157 try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1));
462212158 try self.asmMemoryRegister(.{ ._, .adc }, .{
462312159 .base = .{ .frame = dst_mcv.load_frame.index },
462412160 .mod = .{ .rm = .{
......@@ -4642,7 +12178,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
464212178 });
464312179 try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());
464412180
4645 try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, Immediate.u(1));
12181 try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1));
464612182 try self.asmMemoryRegister(.{ ._, .adc }, .{
464712183 .base = .{ .frame = dst_mcv.load_frame.index },
464812184 .mod = .{ .rm = .{
......@@ -4656,30 +12192,22 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
465612192 try self.asmSetccRegister(.c, .ch);
465712193
465812194 if (slow_inc) {
4659 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));
4660 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), Immediate.u(1));
12195 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
12196 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1));
466112197 } else {
466212198 try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
466312199 try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());
466412200 }
4665 try self.asmRegisterImmediate(
4666 .{ ._, .cmp },
4667 temp_regs[3].to32(),
4668 Immediate.u(limb_len),
4669 );
12201 try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len));
467012202 _ = try self.asmJccReloc(.b, inner_loop);
467112203
467212204 try self.asmRegisterRegister(.{ ._, .@"or" }, .rdx, .rcx);
467312205 const overflow = try self.asmJccReloc(.nz, undefined);
467412206 const overflow_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
4675 try self.asmRegisterImmediate(
4676 .{ ._, .cmp },
4677 temp_regs[2].to32(),
4678 Immediate.u(limb_len),
4679 );
12207 try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[2].to32(), .u(limb_len));
468012208 const no_overflow = try self.asmJccReloc(.nb, undefined);
468112209 if (slow_inc) {
4682 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));
12210 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
468312211 } else {
468412212 try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
468512213 }
......@@ -4691,7 +12219,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
469112219 .scale = .@"8",
469212220 .disp = lhs_mcv.load_frame.off - 8,
469312221 } },
4694 }, Immediate.u(0));
12222 }, .u(0));
469512223 _ = try self.asmJccReloc(.z, overflow_loop);
469612224 self.performReloc(overflow);
469712225 try self.asmMemoryImmediate(.{ ._, .mov }, .{
......@@ -4701,20 +12229,16 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
470112229 .disp = dst_mcv.load_frame.off +
470212230 @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
470312231 } },
4704 }, Immediate.u(1));
12232 }, .u(1));
470512233 self.performReloc(no_overflow);
470612234
470712235 self.performReloc(skip_inner);
470812236 if (slow_inc) {
4709 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));
12237 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
471012238 } else {
471112239 try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
471212240 }
4713 try self.asmRegisterImmediate(
4714 .{ ._, .cmp },
4715 temp_regs[0].to32(),
4716 Immediate.u(limb_len),
4717 );
12241 try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len));
471812242 _ = try self.asmJccReloc(.b, outer_loop);
471912243
472012244 break :result dst_mcv;
......@@ -4727,17 +12251,17 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
472712251 if (src_bits > 64 and src_bits <= 128 and
472812252 dst_info.bits > 64 and dst_info.bits <= 128) switch (dst_info.signedness) {
472912253 .signed => {
4730 const ptr_c_int = try pt.singleMutPtrType(Type.c_int);
4731 const overflow = try self.allocTempRegOrMem(Type.c_int, false);
12254 const ptr_c_int = try pt.singleMutPtrType(.c_int);
12255 const overflow = try self.allocTempRegOrMem(.c_int, false);
473212256 const result = try self.genCall(.{ .lib = .{
473312257 .return_type = .i128_type,
473412258 .param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() },
473512259 .callee = "__muloti4",
4736 } }, &.{ Type.i128, Type.i128, ptr_c_int }, &.{
12260 } }, &.{ .i128, .i128, ptr_c_int }, &.{
473712261 .{ .air_ref = bin_op.lhs },
473812262 .{ .air_ref = bin_op.rhs },
473912263 overflow.address(),
4740 });
12264 }, .{});
474112265
474212266 const dst_mcv = try self.allocRegOrMem(inst, false);
474312267 try self.genSetMem(
......@@ -4749,8 +12273,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
474912273 );
475012274 try self.asmMemoryImmediate(
475112275 .{ ._, .cmp },
4752 try overflow.mem(self, self.memSize(Type.c_int)),
4753 Immediate.s(0),
12276 try overflow.mem(self, .{ .size = self.memSize(.c_int) }),
12277 .s(0),
475412278 );
475512279 try self.genSetMem(
475612280 .{ .frame = dst_mcv.load_frame.index },
......@@ -4769,52 +12293,74 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
476912293 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
477012294
477112295 const tmp_regs =
4772 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
12296 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
477312297 const tmp_locks = self.register_manager.lockRegsAssumeUnused(4, tmp_regs);
477412298 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
477512299
477612300 const lhs_mcv = try self.resolveInst(bin_op.lhs);
477712301 const rhs_mcv = try self.resolveInst(bin_op.rhs);
4778 const mat_lhs_mcv = switch (lhs_mcv) {
4779 .load_symbol => mat_lhs_mcv: {
12302 const mat_lhs_mcv = mat_lhs_mcv: switch (lhs_mcv) {
12303 .register => |lhs_reg| switch (lhs_reg.class()) {
12304 else => lhs_mcv,
12305 .sse => {
12306 const mat_lhs_mcv: MCValue = .{
12307 .register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp),
12308 };
12309 try self.genCopy(dst_ty, mat_lhs_mcv, lhs_mcv, .{});
12310 break :mat_lhs_mcv mat_lhs_mcv;
12311 },
12312 },
12313 .load_symbol => {
478012314 // TODO clean this up!
4781 const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());
12315 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
478212316 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
478312317 },
478412318 else => lhs_mcv,
478512319 };
4786 const mat_lhs_lock = switch (mat_lhs_mcv) {
4787 .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg),
4788 else => null,
12320 const mat_lhs_locks: [2]?RegisterLock = switch (mat_lhs_mcv) {
12321 .register_pair => |mat_lhs_regs| self.register_manager.lockRegs(2, mat_lhs_regs),
12322 .indirect => |reg_off| .{ self.register_manager.lockReg(reg_off.reg), null },
12323 else => @splat(null),
478912324 };
4790 defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock);
4791 const mat_rhs_mcv = switch (rhs_mcv) {
4792 .load_symbol => mat_rhs_mcv: {
12325 defer for (mat_lhs_locks) |mat_lhs_lock| if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock);
12326 const mat_rhs_mcv = mat_rhs_mcv: switch (rhs_mcv) {
12327 .register => |rhs_reg| switch (rhs_reg.class()) {
12328 else => rhs_mcv,
12329 .sse => {
12330 const mat_rhs_mcv: MCValue = .{
12331 .register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp),
12332 };
12333 try self.genCopy(dst_ty, mat_rhs_mcv, rhs_mcv, .{});
12334 break :mat_rhs_mcv mat_rhs_mcv;
12335 },
12336 },
12337 .load_symbol => {
479312338 // TODO clean this up!
4794 const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());
12339 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
479512340 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
479612341 },
479712342 else => rhs_mcv,
479812343 };
4799 const mat_rhs_lock = switch (mat_rhs_mcv) {
4800 .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg),
4801 else => null,
12344 const mat_rhs_locks: [2]?RegisterLock = switch (mat_rhs_mcv) {
12345 .register_pair => |mat_rhs_regs| self.register_manager.lockRegs(2, mat_rhs_regs),
12346 .indirect => |reg_off| .{ self.register_manager.lockReg(reg_off.reg), null },
12347 else => @splat(null),
480212348 };
4803 defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
12349 defer for (mat_rhs_locks) |mat_rhs_lock| if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
480412350
4805 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
12351 if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
480612352 .{ ._, .mov },
480712353 .rax,
4808 try mat_lhs_mcv.mem(self, .qword),
12354 try mat_lhs_mcv.mem(self, .{ .size = .qword }),
480912355 ) else try self.asmRegisterRegister(
481012356 .{ ._, .mov },
481112357 .rax,
481212358 mat_lhs_mcv.register_pair[0],
481312359 );
4814 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
12360 if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
481512361 .{ ._, .mov },
481612362 tmp_regs[0],
4817 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
12363 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
481812364 ) else try self.asmRegisterRegister(
481912365 .{ ._, .mov },
482012366 tmp_regs[0],
......@@ -4824,17 +12370,17 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
482412370 try self.asmSetccRegister(.nz, tmp_regs[1].to8());
482512371 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);
482612372 try self.asmSetccRegister(.o, tmp_regs[2].to8());
4827 if (mat_rhs_mcv.isMemory())
4828 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
12373 if (mat_rhs_mcv.isBase())
12374 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
482912375 else
483012376 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
483112377 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
483212378 try self.asmSetccRegister(.c, tmp_regs[3].to8());
483312379 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[2].to8(), tmp_regs[3].to8());
4834 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
12380 if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
483512381 .{ ._, .mov },
483612382 tmp_regs[0],
4837 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
12383 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
483812384 ) else try self.asmRegisterRegister(
483912385 .{ ._, .mov },
484012386 tmp_regs[0],
......@@ -4848,10 +12394,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
484812394 tmp_regs[3].to8(),
484912395 );
485012396 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
4851 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
12397 if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
485212398 .{ .i_, .mul },
485312399 tmp_regs[0],
4854 try mat_rhs_mcv.mem(self, .qword),
12400 try mat_rhs_mcv.mem(self, .{ .size = .qword }),
485512401 ) else try self.asmRegisterRegister(
485612402 .{ .i_, .mul },
485712403 tmp_regs[0],
......@@ -4906,7 +12452,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
490612452 self.eflags_inst = inst;
490712453 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
490812454 } else {
4909 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
12455 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
491012456 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
491112457 break :result .{ .load_frame = .{ .index = frame_index } };
491212458 },
......@@ -4917,7 +12463,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
491712463 src_ty.fmt(pt), dst_ty.fmt(pt),
491812464 });
491912465
4920 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
12466 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
492112467 if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
492212468 try self.genSetMem(
492312469 .{ .frame = frame_index },
......@@ -4951,7 +12497,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
495112497/// Generates signed or unsigned integer multiplication/division.
495212498/// Clobbers .rax and .rdx registers.
495312499/// Quotient is saved in .rax and remainder in .rdx.
4954fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
12500fn genIntMulDivOpMir(self: *CodeGen, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
495512501 const pt = self.pt;
495612502 const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));
495712503 const bit_size: u32 = @intCast(self.regBitSize(ty));
......@@ -4992,7 +12538,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
499212538 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),
499312539 .memory, .indirect, .load_frame => try self.asmMemory(
499412540 tag,
4995 try mat_rhs.mem(self, Memory.Size.fromSize(abi_size)),
12541 try mat_rhs.mem(self, .{ .size = .fromSize(abi_size) }),
499612542 ),
499712543 else => unreachable,
499812544 }
......@@ -5001,7 +12547,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
500112547
500212548/// Always returns a register.
500312549/// Clobbers .rax and .rdx registers.
5004fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
12550fn genInlineIntDivFloor(self: *CodeGen, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
500512551 const pt = self.pt;
500612552 const zcu = pt.zcu;
500712553 const abi_size: u32 = @intCast(ty.abiSize(zcu));
......@@ -5038,7 +12584,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
503812584 try self.asmRegisterImmediate(
503912585 .{ ._r, .sa },
504012586 registerAlias(divisor, abi_size),
5041 Immediate.u(int_info.bits - 1),
12587 .u(int_info.bits - 1),
504212588 );
504312589 try self.asmRegisterRegister(
504412590 .{ ._, .@"test" },
......@@ -5054,7 +12600,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
505412600 return MCValue{ .register = divisor };
505512601}
505612602
5057fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
12603fn airShlShrBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
505812604 const pt = self.pt;
505912605 const zcu = pt.zcu;
506012606 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -5084,7 +12630,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
508412630 defer self.register_manager.unlockReg(tmp_lock);
508512631
508612632 const lhs_bits: u31 = @intCast(lhs_ty.bitSize(zcu));
5087 const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;
12633 const tmp_ty: Type = if (lhs_bits > 64) .usize else lhs_ty;
508812634 const off = frame_addr.off + (lhs_bits - 1) / 64 * 8;
508912635 try self.genSetReg(
509012636 tmp_reg,
......@@ -5217,8 +12763,8 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
521712763 defer for (reg_locks) |reg_lock| if (reg_lock) |lock|
521812764 self.register_manager.unlockReg(lock);
521912765
5220 const shift_imm =
5221 Immediate.u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu)));
12766 const shift_imm: Immediate =
12767 .u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu)));
522212768 if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate(
522312769 mir_tag,
522412770 registerAlias(dst_reg, abi_size),
......@@ -5264,16 +12810,15 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
526412810 defer self.register_manager.unlockReg(shift_lock);
526512811
526612812 const mask_ty = try pt.vectorType(.{ .len = 16, .child = .u8_type });
5267 const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
12813 const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
526812814 .ty = mask_ty.toIntern(),
526912815 .storage = .{ .elems = &([1]InternPool.Index{
5270 (try rhs_ty.childType(zcu).maxIntScalar(pt, Type.u8)).toIntern(),
12816 (try rhs_ty.childType(zcu).maxIntScalar(pt, .u8)).toIntern(),
527112817 } ++ [1]InternPool.Index{
5272 (try pt.intValue(Type.u8, 0)).toIntern(),
12818 (try pt.intValue(.u8, 0)).toIntern(),
527312819 } ** 15) },
527412820 } })));
5275 const mask_addr_reg =
5276 try self.copyToTmpRegister(Type.usize, mask_mcv.address());
12821 const mask_addr_reg = try self.copyToTmpRegister(.usize, mask_mcv.address());
527712822 const mask_addr_lock = self.register_manager.lockRegAssumeUnused(mask_addr_reg);
527812823 defer self.register_manager.unlockReg(mask_addr_lock);
527912824
......@@ -5322,14 +12867,14 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
532212867 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
532312868}
532412869
5325fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
12870fn airShlSat(self: *CodeGen, inst: Air.Inst.Index) !void {
532612871 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
532712872 _ = bin_op;
532812873 return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
532912874 //return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
533012875}
533112876
5332fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
12877fn airOptionalPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
533312878 const zcu = self.pt.zcu;
533412879 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
533512880 const result: MCValue = result: {
......@@ -5362,7 +12907,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
536212907 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
536312908}
536412909
5365fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
12910fn airOptionalPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
536612911 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
536712912
536812913 const dst_ty = self.typeOfIndex(inst);
......@@ -5375,7 +12920,7 @@ fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
537512920 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
537612921}
537712922
5378fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
12923fn airOptionalPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void {
537912924 const pt = self.pt;
538012925 const zcu = pt.zcu;
538112926 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -5407,7 +12952,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
540712952 try self.genSetMem(
540812953 .{ .reg = dst_mcv.getReg().? },
540912954 pl_abi_size,
5410 Type.bool,
12955 .bool,
541112956 .{ .immediate = 1 },
541212957 .{},
541312958 );
......@@ -5416,7 +12961,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
541612961 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
541712962}
541812963
5419fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
12964fn airUnwrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void {
542012965 const pt = self.pt;
542112966 const zcu = pt.zcu;
542212967 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -5434,7 +12979,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
543412979 break :result operand;
543512980 }
543612981
5437 const err_off = errUnionErrorOffset(payload_ty, zcu);
12982 const err_off = codegen.errUnionErrorOffset(payload_ty, zcu);
543812983 switch (operand) {
543912984 .register => |reg| {
544012985 // TODO reuse operand
......@@ -5446,9 +12991,9 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
544612991 .{ ._r, .sh },
544712992 err_union_ty,
544812993 result,
5449 Type.u8,
12994 .u8,
545012995 .{ .immediate = @as(u6, @intCast(err_off * 8)) },
5451 ) else try self.truncateRegister(Type.anyerror, result.register);
12996 ) else try self.truncateRegister(.anyerror, result.register);
545212997 break :result result;
545312998 },
545412999 .load_frame => |frame_addr| break :result .{ .load_frame = .{
......@@ -5461,7 +13006,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
546113006 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
546213007}
546313008
5464fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
13009fn airUnwrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
546513010 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
546613011 const operand_ty = self.typeOf(ty_op.operand);
546713012 const operand = try self.resolveInst(ty_op.operand);
......@@ -5470,7 +13015,7 @@ fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
547013015}
547113016
547213017// *(E!T) -> E
5473fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
13018fn airUnwrapErrUnionErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
547413019 const pt = self.pt;
547513020 const zcu = pt.zcu;
547613021 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -5492,7 +13037,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
549213037 const eu_ty = src_ty.childType(zcu);
549313038 const pl_ty = eu_ty.errorUnionPayload(zcu);
549413039 const err_ty = eu_ty.errorUnionSet(zcu);
5495 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
13040 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
549613041 const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));
549713042 try self.asmRegisterMemory(
549813043 .{ ._, .mov },
......@@ -5500,7 +13045,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
550013045 .{
550113046 .base = .{ .reg = src_reg },
550213047 .mod = .{ .rm = .{
5503 .size = Memory.Size.fromSize(err_abi_size),
13048 .size = .fromSize(err_abi_size),
550413049 .disp = err_off,
550513050 } },
550613051 },
......@@ -5510,7 +13055,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
551013055}
551113056
551213057// *(E!T) -> *T
5513fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
13058fn airUnwrapErrUnionPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
551413059 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
551513060 const operand_ty = self.typeOf(ty_op.operand);
551613061 const operand = try self.resolveInst(ty_op.operand);
......@@ -5518,7 +13063,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
551813063 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
551913064}
552013065
5521fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
13066fn airErrUnionPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void {
552213067 const pt = self.pt;
552313068 const zcu = pt.zcu;
552413069 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -5535,18 +13080,18 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
553513080 const eu_ty = src_ty.childType(zcu);
553613081 const pl_ty = eu_ty.errorUnionPayload(zcu);
553713082 const err_ty = eu_ty.errorUnionSet(zcu);
5538 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
13083 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
553913084 const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));
554013085 try self.asmMemoryImmediate(
554113086 .{ ._, .mov },
554213087 .{
554313088 .base = .{ .reg = src_reg },
554413089 .mod = .{ .rm = .{
5545 .size = Memory.Size.fromSize(err_abi_size),
13090 .size = .fromSize(err_abi_size),
554613091 .disp = err_off,
554713092 } },
554813093 },
5549 Immediate.u(0),
13094 .u(0),
555013095 );
555113096
555213097 if (self.liveness.isUnused(inst)) break :result .unreach;
......@@ -5559,7 +13104,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
555913104 const dst_lock = self.register_manager.lockReg(dst_reg);
556013105 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
556113106
5562 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
13107 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
556313108 const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
556413109 try self.asmRegisterMemory(
556513110 .{ ._, .lea },
......@@ -5575,7 +13120,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
557513120}
557613121
557713122fn genUnwrapErrUnionPayloadMir(
5578 self: *Self,
13123 self: *CodeGen,
557913124 maybe_inst: ?Air.Inst.Index,
558013125 err_union_ty: Type,
558113126 err_union: MCValue,
......@@ -5587,7 +13132,7 @@ fn genUnwrapErrUnionPayloadMir(
558713132 const result: MCValue = result: {
558813133 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
558913134
5590 const payload_off: u31 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
13135 const payload_off: u31 = @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu));
559113136 switch (err_union) {
559213137 .load_frame => |frame_addr| break :result .{ .load_frame = .{
559313138 .index = frame_addr.index,
......@@ -5598,7 +13143,7 @@ fn genUnwrapErrUnionPayloadMir(
559813143 const eu_lock = self.register_manager.lockReg(reg);
559913144 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
560013145
5601 const payload_in_gp = self.regClassForType(payload_ty).supersetOf(abi.RegisterClass.gp);
13146 const payload_in_gp = self.regSetForType(payload_ty).supersetOf(abi.RegisterClass.gp);
560213147 const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)
560313148 try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)
560413149 else
......@@ -5607,7 +13152,7 @@ fn genUnwrapErrUnionPayloadMir(
560713152 .{ ._r, .sh },
560813153 err_union_ty,
560913154 result_mcv,
5610 Type.u8,
13155 .u8,
561113156 .{ .immediate = @as(u6, @intCast(payload_off * 8)) },
561213157 ) else try self.truncateRegister(payload_ty, result_mcv.register);
561313158 break :result if (payload_in_gp)
......@@ -5625,7 +13170,7 @@ fn genUnwrapErrUnionPayloadMir(
562513170}
562613171
562713172fn genUnwrapErrUnionPayloadPtrMir(
5628 self: *Self,
13173 self: *CodeGen,
562913174 maybe_inst: ?Air.Inst.Index,
563013175 ptr_ty: Type,
563113176 ptr_mcv: MCValue,
......@@ -5636,7 +13181,7 @@ fn genUnwrapErrUnionPayloadPtrMir(
563613181 const payload_ty = err_union_ty.errorUnionPayload(zcu);
563713182
563813183 const result: MCValue = result: {
5639 const payload_off = errUnionPayloadOffset(payload_ty, zcu);
13184 const payload_off = codegen.errUnionPayloadOffset(payload_ty, zcu);
564013185 const result_mcv: MCValue = if (maybe_inst) |inst|
564113186 try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr_mcv)
564213187 else
......@@ -5648,23 +13193,23 @@ fn genUnwrapErrUnionPayloadPtrMir(
564813193 return result;
564913194}
565013195
5651fn airErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {
13196fn airErrReturnTrace(self: *CodeGen, inst: Air.Inst.Index) !void {
565213197 _ = inst;
565313198 return self.fail("TODO implement airErrReturnTrace for {}", .{self.target.cpu.arch});
565413199 //return self.finishAir(inst, result, .{ .none, .none, .none });
565513200}
565613201
5657fn airSetErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {
13202fn airSetErrReturnTrace(self: *CodeGen, inst: Air.Inst.Index) !void {
565813203 _ = inst;
565913204 return self.fail("TODO implement airSetErrReturnTrace for {}", .{self.target.cpu.arch});
566013205}
566113206
5662fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) !void {
13207fn airSaveErrReturnTraceIndex(self: *CodeGen, inst: Air.Inst.Index) !void {
566313208 _ = inst;
566413209 return self.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{self.target.cpu.arch});
566513210}
566613211
5667fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
13212fn airWrapOptional(self: *CodeGen, inst: Air.Inst.Index) !void {
566813213 const pt = self.pt;
566913214 const zcu = pt.zcu;
567013215 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -5696,7 +13241,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
569613241 try self.asmRegisterImmediate(
569713242 .{ ._s, .bt },
569813243 opt_reg,
5699 Immediate.u(@as(u6, @intCast(pl_abi_size * 8))),
13244 .u(@as(u6, @intCast(pl_abi_size * 8))),
570013245 );
570113246 },
570213247
......@@ -5709,7 +13254,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
570913254 .disp = frame_addr.off + pl_abi_size,
571013255 } },
571113256 },
5712 Immediate.u(1),
13257 .u(1),
571313258 ),
571413259 }
571513260 }
......@@ -5719,7 +13264,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
571913264}
572013265
572113266/// T to E!T
5722fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
13267fn airWrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
572313268 const pt = self.pt;
572413269 const zcu = pt.zcu;
572513270 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -5732,9 +13277,9 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
573213277 const result: MCValue = result: {
573313278 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 };
573413279
5735 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
5736 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
5737 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
13280 const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu));
13281 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
13282 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
573813283 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{});
573913284 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }, .{});
574013285 break :result .{ .load_frame = .{ .index = frame_index } };
......@@ -5743,7 +13288,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
574313288}
574413289
574513290/// E to E!T
5746fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
13291fn airWrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void {
574713292 const pt = self.pt;
574813293 const zcu = pt.zcu;
574913294 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -5755,9 +13300,9 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
575513300 const result: MCValue = result: {
575613301 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand);
575713302
5758 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
5759 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
5760 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
13303 const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu));
13304 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
13305 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
576113306 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{});
576213307 const operand = try self.resolveInst(ty_op.operand);
576313308 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand, .{});
......@@ -5766,44 +13311,58 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
576613311 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
576713312}
576813313
5769fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
13314fn airSlicePtr(self: *CodeGen, inst: Air.Inst.Index) !void {
577013315 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
577113316 const result = result: {
577213317 const src_mcv = try self.resolveInst(ty_op.operand);
5773 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
13318 const ptr_mcv: MCValue = switch (src_mcv) {
13319 .register_pair => |regs| .{ .register = regs[0] },
13320 else => src_mcv,
13321 };
13322 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
13323 switch (src_mcv) {
13324 .register_pair => |regs| try self.freeValue(.{ .register = regs[1] }),
13325 else => {},
13326 }
13327 break :result ptr_mcv;
13328 }
577413329
577513330 const dst_mcv = try self.allocRegOrMem(inst, true);
5776 const dst_ty = self.typeOfIndex(inst);
5777 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
13331 try self.genCopy(self.typeOfIndex(inst), dst_mcv, ptr_mcv, .{});
577813332 break :result dst_mcv;
577913333 };
578013334 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
578113335}
578213336
5783fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
13337fn airSliceLen(self: *CodeGen, inst: Air.Inst.Index) !void {
578413338 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5785
5786 const result: MCValue = result: {
13339 const result = result: {
578713340 const src_mcv = try self.resolveInst(ty_op.operand);
5788 switch (src_mcv) {
5789 .load_frame => |frame_addr| {
5790 const len_mcv: MCValue = .{ .load_frame = .{
5791 .index = frame_addr.index,
5792 .off = frame_addr.off + 8,
5793 } };
5794 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;
5795
5796 const dst_mcv = try self.allocRegOrMem(inst, true);
5797 try self.genCopy(Type.usize, dst_mcv, len_mcv, .{});
5798 break :result dst_mcv;
5799 },
13341 const len_mcv: MCValue = switch (src_mcv) {
13342 .register_pair => |regs| .{ .register = regs[1] },
13343 .load_frame => |frame_addr| .{ .load_frame = .{
13344 .index = frame_addr.index,
13345 .off = frame_addr.off + 8,
13346 } },
580013347 else => return self.fail("TODO implement slice_len for {}", .{src_mcv}),
13348 };
13349 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
13350 switch (src_mcv) {
13351 .register_pair => |regs| try self.freeValue(.{ .register = regs[0] }),
13352 .load_frame => {},
13353 else => unreachable,
13354 }
13355 break :result len_mcv;
580113356 }
13357
13358 const dst_mcv = try self.allocRegOrMem(inst, true);
13359 try self.genCopy(self.typeOfIndex(inst), dst_mcv, len_mcv, .{});
13360 break :result dst_mcv;
580213361 };
580313362 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
580413363}
580513364
5806fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
13365fn airPtrSliceLenPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
580713366 const pt = self.pt;
580813367 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
580913368
......@@ -5838,7 +13397,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
583813397 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
583913398}
584013399
5841fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
13400fn airPtrSlicePtrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
584213401 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
584313402
584413403 const dst_ty = self.typeOfIndex(inst);
......@@ -5851,7 +13410,7 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
585113410 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
585213411}
585313412
5854fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Register {
13413fn elemOffset(self: *CodeGen, index_ty: Type, index: MCValue, elem_size: u64) !Register {
585513414 const reg: Register = blk: {
585613415 switch (index) {
585713416 .immediate => |imm| {
......@@ -5871,7 +13430,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
587113430 return reg;
587213431}
587313432
5874fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
13433fn genSliceElemPtr(self: *CodeGen, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
587513434 const pt = self.pt;
587613435 const zcu = pt.zcu;
587713436 const slice_ty = self.typeOf(lhs);
......@@ -5899,7 +13458,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
589913458 defer self.register_manager.unlockReg(offset_reg_lock);
590013459
590113460 const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5902 try self.genSetReg(addr_reg, Type.usize, slice_mcv, .{});
13461 try self.genSetReg(addr_reg, .usize, slice_mcv, .{});
590313462 // TODO we could allocate register here, but need to expect addr register and potentially
590413463 // offset register.
590513464 try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{
......@@ -5908,7 +13467,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
590813467 return MCValue{ .register = addr_reg.to64() };
590913468}
591013469
5911fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
13470fn airSliceElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
591213471 const pt = self.pt;
591313472 const zcu = pt.zcu;
591413473 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -5927,14 +13486,14 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
592713486 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
592813487}
592913488
5930fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
13489fn airSliceElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
593113490 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
593213491 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
593313492 const dst_mcv = try self.genSliceElemPtr(extra.lhs, extra.rhs);
593413493 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
593513494}
593613495
5937fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
13496fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
593813497 const pt = self.pt;
593913498 const zcu = pt.zcu;
594013499 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -5952,7 +13511,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
595213511
595313512 const index_ty = self.typeOf(bin_op.rhs);
595413513 const index_mcv = try self.resolveInst(bin_op.rhs);
5955 const index_lock: ?RegisterLock = switch (index_mcv) {
13514 const index_lock = switch (index_mcv) {
595613515 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
595713516 else => null,
595813517 };
......@@ -5960,48 +13519,111 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
596013519
596113520 try self.spillEflagsIfOccupied();
596213521 if (array_ty.isVector(zcu) and elem_ty.bitSize(zcu) == 1) {
5963 const index_reg = switch (index_mcv) {
5964 .register => |reg| reg,
5965 else => try self.copyToTmpRegister(index_ty, index_mcv),
13522 const array_mat_mcv: MCValue = switch (array_mcv) {
13523 else => array_mcv,
13524 .register_mask => .{ .register = try self.copyToTmpRegister(array_ty, array_mcv) },
13525 };
13526 const array_mat_lock = switch (array_mat_mcv) {
13527 .register => |reg| self.register_manager.lockReg(reg),
13528 else => null,
596613529 };
5967 switch (array_mcv) {
13530 defer if (array_mat_lock) |lock| self.register_manager.unlockReg(lock);
13531
13532 switch (array_mat_mcv) {
596813533 .register => |array_reg| switch (array_reg.class()) {
5969 .general_purpose => try self.asmRegisterRegister(
5970 .{ ._, .bt },
5971 array_reg.to64(),
5972 index_reg.to64(),
5973 ),
5974 .sse => {
5975 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, zcu));
5976 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});
5977 try self.asmMemoryRegister(
13534 .general_purpose => switch (index_mcv) {
13535 .immediate => |index_imm| try self.asmRegisterImmediate(
597813536 .{ ._, .bt },
5979 .{
5980 .base = .{ .frame = frame_index },
5981 .mod = .{ .rm = .{ .size = .qword } },
5982 },
5983 index_reg.to64(),
5984 );
13537 array_reg.to64(),
13538 .u(index_imm),
13539 ),
13540 else => try self.asmRegisterRegister(
13541 .{ ._, .bt },
13542 array_reg.to64(),
13543 switch (index_mcv) {
13544 .register => |index_reg| index_reg,
13545 else => try self.copyToTmpRegister(index_ty, index_mcv),
13546 }.to64(),
13547 ),
13548 },
13549 .sse => {
13550 const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu));
13551 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mat_mcv, .{});
13552 switch (index_mcv) {
13553 .immediate => |index_imm| try self.asmMemoryImmediate(
13554 .{ ._, .bt },
13555 .{
13556 .base = .{ .frame = frame_index },
13557 .mod = .{ .rm = .{
13558 .size = .qword,
13559 .disp = @intCast(index_imm / 64 * 8),
13560 } },
13561 },
13562 .u(index_imm % 64),
13563 ),
13564 else => try self.asmMemoryRegister(
13565 .{ ._, .bt },
13566 .{
13567 .base = .{ .frame = frame_index },
13568 .mod = .{ .rm = .{ .size = .qword } },
13569 },
13570 switch (index_mcv) {
13571 .register => |index_reg| index_reg,
13572 else => try self.copyToTmpRegister(index_ty, index_mcv),
13573 }.to64(),
13574 ),
13575 }
598513576 },
598613577 else => unreachable,
598713578 },
5988 .load_frame => try self.asmMemoryRegister(
5989 .{ ._, .bt },
5990 try array_mcv.mem(self, .qword),
5991 index_reg.to64(),
5992 ),
5993 .memory, .load_symbol, .load_direct, .load_got, .load_tlv => try self.asmMemoryRegister(
5994 .{ ._, .bt },
5995 .{
5996 .base = .{
5997 .reg = try self.copyToTmpRegister(Type.usize, array_mcv.address()),
13579 .load_frame => switch (index_mcv) {
13580 .immediate => |index_imm| try self.asmMemoryImmediate(
13581 .{ ._, .bt },
13582 try array_mat_mcv.mem(self, .{
13583 .size = .qword,
13584 .disp = @intCast(index_imm / 64 * 8),
13585 }),
13586 .u(index_imm % 64),
13587 ),
13588 else => try self.asmMemoryRegister(
13589 .{ ._, .bt },
13590 try array_mat_mcv.mem(self, .{ .size = .qword }),
13591 switch (index_mcv) {
13592 .register => |index_reg| index_reg,
13593 else => try self.copyToTmpRegister(index_ty, index_mcv),
13594 }.to64(),
13595 ),
13596 },
13597 .memory, .load_symbol, .load_direct, .load_got, .load_tlv => switch (index_mcv) {
13598 .immediate => |index_imm| try self.asmMemoryImmediate(
13599 .{ ._, .bt },
13600 .{
13601 .base = .{
13602 .reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()),
13603 },
13604 .mod = .{ .rm = .{
13605 .size = .qword,
13606 .disp = @intCast(index_imm / 64 * 8),
13607 } },
599813608 },
5999 .mod = .{ .rm = .{ .size = .qword } },
6000 },
6001 index_reg.to64(),
6002 ),
13609 .u(index_imm % 64),
13610 ),
13611 else => try self.asmMemoryRegister(
13612 .{ ._, .bt },
13613 .{
13614 .base = .{
13615 .reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()),
13616 },
13617 .mod = .{ .rm = .{ .size = .qword } },
13618 },
13619 switch (index_mcv) {
13620 .register => |index_reg| index_reg,
13621 else => try self.copyToTmpRegister(index_ty, index_mcv),
13622 }.to64(),
13623 ),
13624 },
600313625 else => return self.fail("TODO airArrayElemVal for {s} of {}", .{
6004 @tagName(array_mcv), array_ty.fmt(pt),
13626 @tagName(array_mat_mcv), array_ty.fmt(pt),
600513627 }),
600613628 }
600713629
......@@ -6017,7 +13639,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
601713639
601813640 switch (array_mcv) {
601913641 .register => {
6020 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, zcu));
13642 const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu));
602113643 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});
602213644 try self.asmRegisterMemory(
602313645 .{ ._, .lea },
......@@ -6038,7 +13660,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
603813660 .load_direct,
603913661 .load_got,
604013662 .load_tlv,
6041 => try self.genSetReg(addr_reg, Type.usize, array_mcv.address(), .{}),
13663 => try self.genSetReg(addr_reg, .usize, array_mcv.address(), .{}),
604213664 .lea_symbol, .lea_direct, .lea_tlv => unreachable,
604313665 else => return self.fail("TODO airArrayElemVal_val for {s} of {}", .{
604413666 @tagName(array_mcv), array_ty.fmt(pt),
......@@ -6052,19 +13674,14 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
605213674 // TODO we could allocate register here, but need to expect addr register and potentially
605313675 // offset register.
605413676 const dst_mcv = try self.allocRegOrMem(inst, false);
6055 try self.genBinOpMir(
6056 .{ ._, .add },
6057 Type.usize,
6058 .{ .register = addr_reg },
6059 .{ .register = offset_reg },
6060 );
13677 try self.genBinOpMir(.{ ._, .add }, .usize, .{ .register = addr_reg }, .{ .register = offset_reg });
606113678 try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }, .{});
606213679 break :result dst_mcv;
606313680 };
606413681 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
606513682}
606613683
6067fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
13684fn airPtrElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
606813685 const pt = self.pt;
606913686 const zcu = pt.zcu;
607013687 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -6115,7 +13732,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
611513732 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
611613733}
611713734
6118fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
13735fn airPtrElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
611913736 const pt = self.pt;
612013737 const zcu = pt.zcu;
612113738 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -6161,7 +13778,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
616113778 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
616213779}
616313780
6164fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
13781fn airSetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void {
616513782 const pt = self.pt;
616613783 const zcu = pt.zcu;
616713784 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -6206,7 +13823,7 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
620613823 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
620713824}
620813825
6209fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
13826fn airGetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void {
621013827 const zcu = self.pt.zcu;
621113828 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
621213829
......@@ -6245,13 +13862,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
624513862 .register => {
624613863 const shift: u6 = @intCast(layout.tagOffset() * 8);
624713864 const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
6248 try self.genShiftBinOpMir(
6249 .{ ._r, .sh },
6250 Type.usize,
6251 result,
6252 Type.u8,
6253 .{ .immediate = shift },
6254 );
13865 try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, result, .u8, .{ .immediate = shift });
625513866 break :blk MCValue{
625613867 .register = registerAlias(result.register, @intCast(layout.tag_size)),
625713868 };
......@@ -6263,7 +13874,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
626313874 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
626413875}
626513876
6266fn airClz(self: *Self, inst: Air.Inst.Index) !void {
13877fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
626713878 const pt = self.pt;
626813879 const zcu = pt.zcu;
626913880 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -6296,53 +13907,62 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
629613907 const src_bits: u31 = @intCast(src_ty.bitSize(zcu));
629713908 const has_lzcnt = self.hasFeature(.lzcnt);
629813909 if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) {
6299 const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
13910 const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) {
13911 .load_frame => |src_frame_addr| src_frame_addr,
13912 else => {
13913 const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu));
13914 try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{});
13915 break :src_frame_addr .{ .index = src_frame_addr };
13916 },
13917 };
13918
13919 const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
630013920 const extra_bits = abi_size * 8 - src_bits;
630113921
630213922 const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
630313923 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
630413924 defer self.register_manager.unlockReg(index_lock);
630513925
6306 try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.u(limbs_len));
13926 try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .u(limbs_len));
630713927 switch (extra_bits) {
630813928 1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
630913929 else => try self.asmRegisterImmediate(
631013930 .{ ._, .mov },
631113931 dst_reg.to32(),
6312 Immediate.s(@as(i32, extra_bits) - 1),
13932 .s(@as(i32, extra_bits) - 1),
631313933 ),
631413934 }
631513935 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
631613936 try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32());
631713937 const zero = try self.asmJccReloc(.z, undefined);
631813938 if (self.hasFeature(.slow_incdec)) {
6319 try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), Immediate.u(1));
13939 try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), .u(1));
632013940 } else {
632113941 try self.asmRegister(.{ ._, .dec }, index_reg.to32());
632213942 }
632313943 try self.asmMemoryImmediate(.{ ._, .cmp }, .{
6324 .base = .{ .frame = src_mcv.load_frame.index },
13944 .base = .{ .frame = src_frame_addr.index },
632513945 .mod = .{ .rm = .{
632613946 .size = .qword,
632713947 .index = index_reg.to64(),
632813948 .scale = .@"8",
6329 .disp = src_mcv.load_frame.off,
13949 .disp = src_frame_addr.off,
633013950 } },
6331 }, Immediate.u(0));
13951 }, .u(0));
633213952 _ = try self.asmJccReloc(.e, loop);
6333 try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{
6334 .base = .{ .frame = src_mcv.load_frame.index },
13953 try self.asmRegisterMemory(.{ ._r, .bs }, dst_reg.to64(), .{
13954 .base = .{ .frame = src_frame_addr.index },
633513955 .mod = .{ .rm = .{
633613956 .size = .qword,
633713957 .index = index_reg.to64(),
633813958 .scale = .@"8",
6339 .disp = src_mcv.load_frame.off,
13959 .disp = src_frame_addr.off,
634013960 } },
634113961 });
634213962 self.performReloc(zero);
6343 try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));
13963 try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6));
634413964 try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32());
6345 try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(src_bits - 1));
13965 try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(src_bits - 1));
634613966 try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32());
634713967 break :result dst_mcv;
634813968 }
......@@ -6351,7 +13971,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
635113971 if (src_bits <= 8) {
635213972 const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
635313973 try self.truncateRegister(src_ty, wide_reg);
6354 try self.genBinOpMir(.{ ._, .lzcnt }, Type.u32, dst_mcv, .{ .register = wide_reg });
13974 try self.genBinOpMir(.{ ._, .lzcnt }, .u32, dst_mcv, .{ .register = wide_reg });
635513975 try self.genBinOpMir(
635613976 .{ ._, .sub },
635713977 dst_ty,
......@@ -6371,25 +13991,15 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
637113991 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
637213992 defer self.register_manager.unlockReg(tmp_lock);
637313993
6374 try self.genBinOpMir(
6375 .{ ._, .lzcnt },
6376 Type.u64,
6377 dst_mcv,
6378 if (mat_src_mcv.isMemory())
6379 mat_src_mcv
6380 else
6381 .{ .register = mat_src_mcv.register_pair[0] },
6382 );
13994 try self.genBinOpMir(.{ ._, .lzcnt }, .u64, dst_mcv, if (mat_src_mcv.isBase())
13995 mat_src_mcv
13996 else
13997 .{ .register = mat_src_mcv.register_pair[0] });
638313998 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
6384 try self.genBinOpMir(
6385 .{ ._, .lzcnt },
6386 Type.u64,
6387 tmp_mcv,
6388 if (mat_src_mcv.isMemory())
6389 mat_src_mcv.address().offset(8).deref()
6390 else
6391 .{ .register = mat_src_mcv.register_pair[1] },
6392 );
13999 try self.genBinOpMir(.{ ._, .lzcnt }, .u64, tmp_mcv, if (mat_src_mcv.isBase())
14000 mat_src_mcv.address().offset(8).deref()
14001 else
14002 .{ .register = mat_src_mcv.register_pair[1] });
639314003 try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
639414004
639514005 if (src_bits < 128) try self.genBinOpMir(
......@@ -6404,7 +14014,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
640414014
640514015 assert(src_bits <= 64);
640614016 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);
6407 if (math.isPowerOfTwo(src_bits)) {
14017 if (std.math.isPowerOfTwo(src_bits)) {
640814018 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
640914019 .immediate = src_bits ^ (src_bits - 1),
641014020 });
......@@ -6417,8 +14027,8 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
641714027 defer self.register_manager.unlockReg(wide_lock);
641814028
641914029 try self.truncateRegister(src_ty, wide_reg);
6420 try self.genBinOpMir(.{ ._, .bsr }, Type.u16, dst_mcv, .{ .register = wide_reg });
6421 } else try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
14030 try self.genBinOpMir(.{ ._r, .bs }, .u16, dst_mcv, .{ .register = wide_reg });
14031 } else try self.genBinOpMir(.{ ._r, .bs }, src_ty, dst_mcv, mat_src_mcv);
642214032
642314033 try self.asmCmovccRegisterRegister(
642414034 .z,
......@@ -6429,7 +14039,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
642914039 try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
643014040 } else {
643114041 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
6432 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)),
14042 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)),
643314043 });
643414044 const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg);
643514045 defer self.register_manager.unlockReg(imm_lock);
......@@ -6440,8 +14050,8 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
644014050
644114051 try self.truncateRegister(src_ty, wide_reg);
644214052 try self.genBinOpMir(
6443 .{ ._, .bsr },
6444 if (src_bits <= 8) Type.u16 else src_ty,
14053 .{ ._r, .bs },
14054 if (src_bits <= 8) .u16 else src_ty,
644514055 dst_mcv,
644614056 .{ .register = wide_reg },
644714057 );
......@@ -6460,7 +14070,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
646014070 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
646114071}
646214072
6463fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
14073fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void {
646414074 const pt = self.pt;
646514075 const zcu = pt.zcu;
646614076 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -6493,57 +14103,66 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
649314103 const src_bits: u31 = @intCast(src_ty.bitSize(zcu));
649414104 const has_bmi = self.hasFeature(.bmi);
649514105 if (src_bits > @as(u32, if (has_bmi) 128 else 64)) {
6496 const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
14106 const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) {
14107 .load_frame => |src_frame_addr| src_frame_addr,
14108 else => {
14109 const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu));
14110 try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{});
14111 break :src_frame_addr .{ .index = src_frame_addr };
14112 },
14113 };
14114
14115 const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
649714116 const extra_bits = abi_size * 8 - src_bits;
649814117
649914118 const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
650014119 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
650114120 defer self.register_manager.unlockReg(index_lock);
650214121
6503 try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.s(-1));
14122 try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .s(-1));
650414123 switch (extra_bits) {
650514124 0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
650614125 1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()),
650714126 else => try self.asmRegisterImmediate(
650814127 .{ ._, .mov },
650914128 dst_reg.to32(),
6510 Immediate.s(-@as(i32, extra_bits)),
14129 .s(-@as(i32, extra_bits)),
651114130 ),
651214131 }
651314132 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
651414133 if (self.hasFeature(.slow_incdec)) {
6515 try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));
14134 try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
651614135 } else {
651714136 try self.asmRegister(.{ ._, .inc }, index_reg.to32());
651814137 }
6519 try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), Immediate.u(limbs_len));
14138 try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(limbs_len));
652014139 const zero = try self.asmJccReloc(.nb, undefined);
652114140 try self.asmMemoryImmediate(.{ ._, .cmp }, .{
6522 .base = .{ .frame = src_mcv.load_frame.index },
14141 .base = .{ .frame = src_frame_addr.index },
652314142 .mod = .{ .rm = .{
652414143 .size = .qword,
652514144 .index = index_reg.to64(),
652614145 .scale = .@"8",
6527 .disp = src_mcv.load_frame.off,
14146 .disp = src_frame_addr.off,
652814147 } },
6529 }, Immediate.u(0));
14148 }, .u(0));
653014149 _ = try self.asmJccReloc(.e, loop);
6531 try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{
6532 .base = .{ .frame = src_mcv.load_frame.index },
14150 try self.asmRegisterMemory(.{ ._f, .bs }, dst_reg.to64(), .{
14151 .base = .{ .frame = src_frame_addr.index },
653314152 .mod = .{ .rm = .{
653414153 .size = .qword,
653514154 .index = index_reg.to64(),
653614155 .scale = .@"8",
6537 .disp = src_mcv.load_frame.off,
14156 .disp = src_frame_addr.off,
653814157 } },
653914158 });
654014159 self.performReloc(zero);
6541 try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));
14160 try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6));
654214161 try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32());
654314162 break :result dst_mcv;
654414163 }
654514164
6546 const wide_ty = if (src_bits <= 8) Type.u16 else src_ty;
14165 const wide_ty: Type = if (src_bits <= 8) .u16 else src_ty;
654714166 if (has_bmi) {
654814167 if (src_bits <= 64) {
654914168 const extra_bits = self.regExtraBits(src_ty) + @as(u64, if (src_bits <= 8) 8 else 0);
......@@ -6558,7 +14177,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
655814177 .{ ._, .@"or" },
655914178 wide_ty,
656014179 tmp_mcv,
6561 .{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(64 - extra_bits)) <<
14180 .{ .immediate = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - extra_bits)) <<
656214181 @intCast(src_bits) },
656314182 );
656414183 break :masked tmp_mcv;
......@@ -6571,27 +14190,27 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
657114190 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
657214191 defer self.register_manager.unlockReg(tmp_lock);
657314192
6574 const lo_mat_src_mcv: MCValue = if (mat_src_mcv.isMemory())
14193 const lo_mat_src_mcv: MCValue = if (mat_src_mcv.isBase())
657514194 mat_src_mcv
657614195 else
657714196 .{ .register = mat_src_mcv.register_pair[0] };
6578 const hi_mat_src_mcv: MCValue = if (mat_src_mcv.isMemory())
14197 const hi_mat_src_mcv: MCValue = if (mat_src_mcv.isBase())
657914198 mat_src_mcv.address().offset(8).deref()
658014199 else
658114200 .{ .register = mat_src_mcv.register_pair[1] };
658214201 const masked_mcv = if (src_bits < 128) masked: {
6583 try self.genCopy(Type.u64, dst_mcv, hi_mat_src_mcv, .{});
14202 try self.genCopy(.u64, dst_mcv, hi_mat_src_mcv, .{});
658414203 try self.genBinOpMir(
658514204 .{ ._, .@"or" },
6586 Type.u64,
14205 .u64,
658714206 dst_mcv,
6588 .{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(src_bits - 64) },
14207 .{ .immediate = @as(u64, std.math.maxInt(u64)) << @intCast(src_bits - 64) },
658914208 );
659014209 break :masked dst_mcv;
659114210 } else hi_mat_src_mcv;
6592 try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, dst_mcv, masked_mcv);
14211 try self.genBinOpMir(.{ ._, .tzcnt }, .u64, dst_mcv, masked_mcv);
659314212 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
6594 try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, lo_mat_src_mcv);
14213 try self.genBinOpMir(.{ ._, .tzcnt }, .u64, tmp_mcv, lo_mat_src_mcv);
659514214 try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
659614215 }
659714216 break :result dst_mcv;
......@@ -6602,14 +14221,14 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
660214221 const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);
660314222 defer self.register_manager.unlockReg(width_lock);
660414223
6605 if (src_bits <= 8 or !math.isPowerOfTwo(src_bits)) {
14224 if (src_bits <= 8 or !std.math.isPowerOfTwo(src_bits)) {
660614225 const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
660714226 const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);
660814227 defer self.register_manager.unlockReg(wide_lock);
660914228
661014229 try self.truncateRegister(src_ty, wide_reg);
6611 try self.genBinOpMir(.{ ._, .bsf }, wide_ty, dst_mcv, .{ .register = wide_reg });
6612 } else try self.genBinOpMir(.{ ._, .bsf }, src_ty, dst_mcv, mat_src_mcv);
14230 try self.genBinOpMir(.{ ._f, .bs }, wide_ty, dst_mcv, .{ .register = wide_reg });
14231 } else try self.genBinOpMir(.{ ._f, .bs }, src_ty, dst_mcv, mat_src_mcv);
661314232
661414233 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);
661514234 try self.asmCmovccRegisterRegister(
......@@ -6622,7 +14241,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
662214241 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
662314242}
662414243
6625fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
14244fn airPopCount(self: *CodeGen, inst: Air.Inst.Index) !void {
662614245 const pt = self.pt;
662714246 const zcu = pt.zcu;
662814247 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -6664,13 +14283,13 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
666414283 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
666514284 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
666614285
6667 try self.genPopCount(tmp_regs[0], Type.usize, if (mat_src_mcv.isMemory())
14286 try self.genPopCount(tmp_regs[0], .usize, if (mat_src_mcv.isBase())
666814287 mat_src_mcv
666914288 else
667014289 .{ .register = mat_src_mcv.register_pair[0] }, false);
667114290 const src_info = src_ty.intInfo(zcu);
667214291 const hi_ty = try pt.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1);
6673 try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isMemory())
14292 try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isBase())
667414293 mat_src_mcv.address().offset(8).deref()
667514294 else
667614295 .{ .register = mat_src_mcv.register_pair[1] }, false);
......@@ -6681,7 +14300,7 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
668114300}
668214301
668314302fn genPopCount(
6684 self: *Self,
14303 self: *CodeGen,
668514304 dst_reg: Register,
668614305 src_ty: Type,
668714306 src_mcv: MCValue,
......@@ -6692,7 +14311,7 @@ fn genPopCount(
669214311 const src_abi_size: u32 = @intCast(src_ty.abiSize(pt.zcu));
669314312 if (self.hasFeature(.popcnt)) return self.genBinOpMir(
669414313 .{ ._, .popcnt },
6695 if (src_abi_size > 1) src_ty else Type.u32,
14314 if (src_abi_size > 1) src_ty else .u32,
669614315 .{ .register = dst_reg },
669714316 if (src_abi_size > 1) src_mcv else src: {
669814317 if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv, .{});
......@@ -6701,11 +14320,11 @@ fn genPopCount(
670114320 },
670214321 );
670314322
6704 const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
6705 const imm_0_1 = Immediate.u(mask / 0b1_1);
6706 const imm_00_11 = Immediate.u(mask / 0b01_01);
6707 const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
6708 const imm_0000_0001 = Immediate.u(mask / 0b1111_1111);
14323 const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
14324 const imm_0_1: Immediate = .u(mask / 0b1_1);
14325 const imm_00_11: Immediate = .u(mask / 0b01_01);
14326 const imm_0000_1111: Immediate = .u(mask / 0b0001_0001);
14327 const imm_0000_0001: Immediate = .u(mask / 0b1111_1111);
670914328
671014329 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
671114330 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
......@@ -6722,7 +14341,7 @@ fn genPopCount(
672214341 // dst = operand
672314342 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
672414343 // tmp = operand
6725 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
14344 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1));
672614345 // tmp = operand >> 1
672714346 if (src_abi_size > 4) {
672814347 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
......@@ -6733,7 +14352,7 @@ fn genPopCount(
673314352 // dst = temp1 = operand - ((operand >> 1) & 0x55...55)
673414353 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
673514354 // tmp = temp1
6736 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
14355 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2));
673714356 // dst = temp1 >> 2
673814357 if (src_abi_size > 4) {
673914358 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
......@@ -6749,7 +14368,7 @@ fn genPopCount(
674914368 // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
675014369 try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
675114370 // dst = temp2
6752 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(4));
14371 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(4));
675314372 // tmp = temp2 >> 4
675414373 try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);
675514374 // dst = temp2 + (temp2 >> 4)
......@@ -6767,13 +14386,13 @@ fn genPopCount(
676714386 // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
676814387 // dst = temp3 * 0x01...01
676914388 if (src_abi_size > 1) {
6770 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u((src_abi_size - 1) * 8));
14389 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u((src_abi_size - 1) * 8));
677114390 }
677214391 // dst = (temp3 * 0x01...01) >> (bits - 8)
677314392}
677414393
677514394fn genByteSwap(
6776 self: *Self,
14395 self: *CodeGen,
677714396 inst: Air.Inst.Index,
677814397 src_ty: Type,
677914398 src_mcv: MCValue,
......@@ -6814,13 +14433,18 @@ fn genByteSwap(
681414433 return src_mcv;
681514434 },
681614435 9...16 => {
6817 switch (src_mcv) {
14436 const mat_src_mcv: MCValue = mat_src_mcv: switch (src_mcv) {
14437 .register => {
14438 const frame_index = try self.allocFrameIndex(.initSpill(src_ty, zcu));
14439 try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, src_mcv, .{});
14440 break :mat_src_mcv .{ .load_frame = .{ .index = frame_index } };
14441 },
681814442 .register_pair => |src_regs| if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
681914443 for (src_regs) |src_reg| try self.asmRegister(.{ ._, .bswap }, src_reg.to64());
682014444 return .{ .register_pair = .{ src_regs[1], src_regs[0] } };
6821 },
6822 else => {},
6823 }
14445 } else src_mcv,
14446 else => src_mcv,
14447 };
682414448
682514449 const dst_regs =
682614450 try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp);
......@@ -6828,18 +14452,18 @@ fn genByteSwap(
682814452 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
682914453
683014454 for (dst_regs, 0..) |dst_reg, limb_index| {
6831 if (src_mcv.isMemory()) {
14455 if (mat_src_mcv.isBase()) {
683214456 try self.asmRegisterMemory(
683314457 .{ ._, if (has_movbe) .movbe else .mov },
683414458 dst_reg.to64(),
6835 try src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .qword),
14459 try mat_src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .{ .size = .qword }),
683614460 );
683714461 if (!has_movbe) try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
683814462 } else {
683914463 try self.asmRegisterRegister(
684014464 .{ ._, .mov },
684114465 dst_reg.to64(),
6842 src_mcv.register_pair[limb_index].to64(),
14466 mat_src_mcv.register_pair[limb_index].to64(),
684314467 );
684414468 try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
684514469 }
......@@ -6847,20 +14471,16 @@ fn genByteSwap(
684714471 return .{ .register_pair = .{ dst_regs[1], dst_regs[0] } };
684814472 },
684914473 else => {
6850 const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
14474 const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
685114475
685214476 const temp_regs =
6853 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
14477 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
685414478 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
685514479 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
685614480
685714481 const dst_mcv = try self.allocRegOrMem(inst, false);
685814482 try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());
6859 try self.asmRegisterImmediate(
6860 .{ ._, .mov },
6861 temp_regs[1].to32(),
6862 Immediate.u(limbs_len - 1),
6863 );
14483 try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1));
686414484
686514485 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
686614486 try self.asmRegisterMemory(
......@@ -6912,8 +14532,8 @@ fn genByteSwap(
691214532 } },
691314533 }, temp_regs[2].to64());
691414534 if (self.hasFeature(.slow_incdec)) {
6915 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));
6916 try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));
14535 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
14536 try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
691714537 } else {
691814538 try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
691914539 try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
......@@ -6942,7 +14562,7 @@ fn genByteSwap(
694214562 return dst_mcv;
694314563}
694414564
6945fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
14565fn airByteSwap(self: *CodeGen, inst: Air.Inst.Index) !void {
694614566 const pt = self.pt;
694714567 const zcu = pt.zcu;
694814568 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -6959,13 +14579,13 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
695914579 } },
696014580 src_ty,
696114581 dst_mcv,
6962 if (src_bits > 256) Type.u16 else Type.u8,
14582 if (src_bits > 256) .u16 else .u8,
696314583 .{ .immediate = src_ty.abiSize(zcu) * 8 - src_bits },
696414584 );
696514585 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
696614586}
696714587
6968fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
14588fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void {
696914589 const pt = self.pt;
697014590 const zcu = pt.zcu;
697114591 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -6994,10 +14614,10 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
699414614 else
699514615 undefined;
699614616
6997 const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8);
6998 const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
6999 const imm_00_11 = Immediate.u(mask / 0b01_01);
7000 const imm_0_1 = Immediate.u(mask / 0b1_1);
14617 const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8);
14618 const imm_0000_1111: Immediate = .u(mask / 0b0001_0001);
14619 const imm_00_11: Immediate = .u(mask / 0b01_01);
14620 const imm_0_1: Immediate = .u(mask / 0b1_1);
700114621
700214622 for (dst_mcv.getRegs()) |dst_reg| {
700314623 const dst = registerAlias(dst_reg, limb_abi_size);
......@@ -7005,7 +14625,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
700514625 // dst = temp1 = bswap(operand)
700614626 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
700714627 // tmp = temp1
7008 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(4));
14628 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(4));
700914629 // dst = temp1 >> 4
701014630 if (limb_abi_size > 4) {
701114631 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
......@@ -7017,13 +14637,13 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
701714637 }
701814638 // tmp = temp1 & 0x0F...0F
701914639 // dst = (temp1 >> 4) & 0x0F...0F
7020 try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, Immediate.u(4));
14640 try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, .u(4));
702114641 // tmp = (temp1 & 0x0F...0F) << 4
702214642 try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);
702314643 // dst = temp2 = ((temp1 >> 4) & 0x0F...0F) | ((temp1 & 0x0F...0F) << 4)
702414644 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
702514645 // tmp = temp2
7026 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
14646 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2));
702714647 // dst = temp2 >> 2
702814648 if (limb_abi_size > 4) {
702914649 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
......@@ -7050,7 +14670,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
705014670 // tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
705114671 try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
705214672 // dst = temp3
7053 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
14673 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1));
705414674 // tmp = temp3 >> 1
705514675 if (limb_abi_size > 4) {
705614676 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
......@@ -7083,12 +14703,12 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
708314703 if (extra_bits > 0) try self.genShiftBinOpMir(switch (signedness) {
708414704 .signed => .{ ._r, .sa },
708514705 .unsigned => .{ ._r, .sh },
7086 }, src_ty, dst_mcv, Type.u8, .{ .immediate = extra_bits });
14706 }, src_ty, dst_mcv, .u8, .{ .immediate = extra_bits });
708714707
708814708 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
708914709}
709014710
7091fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) !void {
14711fn floatSign(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) !void {
709214712 const pt = self.pt;
709314713 const zcu = pt.zcu;
709414714 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
......@@ -7150,12 +14770,12 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
715014770 .abs => try vec_ty.maxInt(pt, vec_ty),
715114771 else => unreachable,
715214772 });
7153 const sign_mem: Memory = if (sign_mcv.isMemory())
7154 try sign_mcv.mem(self, Memory.Size.fromSize(abi_size))
14773 const sign_mem: Memory = if (sign_mcv.isBase())
14774 try sign_mcv.mem(self, .{ .size = .fromSize(abi_size) })
715514775 else
715614776 .{
7157 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
7158 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } },
14777 .base = .{ .reg = try self.copyToTmpRegister(.usize, sign_mcv.address()) },
14778 .mod = .{ .rm = .{ .size = .fromSize(abi_size) } },
715914779 };
716014780
716114781 if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(
......@@ -7216,7 +14836,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
721614836 return self.finishAir(inst, result, .{ operand, .none, .none });
721714837}
721814838
7219fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
14839fn airFloatSign(self: *CodeGen, inst: Air.Inst.Index) !void {
722014840 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
722114841 const ty = self.typeOf(un_op);
722214842 return self.floatSign(inst, un_op, ty);
......@@ -7241,7 +14861,7 @@ const RoundMode = packed struct(u5) {
724114861 } = .normal,
724214862};
724314863
7244fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void {
14864fn airRound(self: *CodeGen, inst: Air.Inst.Index, mode: RoundMode) !void {
724514865 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
724614866 const ty = self.typeOf(un_op);
724714867
......@@ -7265,7 +14885,7 @@ fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void {
726514885 return self.finishAir(inst, result, .{ un_op, .none, .none });
726614886}
726714887
7268fn getRoundTag(self: *Self, ty: Type) ?Mir.Inst.FixedTag {
14888fn getRoundTag(self: *CodeGen, ty: Type) ?Mir.Inst.FixedTag {
726914889 const pt = self.pt;
727014890 const zcu = pt.zcu;
727114891 return if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(zcu)) {
......@@ -7298,7 +14918,7 @@ fn getRoundTag(self: *Self, ty: Type) ?Mir.Inst.FixedTag {
729814918 } else null;
729914919}
730014920
7301fn genRoundLibcall(self: *Self, ty: Type, src_mcv: MCValue, mode: RoundMode) !MCValue {
14921fn genRoundLibcall(self: *CodeGen, ty: Type, src_mcv: MCValue, mode: RoundMode) !MCValue {
730214922 const pt = self.pt;
730314923 const zcu = pt.zcu;
730414924 if (self.getRoundTag(ty)) |_| return .none;
......@@ -7320,10 +14940,10 @@ fn genRoundLibcall(self: *Self, ty: Type, src_mcv: MCValue, mode: RoundMode) !MC
732014940 },
732114941 floatLibcAbiSuffix(ty),
732214942 }) catch unreachable,
7323 } }, &.{ty}, &.{src_mcv});
14943 } }, &.{ty}, &.{src_mcv}, .{});
732414944}
732514945
7326fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: RoundMode) !void {
14946fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: RoundMode) !void {
732714947 const pt = self.pt;
732814948 const mir_tag = self.getRoundTag(ty) orelse {
732914949 const result = try self.genRoundLibcall(ty, src_mcv, mode);
......@@ -7332,12 +14952,12 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
733214952 const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));
733314953 const dst_alias = registerAlias(dst_reg, abi_size);
733414954 switch (mir_tag[0]) {
7335 .v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
14955 .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
733614956 mir_tag,
733714957 dst_alias,
733814958 dst_alias,
7339 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7340 Immediate.u(@as(u5, @bitCast(mode))),
14959 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
14960 .u(@as(u5, @bitCast(mode))),
734114961 ) else try self.asmRegisterRegisterRegisterImmediate(
734214962 mir_tag,
734314963 dst_alias,
......@@ -7346,13 +14966,13 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
734614966 src_mcv.getReg().?
734714967 else
734814968 try self.copyToTmpRegister(ty, src_mcv), abi_size),
7349 Immediate.u(@as(u5, @bitCast(mode))),
14969 .u(@as(u5, @bitCast(mode))),
735014970 ),
7351 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
14971 else => if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate(
735214972 mir_tag,
735314973 dst_alias,
7354 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7355 Immediate.u(@as(u5, @bitCast(mode))),
14974 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
14975 .u(@as(u5, @bitCast(mode))),
735614976 ) else try self.asmRegisterRegisterImmediate(
735714977 mir_tag,
735814978 dst_alias,
......@@ -7360,12 +14980,12 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
736014980 src_mcv.getReg().?
736114981 else
736214982 try self.copyToTmpRegister(ty, src_mcv), abi_size),
7363 Immediate.u(@as(u5, @bitCast(mode))),
14983 .u(@as(u5, @bitCast(mode))),
736414984 ),
736514985 }
736614986}
736714987
7368fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
14988fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void {
736914989 const pt = self.pt;
737014990 const zcu = pt.zcu;
737114991 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -7393,7 +15013,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
739315013 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
739415014 .l,
739515015 registerAlias(dst_mcv.register, cmov_abi_size),
7396 try src_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
15016 try src_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }),
739715017 ),
739815018 else => {
739915019 const val_reg = try self.copyToTmpRegister(ty, src_mcv);
......@@ -7433,7 +15053,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
743315053 defer self.register_manager.unlockReg(tmp_lock);
743415054
743515055 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
7436 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));
15056 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
743715057 try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[0], tmp_reg);
743815058 try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[1], tmp_reg);
743915059 try self.asmRegisterRegister(.{ ._, .sub }, dst_regs[0], tmp_reg);
......@@ -7443,10 +15063,10 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
744315063 },
744415064 else => {
744515065 const abi_size: u31 = @intCast(ty.abiSize(zcu));
7446 const limb_len = math.divCeil(u31, abi_size, 8) catch unreachable;
15066 const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;
744715067
744815068 const tmp_regs =
7449 try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);
15069 try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp);
745015070 const tmp_locks = self.register_manager.lockRegsAssumeUnused(3, tmp_regs);
745115071 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
745215072
......@@ -7459,8 +15079,8 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
745915079
746015080 try self.asmMemoryImmediate(
746115081 .{ ._, .cmp },
7462 try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .qword),
7463 Immediate.u(0),
15082 try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .{ .size = .qword }),
15083 .u(0),
746415084 );
746515085 const positive = try self.asmJccReloc(.ns, undefined);
746615086
......@@ -7469,7 +15089,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
746915089
747015090 const neg_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
747115091 try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[2].to32(), tmp_regs[2].to32());
7472 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), Immediate.u(1));
15092 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), .u(1));
747315093 try self.asmRegisterMemory(.{ ._, .sbb }, tmp_regs[2].to64(), .{
747415094 .base = .{ .frame = dst_mcv.load_frame.index },
747515095 .mod = .{ .rm = .{
......@@ -7491,11 +15111,11 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
749115111 }, tmp_regs[2].to64());
749215112
749315113 if (self.hasFeature(.slow_incdec)) {
7494 try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), Immediate.u(1));
15114 try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), .u(1));
749515115 } else {
749615116 try self.asmRegister(.{ ._, .inc }, tmp_regs[0].to32());
749715117 }
7498 try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), Immediate.u(limb_len));
15118 try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), .u(limb_len));
749915119 _ = try self.asmJccReloc(.b, neg_loop);
750015120
750115121 self.performReloc(positive);
......@@ -7547,12 +15167,12 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
754715167 const dst_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
754815168 src_mcv.getReg().?
754915169 else
7550 try self.register_manager.allocReg(inst, self.regClassForType(ty));
15170 try self.register_manager.allocReg(inst, self.regSetForType(ty));
755115171 const dst_alias = registerAlias(dst_reg, abi_size);
7552 if (src_mcv.isMemory()) try self.asmRegisterMemory(
15172 if (src_mcv.isBase()) try self.asmRegisterMemory(
755315173 mir_tag,
755415174 dst_alias,
7555 try src_mcv.mem(self, self.memSize(ty)),
15175 try src_mcv.mem(self, .{ .size = self.memSize(ty) }),
755615176 ) else try self.asmRegisterRegister(
755715177 mir_tag,
755815178 dst_alias,
......@@ -7566,7 +15186,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
756615186 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
756715187}
756815188
7569fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
15189fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void {
757015190 const pt = self.pt;
757115191 const zcu = pt.zcu;
757215192 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
......@@ -7591,7 +15211,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
759115211 floatLibcAbiPrefix(ty),
759215212 floatLibcAbiSuffix(ty),
759315213 }) catch unreachable,
7594 } }, &.{ty}, &.{.{ .air_ref = un_op }});
15214 } }, &.{ty}, &.{.{ .air_ref = un_op }}, .{});
759515215 }
759615216 },
759715217 else => {},
......@@ -7620,7 +15240,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
762015240 .{ .v_, .cvtps2ph },
762115241 dst_reg,
762215242 dst_reg,
7623 Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
15243 .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
762415244 );
762515245 break :result dst_mcv;
762615246 },
......@@ -7650,18 +15270,18 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
765015270 .{ .v_, .cvtps2ph },
765115271 dst_reg,
765215272 dst_reg,
7653 Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
15273 .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
765415274 );
765515275 break :result dst_mcv;
765615276 },
765715277 2...8 => {
765815278 const wide_reg = registerAlias(dst_reg, abi_size * 2);
7659 if (src_mcv.isMemory()) try self.asmRegisterMemory(
15279 if (src_mcv.isBase()) try self.asmRegisterMemory(
766015280 .{ .v_ps, .cvtph2 },
766115281 wide_reg,
7662 try src_mcv.mem(self, Memory.Size.fromSize(
15282 try src_mcv.mem(self, .{ .size = .fromSize(
766315283 @intCast(@divExact(wide_reg.bitSize(), 16)),
7664 )),
15284 ) }),
766515285 ) else try self.asmRegisterRegister(
766615286 .{ .v_ps, .cvtph2 },
766715287 wide_reg,
......@@ -7675,7 +15295,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
767515295 .{ .v_, .cvtps2ph },
767615296 dst_reg,
767715297 wide_reg,
7678 Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
15298 .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
767915299 );
768015300 break :result dst_mcv;
768115301 },
......@@ -7699,15 +15319,13 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
769915319 else => unreachable,
770015320 },
770115321 else => unreachable,
7702 }) orelse return self.fail("TODO implement airSqrt for {}", .{
7703 ty.fmt(pt),
7704 });
15322 }) orelse return self.fail("TODO implement airSqrt for {}", .{ty.fmt(pt)});
770515323 switch (mir_tag[0]) {
7706 .v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
15324 .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
770715325 mir_tag,
770815326 dst_reg,
770915327 dst_reg,
7710 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
15328 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
771115329 ) else try self.asmRegisterRegisterRegister(
771215330 mir_tag,
771315331 dst_reg,
......@@ -7717,10 +15335,10 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
771715335 else
771815336 try self.copyToTmpRegister(ty, src_mcv), abi_size),
771915337 ),
7720 else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
15338 else => if (src_mcv.isBase()) try self.asmRegisterMemory(
772115339 mir_tag,
772215340 dst_reg,
7723 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
15341 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
772415342 ) else try self.asmRegisterRegister(
772515343 mir_tag,
772615344 dst_reg,
......@@ -7735,7 +15353,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
773515353 return self.finishAir(inst, result, .{ un_op, .none, .none });
773615354}
773715355
7738fn airUnaryMath(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
15356fn airUnaryMath(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
773915357 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
774015358 const ty = self.typeOf(un_op);
774115359 var callee_buf: ["__round?".len]u8 = undefined;
......@@ -7759,12 +15377,12 @@ fn airUnaryMath(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
775915377 },
776015378 floatLibcAbiSuffix(ty),
776115379 }) catch unreachable,
7762 } }, &.{ty}, &.{.{ .air_ref = un_op }});
15380 } }, &.{ty}, &.{.{ .air_ref = un_op }}, .{});
776315381 return self.finishAir(inst, result, .{ un_op, .none, .none });
776415382}
776515383
776615384fn reuseOperand(
7767 self: *Self,
15385 self: *CodeGen,
776815386 inst: Air.Inst.Index,
776915387 operand: Air.Inst.Ref,
777015388 op_index: Liveness.OperandInt,
......@@ -7774,7 +15392,7 @@ fn reuseOperand(
777415392}
777515393
777615394fn reuseOperandAdvanced(
7777 self: *Self,
15395 self: *CodeGen,
777815396 inst: Air.Inst.Index,
777915397 operand: Air.Inst.Ref,
778015398 op_index: Liveness.OperandInt,
......@@ -7785,7 +15403,7 @@ fn reuseOperandAdvanced(
778515403 return false;
778615404
778715405 switch (mcv) {
7788 .register, .register_pair, .register_overflow => for (mcv.getRegs()) |reg| {
15406 .register, .register_pair, .register_overflow, .register_mask => for (mcv.getRegs()) |reg| {
778915407 // If it's in the registers table, need to associate the register(s) with the
779015408 // new instruction.
779115409 if (maybe_tracked_inst) |tracked_inst| {
......@@ -7805,19 +15423,19 @@ fn reuseOperandAdvanced(
780515423 }
780615424
780715425 // Prevent the operand deaths processing code from deallocating it.
7808 self.liveness.clearOperandDeath(inst, op_index);
15426 self.reused_operands.set(op_index);
780915427 const op_inst = operand.toIndex().?;
781015428 self.getResolvedInstValue(op_inst).reuse(self, maybe_tracked_inst, op_inst);
781115429
781215430 return true;
781315431}
781415432
7815fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
15433fn packedLoad(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
781615434 const pt = self.pt;
781715435 const zcu = pt.zcu;
781815436
781915437 const ptr_info = ptr_ty.ptrInfo(zcu);
7820 const val_ty = Type.fromInterned(ptr_info.child);
15438 const val_ty: Type = .fromInterned(ptr_info.child);
782115439 if (!val_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;
782215440 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));
782315441
......@@ -7851,9 +15469,9 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
785115469 defer self.register_manager.unlockReg(tmp_lock);
785215470
785315471 const hi_mcv = dst_mcv.address().offset(@intCast(val_bit_size / 64 * 8)).deref();
7854 try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});
15472 try self.genSetReg(tmp_reg, .usize, hi_mcv, .{});
785515473 try self.truncateRegister(val_ty, tmp_reg);
7856 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});
15474 try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{});
785715475 }
785815476 }
785915477 return;
......@@ -7885,12 +15503,12 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
788515503 try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
788615504 .base = .{ .reg = ptr_reg },
788715505 .mod = .{ .rm = .{
7888 .size = Memory.Size.fromSize(load_abi_size),
15506 .size = .fromSize(load_abi_size),
788915507 .disp = val_byte_off,
789015508 } },
789115509 });
789215510 try self.spillEflagsIfOccupied();
7893 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
15511 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, .u(val_bit_off));
789415512 } else {
789515513 const tmp_reg =
789615514 registerAlias(try self.register_manager.allocReg(null, abi.RegisterClass.gp), val_abi_size);
......@@ -7901,31 +15519,26 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
790115519 try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{
790215520 .base = .{ .reg = ptr_reg },
790315521 .mod = .{ .rm = .{
7904 .size = Memory.Size.fromSize(val_abi_size),
15522 .size = .fromSize(val_abi_size),
790515523 .disp = val_byte_off,
790615524 } },
790715525 });
790815526 try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
790915527 .base = .{ .reg = ptr_reg },
791015528 .mod = .{ .rm = .{
7911 .size = Memory.Size.fromSize(val_abi_size),
15529 .size = .fromSize(val_abi_size),
791215530 .disp = val_byte_off + limb_abi_size,
791315531 } },
791415532 });
791515533 try self.spillEflagsIfOccupied();
7916 try self.asmRegisterRegisterImmediate(
7917 .{ ._rd, .sh },
7918 dst_alias,
7919 tmp_reg,
7920 Immediate.u(val_bit_off),
7921 );
15534 try self.asmRegisterRegisterImmediate(.{ ._rd, .sh }, dst_alias, tmp_reg, .u(val_bit_off));
792215535 }
792315536
792415537 if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
792515538 try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg }, .{});
792615539}
792715540
7928fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
15541fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
792915542 const pt = self.pt;
793015543 const zcu = pt.zcu;
793115544 const dst_ty = ptr_ty.childType(zcu);
......@@ -7937,7 +15550,10 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro
793715550 .undef,
793815551 .eflags,
793915552 .register_pair,
15553 .register_triple,
15554 .register_quadruple,
794015555 .register_overflow,
15556 .register_mask,
794115557 .elementwise_regs_then_frame,
794215558 .reserved_frame,
794315559 => unreachable, // not a valid pointer
......@@ -7968,7 +15584,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro
796815584 }
796915585}
797015586
7971fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
15587fn airLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
797215588 const pt = self.pt;
797315589 const zcu = pt.zcu;
797415590 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -7983,12 +15599,12 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
798315599 const ptr_ty = self.typeOf(ty_op.operand);
798415600 const elem_size = elem_ty.abiSize(zcu);
798515601
7986 const elem_rc = self.regClassForType(elem_ty);
7987 const ptr_rc = self.regClassForType(ptr_ty);
15602 const elem_rs = self.regSetForType(elem_ty);
15603 const ptr_rs = self.regSetForType(ptr_ty);
798815604
798915605 const ptr_mcv = try self.resolveInst(ty_op.operand);
7990 const dst_mcv = if (elem_size <= 8 and elem_rc.supersetOf(ptr_rc) and
7991 self.reuseOperand(inst, ty_op.operand, 0, ptr_mcv))
15606 const dst_mcv = if (elem_size <= 8 and std.math.isPowerOfTwo(elem_size) and
15607 elem_rs.supersetOf(ptr_rs) and self.reuseOperand(inst, ty_op.operand, 0, ptr_mcv))
799215608 // The MCValue that holds the pointer can be re-used as the value.
799315609 ptr_mcv
799415610 else
......@@ -8010,13 +15626,13 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
801015626 const high_reg = if (high_mcv.isRegister())
801115627 high_mcv.getReg().?
801215628 else
8013 try self.copyToTmpRegister(Type.usize, high_mcv);
15629 try self.copyToTmpRegister(.usize, high_mcv);
801415630 const high_lock = self.register_manager.lockReg(high_reg);
801515631 defer if (high_lock) |lock| self.register_manager.unlockReg(lock);
801615632
801715633 try self.truncateRegister(elem_ty, high_reg);
801815634 if (!high_mcv.isRegister()) try self.genCopy(
8019 if (elem_size <= 8) elem_ty else Type.usize,
15635 if (elem_size <= 8) elem_ty else .usize,
802015636 high_mcv,
802115637 .{ .register = high_reg },
802215638 .{},
......@@ -8027,11 +15643,11 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
802715643 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
802815644}
802915645
8030fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
15646fn packedStore(self: *CodeGen, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
803115647 const pt = self.pt;
803215648 const zcu = pt.zcu;
803315649 const ptr_info = ptr_ty.ptrInfo(zcu);
8034 const src_ty = Type.fromInterned(ptr_info.child);
15650 const src_ty: Type = .fromInterned(ptr_info.child);
803515651 if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;
803615652
803715653 const limb_abi_size: u16 = @min(ptr_info.packed_offset.host_size, 8);
......@@ -8051,6 +15667,15 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
805115667 const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);
805215668 defer self.register_manager.unlockReg(ptr_lock);
805315669
15670 const mat_src_mcv: MCValue = mat_src_mcv: switch (src_mcv) {
15671 .register => if (src_bit_size > 64) {
15672 const frame_index = try self.allocFrameIndex(.initSpill(src_ty, self.pt.zcu));
15673 try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, src_mcv, .{});
15674 break :mat_src_mcv .{ .load_frame = .{ .index = frame_index } };
15675 } else src_mcv,
15676 else => src_mcv,
15677 };
15678
805415679 var limb_i: u16 = 0;
805515680 while (limb_i * limb_abi_bits < src_bit_off + src_bit_size) : (limb_i += 1) {
805615681 const part_bit_off = if (limb_i == 0) src_bit_off else 0;
......@@ -8059,21 +15684,21 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
805915684 const limb_mem: Memory = .{
806015685 .base = .{ .reg = ptr_reg },
806115686 .mod = .{ .rm = .{
8062 .size = Memory.Size.fromSize(limb_abi_size),
15687 .size = .fromSize(limb_abi_size),
806315688 .disp = src_byte_off + limb_i * limb_abi_size,
806415689 } },
806515690 };
806615691
8067 const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(64 - part_bit_size)) <<
15692 const part_mask = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - part_bit_size)) <<
806815693 @intCast(part_bit_off);
8069 const part_mask_not = part_mask ^ (@as(u64, math.maxInt(u64)) >> @intCast(64 - limb_abi_bits));
15694 const part_mask_not = part_mask ^ (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_bits));
807015695 if (limb_abi_size <= 4) {
8071 try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.u(part_mask_not));
8072 } else if (math.cast(i32, @as(i64, @bitCast(part_mask_not)))) |small| {
8073 try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.s(small));
15696 try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .u(part_mask_not));
15697 } else if (std.math.cast(i32, @as(i64, @bitCast(part_mask_not)))) |small| {
15698 try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .s(small));
807415699 } else {
807515700 const part_mask_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
8076 try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, Immediate.u(part_mask_not));
15701 try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, .u(part_mask_not));
807715702 try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);
807815703 }
807915704
......@@ -8083,20 +15708,20 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
808315708 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
808415709 defer self.register_manager.unlockReg(tmp_lock);
808515710
8086 try self.genSetReg(tmp_reg, limb_ty, src_mcv, .{});
15711 try self.genSetReg(tmp_reg, limb_ty, mat_src_mcv, .{});
808715712 switch (limb_i) {
808815713 0 => try self.genShiftBinOpMir(
808915714 .{ ._l, .sh },
809015715 limb_ty,
809115716 tmp_mcv,
8092 Type.u8,
15717 .u8,
809315718 .{ .immediate = src_bit_off },
809415719 ),
809515720 1 => try self.genShiftBinOpMir(
809615721 .{ ._r, .sh },
809715722 limb_ty,
809815723 tmp_mcv,
8099 Type.u8,
15724 .u8,
810015725 .{ .immediate = limb_abi_bits - src_bit_off },
810115726 ),
810215727 else => unreachable,
......@@ -8114,8 +15739,8 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
811415739 defer self.register_manager.unlockReg(tmp_lock);
811515740
811615741 try self.genSetReg(tmp_reg, limb_ty, switch (limb_i) {
8117 0 => src_mcv,
8118 else => src_mcv.address().offset(limb_i * limb_abi_size).deref(),
15742 0 => mat_src_mcv,
15743 else => mat_src_mcv.address().offset(limb_i * limb_abi_size).deref(),
811915744 }, .{});
812015745 try self.genBinOpMir(.{ ._, .@"and" }, limb_ty, tmp_mcv, .{ .immediate = part_mask });
812115746 try self.asmMemoryRegister(
......@@ -8128,7 +15753,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
812815753}
812915754
813015755fn store(
8131 self: *Self,
15756 self: *CodeGen,
813215757 ptr_ty: Type,
813315758 ptr_mcv: MCValue,
813415759 src_mcv: MCValue,
......@@ -8145,7 +15770,10 @@ fn store(
814515770 .undef,
814615771 .eflags,
814715772 .register_pair,
15773 .register_triple,
15774 .register_quadruple,
814815775 .register_overflow,
15776 .register_mask,
814915777 .elementwise_regs_then_frame,
815015778 .reserved_frame,
815115779 => unreachable, // not a valid pointer
......@@ -8176,7 +15804,7 @@ fn store(
817615804 }
817715805}
817815806
8179fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
15807fn airStore(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
818015808 const pt = self.pt;
818115809 const zcu = pt.zcu;
818215810 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -8202,32 +15830,21 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
820215830 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
820315831}
820415832
8205fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
15833fn airStructFieldPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
820615834 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
820715835 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
820815836 const result = try self.fieldPtr(inst, extra.struct_operand, extra.field_index);
820915837 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
821015838}
821115839
8212fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
15840fn airStructFieldPtrIndex(self: *CodeGen, inst: Air.Inst.Index, field_index: u8) !void {
821315841 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
8214 const result = try self.fieldPtr(inst, ty_op.operand, index);
15842 const result = try self.fieldPtr(inst, ty_op.operand, field_index);
821515843 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
821615844}
821715845
8218fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
8219 const pt = self.pt;
8220 const zcu = pt.zcu;
15846fn fieldPtr(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_index: u32) !MCValue {
822115847 const ptr_field_ty = self.typeOfIndex(inst);
8222 const ptr_container_ty = self.typeOf(operand);
8223 const container_ty = ptr_container_ty.childType(zcu);
8224
8225 const field_off: i32 = switch (container_ty.containerLayout(zcu)) {
8226 .auto, .@"extern" => @intCast(container_ty.structFieldOffset(index, zcu)),
8227 .@"packed" => @divExact(@as(i32, ptr_container_ty.ptrInfo(zcu).packed_offset.bit_offset) +
8228 (if (zcu.typeToStruct(container_ty)) |struct_obj| pt.structPackedFieldBitOffset(struct_obj, index) else 0) -
8229 ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8),
8230 };
823115848
823215849 const src_mcv = try self.resolveInst(operand);
823315850 const dst_mcv = if (switch (src_mcv) {
......@@ -8235,10 +15852,22 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
823515852 .register, .register_offset => self.reuseOperand(inst, operand, 0, src_mcv),
823615853 else => false,
823715854 }) src_mcv else try self.copyToRegisterWithInstTracking(inst, ptr_field_ty, src_mcv);
8238 return dst_mcv.offset(field_off);
15855 return dst_mcv.offset(self.fieldOffset(self.typeOf(operand), ptr_field_ty, field_index));
15856}
15857
15858fn fieldOffset(self: *CodeGen, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u32) i32 {
15859 const pt = self.pt;
15860 const zcu = pt.zcu;
15861 const agg_ty = ptr_agg_ty.childType(zcu);
15862 return switch (agg_ty.containerLayout(zcu)) {
15863 .auto, .@"extern" => @intCast(agg_ty.structFieldOffset(field_index, zcu)),
15864 .@"packed" => @divExact(@as(i32, ptr_agg_ty.ptrInfo(zcu).packed_offset.bit_offset) +
15865 (if (zcu.typeToStruct(agg_ty)) |loaded_struct| pt.structPackedFieldBitOffset(loaded_struct, field_index) else 0) -
15866 ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8),
15867 };
823915868}
824015869
8241fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
15870fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void {
824215871 const pt = self.pt;
824315872 const zcu = pt.zcu;
824415873 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -8248,16 +15877,19 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
824815877 const index = extra.field_index;
824915878
825015879 const container_ty = self.typeOf(operand);
8251 const container_rc = self.regClassForType(container_ty);
15880 const container_rc = self.regSetForType(container_ty);
825215881 const field_ty = container_ty.fieldType(index, zcu);
825315882 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
8254 const field_rc = self.regClassForType(field_ty);
15883 const field_rc = self.regSetForType(field_ty);
825515884 const field_is_gp = field_rc.supersetOf(abi.RegisterClass.gp);
825615885
825715886 const src_mcv = try self.resolveInst(operand);
825815887 const field_off: u32 = switch (container_ty.containerLayout(zcu)) {
825915888 .auto, .@"extern" => @intCast(container_ty.structFieldOffset(extra.field_index, zcu) * 8),
8260 .@"packed" => if (zcu.typeToStruct(container_ty)) |struct_obj| pt.structPackedFieldBitOffset(struct_obj, extra.field_index) else 0,
15889 .@"packed" => if (zcu.typeToStruct(container_ty)) |loaded_struct|
15890 pt.structPackedFieldBitOffset(loaded_struct, extra.field_index)
15891 else
15892 0,
826115893 };
826215894
826315895 switch (src_mcv) {
......@@ -8272,20 +15904,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
827215904 else if (field_off == 0)
827315905 (try self.copyToRegisterWithInstTracking(inst, field_ty, src_mcv)).register
827415906 else
8275 try self.copyToTmpRegister(Type.usize, .{ .register = src_reg });
15907 try self.copyToTmpRegister(.usize, .{ .register = src_reg });
827615908 const dst_mcv: MCValue = .{ .register = dst_reg };
827715909 const dst_lock = self.register_manager.lockReg(dst_reg);
827815910 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
827915911
828015912 if (field_off > 0) {
828115913 try self.spillEflagsIfOccupied();
8282 try self.genShiftBinOpMir(
8283 .{ ._r, .sh },
8284 Type.usize,
8285 dst_mcv,
8286 Type.u8,
8287 .{ .immediate = field_off },
8288 );
15914 try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, dst_mcv, .u8, .{ .immediate = field_off });
828915915 }
829015916 if (abi.RegisterClass.gp.isSet(RegisterManager.indexOfRegIntoTracked(dst_reg).?) and
829115917 container_ty.abiSize(zcu) * 8 > field_ty.bitSize(zcu))
......@@ -8309,7 +15935,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
830915935 const dst_regs: [2]Register = if (field_rc.supersetOf(container_rc) and
831015936 self.reuseOperand(inst, operand, 0, src_mcv)) src_regs else dst: {
831115937 const dst_regs =
8312 try self.register_manager.allocRegs(2, .{null} ** 2, field_rc);
15938 try self.register_manager.allocRegs(2, @splat(null), field_rc);
831315939 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
831415940 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
831515941
......@@ -8323,13 +15949,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
832315949
832415950 if (field_off > 0) {
832515951 try self.spillEflagsIfOccupied();
8326 try self.genShiftBinOpMir(
8327 .{ ._r, .sh },
8328 Type.u128,
8329 dst_mcv,
8330 Type.u8,
8331 .{ .immediate = field_off },
8332 );
15952 try self.genShiftBinOpMir(.{ ._r, .sh }, .u128, dst_mcv, .u8, .{ .immediate = field_off });
833315953 }
833415954
833515955 if (field_bit_size <= 64) {
......@@ -8353,20 +15973,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
835315973 try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
835415974 };
835515975
8356 const dst_reg = try self.copyToTmpRegister(Type.usize, .{ .register = src_reg });
15976 const dst_reg = try self.copyToTmpRegister(.usize, .{ .register = src_reg });
835715977 const dst_mcv = MCValue{ .register = dst_reg };
835815978 const dst_lock = self.register_manager.lockReg(dst_reg);
835915979 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
836015980
836115981 if (field_off % 64 > 0) {
836215982 try self.spillEflagsIfOccupied();
8363 try self.genShiftBinOpMir(
8364 .{ ._r, .sh },
8365 Type.usize,
8366 dst_mcv,
8367 Type.u8,
8368 .{ .immediate = field_off % 64 },
8369 );
15983 try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, dst_mcv, .u8, .{ .immediate = field_off % 64 });
837015984 }
837115985 if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg);
837215986
......@@ -8381,11 +15995,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
838115995 0 => if (self.reuseOperand(inst, extra.struct_operand, 0, src_mcv)) {
838215996 self.eflags_inst = null; // actually stop tracking the overflow part
838315997 break :result .{ .register = ro.reg };
8384 } else break :result try self.copyToRegisterWithInstTracking(
8385 inst,
8386 Type.usize,
8387 .{ .register = ro.reg },
8388 ),
15998 } else break :result try self.copyToRegisterWithInstTracking(inst, .usize, .{ .register = ro.reg }),
838915999 // Get overflow bit.
839016000 1 => if (self.reuseOperandAdvanced(inst, extra.struct_operand, 0, src_mcv, null)) {
839116001 self.eflags_inst = inst; // actually keep tracking the overflow part
......@@ -8436,16 +16046,16 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
843616046 try self.genCopy(field_ty, dst_mcv, off_mcv, .{});
843716047 break :dst dst_mcv;
843816048 };
8439 if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) {
16049 if (field_abi_size * 8 > field_bit_size and dst_mcv.isBase()) {
844016050 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
844116051 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
844216052 defer self.register_manager.unlockReg(tmp_lock);
844316053
844416054 const hi_mcv =
844516055 dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();
8446 try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});
16056 try self.genSetReg(tmp_reg, .usize, hi_mcv, .{});
844716057 try self.truncateRegister(field_ty, tmp_reg);
8448 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});
16058 try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{});
844916059 }
845016060 break :result dst_mcv;
845116061 }
......@@ -8471,12 +16081,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
847116081 try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
847216082 .base = .{ .frame = frame_addr.index },
847316083 .mod = .{ .rm = .{
8474 .size = Memory.Size.fromSize(load_abi_size),
16084 .size = .fromSize(load_abi_size),
847516085 .disp = frame_addr.off + field_byte_off,
847616086 } },
847716087 });
847816088 try self.spillEflagsIfOccupied();
8479 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
16089 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, .u(field_bit_off));
848016090 } else {
848116091 const tmp_reg = registerAlias(
848216092 try self.register_manager.allocReg(null, abi.RegisterClass.gp),
......@@ -8492,7 +16102,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
849216102 .{
849316103 .base = .{ .frame = frame_addr.index },
849416104 .mod = .{ .rm = .{
8495 .size = Memory.Size.fromSize(field_abi_size),
16105 .size = .fromSize(field_abi_size),
849616106 .disp = frame_addr.off + field_byte_off,
849716107 } },
849816108 },
......@@ -8500,7 +16110,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
850016110 try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
850116111 .base = .{ .frame = frame_addr.index },
850216112 .mod = .{ .rm = .{
8503 .size = Memory.Size.fromSize(field_abi_size),
16113 .size = .fromSize(field_abi_size),
850416114 .disp = frame_addr.off + field_byte_off + limb_abi_size,
850516115 } },
850616116 });
......@@ -8509,7 +16119,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
850916119 .{ ._rd, .sh },
851016120 dst_alias,
851116121 tmp_reg,
8512 Immediate.u(field_bit_off),
16122 .u(field_bit_off),
851316123 );
851416124 }
851516125
......@@ -8527,32 +16137,22 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
852716137 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
852816138}
852916139
8530fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
8531 const pt = self.pt;
8532 const zcu = pt.zcu;
16140fn airFieldParentPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
853316141 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
853416142 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
853516143
8536 const inst_ty = self.typeOfIndex(inst);
8537 const parent_ty = inst_ty.childType(zcu);
8538 const field_off: i32 = switch (parent_ty.containerLayout(zcu)) {
8539 .auto, .@"extern" => @intCast(parent_ty.structFieldOffset(extra.field_index, zcu)),
8540 .@"packed" => @divExact(@as(i32, inst_ty.ptrInfo(zcu).packed_offset.bit_offset) +
8541 (if (zcu.typeToStruct(parent_ty)) |struct_obj| pt.structPackedFieldBitOffset(struct_obj, extra.field_index) else 0) -
8542 self.typeOf(extra.field_ptr).ptrInfo(zcu).packed_offset.bit_offset, 8),
8543 };
8544
16144 const ptr_agg_ty = self.typeOfIndex(inst);
854516145 const src_mcv = try self.resolveInst(extra.field_ptr);
854616146 const dst_mcv = if (src_mcv.isRegisterOffset() and
854716147 self.reuseOperand(inst, extra.field_ptr, 0, src_mcv))
854816148 src_mcv
854916149 else
8550 try self.copyToRegisterWithInstTracking(inst, inst_ty, src_mcv);
8551 const result = dst_mcv.offset(-field_off);
16150 try self.copyToRegisterWithInstTracking(inst, ptr_agg_ty, src_mcv);
16151 const result = dst_mcv.offset(-self.fieldOffset(ptr_agg_ty, self.typeOf(extra.field_ptr), extra.field_index));
855216152 return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });
855316153}
855416154
8555fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
16155fn genUnOp(self: *CodeGen, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
855616156 const pt = self.pt;
855716157 const zcu = pt.zcu;
855816158 const src_ty = self.typeOf(src_air);
......@@ -8596,8 +16196,8 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
859616196 switch (tag) {
859716197 .not => {
859816198 const limb_abi_size: u16 = @min(abi_size, 8);
8599 const int_info = if (src_ty.ip_index == .bool_type)
8600 std.builtin.Type.Int{ .signedness = .unsigned, .bits = 1 }
16199 const int_info: InternPool.Key.IntType = if (src_ty.ip_index == .bool_type)
16200 .{ .signedness = .unsigned, .bits = 1 }
860116201 else
860216202 src_ty.intInfo(zcu);
860316203 var byte_off: i32 = 0;
......@@ -8613,7 +16213,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
861316213 };
861416214
861516215 if (int_info.signedness == .unsigned and self.regExtraBits(limb_ty) > 0) {
8616 const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - limb_bits);
16216 const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_bits);
861716217 try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });
861816218 } else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
861916219 }
......@@ -8630,9 +16230,9 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
863016230 defer self.register_manager.unlockReg(tmp_lock);
863116231
863216232 const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
8633 try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});
16233 try self.genSetReg(tmp_reg, .usize, hi_mcv, .{});
863416234 try self.truncateRegister(src_ty, tmp_reg);
8635 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});
16235 try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{});
863616236 }
863716237 }
863816238 },
......@@ -8641,7 +16241,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
864116241 return dst_mcv;
864216242}
864316243
8644fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
16244fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
864516245 const pt = self.pt;
864616246 const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));
864716247 if (abi_size > 8) return self.fail("TODO implement {} for {}", .{ mir_tag, dst_ty.fmt(pt) });
......@@ -8654,6 +16254,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
865416254 .register_offset,
865516255 .eflags,
865616256 .register_overflow,
16257 .register_mask,
865716258 .lea_symbol,
865816259 .lea_direct,
865916260 .lea_got,
......@@ -8664,27 +16265,27 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
866416265 .air_ref,
866516266 => unreachable, // unmodifiable destination
866616267 .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),
8667 .register_pair => unreachable, // unimplemented
16268 .register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented
866816269 .memory, .load_symbol, .load_got, .load_direct, .load_tlv => {
866916270 const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
867016271 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
867116272 defer self.register_manager.unlockReg(addr_reg_lock);
867216273
8673 try self.genSetReg(addr_reg, Type.usize, dst_mcv.address(), .{});
16274 try self.genSetReg(addr_reg, .usize, dst_mcv.address(), .{});
867416275 try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{
8675 .size = Memory.Size.fromSize(abi_size),
16276 .size = .fromSize(abi_size),
867616277 } } });
867716278 },
867816279 .indirect, .load_frame => try self.asmMemory(
867916280 mir_tag,
8680 try dst_mcv.mem(self, Memory.Size.fromSize(abi_size)),
16281 try dst_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
868116282 ),
868216283 }
868316284}
868416285
868516286/// Clobbers .rcx for non-immediate shift value.
868616287fn genShiftBinOpMir(
8687 self: *Self,
16288 self: *CodeGen,
868816289 tag: Mir.Inst.FixedTag,
868916290 lhs_ty: Type,
869016291 lhs_mcv: MCValue,
......@@ -8698,7 +16299,7 @@ fn genShiftBinOpMir(
869816299 try self.spillEflagsIfOccupied();
869916300
870016301 if (abi_size > 16) {
8701 const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
16302 const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
870216303 assert(shift_abi_size >= 1 and shift_abi_size <= 2);
870316304
870416305 const rcx_lock: ?RegisterLock = switch (rhs_mcv) {
......@@ -8719,18 +16320,18 @@ fn genShiftBinOpMir(
871916320 };
872016321 defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
872116322
8722 const temp_regs = try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
16323 const temp_regs = try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
872316324 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
872416325 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
872516326
872616327 switch (tag[0]) {
872716328 ._l => {
8728 try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), Immediate.u(limbs_len - 1));
16329 try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1));
872916330 switch (rhs_mcv) {
873016331 .immediate => |shift_imm| try self.asmRegisterImmediate(
873116332 .{ ._, .mov },
873216333 temp_regs[0].to32(),
8733 Immediate.u(limbs_len - (shift_imm >> 6) - 1),
16334 .u(limbs_len - (shift_imm >> 6) - 1),
873416335 ),
873516336 else => {
873616337 try self.asmRegisterRegister(
......@@ -8738,16 +16339,8 @@ fn genShiftBinOpMir(
873816339 temp_regs[2].to32(),
873916340 registerAlias(.rcx, shift_abi_size),
874016341 );
8741 try self.asmRegisterImmediate(
8742 .{ ._, .@"and" },
8743 .cl,
8744 Immediate.u(math.maxInt(u6)),
8745 );
8746 try self.asmRegisterImmediate(
8747 .{ ._r, .sh },
8748 temp_regs[2].to32(),
8749 Immediate.u(6),
8750 );
16342 try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6)));
16343 try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[2].to32(), .u(6));
875116344 try self.asmRegisterRegister(
875216345 .{ ._, .mov },
875316346 temp_regs[0].to32(),
......@@ -8767,7 +16360,7 @@ fn genShiftBinOpMir(
876716360 .immediate => |shift_imm| try self.asmRegisterImmediate(
876816361 .{ ._, .mov },
876916362 temp_regs[0].to32(),
8770 Immediate.u(shift_imm >> 6),
16363 .u(shift_imm >> 6),
877116364 ),
877216365 else => {
877316366 try self.asmRegisterRegister(
......@@ -8775,16 +16368,8 @@ fn genShiftBinOpMir(
877516368 temp_regs[0].to32(),
877616369 registerAlias(.rcx, shift_abi_size),
877716370 );
8778 try self.asmRegisterImmediate(
8779 .{ ._, .@"and" },
8780 .cl,
8781 Immediate.u(math.maxInt(u6)),
8782 );
8783 try self.asmRegisterImmediate(
8784 .{ ._r, .sh },
8785 temp_regs[0].to32(),
8786 Immediate.u(6),
8787 );
16371 try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6)));
16372 try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[0].to32(), .u(6));
878816373 },
878916374 }
879016375 },
......@@ -8813,7 +16398,7 @@ fn genShiftBinOpMir(
881316398 try self.asmRegisterImmediate(
881416399 .{ ._, .cmp },
881516400 temp_regs[0].to32(),
8816 Immediate.u(limbs_len - 1),
16401 .u(limbs_len - 1),
881716402 );
881816403 break :skip try self.asmJccReloc(.nb, undefined);
881916404 },
......@@ -8843,7 +16428,7 @@ fn genShiftBinOpMir(
884316428 }, .sh },
884416429 temp_regs[2].to64(),
884516430 temp_regs[3].to64(),
8846 Immediate.u(shift_imm & math.maxInt(u6)),
16431 .u(shift_imm & std.math.maxInt(u6)),
884716432 ),
884816433 else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {
884916434 ._l => ._ld,
......@@ -8864,8 +16449,8 @@ fn genShiftBinOpMir(
886416449 switch (tag[0]) {
886516450 ._l => {
886616451 if (slow_inc_dec) {
8867 try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));
8868 try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), Immediate.u(1));
16452 try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
16453 try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), .u(1));
886916454 } else {
887016455 try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
887116456 try self.asmRegister(.{ ._, .dec }, temp_regs[0].to32());
......@@ -8874,8 +16459,8 @@ fn genShiftBinOpMir(
887416459 },
887516460 ._r => {
887616461 if (slow_inc_dec) {
8877 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), Immediate.u(1));
8878 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));
16462 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1));
16463 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
887916464 } else {
888016465 try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());
888116466 try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
......@@ -8883,7 +16468,7 @@ fn genShiftBinOpMir(
888316468 try self.asmRegisterImmediate(
888416469 .{ ._, .cmp },
888516470 temp_regs[0].to32(),
8886 Immediate.u(limbs_len - 1),
16471 .u(limbs_len - 1),
888716472 );
888816473 _ = try self.asmJccReloc(.b, loop);
888916474 },
......@@ -8898,7 +16483,7 @@ fn genShiftBinOpMir(
889816483 .immediate => |shift_imm| try self.asmRegisterImmediate(
889916484 tag,
890016485 temp_regs[2].to64(),
8901 Immediate.u(shift_imm & math.maxInt(u6)),
16486 .u(shift_imm & std.math.maxInt(u6)),
890216487 ),
890316488 else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),
890416489 }
......@@ -8914,7 +16499,7 @@ fn genShiftBinOpMir(
891416499 if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(
891516500 tag,
891616501 temp_regs[2].to64(),
8917 Immediate.u(63),
16502 .u(63),
891816503 );
891916504 if (switch (rhs_mcv) {
892016505 .immediate => |shift_imm| shift_imm >> 6 > 0,
......@@ -8935,7 +16520,7 @@ fn genShiftBinOpMir(
893516520 try self.asmRegisterImmediate(
893616521 .{ ._, .cmp },
893716522 temp_regs[1].to32(),
8938 Immediate.u(limbs_len - 1),
16523 .u(limbs_len - 1),
893916524 );
894016525 break :skip try self.asmJccReloc(.nb, undefined);
894116526 },
......@@ -8945,12 +16530,12 @@ fn genShiftBinOpMir(
894516530 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
894616531 switch (tag[0]) {
894716532 ._l => if (slow_inc_dec) {
8948 try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));
16533 try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
894916534 } else {
895016535 try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
895116536 },
895216537 ._r => if (slow_inc_dec) {
8953 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), Immediate.u(1));
16538 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1));
895416539 } else {
895516540 try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());
895616541 },
......@@ -8972,14 +16557,14 @@ fn genShiftBinOpMir(
897216557 .scale = .@"8",
897316558 .disp = lhs_mcv.load_frame.off,
897416559 } },
8975 }, Immediate.u(0));
16560 }, .u(0));
897616561 switch (tag[0]) {
897716562 ._l => _ = try self.asmJccReloc(.nz, loop),
897816563 ._r => {
897916564 try self.asmRegisterImmediate(
898016565 .{ ._, .cmp },
898116566 temp_regs[1].to32(),
8982 Immediate.u(limbs_len - 1),
16567 .u(limbs_len - 1),
898316568 );
898416569 _ = try self.asmJccReloc(.b, loop);
898516570 },
......@@ -9021,12 +16606,12 @@ fn genShiftBinOpMir(
902116606 info.double_tag,
902216607 lhs_regs[info.indices[1]],
902316608 lhs_regs[info.indices[0]],
9024 Immediate.u(shift_imm),
16609 .u(shift_imm),
902516610 );
902616611 try self.asmRegisterImmediate(
902716612 tag,
902816613 lhs_regs[info.indices[0]],
9029 Immediate.u(shift_imm),
16614 .u(shift_imm),
903016615 );
903116616 return;
903216617 } else {
......@@ -9039,7 +16624,7 @@ fn genShiftBinOpMir(
903916624 if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(
904016625 tag,
904116626 lhs_regs[info.indices[0]],
9042 Immediate.u(63),
16627 .u(63),
904316628 ) else try self.asmRegisterRegister(
904416629 .{ ._, .xor },
904516630 lhs_regs[info.indices[0]],
......@@ -9048,7 +16633,7 @@ fn genShiftBinOpMir(
904816633 if (shift_imm > 64) try self.asmRegisterImmediate(
904916634 tag,
905016635 lhs_regs[info.indices[1]],
9051 Immediate.u(shift_imm - 64),
16636 .u(shift_imm - 64),
905216637 );
905316638 return;
905416639 },
......@@ -9059,7 +16644,7 @@ fn genShiftBinOpMir(
905916644
906016645 if (tag[0] == ._r and tag[1] == .sa) {
906116646 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, lhs_regs[info.indices[0]]);
9062 try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
16647 try self.asmRegisterImmediate(tag, tmp_reg, .u(63));
906316648 } else try self.asmRegisterRegister(
906416649 .{ ._, .xor },
906516650 tmp_reg.to32(),
......@@ -9076,11 +16661,7 @@ fn genShiftBinOpMir(
907616661 lhs_regs[info.indices[0]],
907716662 registerAlias(shift_reg, 1),
907816663 );
9079 try self.asmRegisterImmediate(
9080 .{ ._, .cmp },
9081 registerAlias(shift_reg, 1),
9082 Immediate.u(64),
9083 );
16664 try self.asmRegisterImmediate(.{ ._, .cmp }, registerAlias(shift_reg, 1), .u(64));
908416665 try self.asmCmovccRegisterRegister(
908516666 .ae,
908616667 lhs_regs[info.indices[1]],
......@@ -9119,7 +16700,7 @@ fn genShiftBinOpMir(
911916700 } },
912016701 },
912116702 tmp_reg,
9122 Immediate.u(shift_imm),
16703 .u(shift_imm),
912316704 );
912416705 try self.asmMemoryImmediate(
912516706 tag,
......@@ -9130,7 +16711,7 @@ fn genShiftBinOpMir(
913016711 .disp = dst_frame_addr.off + info.indices[0] * 8,
913116712 } },
913216713 },
9133 Immediate.u(shift_imm),
16714 .u(shift_imm),
913416715 );
913516716 return;
913616717 } else {
......@@ -9149,7 +16730,7 @@ fn genShiftBinOpMir(
914916730 if (shift_imm > 64) try self.asmRegisterImmediate(
915016731 tag,
915116732 tmp_reg,
9152 Immediate.u(shift_imm - 64),
16733 .u(shift_imm - 64),
915316734 );
915416735 try self.asmMemoryRegister(
915516736 .{ ._, .mov },
......@@ -9171,7 +16752,7 @@ fn genShiftBinOpMir(
917116752 .disp = dst_frame_addr.off + info.indices[0] * 8,
917216753 } },
917316754 },
9174 Immediate.u(63),
16755 .u(63),
917516756 ) else {
917616757 try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());
917716758 try self.asmMemoryRegister(
......@@ -9223,7 +16804,7 @@ fn genShiftBinOpMir(
922316804 );
922416805 if (tag[0] == ._r and tag[1] == .sa) {
922516806 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);
9226 try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
16807 try self.asmRegisterImmediate(tag, tmp_reg, .u(63));
922716808 } else try self.asmRegisterRegister(
922816809 .{ ._, .xor },
922916810 tmp_reg.to32(),
......@@ -9239,7 +16820,7 @@ fn genShiftBinOpMir(
923916820 try self.asmRegisterImmediate(
924016821 .{ ._, .cmp },
924116822 registerAlias(shift_reg, 1),
9242 Immediate.u(64),
16823 .u(64),
924316824 );
924416825 try self.asmCmovccRegisterRegister(.ae, second_reg, first_reg);
924516826 try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg);
......@@ -9277,7 +16858,7 @@ fn genShiftBinOpMir(
927716858 .immediate => |shift_imm| return self.asmRegisterImmediate(
927816859 tag,
927916860 registerAlias(lhs_reg, abi_size),
9280 Immediate.u(shift_imm),
16861 .u(shift_imm),
928116862 ),
928216863 .register => |shift_reg| return self.asmRegisterRegister(
928316864 tag,
......@@ -9291,8 +16872,8 @@ fn genShiftBinOpMir(
929116872 .memory => |addr| .{
929216873 .base = .{ .reg = .ds },
929316874 .mod = .{ .rm = .{
9294 .size = Memory.Size.fromSize(abi_size),
9295 .disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
16875 .size = .fromSize(abi_size),
16876 .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
929616877 return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
929716878 @tagName(lhs_mcv),
929816879 @tagName(shift_mcv),
......@@ -9302,25 +16883,21 @@ fn genShiftBinOpMir(
930216883 .indirect => |reg_off| .{
930316884 .base = .{ .reg = reg_off.reg },
930416885 .mod = .{ .rm = .{
9305 .size = Memory.Size.fromSize(abi_size),
16886 .size = .fromSize(abi_size),
930616887 .disp = reg_off.off,
930716888 } },
930816889 },
930916890 .load_frame => |frame_addr| .{
931016891 .base = .{ .frame = frame_addr.index },
931116892 .mod = .{ .rm = .{
9312 .size = Memory.Size.fromSize(abi_size),
16893 .size = .fromSize(abi_size),
931316894 .disp = frame_addr.off,
931416895 } },
931516896 },
931616897 else => unreachable,
931716898 };
931816899 switch (shift_mcv) {
9319 .immediate => |shift_imm| return self.asmMemoryImmediate(
9320 tag,
9321 lhs_mem,
9322 Immediate.u(shift_imm),
9323 ),
16900 .immediate => |shift_imm| return self.asmMemoryImmediate(tag, lhs_mem, .u(shift_imm)),
932416901 .register => |shift_reg| return self.asmMemoryRegister(
932516902 tag,
932616903 lhs_mem,
......@@ -9341,7 +16918,7 @@ fn genShiftBinOpMir(
934116918/// Clobbers .rcx for non-immediate rhs, therefore care is needed to spill .rcx upfront.
934216919/// Asserts .rcx is free.
934316920fn genShiftBinOp(
9344 self: *Self,
16921 self: *CodeGen,
934516922 air_tag: Air.Inst.Tag,
934616923 maybe_inst: ?Air.Inst.Index,
934716924 lhs_mcv: MCValue,
......@@ -9359,7 +16936,18 @@ fn genShiftBinOp(
935916936 const rcx_lock = self.register_manager.lockReg(.rcx);
936016937 defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
936116938
9362 const lhs_lock = switch (lhs_mcv) {
16939 const mat_lhs_mcv: MCValue, const can_reuse_lhs = switch (lhs_mcv) {
16940 .register => |lhs_reg| switch (lhs_reg.class()) {
16941 .general_purpose => .{ lhs_mcv, true },
16942 else => lhs: {
16943 const mat_lhs_mcv = try self.allocTempRegOrMem(lhs_ty, true);
16944 try self.genCopy(lhs_ty, mat_lhs_mcv, lhs_mcv, .{});
16945 break :lhs .{ mat_lhs_mcv, false };
16946 },
16947 },
16948 else => .{ lhs_mcv, true },
16949 };
16950 const lhs_lock = switch (mat_lhs_mcv) {
936316951 .register => |reg| self.register_manager.lockReg(reg),
936416952 else => null,
936516953 };
......@@ -9372,12 +16960,12 @@ fn genShiftBinOp(
937216960 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
937316961
937416962 const dst_mcv: MCValue = dst: {
9375 if (maybe_inst) |inst| {
16963 if (can_reuse_lhs) if (maybe_inst) |inst| {
937616964 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9377 if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;
9378 }
16965 if (self.reuseOperand(inst, bin_op.lhs, 0, mat_lhs_mcv)) break :dst mat_lhs_mcv;
16966 };
937916967 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
9380 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv, .{});
16968 try self.genCopy(lhs_ty, dst_mcv, mat_lhs_mcv, .{});
938116969 break :dst dst_mcv;
938216970 };
938316971
......@@ -9400,7 +16988,7 @@ fn genShiftBinOp(
940016988/// Clobbers .rax and .rdx therefore care is needed to spill .rax and .rdx upfront.
940116989/// Asserts .rax and .rdx are free.
940216990fn genMulDivBinOp(
9403 self: *Self,
16991 self: *CodeGen,
940416992 tag: Air.Inst.Tag,
940516993 maybe_inst: ?Air.Inst.Index,
940616994 dst_ty: Type,
......@@ -9430,7 +17018,7 @@ fn genMulDivBinOp(
943017018 const mat_lhs_mcv = switch (lhs_mcv) {
943117019 .load_symbol => mat_lhs_mcv: {
943217020 // TODO clean this up!
9433 const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());
17021 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
943417022 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
943517023 },
943617024 else => lhs_mcv,
......@@ -9443,7 +17031,7 @@ fn genMulDivBinOp(
944317031 const mat_rhs_mcv = switch (rhs_mcv) {
944417032 .load_symbol => mat_rhs_mcv: {
944517033 // TODO clean this up!
9446 const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());
17034 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
944717035 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
944817036 },
944917037 else => rhs_mcv,
......@@ -9458,28 +17046,28 @@ fn genMulDivBinOp(
945817046 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
945917047 defer self.register_manager.unlockReg(tmp_lock);
946017048
9461 if (mat_lhs_mcv.isMemory())
9462 try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .qword))
17049 if (mat_lhs_mcv.isBase())
17050 try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .{ .size = .qword }))
946317051 else
946417052 try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);
9465 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
17053 if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
946617054 .{ ._, .mov },
946717055 tmp_reg,
9468 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
17056 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
946917057 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);
947017058 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);
9471 if (mat_rhs_mcv.isMemory())
9472 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
17059 if (mat_rhs_mcv.isBase())
17060 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
947317061 else
947417062 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
947517063 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
9476 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
17064 if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
947717065 .{ ._, .mov },
947817066 tmp_reg,
9479 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
17067 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
948017068 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);
9481 if (mat_rhs_mcv.isMemory())
9482 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .qword))
17069 if (mat_rhs_mcv.isBase())
17070 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
948317071 else
948417072 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);
948517073 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
......@@ -9495,7 +17083,7 @@ fn genMulDivBinOp(
949517083 switch (tag) {
949617084 .mul, .mul_wrap => {
949717085 const slow_inc = self.hasFeature(.slow_incdec);
9498 const limb_len = math.divCeil(u32, src_abi_size, 8) catch unreachable;
17086 const limb_len = std.math.divCeil(u32, src_abi_size, 8) catch unreachable;
949917087
950017088 try self.spillRegisters(&.{ .rax, .rcx, .rdx });
950117089 const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });
......@@ -9511,7 +17099,7 @@ fn genMulDivBinOp(
951117099 );
951217100
951317101 const temp_regs =
9514 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
17102 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
951517103 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
951617104 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
951717105
......@@ -9536,7 +17124,7 @@ fn genMulDivBinOp(
953617124 try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);
953717125
953817126 const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
9539 try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, Immediate.u(1));
17127 try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1));
954017128 try self.asmMemoryRegister(.{ ._, .adc }, .{
954117129 .base = .{ .frame = dst_mcv.load_frame.index },
954217130 .mod = .{ .rm = .{
......@@ -9559,7 +17147,7 @@ fn genMulDivBinOp(
955917147 });
956017148 try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());
956117149
9562 try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, Immediate.u(1));
17150 try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1));
956317151 try self.asmMemoryRegister(.{ ._, .adc }, .{
956417152 .base = .{ .frame = dst_mcv.load_frame.index },
956517153 .mod = .{ .rm = .{
......@@ -9572,30 +17160,22 @@ fn genMulDivBinOp(
957217160 try self.asmSetccRegister(.c, .ch);
957317161
957417162 if (slow_inc) {
9575 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));
9576 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), Immediate.u(1));
17163 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
17164 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1));
957717165 } else {
957817166 try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
957917167 try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());
958017168 }
9581 try self.asmRegisterImmediate(
9582 .{ ._, .cmp },
9583 temp_regs[3].to32(),
9584 Immediate.u(limb_len),
9585 );
17169 try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len));
958617170 _ = try self.asmJccReloc(.b, inner_loop);
958717171
958817172 self.performReloc(skip_inner);
958917173 if (slow_inc) {
9590 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));
17174 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
959117175 } else {
959217176 try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
959317177 }
9594 try self.asmRegisterImmediate(
9595 .{ ._, .cmp },
9596 temp_regs[0].to32(),
9597 Immediate.u(limb_len),
9598 );
17178 try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len));
959917179 _ = try self.asmJccReloc(.b, outer_loop);
960017180
960117181 return dst_mcv;
......@@ -9639,13 +17219,13 @@ fn genMulDivBinOp(
963917219 manyptr_u32_ty,
964017220 manyptr_const_u32_ty,
964117221 manyptr_const_u32_ty,
9642 Type.usize,
17222 .usize,
964317223 }, &.{
964417224 dst_mcv.address(),
964517225 lhs_mcv.address(),
964617226 rhs_mcv.address(),
9647 .{ .immediate = src_info.bits },
9648 });
17227 .{ .immediate = 8 * src_abi_size },
17228 }, .{});
964917229 return dst_mcv;
965017230 },
965117231 },
......@@ -9662,7 +17242,8 @@ fn genMulDivBinOp(
966217242 const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx });
966317243 defer for (reg_locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
966417244
9665 const signedness = ty.intInfo(zcu).signedness;
17245 const int_info = ty.intInfo(zcu);
17246 const signedness = int_info.signedness;
966617247 switch (tag) {
966717248 .mul,
966817249 .mul_wrap,
......@@ -9695,6 +17276,15 @@ fn genMulDivBinOp(
969517276 },
969617277 }, ty, lhs_mcv, rhs_mcv);
969717278
17279 switch (tag) {
17280 .mul, .rem, .div_trunc, .div_exact => {},
17281 .mul_wrap => if (dst_ty.intInfo(zcu).bits < 8 * dst_abi_size) try self.truncateRegister(
17282 dst_ty,
17283 if (dst_abi_size <= 8) .rax else .rdx,
17284 ),
17285 else => unreachable,
17286 }
17287
969817288 if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) {
969917289 .mul, .mul_wrap, .div_trunc, .div_exact => .rax,
970017290 .rem => .rdx,
......@@ -9814,7 +17404,7 @@ fn genMulDivBinOp(
981417404}
981517405
981617406fn genBinOp(
9817 self: *Self,
17407 self: *CodeGen,
981817408 maybe_inst: ?Air.Inst.Index,
981917409 air_tag: Air.Inst.Tag,
982017410 lhs_air: Air.Inst.Ref,
......@@ -9869,7 +17459,7 @@ fn genBinOp(
986917459 .return_type = lhs_ty.toIntern(),
987017460 .param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() },
987117461 .callee = callee,
9872 } }, &.{ lhs_ty, rhs_ty }, &.{ .{ .air_ref = lhs_air }, .{ .air_ref = rhs_air } });
17462 } }, &.{ lhs_ty, rhs_ty }, &.{ .{ .air_ref = lhs_air }, .{ .air_ref = rhs_air } }, .{});
987317463 return switch (air_tag) {
987417464 .mod => result: {
987517465 const adjusted: MCValue = if (type_needs_libcall) adjusted: {
......@@ -9883,7 +17473,7 @@ fn genBinOp(
988317473 .callee = std.fmt.bufPrint(&add_callee_buf, "__add{c}f3", .{
988417474 floatCompilerRtAbiName(float_bits),
988517475 }) catch unreachable,
9886 } }, &.{ lhs_ty, rhs_ty }, &.{ result, .{ .air_ref = rhs_air } });
17476 } }, &.{ lhs_ty, rhs_ty }, &.{ result, .{ .air_ref = rhs_air } }, .{});
988717477 } else switch (float_bits) {
988817478 16, 32, 64 => adjusted: {
988917479 const dst_reg = switch (result) {
......@@ -9906,12 +17496,12 @@ fn genBinOp(
990617496 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
990717497 defer self.register_manager.unlockReg(tmp_lock);
990817498
9909 if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
17499 if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
991017500 .{ .vp_w, .insr },
991117501 dst_reg,
991217502 dst_reg,
9913 try rhs_mcv.mem(self, .word),
9914 Immediate.u(1),
17503 try rhs_mcv.mem(self, .{ .size = .word }),
17504 .u(1),
991517505 ) else try self.asmRegisterRegisterRegister(
991617506 .{ .vp_, .unpcklwd },
991717507 dst_reg,
......@@ -9931,11 +17521,11 @@ fn genBinOp(
993117521 64 => .{ .v_sd, .add },
993217522 else => unreachable,
993317523 };
9934 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
17524 if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
993517525 mir_tag,
993617526 dst_reg,
993717527 dst_reg,
9938 try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
17528 try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }),
993917529 ) else try self.asmRegisterRegisterRegister(
994017530 mir_tag,
994117531 dst_reg,
......@@ -9951,10 +17541,10 @@ fn genBinOp(
995117541 64 => .{ ._sd, .add },
995217542 else => unreachable,
995317543 };
9954 if (src_mcv.isMemory()) try self.asmRegisterMemory(
17544 if (src_mcv.isBase()) try self.asmRegisterMemory(
995517545 mir_tag,
995617546 dst_reg,
9957 try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
17547 try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }),
995817548 ) else try self.asmRegisterRegister(
995917549 mir_tag,
996017550 dst_reg,
......@@ -9969,7 +17559,7 @@ fn genBinOp(
996917559 .{ .v_, .cvtps2ph },
997017560 dst_reg,
997117561 dst_reg,
9972 Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
17562 .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
997317563 );
997417564 break :adjusted .{ .register = dst_reg };
997517565 },
......@@ -9982,7 +17572,7 @@ fn genBinOp(
998217572 .return_type = lhs_ty.toIntern(),
998317573 .param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() },
998417574 .callee = callee,
9985 } }, &.{ lhs_ty, rhs_ty }, &.{ adjusted, .{ .air_ref = rhs_air } });
17575 } }, &.{ lhs_ty, rhs_ty }, &.{ adjusted, .{ .air_ref = rhs_air } }, .{});
998617576 },
998717577 .div_trunc, .div_floor => try self.genRoundLibcall(lhs_ty, result, .{
998817578 .mode = switch (air_tag) {
......@@ -10006,7 +17596,7 @@ fn genBinOp(
1000617596 };
1000717597 if (sse_op and ((lhs_ty.scalarType(zcu).isRuntimeFloat() and
1000817598 lhs_ty.scalarType(zcu).floatBits(self.target.*) == 80) or
10009 lhs_ty.abiSize(zcu) > @as(u6, if (self.hasFeature(.avx)) 32 else 16)))
17599 lhs_ty.abiSize(zcu) > self.vectorSize(.float)))
1001017600 return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(air_tag), lhs_ty.fmt(pt) });
1001117601
1001217602 const maybe_mask_reg = switch (air_tag) {
......@@ -10085,14 +17675,14 @@ fn genBinOp(
1008517675 const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs);
1008617676 break :locks .{ locks[0], locks[1] };
1008717677 },
10088 else => .{null} ** 2,
17678 else => @splat(null),
1008917679 };
1009017680 defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
1009117681
1009217682 const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) {
1009317683 .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },
1009417684 .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),
10095 else => .{null} ** 2,
17685 else => @splat(null),
1009617686 };
1009717687 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1009817688
......@@ -10125,7 +17715,7 @@ fn genBinOp(
1012517715 const dst_locks: [2]?RegisterLock = switch (dst_mcv) {
1012617716 .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null },
1012717717 .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs),
10128 else => .{null} ** 2,
17718 else => @splat(null),
1012917719 };
1013017720 defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);
1013117721
......@@ -10141,7 +17731,7 @@ fn genBinOp(
1014117731 const src_locks: [2]?RegisterLock = switch (src_mcv) {
1014217732 .register => |src_reg| .{ self.register_manager.lockReg(src_reg), null },
1014317733 .register_pair => |src_regs| self.register_manager.lockRegs(2, src_regs),
10144 else => .{null} ** 2,
17734 else => @splat(null),
1014517735 };
1014617736 defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock);
1014717737
......@@ -10199,11 +17789,7 @@ fn genBinOp(
1019917789 const dst_regs = switch (dst_mcv) {
1020017790 .register_pair => |dst_regs| dst_regs,
1020117791 else => dst: {
10202 const dst_regs = try self.register_manager.allocRegs(
10203 2,
10204 .{null} ** 2,
10205 abi.RegisterClass.gp,
10206 );
17792 const dst_regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
1020717793 const dst_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
1020817794 defer for (dst_regs_locks) |lock| self.register_manager.unlockReg(lock);
1020917795
......@@ -10227,26 +17813,26 @@ fn genBinOp(
1022717813 };
1022817814
1022917815 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
10230 if (src_mcv.isMemory()) {
17816 if (src_mcv.isBase()) {
1023117817 try self.asmRegisterMemory(
1023217818 .{ ._, .cmp },
1023317819 dst_regs[0],
10234 try src_mcv.mem(self, .qword),
17820 try src_mcv.mem(self, .{ .size = .qword }),
1023517821 );
1023617822 try self.asmRegisterMemory(
1023717823 .{ ._, .sbb },
1023817824 tmp_reg,
10239 try src_mcv.address().offset(8).deref().mem(self, .qword),
17825 try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
1024017826 );
1024117827 try self.asmCmovccRegisterMemory(
1024217828 cc,
1024317829 dst_regs[0],
10244 try src_mcv.mem(self, .qword),
17830 try src_mcv.mem(self, .{ .size = .qword }),
1024517831 );
1024617832 try self.asmCmovccRegisterMemory(
1024717833 cc,
1024817834 dst_regs[1],
10249 try src_mcv.address().offset(8).deref().mem(self, .qword),
17835 try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
1025017836 );
1025117837 } else {
1025217838 try self.asmRegisterRegister(
......@@ -10278,7 +17864,7 @@ fn genBinOp(
1027817864 .lea_tlv,
1027917865 .lea_frame,
1028017866 => true,
10281 .memory => |addr| math.cast(i32, @as(i64, @bitCast(addr))) == null,
17867 .memory => |addr| std.math.cast(i32, @as(i64, @bitCast(addr))) == null,
1028217868 else => false,
1028317869 .register_pair,
1028417870 .register_overflow,
......@@ -10324,8 +17910,11 @@ fn genBinOp(
1032417910 .immediate,
1032517911 .eflags,
1032617912 .register_pair,
17913 .register_triple,
17914 .register_quadruple,
1032717915 .register_offset,
1032817916 .register_overflow,
17917 .register_mask,
1032917918 .load_symbol,
1033017919 .lea_symbol,
1033117920 .load_direct,
......@@ -10351,21 +17940,21 @@ fn genBinOp(
1035117940 .memory => |addr| .{
1035217941 .base = .{ .reg = .ds },
1035317942 .mod = .{ .rm = .{
10354 .size = Memory.Size.fromSize(cmov_abi_size),
17943 .size = .fromSize(cmov_abi_size),
1035517944 .disp = @intCast(@as(i64, @bitCast(addr))),
1035617945 } },
1035717946 },
1035817947 .indirect => |reg_off| .{
1035917948 .base = .{ .reg = reg_off.reg },
1036017949 .mod = .{ .rm = .{
10361 .size = Memory.Size.fromSize(cmov_abi_size),
17950 .size = .fromSize(cmov_abi_size),
1036217951 .disp = reg_off.off,
1036317952 } },
1036417953 },
1036517954 .load_frame => |frame_addr| .{
1036617955 .base = .{ .frame = frame_addr.index },
1036717956 .mod = .{ .rm = .{
10368 .size = Memory.Size.fromSize(cmov_abi_size),
17957 .size = .fromSize(cmov_abi_size),
1036917958 .disp = frame_addr.off,
1037017959 } },
1037117960 },
......@@ -10400,21 +17989,22 @@ fn genBinOp(
1040017989 .float => switch (lhs_ty.floatBits(self.target.*)) {
1040117990 16 => {
1040217991 assert(self.hasFeature(.f16c));
10403 const tmp_reg =
10404 (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
17992 const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
17993
17994 const tmp_reg = (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
1040517995 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
1040617996 defer self.register_manager.unlockReg(tmp_lock);
1040717997
10408 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
17998 if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
1040917999 .{ .vp_w, .insr },
1041018000 dst_reg,
10411 dst_reg,
10412 try src_mcv.mem(self, .word),
10413 Immediate.u(1),
18001 lhs_reg,
18002 try src_mcv.mem(self, .{ .size = .word }),
18003 .u(1),
1041418004 ) else try self.asmRegisterRegisterRegister(
1041518005 .{ .vp_, .unpcklwd },
1041618006 dst_reg,
10417 dst_reg,
18007 lhs_reg,
1041818008 (if (src_mcv.isRegister())
1041918009 src_mcv.getReg().?
1042018010 else
......@@ -10442,7 +18032,7 @@ fn genBinOp(
1044218032 dst_reg,
1044318033 dst_reg,
1044418034 dst_reg,
10445 Immediate.u(@as(u5, @bitCast(RoundMode{
18035 .u(@as(u5, @bitCast(RoundMode{
1044618036 .mode = switch (air_tag) {
1044718037 .div_trunc => .zero,
1044818038 .div_floor => .down,
......@@ -10457,7 +18047,7 @@ fn genBinOp(
1045718047 .{ .v_, .cvtps2ph },
1045818048 dst_reg,
1045918049 dst_reg,
10460 Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
18050 .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
1046118051 );
1046218052 return dst_mcv;
1046318053 },
......@@ -10842,25 +18432,24 @@ fn genBinOp(
1084218432 .float => switch (lhs_ty.childType(zcu).floatBits(self.target.*)) {
1084318433 16 => tag: {
1084418434 assert(self.hasFeature(.f16c));
18435 const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
1084518436 switch (lhs_ty.vectorLen(zcu)) {
1084618437 1 => {
10847 const tmp_reg = (try self.register_manager.allocReg(
10848 null,
10849 abi.RegisterClass.sse,
10850 )).to128();
18438 const tmp_reg =
18439 (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
1085118440 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
1085218441 defer self.register_manager.unlockReg(tmp_lock);
1085318442
10854 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
18443 if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
1085518444 .{ .vp_w, .insr },
1085618445 dst_reg,
10857 dst_reg,
10858 try src_mcv.mem(self, .word),
10859 Immediate.u(1),
18446 lhs_reg,
18447 try src_mcv.mem(self, .{ .size = .word }),
18448 .u(1),
1086018449 ) else try self.asmRegisterRegisterRegister(
1086118450 .{ .vp_, .unpcklwd },
1086218451 dst_reg,
10863 dst_reg,
18452 lhs_reg,
1086418453 (if (src_mcv.isRegister())
1086518454 src_mcv.getReg().?
1086618455 else
......@@ -10886,7 +18475,7 @@ fn genBinOp(
1088618475 .{ .v_, .cvtps2ph },
1088718476 dst_reg,
1088818477 dst_reg,
10889 Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
18478 .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
1089018479 );
1089118480 return dst_mcv;
1089218481 },
......@@ -10898,15 +18487,16 @@ fn genBinOp(
1089818487 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
1089918488 defer self.register_manager.unlockReg(tmp_lock);
1090018489
10901 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18490 if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
1090218491 .{ .vp_d, .insr },
1090318492 dst_reg,
10904 try src_mcv.mem(self, .dword),
10905 Immediate.u(1),
18493 lhs_reg,
18494 try src_mcv.mem(self, .{ .size = .dword }),
18495 .u(1),
1090618496 ) else try self.asmRegisterRegisterRegister(
1090718497 .{ .v_ps, .unpckl },
1090818498 dst_reg,
10909 dst_reg,
18499 lhs_reg,
1091018500 (if (src_mcv.isRegister())
1091118501 src_mcv.getReg().?
1091218502 else
......@@ -10937,7 +18527,7 @@ fn genBinOp(
1093718527 .{ .v_, .cvtps2ph },
1093818528 dst_reg,
1093918529 dst_reg,
10940 Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
18530 .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
1094118531 );
1094218532 return dst_mcv;
1094318533 },
......@@ -10949,11 +18539,11 @@ fn genBinOp(
1094918539 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
1095018540 defer self.register_manager.unlockReg(tmp_lock);
1095118541
10952 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg);
10953 if (src_mcv.isMemory()) try self.asmRegisterMemory(
18542 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, lhs_reg);
18543 if (src_mcv.isBase()) try self.asmRegisterMemory(
1095418544 .{ .v_ps, .cvtph2 },
1095518545 tmp_reg,
10956 try src_mcv.mem(self, .qword),
18546 try src_mcv.mem(self, .{ .size = .qword }),
1095718547 ) else try self.asmRegisterRegister(
1095818548 .{ .v_ps, .cvtph2 },
1095918549 tmp_reg,
......@@ -10980,7 +18570,7 @@ fn genBinOp(
1098018570 .{ .v_, .cvtps2ph },
1098118571 dst_reg,
1098218572 dst_reg,
10983 Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
18573 .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
1098418574 );
1098518575 return dst_mcv;
1098618576 },
......@@ -10992,11 +18582,11 @@ fn genBinOp(
1099218582 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
1099318583 defer self.register_manager.unlockReg(tmp_lock);
1099418584
10995 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), dst_reg);
10996 if (src_mcv.isMemory()) try self.asmRegisterMemory(
18585 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), lhs_reg);
18586 if (src_mcv.isBase()) try self.asmRegisterMemory(
1099718587 .{ .v_ps, .cvtph2 },
1099818588 tmp_reg,
10999 try src_mcv.mem(self, .xword),
18589 try src_mcv.mem(self, .{ .size = .xword }),
1100018590 ) else try self.asmRegisterRegister(
1100118591 .{ .v_ps, .cvtph2 },
1100218592 tmp_reg,
......@@ -11023,7 +18613,7 @@ fn genBinOp(
1102318613 .{ .v_, .cvtps2ph },
1102418614 dst_reg,
1102518615 dst_reg.to256(),
11026 Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
18616 .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
1102718617 );
1102818618 return dst_mcv;
1102918619 },
......@@ -11153,16 +18743,15 @@ fn genBinOp(
1115318743
1115418744 switch (mir_tag[1]) {
1115518745 else => if (self.hasFeature(.avx)) {
11156 const lhs_reg =
11157 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
11158 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
18746 const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
18747 if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
1115918748 mir_tag,
1116018749 dst_reg,
1116118750 lhs_reg,
11162 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11163 else => Memory.Size.fromSize(abi_size),
11164 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
11165 }),
18751 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
18752 else => .fromSize(abi_size),
18753 .vector => .fromBitSize(dst_reg.bitSize()),
18754 } }),
1116618755 ) else try self.asmRegisterRegisterRegister(
1116718756 mir_tag,
1116818757 dst_reg,
......@@ -11174,13 +18763,13 @@ fn genBinOp(
1117418763 );
1117518764 } else {
1117618765 assert(copied_to_dst);
11177 if (src_mcv.isMemory()) try self.asmRegisterMemory(
18766 if (src_mcv.isBase()) try self.asmRegisterMemory(
1117818767 mir_tag,
1117918768 dst_reg,
11180 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11181 else => Memory.Size.fromSize(abi_size),
11182 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
11183 }),
18769 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
18770 else => .fromSize(abi_size),
18771 .vector => .fromBitSize(dst_reg.bitSize()),
18772 } }),
1118418773 ) else try self.asmRegisterRegister(
1118518774 mir_tag,
1118618775 dst_reg,
......@@ -11191,7 +18780,7 @@ fn genBinOp(
1119118780 );
1119218781 },
1119318782 .cmp => {
11194 const imm = Immediate.u(switch (air_tag) {
18783 const imm: Immediate = .u(switch (air_tag) {
1119518784 .cmp_eq => 0,
1119618785 .cmp_lt, .cmp_gt => 1,
1119718786 .cmp_lte, .cmp_gte => 2,
......@@ -11201,14 +18790,14 @@ fn genBinOp(
1120118790 if (self.hasFeature(.avx)) {
1120218791 const lhs_reg =
1120318792 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
11204 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
18793 if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
1120518794 mir_tag,
1120618795 dst_reg,
1120718796 lhs_reg,
11208 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11209 else => Memory.Size.fromSize(abi_size),
11210 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
11211 }),
18797 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
18798 else => .fromSize(abi_size),
18799 .vector => .fromBitSize(dst_reg.bitSize()),
18800 } }),
1121218801 imm,
1121318802 ) else try self.asmRegisterRegisterRegisterImmediate(
1121418803 mir_tag,
......@@ -11222,13 +18811,13 @@ fn genBinOp(
1122218811 );
1122318812 } else {
1122418813 assert(copied_to_dst);
11225 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18814 if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate(
1122618815 mir_tag,
1122718816 dst_reg,
11228 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11229 else => Memory.Size.fromSize(abi_size),
11230 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
11231 }),
18817 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
18818 else => .fromSize(abi_size),
18819 .vector => .fromBitSize(dst_reg.bitSize()),
18820 } }),
1123218821 imm,
1123318822 ) else try self.asmRegisterRegisterImmediate(
1123418823 mir_tag,
......@@ -11289,7 +18878,7 @@ fn genBinOp(
1128918878 mask_reg,
1129018879 rhs_copy_reg,
1129118880 rhs_copy_reg,
11292 Immediate.u(3), // unord
18881 .u(3), // unord
1129318882 );
1129418883 try self.asmRegisterRegisterRegisterRegister(
1129518884 @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
......@@ -11356,7 +18945,7 @@ fn genBinOp(
1135618945 }),
1135718946 mask_reg,
1135818947 mask_reg,
11359 Immediate.u(if (has_blend) 3 else 7), // unord, ord
18948 .u(if (has_blend) 3 else 7), // unord, ord
1136018949 );
1136118950 if (has_blend) try self.asmRegisterRegisterRegister(
1136218951 @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
......@@ -11433,12 +19022,12 @@ fn genBinOp(
1143319022 => {
1143419023 const unsigned_ty = try lhs_ty.toUnsigned(pt);
1143519024 const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty));
11436 const not_mem: Memory = if (not_mcv.isMemory())
11437 try not_mcv.mem(self, Memory.Size.fromSize(abi_size))
19025 const not_mem: Memory = if (not_mcv.isBase())
19026 try not_mcv.mem(self, .{ .size = .fromSize(abi_size) })
1143819027 else
1143919028 .{ .base = .{
11440 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
11441 }, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };
19029 .reg = try self.copyToTmpRegister(.usize, not_mcv.address()),
19030 }, .mod = .{ .rm = .{ .size = .fromSize(abi_size) } } };
1144219031 switch (mir_tag[0]) {
1144319032 .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
1144419033 .{ .vp_, .xor },
......@@ -11495,7 +19084,7 @@ fn genBinOp(
1149519084}
1149619085
1149719086fn genBinOpMir(
11498 self: *Self,
19087 self: *CodeGen,
1149919088 mir_tag: Mir.Inst.FixedTag,
1150019089 ty: Type,
1150119090 dst_mcv: MCValue,
......@@ -11513,6 +19102,7 @@ fn genBinOpMir(
1151319102 .immediate,
1151419103 .eflags,
1151519104 .register_overflow,
19105 .register_mask,
1151619106 .lea_direct,
1151719107 .lea_got,
1151819108 .lea_tlv,
......@@ -11522,9 +19112,9 @@ fn genBinOpMir(
1152219112 .reserved_frame,
1152319113 .air_ref,
1152419114 => unreachable, // unmodifiable destination
11525 .register, .register_pair, .register_offset => {
19115 .register, .register_pair, .register_triple, .register_quadruple, .register_offset => {
1152619116 switch (dst_mcv) {
11527 .register, .register_pair => {},
19117 .register, .register_pair, .register_triple, .register_quadruple => {},
1152819118 .register_offset => |ro| assert(ro.off == 0),
1152919119 else => unreachable,
1153019120 }
......@@ -11553,10 +19143,15 @@ fn genBinOpMir(
1155319143 .dead,
1155419144 .undef,
1155519145 .register_overflow,
19146 .register_mask,
1155619147 .elementwise_regs_then_frame,
1155719148 .reserved_frame,
1155819149 => unreachable,
11559 .register, .register_pair => try self.asmRegisterRegister(
19150 .register,
19151 .register_pair,
19152 .register_triple,
19153 .register_quadruple,
19154 => try self.asmRegisterRegister(
1156019155 mir_limb_tag,
1156119156 dst_alias,
1156219157 registerAlias(src_mcv.getRegs()[dst_reg_i], limb_abi_size),
......@@ -11567,29 +19162,29 @@ fn genBinOpMir(
1156719162 8 => try self.asmRegisterImmediate(
1156819163 mir_limb_tag,
1156919164 dst_alias,
11570 if (math.cast(i8, @as(i64, @bitCast(imm)))) |small|
11571 Immediate.s(small)
19165 if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |small|
19166 .s(small)
1157219167 else
11573 Immediate.u(@as(u8, @intCast(imm))),
19168 .u(@as(u8, @intCast(imm))),
1157419169 ),
1157519170 16 => try self.asmRegisterImmediate(
1157619171 mir_limb_tag,
1157719172 dst_alias,
11578 if (math.cast(i16, @as(i64, @bitCast(imm)))) |small|
11579 Immediate.s(small)
19173 if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |small|
19174 .s(small)
1158019175 else
11581 Immediate.u(@as(u16, @intCast(imm))),
19176 .u(@as(u16, @intCast(imm))),
1158219177 ),
1158319178 32 => try self.asmRegisterImmediate(
1158419179 mir_limb_tag,
1158519180 dst_alias,
11586 if (math.cast(i32, @as(i64, @bitCast(imm)))) |small|
11587 Immediate.s(small)
19181 if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small|
19182 .s(small)
1158819183 else
11589 Immediate.u(@as(u32, @intCast(imm))),
19184 .u(@as(u32, @intCast(imm))),
1159019185 ),
11591 64 => if (math.cast(i32, @as(i64, @bitCast(imm)))) |small|
11592 try self.asmRegisterImmediate(mir_limb_tag, dst_alias, Immediate.s(small))
19186 64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small|
19187 try self.asmRegisterImmediate(mir_limb_tag, dst_alias, .s(small))
1159319188 else
1159419189 try self.asmRegisterRegister(mir_limb_tag, dst_alias, registerAlias(
1159519190 try self.copyToTmpRegister(ty, src_mcv),
......@@ -11618,21 +19213,21 @@ fn genBinOpMir(
1161819213 .memory => |addr| .{
1161919214 .base = .{ .reg = .ds },
1162019215 .mod = .{ .rm = .{
11621 .size = Memory.Size.fromSize(limb_abi_size),
11622 .disp = math.cast(i32, addr + off) orelse break :direct,
19216 .size = .fromSize(limb_abi_size),
19217 .disp = std.math.cast(i32, addr + off) orelse break :direct,
1162319218 } },
1162419219 },
1162519220 .indirect => |reg_off| .{
1162619221 .base = .{ .reg = reg_off.reg },
1162719222 .mod = .{ .rm = .{
11628 .size = Memory.Size.fromSize(limb_abi_size),
19223 .size = .fromSize(limb_abi_size),
1162919224 .disp = reg_off.off + off,
1163019225 } },
1163119226 },
1163219227 .load_frame => |frame_addr| .{
1163319228 .base = .{ .frame = frame_addr.index },
1163419229 .mod = .{ .rm = .{
11635 .size = Memory.Size.fromSize(limb_abi_size),
19230 .size = .fromSize(limb_abi_size),
1163619231 .disp = frame_addr.off + off,
1163719232 } },
1163819233 },
......@@ -11695,7 +19290,7 @@ fn genBinOpMir(
1169519290 const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
1169619291 errdefer self.register_manager.unlockReg(dst_addr_lock);
1169719292
11698 try self.genSetReg(dst_addr_reg, Type.usize, dst_mcv.address(), .{});
19293 try self.genSetReg(dst_addr_reg, .usize, dst_mcv.address(), .{});
1169919294 break :dst .{ .addr_reg = dst_addr_reg, .addr_lock = dst_addr_lock };
1170019295 },
1170119296 .load_frame => null,
......@@ -11712,6 +19307,7 @@ fn genBinOpMir(
1171219307 .dead,
1171319308 .undef,
1171419309 .register_overflow,
19310 .register_mask,
1171519311 .elementwise_regs_then_frame,
1171619312 .reserved_frame,
1171719313 .air_ref,
......@@ -11720,6 +19316,8 @@ fn genBinOpMir(
1172019316 .eflags,
1172119317 .register,
1172219318 .register_pair,
19319 .register_triple,
19320 .register_quadruple,
1172319321 .register_offset,
1172419322 .indirect,
1172519323 .lea_direct,
......@@ -11731,8 +19329,8 @@ fn genBinOpMir(
1173119329 => null,
1173219330 .memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {
1173319331 switch (resolved_src_mcv) {
11734 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr))) != null and
11735 math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null)
19332 .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr))) != null and
19333 std.math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null)
1173619334 break :src null,
1173719335 .load_symbol, .load_got, .load_direct, .load_tlv => {},
1173819336 else => unreachable,
......@@ -11743,7 +19341,7 @@ fn genBinOpMir(
1174319341 const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
1174419342 errdefer self.register_manager.unlockReg(src_addr_lock);
1174519343
11746 try self.genSetReg(src_addr_reg, Type.usize, resolved_src_mcv.address(), .{});
19344 try self.genSetReg(src_addr_reg, .usize, resolved_src_mcv.address(), .{});
1174719345 break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
1174819346 },
1174919347 };
......@@ -11751,9 +19349,9 @@ fn genBinOpMir(
1175119349
1175219350 const ty_signedness =
1175319351 if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned;
11754 const limb_ty = if (abi_size <= 8) ty else switch (ty_signedness) {
11755 .signed => Type.usize,
11756 .unsigned => Type.isize,
19352 const limb_ty: Type = if (abi_size <= 8) ty else switch (ty_signedness) {
19353 .signed => .usize,
19354 .unsigned => .isize,
1175719355 };
1175819356 var limb_i: usize = 0;
1175919357 var off: i32 = 0;
......@@ -11781,21 +19379,21 @@ fn genBinOpMir(
1178119379 => .{
1178219380 .base = .{ .reg = dst_info.?.addr_reg },
1178319381 .mod = .{ .rm = .{
11784 .size = Memory.Size.fromSize(limb_abi_size),
19382 .size = .fromSize(limb_abi_size),
1178519383 .disp = off,
1178619384 } },
1178719385 },
1178819386 .indirect => |reg_off| .{
1178919387 .base = .{ .reg = reg_off.reg },
1179019388 .mod = .{ .rm = .{
11791 .size = Memory.Size.fromSize(limb_abi_size),
19389 .size = .fromSize(limb_abi_size),
1179219390 .disp = reg_off.off + off,
1179319391 } },
1179419392 },
1179519393 .load_frame => |frame_addr| .{
1179619394 .base = .{ .frame = frame_addr.index },
1179719395 .mod = .{ .rm = .{
11798 .size = Memory.Size.fromSize(limb_abi_size),
19396 .size = .fromSize(limb_abi_size),
1179919397 .disp = frame_addr.off + off,
1180019398 } },
1180119399 },
......@@ -11807,6 +19405,7 @@ fn genBinOpMir(
1180719405 .dead,
1180819406 .undef,
1180919407 .register_overflow,
19408 .register_mask,
1181019409 .elementwise_regs_then_frame,
1181119410 .reserved_frame,
1181219411 .air_ref,
......@@ -11823,33 +19422,29 @@ fn genBinOpMir(
1182319422 8 => try self.asmMemoryImmediate(
1182419423 mir_limb_tag,
1182519424 dst_limb_mem,
11826 if (math.cast(i8, @as(i64, @bitCast(imm)))) |small|
11827 Immediate.s(small)
19425 if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |small|
19426 .s(small)
1182819427 else
11829 Immediate.u(@as(u8, @intCast(imm))),
19428 .u(@as(u8, @intCast(imm))),
1183019429 ),
1183119430 16 => try self.asmMemoryImmediate(
1183219431 mir_limb_tag,
1183319432 dst_limb_mem,
11834 if (math.cast(i16, @as(i64, @bitCast(imm)))) |small|
11835 Immediate.s(small)
19433 if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |small|
19434 .s(small)
1183619435 else
11837 Immediate.u(@as(u16, @intCast(imm))),
19436 .u(@as(u16, @intCast(imm))),
1183819437 ),
1183919438 32 => try self.asmMemoryImmediate(
1184019439 mir_limb_tag,
1184119440 dst_limb_mem,
11842 if (math.cast(i32, @as(i64, @bitCast(imm)))) |small|
11843 Immediate.s(small)
19441 if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small|
19442 .s(small)
1184419443 else
11845 Immediate.u(@as(u32, @intCast(imm))),
19444 .u(@as(u32, @intCast(imm))),
1184619445 ),
11847 64 => if (math.cast(i32, @as(i64, @bitCast(imm)))) |small|
11848 try self.asmMemoryImmediate(
11849 mir_limb_tag,
11850 dst_limb_mem,
11851 Immediate.s(small),
11852 )
19446 64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small|
19447 try self.asmMemoryImmediate(mir_limb_tag, dst_limb_mem, .s(small))
1185319448 else
1185419449 try self.asmMemoryRegister(
1185519450 mir_limb_tag,
......@@ -11864,6 +19459,8 @@ fn genBinOpMir(
1186419459 },
1186519460 .register,
1186619461 .register_pair,
19462 .register_triple,
19463 .register_quadruple,
1186719464 .register_offset,
1186819465 .eflags,
1186919466 .memory,
......@@ -11882,7 +19479,7 @@ fn genBinOpMir(
1188219479 const src_limb_mcv: MCValue = if (src_info) |info| .{
1188319480 .indirect = .{ .reg = info.addr_reg, .off = off },
1188419481 } else switch (resolved_src_mcv) {
11885 .register, .register_pair => .{
19482 .register, .register_pair, .register_triple, .register_quadruple => .{
1188619483 .register = resolved_src_mcv.getRegs()[limb_i],
1188719484 },
1188819485 .eflags,
......@@ -11925,7 +19522,7 @@ fn genBinOpMir(
1192519522
1192619523/// Performs multi-operand integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.
1192719524/// Does not support byte-size operands.
11928fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
19525fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
1192919526 const pt = self.pt;
1193019527 const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));
1193119528 try self.spillEflagsIfOccupied();
......@@ -11938,6 +19535,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
1193819535 .eflags,
1193919536 .register_offset,
1194019537 .register_overflow,
19538 .register_mask,
1194119539 .lea_symbol,
1194219540 .lea_direct,
1194319541 .lea_got,
......@@ -11948,10 +19546,19 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
1194819546 .air_ref,
1194919547 => unreachable, // unmodifiable destination
1195019548 .register => |dst_reg| {
11951 const dst_alias = registerAlias(dst_reg, abi_size);
19549 const alias_size = switch (abi_size) {
19550 1 => 4,
19551 else => abi_size,
19552 };
19553 const dst_alias = registerAlias(dst_reg, alias_size);
1195219554 const dst_lock = self.register_manager.lockReg(dst_reg);
1195319555 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
1195419556
19557 switch (abi_size) {
19558 1 => try self.asmRegisterRegister(.{ ._, .movzx }, dst_reg.to32(), dst_reg.to8()),
19559 else => {},
19560 }
19561
1195519562 const resolved_src_mcv = switch (src_mcv) {
1195619563 else => src_mcv,
1195719564 .air_ref => |src_ref| try self.resolveInst(src_ref),
......@@ -11962,24 +19569,28 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
1196219569 .dead,
1196319570 .undef,
1196419571 .register_pair,
19572 .register_triple,
19573 .register_quadruple,
1196519574 .register_overflow,
19575 .register_mask,
1196619576 .elementwise_regs_then_frame,
1196719577 .reserved_frame,
1196819578 .air_ref,
1196919579 => unreachable,
11970 .register => |src_reg| try self.asmRegisterRegister(
11971 .{ .i_, .mul },
11972 dst_alias,
11973 registerAlias(src_reg, abi_size),
11974 ),
19580 .register => |src_reg| {
19581 switch (abi_size) {
19582 1 => try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()),
19583 else => {},
19584 }
19585 try self.asmRegisterRegister(
19586 .{ .i_, .mul },
19587 dst_alias,
19588 registerAlias(src_reg, alias_size),
19589 );
19590 },
1197519591 .immediate => |imm| {
11976 if (math.cast(i32, imm)) |small| {
11977 try self.asmRegisterRegisterImmediate(
11978 .{ .i_, .mul },
11979 dst_alias,
11980 dst_alias,
11981 Immediate.s(small),
11982 );
19592 if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| {
19593 try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst_alias, dst_alias, .s(small));
1198319594 } else {
1198419595 const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);
1198519596 return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });
......@@ -11996,50 +19607,60 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
1199619607 .load_tlv,
1199719608 .lea_tlv,
1199819609 .lea_frame,
11999 => try self.asmRegisterRegister(
12000 .{ .i_, .mul },
12001 dst_alias,
12002 registerAlias(try self.copyToTmpRegister(dst_ty, resolved_src_mcv), abi_size),
12003 ),
12004 .memory, .indirect, .load_frame => try self.asmRegisterMemory(
12005 .{ .i_, .mul },
12006 dst_alias,
12007 switch (resolved_src_mcv) {
12008 .memory => |addr| .{
12009 .base = .{ .reg = .ds },
12010 .mod = .{ .rm = .{
12011 .size = Memory.Size.fromSize(abi_size),
12012 .disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
12013 return self.asmRegisterRegister(
12014 .{ .i_, .mul },
12015 dst_alias,
12016 registerAlias(
12017 try self.copyToTmpRegister(dst_ty, resolved_src_mcv),
12018 abi_size,
19610 => {
19611 const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);
19612 switch (abi_size) {
19613 1 => try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()),
19614 else => {},
19615 }
19616 try self.asmRegisterRegister(.{ .i_, .mul }, dst_alias, registerAlias(src_reg, alias_size));
19617 },
19618 .memory, .indirect, .load_frame => switch (abi_size) {
19619 1 => {
19620 const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);
19621 try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8());
19622 try self.asmRegisterRegister(.{ .i_, .mul }, dst_alias, registerAlias(src_reg, alias_size));
19623 },
19624 else => try self.asmRegisterMemory(
19625 .{ .i_, .mul },
19626 dst_alias,
19627 switch (resolved_src_mcv) {
19628 .memory => |addr| .{
19629 .base = .{ .reg = .ds },
19630 .mod = .{ .rm = .{
19631 .size = .fromSize(abi_size),
19632 .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
19633 return self.asmRegisterRegister(
19634 .{ .i_, .mul },
19635 dst_alias,
19636 registerAlias(
19637 try self.copyToTmpRegister(dst_ty, resolved_src_mcv),
19638 abi_size,
19639 ),
1201919640 ),
12020 ),
12021 } },
12022 },
12023 .indirect => |reg_off| .{
12024 .base = .{ .reg = reg_off.reg },
12025 .mod = .{ .rm = .{
12026 .size = Memory.Size.fromSize(abi_size),
12027 .disp = reg_off.off,
12028 } },
12029 },
12030 .load_frame => |frame_addr| .{
12031 .base = .{ .frame = frame_addr.index },
12032 .mod = .{ .rm = .{
12033 .size = Memory.Size.fromSize(abi_size),
12034 .disp = frame_addr.off,
12035 } },
19641 } },
19642 },
19643 .indirect => |reg_off| .{
19644 .base = .{ .reg = reg_off.reg },
19645 .mod = .{ .rm = .{
19646 .size = .fromSize(abi_size),
19647 .disp = reg_off.off,
19648 } },
19649 },
19650 .load_frame => |frame_addr| .{
19651 .base = .{ .frame = frame_addr.index },
19652 .mod = .{ .rm = .{
19653 .size = .fromSize(abi_size),
19654 .disp = frame_addr.off,
19655 } },
19656 },
19657 else => unreachable,
1203619658 },
12037 else => unreachable,
12038 },
12039 ),
19659 ),
19660 },
1204019661 }
1204119662 },
12042 .register_pair => unreachable, // unimplemented
19663 .register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented
1204319664 .memory, .indirect, .load_symbol, .load_direct, .load_got, .load_tlv, .load_frame => {
1204419665 const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
1204519666 const tmp_mcv = MCValue{ .register = tmp_reg };
......@@ -12052,7 +19673,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
1205219673 }
1205319674}
1205419675
12055fn airArg(self: *Self, inst: Air.Inst.Index) !void {
19676fn airArg(self: *CodeGen, inst: Air.Inst.Index) !void {
1205619677 const pt = self.pt;
1205719678 const zcu = pt.zcu;
1205819679 // skip zero-bit arguments as they don't have a corresponding arg instruction
......@@ -12078,8 +19699,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1207819699 try self.spillEflagsIfOccupied();
1207919700
1208019701 const fn_info = zcu.typeToFunc(self.fn_type).?;
12081 const cc = abi.resolveCallingConvention(fn_info.cc, self.target.*);
12082 const param_int_regs = abi.getCAbiIntParamRegs(cc);
19702 const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc);
1208319703 var prev_reg: Register = undefined;
1208419704 for (
1208519705 param_int_regs[param_int_regs.len - regs_frame_addr.regs ..],
......@@ -12087,11 +19707,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1208719707 ) |dst_reg, elem_index| {
1208819708 assert(self.register_manager.isRegFree(dst_reg));
1208919709 if (elem_index > 0) {
12090 try self.asmRegisterImmediate(
12091 .{ ._l, .sh },
12092 dst_reg.to8(),
12093 Immediate.u(elem_index),
12094 );
19710 try self.asmRegisterImmediate(.{ ._l, .sh }, dst_reg.to8(), .u(elem_index));
1209519711 try self.asmRegisterRegister(
1209619712 .{ ._, .@"or" },
1209719713 dst_reg.to8(),
......@@ -12110,7 +19726,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1211019726 const dst_mcv = try self.allocRegOrMem(inst, false);
1211119727 if (regs_frame_addr.regs > 0) try self.asmMemoryRegister(
1211219728 .{ ._, .mov },
12113 try dst_mcv.mem(self, .byte),
19729 try dst_mcv.mem(self, .{ .size = .byte }),
1211419730 prev_reg.to8(),
1211519731 );
1211619732 try self.genInlineMemset(
......@@ -12127,7 +19743,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1212719743 try self.asmRegisterImmediate(
1212819744 .{ ._, .mov },
1212919745 index_reg.to32(),
12130 Immediate.u(regs_frame_addr.regs),
19746 .u(regs_frame_addr.regs),
1213119747 );
1213219748 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
1213319749 try self.asmMemoryImmediate(.{ ._, .cmp }, .{
......@@ -12142,19 +19758,19 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1214219758 const unset = try self.asmJccReloc(.e, undefined);
1214319759 try self.asmMemoryRegister(
1214419760 .{ ._s, .bt },
12145 try dst_mcv.mem(self, .dword),
19761 try dst_mcv.mem(self, .{ .size = .dword }),
1214619762 index_reg.to32(),
1214719763 );
1214819764 self.performReloc(unset);
1214919765 if (self.hasFeature(.slow_incdec)) {
12150 try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));
19766 try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
1215119767 } else {
1215219768 try self.asmRegister(.{ ._, .inc }, index_reg.to32());
1215319769 }
1215419770 try self.asmRegisterImmediate(
1215519771 .{ ._, .cmp },
1215619772 index_reg.to32(),
12157 Immediate.u(arg_ty.vectorLen(zcu)),
19773 .u(arg_ty.vectorLen(zcu)),
1215819774 );
1215919775 _ = try self.asmJccReloc(.b, loop);
1216019776
......@@ -12166,7 +19782,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1216619782 return self.finishAir(inst, result, .{ .none, .none, .none });
1216719783}
1216819784
12169fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {
19785fn airDbgArg(self: *CodeGen, inst: Air.Inst.Index) !void {
1217019786 // skip zero-bit arguments as they don't have a corresponding arg instruction
1217119787 var arg_index = self.arg_index;
1217219788 while (self.args[arg_index] == .none) arg_index += 1;
......@@ -12180,16 +19796,14 @@ fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {
1218019796 for (self.args[self.arg_index..]) |arg| {
1218119797 if (arg != .none) break;
1218219798 } else try self.airDbgVarArgs();
12183 self.finishAirBookkeeping();
1218419799}
1218519800
12186fn airDbgVarArgs(self: *Self) !void {
12187 if (self.pt.zcu.typeToFunc(self.fn_type).?.is_var_args)
12188 try self.asmPseudo(.pseudo_dbg_var_args_none);
19801fn airDbgVarArgs(self: *CodeGen) !void {
19802 if (self.pt.zcu.typeToFunc(self.fn_type).?.is_var_args) try self.asmPseudo(.pseudo_dbg_var_args_none);
1218919803}
1219019804
1219119805fn genLocalDebugInfo(
12192 self: *Self,
19806 self: *CodeGen,
1219319807 inst: Air.Inst.Index,
1219419808 mcv: MCValue,
1219519809) !void {
......@@ -12200,9 +19814,9 @@ fn genLocalDebugInfo(
1220019814 switch (mcv) {
1220119815 .none => try self.asmAir(.dbg_local, inst),
1220219816 .unreach, .dead, .elementwise_regs_then_frame, .reserved_frame, .air_ref => unreachable,
12203 .immediate => |imm| try self.asmAirImmediate(.dbg_local, inst, Immediate.u(imm)),
19817 .immediate => |imm| try self.asmAirImmediate(.dbg_local, inst, .u(imm)),
1220419818 .lea_frame => |frame_addr| try self.asmAirFrameAddress(.dbg_local, inst, frame_addr),
12205 .lea_symbol => |sym_off| try self.asmAirImmediate(.dbg_local, inst, Immediate.rel(sym_off)),
19819 .lea_symbol => |sym_off| try self.asmAirImmediate(.dbg_local, inst, .rel(sym_off)),
1220619820 else => {
1220719821 const ty = switch (tag) {
1220819822 else => unreachable,
......@@ -12211,7 +19825,7 @@ fn genLocalDebugInfo(
1221119825 self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op.operand,
1221219826 ),
1221319827 };
12214 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, self.pt.zcu));
19828 const frame_index = try self.allocFrameIndex(.initSpill(ty, self.pt.zcu));
1221519829 try self.genSetMem(.{ .frame = frame_index }, 0, ty, mcv, .{});
1221619830 try self.asmAirMemory(.dbg_local, inst, .{
1221719831 .base = .{ .frame = frame_index },
......@@ -12245,29 +19859,19 @@ fn genLocalDebugInfo(
1224519859 }
1224619860}
1224719861
12248fn airTrap(self: *Self) !void {
12249 try self.asmOpOnly(.{ ._, .ud2 });
12250 self.finishAirBookkeeping();
12251}
12252
12253fn airBreakpoint(self: *Self) !void {
12254 try self.asmOpOnly(.{ ._, .int3 });
12255 self.finishAirBookkeeping();
12256}
12257
12258fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
19862fn airRetAddr(self: *CodeGen, inst: Air.Inst.Index) !void {
1225919863 const dst_mcv = try self.allocRegOrMem(inst, true);
12260 try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});
19864 try self.genCopy(.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});
1226119865 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
1226219866}
1226319867
12264fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
19868fn airFrameAddress(self: *CodeGen, inst: Air.Inst.Index) !void {
1226519869 const dst_mcv = try self.allocRegOrMem(inst, true);
12266 try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }, .{});
19870 try self.genCopy(.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }, .{});
1226719871 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
1226819872}
1226919873
12270fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
19874fn airCall(self: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier, opts: CopyOptions) !void {
1227119875 if (modifier == .always_tail) return self.fail("TODO implement tail calls for x86_64", .{});
1227219876
1227319877 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
......@@ -12291,7 +19895,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
1229119895 defer allocator.free(arg_vals);
1229219896 for (arg_vals, arg_refs) |*arg_val, arg_ref| arg_val.* = .{ .air_ref = arg_ref };
1229319897
12294 const ret = try self.genCall(.{ .air = pl_op.operand }, arg_tys, arg_vals);
19898 const ret = try self.genCall(.{ .air = pl_op.operand }, arg_tys, arg_vals, opts);
1229519899
1229619900 var bt = self.liveness.iterateBigTomb(inst);
1229719901 try self.feed(&bt, pl_op.operand);
......@@ -12301,7 +19905,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
1230119905 return self.finishAirResult(inst, result);
1230219906}
1230319907
12304fn genCall(self: *Self, info: union(enum) {
19908fn genCall(self: *CodeGen, info: union(enum) {
1230519909 air: Air.Inst.Ref,
1230619910 lib: struct {
1230719911 return_type: InternPool.Index,
......@@ -12309,7 +19913,7 @@ fn genCall(self: *Self, info: union(enum) {
1230919913 lib: ?[]const u8 = null,
1231019914 callee: []const u8,
1231119915 },
12312}, arg_types: []const Type, args: []const MCValue) !MCValue {
19916}, arg_types: []const Type, args: []const MCValue, opts: CopyOptions) !MCValue {
1231319917 const pt = self.pt;
1231419918 const zcu = pt.zcu;
1231519919 const ip = &zcu.intern_pool;
......@@ -12330,7 +19934,6 @@ fn genCall(self: *Self, info: union(enum) {
1233019934 }),
1233119935 };
1233219936 const fn_info = zcu.typeToFunc(fn_ty).?;
12333 const resolved_cc = abi.resolveCallingConvention(fn_info.cc, self.target.*);
1233419937
1233519938 const ExpectedContents = extern struct {
1233619939 var_args: [16][@sizeOf(Type)]u8 align(@alignOf(Type)),
......@@ -12348,7 +19951,7 @@ fn genCall(self: *Self, info: union(enum) {
1234819951 const frame_indices = try allocator.alloc(FrameIndex, args.len);
1234919952 defer allocator.free(frame_indices);
1235019953
12351 var reg_locks = std.ArrayList(?RegisterLock).init(allocator);
19954 var reg_locks: std.ArrayList(?RegisterLock) = .init(allocator);
1235219955 defer reg_locks.deinit();
1235319956 try reg_locks.ensureTotalCapacity(16);
1235419957 defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
......@@ -12358,7 +19961,7 @@ fn genCall(self: *Self, info: union(enum) {
1235819961
1235919962 // We need a properly aligned and sized call frame to be able to call this function.
1236019963 {
12361 const needed_call_frame = FrameAlloc.init(.{
19964 const needed_call_frame: FrameAlloc = .init(.{
1236219965 .size = call_info.stack_byte_count,
1236319966 .alignment = call_info.stack_align,
1236419967 });
......@@ -12372,7 +19975,7 @@ fn genCall(self: *Self, info: union(enum) {
1237219975 }
1237319976
1237419977 try self.spillEflagsIfOccupied();
12375 try self.spillCallerPreservedRegs(resolved_cc);
19978 try self.spillCallerPreservedRegs(fn_info.cc);
1237619979
1237719980 // set stack arguments first because this can clobber registers
1237819981 // also clobber spill arguments as we go
......@@ -12393,13 +19996,13 @@ fn genCall(self: *Self, info: union(enum) {
1239319996 try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs));
1239419997 },
1239519998 .indirect => |reg_off| {
12396 frame_index.* = try self.allocFrameIndex(FrameAlloc.initType(arg_ty, zcu));
12397 try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, .{});
19999 frame_index.* = try self.allocFrameIndex(.initType(arg_ty, zcu));
20000 try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts);
1239820001 try self.register_manager.getReg(reg_off.reg, null);
1239920002 try reg_locks.append(self.register_manager.lockReg(reg_off.reg));
1240020003 },
1240120004 .load_frame => {
12402 try self.genCopy(arg_ty, dst_arg, src_arg, .{});
20005 try self.genCopy(arg_ty, dst_arg, src_arg, opts);
1240320006 try self.freeValue(src_arg);
1240420007 },
1240520008 .elementwise_regs_then_frame => |regs_frame_addr| {
......@@ -12407,14 +20010,11 @@ fn genCall(self: *Self, info: union(enum) {
1240720010 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
1240820011 defer self.register_manager.unlockReg(index_lock);
1240920012
12410 const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .dword) else .{
12411 .base = .{ .reg = try self.copyToTmpRegister(
12412 Type.usize,
12413 switch (src_arg) {
12414 else => src_arg,
12415 .air_ref => |src_ref| try self.resolveInst(src_ref),
12416 }.address(),
12417 ) },
20013 const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{
20014 .base = .{ .reg = try self.copyToTmpRegister(.usize, switch (src_arg) {
20015 else => src_arg,
20016 .air_ref => |src_ref| try self.resolveInst(src_ref),
20017 }.address()) },
1241820018 .mod = .{ .rm = .{ .size = .dword } },
1241920019 };
1242020020 const src_lock = switch (src_mem.base) {
......@@ -12426,7 +20026,7 @@ fn genCall(self: *Self, info: union(enum) {
1242620026 try self.asmRegisterImmediate(
1242720027 .{ ._, .mov },
1242820028 index_reg.to32(),
12429 Immediate.u(regs_frame_addr.regs),
20029 .u(regs_frame_addr.regs),
1243020030 );
1243120031 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
1243220032 try self.asmMemoryRegister(.{ ._, .bt }, src_mem, index_reg.to32());
......@@ -12440,18 +20040,18 @@ fn genCall(self: *Self, info: union(enum) {
1244020040 } },
1244120041 });
1244220042 if (self.hasFeature(.slow_incdec)) {
12443 try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));
20043 try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
1244420044 } else {
1244520045 try self.asmRegister(.{ ._, .inc }, index_reg.to32());
1244620046 }
1244720047 try self.asmRegisterImmediate(
1244820048 .{ ._, .cmp },
1244920049 index_reg.to32(),
12450 Immediate.u(arg_ty.vectorLen(zcu)),
20050 .u(arg_ty.vectorLen(zcu)),
1245120051 );
1245220052 _ = try self.asmJccReloc(.b, loop);
1245320053
12454 const param_int_regs = abi.getCAbiIntParamRegs(resolved_cc);
20054 const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc);
1245520055 for (param_int_regs[param_int_regs.len - regs_frame_addr.regs ..]) |dst_reg| {
1245620056 try self.register_manager.getReg(dst_reg, null);
1245720057 try reg_locks.append(self.register_manager.lockReg(dst_reg));
......@@ -12464,9 +20064,9 @@ fn genCall(self: *Self, info: union(enum) {
1246420064 switch (call_info.return_value.long) {
1246520065 .none, .unreach => {},
1246620066 .indirect => |reg_off| {
12467 const ret_ty = Type.fromInterned(fn_info.return_type);
12468 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ret_ty, zcu));
12469 try self.genSetReg(reg_off.reg, Type.usize, .{
20067 const ret_ty: Type = .fromInterned(fn_info.return_type);
20068 const frame_index = try self.allocFrameIndex(.initSpill(ret_ty, zcu));
20069 try self.genSetReg(reg_off.reg, .usize, .{
1247020070 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
1247120071 }, .{});
1247220072 call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } };
......@@ -12479,29 +20079,27 @@ fn genCall(self: *Self, info: union(enum) {
1247920079 switch (dst_arg) {
1248020080 .none, .load_frame => {},
1248120081 .register => |dst_reg| switch (fn_info.cc) {
12482 else => try self.genSetReg(
12483 registerAlias(dst_reg, @intCast(arg_ty.abiSize(zcu))),
12484 arg_ty,
12485 src_arg,
12486 .{},
12487 ),
20082 else => try self.genSetReg(registerAlias(
20083 dst_reg,
20084 @intCast(arg_ty.abiSize(zcu)),
20085 ), arg_ty, src_arg, opts),
1248820086 .x86_64_sysv, .x86_64_win => {
1248920087 const promoted_ty = self.promoteInt(arg_ty);
1249020088 const promoted_abi_size: u32 = @intCast(promoted_ty.abiSize(zcu));
1249120089 const dst_alias = registerAlias(dst_reg, promoted_abi_size);
12492 try self.genSetReg(dst_alias, promoted_ty, src_arg, .{});
20090 try self.genSetReg(dst_alias, promoted_ty, src_arg, opts);
1249320091 if (promoted_ty.toIntern() != arg_ty.toIntern())
1249420092 try self.truncateRegister(arg_ty, dst_alias);
1249520093 },
1249620094 },
12497 .register_pair => try self.genCopy(arg_ty, dst_arg, src_arg, .{}),
12498 .indirect => |reg_off| try self.genSetReg(reg_off.reg, Type.usize, .{
20095 .register_pair => try self.genCopy(arg_ty, dst_arg, src_arg, opts),
20096 .indirect => |reg_off| try self.genSetReg(reg_off.reg, .usize, .{
1249920097 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
1250020098 }, .{}),
1250120099 .elementwise_regs_then_frame => |regs_frame_addr| {
12502 const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .dword) else .{
20100 const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{
1250320101 .base = .{ .reg = try self.copyToTmpRegister(
12504 Type.usize,
20102 .usize,
1250520103 switch (src_arg) {
1250620104 else => src_arg,
1250720105 .air_ref => |src_ref| try self.resolveInst(src_ref),
......@@ -12515,25 +20113,20 @@ fn genCall(self: *Self, info: union(enum) {
1251520113 };
1251620114 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
1251720115
12518 const param_int_regs = abi.getCAbiIntParamRegs(resolved_cc);
20116 const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc);
1251920117 for (
1252020118 param_int_regs[param_int_regs.len - regs_frame_addr.regs ..],
1252120119 0..,
1252220120 ) |dst_reg, elem_index| {
1252320121 try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
12524 try self.asmMemoryImmediate(
12525 .{ ._, .bt },
12526 src_mem,
12527 Immediate.u(elem_index),
12528 );
20122 try self.asmMemoryImmediate(.{ ._, .bt }, src_mem, .u(elem_index));
1252920123 try self.asmSetccRegister(.c, dst_reg.to8());
1253020124 }
1253120125 },
1253220126 else => unreachable,
1253320127 };
1253420128
12535 if (fn_info.is_var_args)
12536 try self.asmRegisterImmediate(.{ ._, .mov }, .al, Immediate.u(call_info.fp_count));
20129 if (fn_info.is_var_args) try self.asmRegisterImmediate(.{ ._, .mov }, .al, .u(call_info.fp_count));
1253720130
1253820131 // Due to incremental compilation, how function calls are generated depends
1253920132 // on linking.
......@@ -12551,17 +20144,18 @@ fn genCall(self: *Self, info: union(enum) {
1255120144 if (self.bin_file.cast(.elf)) |elf_file| {
1255220145 const zo = elf_file.zigObjectPtr().?;
1255320146 const sym_index = try zo.getOrCreateMetadataForNav(zcu, func.owner_nav);
12554 try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym_index }));
20147 try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym_index }));
1255520148 } else if (self.bin_file.cast(.coff)) |coff_file| {
1255620149 const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav);
1255720150 const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
12558 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index }, .{});
12559 try self.asmRegister(.{ ._, .call }, .rax);
20151 const scratch_reg = abi.getCAbiLinkerScratchReg(fn_info.cc);
20152 try self.genSetReg(scratch_reg, .usize, .{ .lea_got = sym_index }, .{});
20153 try self.asmRegister(.{ ._, .call }, scratch_reg);
1256020154 } else if (self.bin_file.cast(.macho)) |macho_file| {
1256120155 const zo = macho_file.getZigObject().?;
1256220156 const sym_index = try zo.getOrCreateMetadataForNav(macho_file, func.owner_nav);
1256320157 const sym = zo.symbols.items[sym_index];
12564 try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym.nlist_idx }));
20158 try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym.nlist_idx }));
1256520159 } else if (self.bin_file.cast(.plan9)) |p9| {
1256620160 const atom_index = try p9.seeNav(pt, func.owner_nav);
1256720161 const atom = p9.getAtom(atom_index);
......@@ -12579,13 +20173,13 @@ fn genCall(self: *Self, info: union(enum) {
1257920173 @"extern".name.toSlice(ip),
1258020174 @"extern".lib_name.toSlice(ip),
1258120175 );
12582 try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));
20176 try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
1258320177 } else if (self.bin_file.cast(.macho)) |macho_file| {
1258420178 const target_sym_index = try macho_file.getGlobalSymbol(
1258520179 @"extern".name.toSlice(ip),
1258620180 @"extern".lib_name.toSlice(ip),
1258720181 );
12588 try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));
20182 try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
1258920183 } else try self.genExternSymbolRef(
1259020184 .call,
1259120185 @"extern".lib_name.toSlice(ip),
......@@ -12595,21 +20189,22 @@ fn genCall(self: *Self, info: union(enum) {
1259520189 }
1259620190 } else {
1259720191 assert(self.typeOf(callee).zigTypeTag(zcu) == .pointer);
12598 try self.genSetReg(.rax, Type.usize, .{ .air_ref = callee }, .{});
12599 try self.asmRegister(.{ ._, .call }, .rax);
20192 const scratch_reg = abi.getCAbiLinkerScratchReg(fn_info.cc);
20193 try self.genSetReg(scratch_reg, .usize, .{ .air_ref = callee }, .{});
20194 try self.asmRegister(.{ ._, .call }, scratch_reg);
1260020195 },
1260120196 .lib => |lib| if (self.bin_file.cast(.elf)) |elf_file| {
1260220197 const target_sym_index = try elf_file.getGlobalSymbol(lib.callee, lib.lib);
12603 try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));
20198 try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
1260420199 } else if (self.bin_file.cast(.macho)) |macho_file| {
1260520200 const target_sym_index = try macho_file.getGlobalSymbol(lib.callee, lib.lib);
12606 try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));
20201 try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
1260720202 } else try self.genExternSymbolRef(.call, lib.lib, lib.callee),
1260820203 }
1260920204 return call_info.return_value.short;
1261020205}
1261120206
12612fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
20207fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
1261320208 const pt = self.pt;
1261420209 const zcu = pt.zcu;
1261520210 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
......@@ -12619,13 +20214,15 @@ fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1261920214 .none => {},
1262020215 .register,
1262120216 .register_pair,
20217 .register_triple,
20218 .register_quadruple,
1262220219 => try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }),
1262320220 .indirect => |reg_off| {
1262420221 try self.register_manager.getReg(reg_off.reg, null);
1262520222 const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
1262620223 defer self.register_manager.unlockReg(lock);
1262720224
12628 try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long, .{});
20225 try self.genSetReg(reg_off.reg, .usize, self.ret_mcv.long, .{});
1262920226 try self.genSetMem(
1263020227 .{ .reg = reg_off.reg },
1263120228 reg_off.off,
......@@ -12642,10 +20239,10 @@ fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1264220239 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
1264320240 // which is available if the jump is 127 bytes or less forward.
1264420241 const jmp_reloc = try self.asmJmpReloc(undefined);
12645 try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
20242 try self.epilogue_relocs.append(self.gpa, jmp_reloc);
1264620243}
1264720244
12648fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
20245fn airRetLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
1264920246 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1265020247 const ptr = try self.resolveInst(un_op);
1265120248
......@@ -12662,10 +20259,10 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
1266220259 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
1266320260 // which is available if the jump is 127 bytes or less forward.
1266420261 const jmp_reloc = try self.asmJmpReloc(undefined);
12665 try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
20262 try self.epilogue_relocs.append(self.gpa, jmp_reloc);
1266620263}
1266720264
12668fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
20265fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !void {
1266920266 const pt = self.pt;
1267020267 const zcu = pt.zcu;
1267120268 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -12686,7 +20283,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1268620283 self.register_manager.lockRegAssumeUnused(lhs_ro.reg),
1268720284 null,
1268820285 },
12689 else => .{null} ** 2,
20286 else => @splat(null),
1269020287 };
1269120288 defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
1269220289
......@@ -12695,7 +20292,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1269520292 .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },
1269620293 .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),
1269720294 .register_offset => |rhs_ro| .{ self.register_manager.lockReg(rhs_ro.reg), null },
12698 else => .{null} ** 2,
20295 else => @splat(null),
1269920296 };
1270020297 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1270120298
......@@ -12723,8 +20320,8 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1272320320 },
1272420321 floatCompilerRtAbiName(float_bits),
1272520322 }) catch unreachable,
12726 } }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } });
12727 try self.genBinOpMir(.{ ._, .@"test" }, Type.i32, ret, ret);
20323 } }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } }, .{});
20324 try self.genBinOpMir(.{ ._, .@"test" }, .i32, ret, ret);
1272820325 break :result switch (op) {
1272920326 .eq => .e,
1273020327 .neq => .ne,
......@@ -12745,23 +20342,23 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1274520342 const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg);
1274620343 defer self.register_manager.unlockReg(temp_lhs_lock);
1274720344
12748 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
20345 if (lhs_mcv.isBase()) try self.asmRegisterMemory(
1274920346 .{ ._, .mov },
1275020347 temp_lhs_reg.to8(),
12751 try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte),
20348 try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }),
1275220349 ) else {
1275320350 try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{});
1275420351 try self.asmRegisterImmediate(
1275520352 .{ ._r, .sh },
1275620353 registerAlias(temp_lhs_reg, opt_abi_size),
12757 Immediate.u(payload_abi_size * 8),
20354 .u(payload_abi_size * 8),
1275820355 );
1275920356 }
1276020357
1276120358 const payload_compare = payload_compare: {
12762 if (rhs_mcv.isMemory()) {
20359 if (rhs_mcv.isBase()) {
1276320360 const rhs_mem =
12764 try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte);
20361 try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte });
1276520362 try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8());
1276620363 const payload_compare = try self.asmJccReloc(.nz, undefined);
1276720364 try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem);
......@@ -12775,7 +20372,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1277520372 try self.asmRegisterImmediate(
1277620373 .{ ._r, .sh },
1277720374 registerAlias(temp_rhs_reg, opt_abi_size),
12778 Immediate.u(payload_abi_size * 8),
20375 .u(payload_abi_size * 8),
1277920376 );
1278020377 try self.asmRegisterRegister(
1278120378 .{ ._, .@"test" },
......@@ -12810,13 +20407,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1281020407 } else .may_flip;
1281120408
1281220409 const flipped = switch (may_flip) {
12813 .may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isMemory(),
20410 .may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isBase(),
1281420411 .must_flip => true,
1281520412 .must_not_flip => false,
1281620413 };
1281720414 const unmat_dst_mcv = if (flipped) rhs_mcv else lhs_mcv;
1281820415 const dst_mcv = if (unmat_dst_mcv.isRegister() or
12819 (abi_size <= 8 and unmat_dst_mcv.isMemory())) unmat_dst_mcv else dst: {
20416 (abi_size <= 8 and unmat_dst_mcv.isBase())) unmat_dst_mcv else dst: {
1282020417 const dst_mcv = try self.allocTempRegOrMem(ty, true);
1282120418 try self.genCopy(ty, dst_mcv, unmat_dst_mcv, .{});
1282220419 break :dst dst_mcv;
......@@ -12830,7 +20427,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1283020427 if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
1283120428 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
1283220429
12833 break :result Condition.fromCompareOperator(
20430 break :result .fromCompareOperator(
1283420431 if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned,
1283520432 result_op: {
1283620433 const flipped_op = if (flipped) op.reverse() else op;
......@@ -12851,9 +20448,9 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1285120448 .undef,
1285220449 .immediate,
1285320450 .eflags,
12854 .register,
1285520451 .register_offset,
1285620452 .register_overflow,
20453 .register_mask,
1285720454 .indirect,
1285820455 .lea_direct,
1285920456 .lea_got,
......@@ -12864,13 +20461,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1286420461 .reserved_frame,
1286520462 .air_ref,
1286620463 => unreachable,
12867 .register_pair, .load_frame => null,
20464 .register, .register_pair, .register_triple, .register_quadruple, .load_frame => null,
1286820465 .memory, .load_symbol, .load_got, .load_direct, .load_tlv => dst: {
1286920466 switch (resolved_dst_mcv) {
12870 .memory => |addr| if (math.cast(
20467 .memory => |addr| if (std.math.cast(
1287120468 i32,
1287220469 @as(i64, @bitCast(addr)),
12873 ) != null and math.cast(
20470 ) != null and std.math.cast(
1287420471 i32,
1287520472 @as(i64, @bitCast(addr)) + abi_size - 8,
1287620473 ) != null) break :dst null,
......@@ -12886,20 +20483,14 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1288620483 self.register_manager.lockRegAssumeUnused(dst_addr_reg);
1288720484 errdefer self.register_manager.unlockReg(dst_addr_lock);
1288820485
12889 try self.genSetReg(
12890 dst_addr_reg,
12891 Type.usize,
12892 resolved_dst_mcv.address(),
12893 .{},
12894 );
20486 try self.genSetReg(dst_addr_reg, .usize, resolved_dst_mcv.address(), .{});
1289520487 break :dst .{
1289620488 .addr_reg = dst_addr_reg,
1289720489 .addr_lock = dst_addr_lock,
1289820490 };
1289920491 },
1290020492 };
12901 defer if (dst_info) |info|
12902 self.register_manager.unlockReg(info.addr_lock);
20493 defer if (dst_info) |info| self.register_manager.unlockReg(info.addr_lock);
1290320494
1290420495 const resolved_src_mcv = switch (src_mcv) {
1290520496 else => src_mcv,
......@@ -12915,6 +20506,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1291520506 .register,
1291620507 .register_offset,
1291720508 .register_overflow,
20509 .register_mask,
1291820510 .indirect,
1291920511 .lea_symbol,
1292020512 .lea_direct,
......@@ -12925,13 +20517,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1292520517 .reserved_frame,
1292620518 .air_ref,
1292720519 => unreachable,
12928 .register_pair, .load_frame => null,
20520 .register_pair, .register_triple, .register_quadruple, .load_frame => null,
1292920521 .memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {
1293020522 switch (resolved_src_mcv) {
12931 .memory => |addr| if (math.cast(
20523 .memory => |addr| if (std.math.cast(
1293220524 i32,
1293320525 @as(i64, @bitCast(addr)),
12934 ) != null and math.cast(
20526 ) != null and std.math.cast(
1293520527 i32,
1293620528 @as(i64, @bitCast(addr)) + abi_size - 8,
1293720529 ) != null) break :src null,
......@@ -12947,12 +20539,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1294720539 self.register_manager.lockRegAssumeUnused(src_addr_reg);
1294820540 errdefer self.register_manager.unlockReg(src_addr_lock);
1294920541
12950 try self.genSetReg(
12951 src_addr_reg,
12952 Type.usize,
12953 resolved_src_mcv.address(),
12954 .{},
12955 );
20542 try self.genSetReg(src_addr_reg, .usize, resolved_src_mcv.address(), .{});
1295620543 break :src .{
1295720544 .addr_reg = src_addr_reg,
1295820545 .addr_lock = src_addr_lock,
......@@ -12962,25 +20549,24 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1296220549 defer if (src_info) |info|
1296320550 self.register_manager.unlockReg(info.addr_lock);
1296420551
12965 const regs = try self.register_manager.allocRegs(
12966 2,
12967 .{null} ** 2,
12968 abi.RegisterClass.gp,
12969 );
20552 const regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
1297020553 const acc_reg = regs[0].to64();
1297120554 const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
1297220555 defer for (locks) |lock| self.register_manager.unlockReg(lock);
1297320556
12974 const limbs_len = math.divCeil(u16, abi_size, 8) catch unreachable;
20557 const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable;
1297520558 var limb_i: u16 = 0;
1297620559 while (limb_i < limbs_len) : (limb_i += 1) {
1297720560 const off = limb_i * 8;
1297820561 const tmp_reg = regs[@min(limb_i, 1)].to64();
1297920562
12980 try self.genSetReg(tmp_reg, Type.usize, if (dst_info) |info| .{
20563 try self.genSetReg(tmp_reg, .usize, if (dst_info) |info| .{
1298120564 .indirect = .{ .reg = info.addr_reg, .off = off },
1298220565 } else switch (resolved_dst_mcv) {
12983 .register_pair => |dst_regs| .{ .register = dst_regs[limb_i] },
20566 inline .register_pair,
20567 .register_triple,
20568 .register_quadruple,
20569 => |dst_regs| .{ .register = dst_regs[limb_i] },
1298420570 .memory => |dst_addr| .{
1298520571 .memory = @bitCast(@as(i64, @bitCast(dst_addr)) + off),
1298620572 },
......@@ -12997,14 +20583,15 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1299720583
1299820584 try self.genBinOpMir(
1299920585 .{ ._, .xor },
13000 Type.usize,
20586 .usize,
1300120587 .{ .register = tmp_reg },
1300220588 if (src_info) |info| .{
1300320589 .indirect = .{ .reg = info.addr_reg, .off = off },
1300420590 } else switch (resolved_src_mcv) {
13005 .register_pair => |src_regs| .{
13006 .register = src_regs[limb_i],
13007 },
20591 inline .register_pair,
20592 .register_triple,
20593 .register_quadruple,
20594 => |src_regs| .{ .register = src_regs[limb_i] },
1300820595 .memory => |src_addr| .{
1300920596 .memory = @bitCast(@as(i64, @bitCast(src_addr)) + off),
1301020597 },
......@@ -13062,12 +20649,12 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1306220649 const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg);
1306320650 defer self.register_manager.unlockReg(tmp2_lock);
1306420651
13065 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
20652 if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
1306620653 .{ .vp_w, .insr },
1306720654 tmp1_reg,
1306820655 dst_reg.to128(),
13069 try src_mcv.mem(self, .word),
13070 Immediate.u(1),
20656 try src_mcv.mem(self, .{ .size = .word }),
20657 .u(1),
1307120658 ) else try self.asmRegisterRegisterRegister(
1307220659 .{ .vp_, .unpcklwd },
1307320660 tmp1_reg,
......@@ -13112,19 +20699,19 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1311220699 return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });
1311320700}
1311420701
13115fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
20702fn airCmpVector(self: *CodeGen, inst: Air.Inst.Index) !void {
1311620703 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1311720704 const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
1311820705 const dst_mcv = try self.genBinOp(
1311920706 inst,
13120 Air.Inst.Tag.fromCmpOp(extra.compareOperator(), false),
20707 .fromCmpOp(extra.compareOperator(), false),
1312120708 extra.lhs,
1312220709 extra.rhs,
1312320710 );
1312420711 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
1312520712}
1312620713
13127fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
20714fn airCmpLtErrorsLen(self: *CodeGen, inst: Air.Inst.Index) !void {
1312820715 const pt = self.pt;
1312920716 const zcu = pt.zcu;
1313020717 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
......@@ -13149,7 +20736,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
1314920736 registerAlias(dst_reg, op_abi_size),
1315020737 .{
1315120738 .base = .{ .reg = addr_reg },
13152 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(op_abi_size) } },
20739 .mod = .{ .rm = .{ .size = .fromSize(op_abi_size) } },
1315320740 },
1315420741 );
1315520742
......@@ -13157,7 +20744,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
1315720744 return self.finishAir(inst, .{ .eflags = .b }, .{ un_op, .none, .none });
1315820745}
1315920746
13160fn airTry(self: *Self, inst: Air.Inst.Index) !void {
20747fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void {
1316120748 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
1316220749 const extra = self.air.extraData(Air.Try, pl_op.payload);
1316320750 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
......@@ -13166,7 +20753,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void {
1316620753 return self.finishAir(inst, result, .{ .none, .none, .none });
1316720754}
1316820755
13169fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {
20756fn airTryPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1317020757 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1317120758 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
1317220759 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
......@@ -13176,7 +20763,7 @@ fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {
1317620763}
1317720764
1317820765fn genTry(
13179 self: *Self,
20766 self: *CodeGen,
1318020767 inst: Air.Inst.Index,
1318120768 operand: Air.Inst.Ref,
1318220769 body: []const Air.Inst.Index,
......@@ -13191,7 +20778,7 @@ fn genTry(
1319120778 else
1319220779 try self.isErr(null, operand_ty, operand_mcv);
1319320780
13194 const reloc = try self.genCondBrMir(Type.anyerror, is_err_mcv);
20781 const reloc = try self.genCondBrMir(.anyerror, is_err_mcv);
1319520782
1319620783 if (self.liveness.operandDies(inst, 0)) {
1319720784 if (operand.toIndex()) |operand_inst| try self.processDeath(operand_inst);
......@@ -13222,7 +20809,7 @@ fn genTry(
1322220809 return result;
1322320810}
1322420811
13225fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
20812fn airDbgStmt(self: *CodeGen, inst: Air.Inst.Index) !void {
1322620813 const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
1322720814 _ = try self.addInst(.{
1322820815 .tag = .pseudo,
......@@ -13232,18 +20819,16 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
1323220819 .column = dbg_stmt.column,
1323320820 } },
1323420821 });
13235 self.finishAirBookkeeping();
1323620822}
1323720823
13238fn airDbgEmptyStmt(self: *Self) !void {
20824fn airDbgEmptyStmt(self: *CodeGen) !void {
1323920825 if (self.mir_instructions.len > 0 and
1324020826 self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] == .pseudo_dbg_line_stmt_line_column)
1324120827 self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] = .pseudo_dbg_line_line_column;
1324220828 try self.asmOpOnly(.{ ._, .nop });
13243 self.finishAirBookkeeping();
1324420829}
1324520830
13246fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
20831fn airDbgInlineBlock(self: *CodeGen, inst: Air.Inst.Index) !void {
1324720832 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1324820833 const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
1324920834 const old_inline_func = self.inline_func;
......@@ -13262,13 +20847,13 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
1326220847 });
1326320848}
1326420849
13265fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
20850fn airDbgVar(self: *CodeGen, inst: Air.Inst.Index) !void {
1326620851 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
1326720852 try self.genLocalDebugInfo(inst, try self.resolveInst(pl_op.operand));
1326820853 return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
1326920854}
1327020855
13271fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
20856fn genCondBrMir(self: *CodeGen, ty: Type, mcv: MCValue) !Mir.Inst.Index {
1327220857 const pt = self.pt;
1327320858 const abi_size = ty.abiSize(pt.zcu);
1327420859 switch (mcv) {
......@@ -13278,7 +20863,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
1327820863 },
1327920864 .register => |reg| {
1328020865 try self.spillEflagsIfOccupied();
13281 try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), Immediate.u(1));
20866 try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), .u(1));
1328220867 return self.asmJccReloc(.z, undefined);
1328320868 },
1328420869 .immediate,
......@@ -13295,7 +20880,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
1329520880 }
1329620881}
1329720882
13298fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
20883fn airCondBr(self: *CodeGen, inst: Air.Inst.Index) !void {
1329920884 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
1330020885 const cond = try self.resolveInst(pl_op.operand);
1330120886 const cond_ty = self.typeOf(pl_op.operand);
......@@ -13338,10 +20923,9 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
1333820923 });
1333920924
1334020925 // We already took care of pl_op.operand earlier, so there's nothing left to do.
13341 self.finishAirBookkeeping();
1334220926}
1334320927
13344fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
20928fn isNull(self: *CodeGen, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
1334520929 const pt = self.pt;
1334620930 const zcu = pt.zcu;
1334720931 switch (opt_mcv) {
......@@ -13353,10 +20937,10 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
1335320937
1335420938 const pl_ty = opt_ty.optionalChild(zcu);
1335520939
13356 const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
20940 const some_info: struct { off: u31, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
1335720941 .{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }
1335820942 else
13359 .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool };
20943 .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = .bool };
1336020944
1336120945 self.eflags_inst = inst;
1336220946 switch (opt_mcv) {
......@@ -13366,9 +20950,11 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
1336620950 .undef,
1336720951 .immediate,
1336820952 .eflags,
13369 .register_pair,
20953 .register_triple,
20954 .register_quadruple,
1337020955 .register_offset,
1337120956 .register_overflow,
20957 .register_mask,
1337220958 .lea_direct,
1337320959 .lea_got,
1337420960 .lea_tlv,
......@@ -13396,7 +20982,25 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
1339620982 try self.asmRegisterImmediate(
1339720983 .{ ._, .bt },
1339820984 registerAlias(opt_reg, opt_abi_size),
13399 Immediate.u(@as(u6, @intCast(some_info.off * 8))),
20985 .u(@as(u6, @intCast(some_info.off * 8))),
20986 );
20987 return .{ .eflags = .nc };
20988 },
20989
20990 .register_pair => |opt_regs| {
20991 if (some_info.off == 0) {
20992 const some_abi_size: u32 = @intCast(some_info.ty.abiSize(zcu));
20993 const alias_reg = registerAlias(opt_regs[0], some_abi_size);
20994 assert(some_abi_size * 8 == alias_reg.bitSize());
20995 try self.asmRegisterRegister(.{ ._, .@"test" }, alias_reg, alias_reg);
20996 return .{ .eflags = .z };
20997 }
20998 assert(some_info.ty.ip_index == .bool_type);
20999 const opt_abi_size: u32 = @intCast(opt_ty.abiSize(zcu));
21000 try self.asmRegisterImmediate(
21001 .{ ._, .bt },
21002 registerAlias(opt_regs[some_info.off / 8], opt_abi_size),
21003 .u(@as(u6, @truncate(some_info.off * 8))),
1340021004 );
1340121005 return .{ .eflags = .nc };
1340221006 },
......@@ -13411,18 +21015,18 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
1341121015 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
1341221016 defer self.register_manager.unlockReg(addr_reg_lock);
1341321017
13414 try self.genSetReg(addr_reg, Type.usize, opt_mcv.address(), .{});
21018 try self.genSetReg(addr_reg, .usize, opt_mcv.address(), .{});
1341521019 const some_abi_size: u32 = @intCast(some_info.ty.abiSize(zcu));
1341621020 try self.asmMemoryImmediate(
1341721021 .{ ._, .cmp },
1341821022 .{
1341921023 .base = .{ .reg = addr_reg },
1342021024 .mod = .{ .rm = .{
13421 .size = Memory.Size.fromSize(some_abi_size),
21025 .size = .fromSize(some_abi_size),
1342221026 .disp = some_info.off,
1342321027 } },
1342421028 },
13425 Immediate.u(0),
21029 .u(0),
1342621030 );
1342721031 return .{ .eflags = .e };
1342821032 },
......@@ -13435,27 +21039,27 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
1343521039 .indirect => |reg_off| .{
1343621040 .base = .{ .reg = reg_off.reg },
1343721041 .mod = .{ .rm = .{
13438 .size = Memory.Size.fromSize(some_abi_size),
21042 .size = .fromSize(some_abi_size),
1343921043 .disp = reg_off.off + some_info.off,
1344021044 } },
1344121045 },
1344221046 .load_frame => |frame_addr| .{
1344321047 .base = .{ .frame = frame_addr.index },
1344421048 .mod = .{ .rm = .{
13445 .size = Memory.Size.fromSize(some_abi_size),
21049 .size = .fromSize(some_abi_size),
1344621050 .disp = frame_addr.off + some_info.off,
1344721051 } },
1344821052 },
1344921053 else => unreachable,
1345021054 },
13451 Immediate.u(0),
21055 .u(0),
1345221056 );
1345321057 return .{ .eflags = .e };
1345421058 },
1345521059 }
1345621060}
1345721061
13458fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
21062fn isNullPtr(self: *CodeGen, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
1345921063 const pt = self.pt;
1346021064 const zcu = pt.zcu;
1346121065 const opt_ty = ptr_ty.childType(zcu);
......@@ -13466,7 +21070,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
1346621070 const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
1346721071 .{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }
1346821072 else
13469 .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool };
21073 .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = .bool };
1347021074
1347121075 const ptr_reg = switch (ptr_mcv) {
1347221076 .register => |reg| reg,
......@@ -13481,18 +21085,18 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
1348121085 .{
1348221086 .base = .{ .reg = ptr_reg },
1348321087 .mod = .{ .rm = .{
13484 .size = Memory.Size.fromSize(some_abi_size),
21088 .size = .fromSize(some_abi_size),
1348521089 .disp = some_info.off,
1348621090 } },
1348721091 },
13488 Immediate.u(0),
21092 .u(0),
1348921093 );
1349021094
1349121095 self.eflags_inst = inst;
1349221096 return .{ .eflags = .e };
1349321097}
1349421098
13495fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
21099fn isErr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
1349621100 const pt = self.pt;
1349721101 const zcu = pt.zcu;
1349821102 const err_ty = eu_ty.errorUnionSet(zcu);
......@@ -13500,7 +21104,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)
1350021104
1350121105 try self.spillEflagsIfOccupied();
1350221106
13503 const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
21107 const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
1350421108 switch (eu_mcv) {
1350521109 .register => |reg| {
1350621110 const eu_lock = self.register_manager.lockReg(reg);
......@@ -13512,22 +21116,17 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)
1351221116 .{ ._r, .sh },
1351321117 eu_ty,
1351421118 .{ .register = tmp_reg },
13515 Type.u8,
21119 .u8,
1351621120 .{ .immediate = @as(u6, @intCast(err_off * 8)) },
1351721121 );
1351821122 } else {
13519 try self.truncateRegister(Type.anyerror, tmp_reg);
21123 try self.truncateRegister(.anyerror, tmp_reg);
1352021124 }
13521 try self.genBinOpMir(
13522 .{ ._, .cmp },
13523 Type.anyerror,
13524 .{ .register = tmp_reg },
13525 .{ .immediate = 0 },
13526 );
21125 try self.genBinOpMir(.{ ._, .cmp }, .anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });
1352721126 },
1352821127 .load_frame => |frame_addr| try self.genBinOpMir(
1352921128 .{ ._, .cmp },
13530 Type.anyerror,
21129 .anyerror,
1353121130 .{ .load_frame = .{
1353221131 .index = frame_addr.index,
1353321132 .off = frame_addr.off + err_off,
......@@ -13541,7 +21140,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)
1354121140 return MCValue{ .eflags = .a };
1354221141}
1354321142
13544fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
21143fn isErrPtr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
1354521144 const pt = self.pt;
1354621145 const zcu = pt.zcu;
1354721146 const eu_ty = ptr_ty.childType(zcu);
......@@ -13557,24 +21156,24 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV
1355721156 const ptr_lock = self.register_manager.lockReg(ptr_reg);
1355821157 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
1355921158
13560 const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
21159 const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
1356121160 try self.asmMemoryImmediate(
1356221161 .{ ._, .cmp },
1356321162 .{
1356421163 .base = .{ .reg = ptr_reg },
1356521164 .mod = .{ .rm = .{
13566 .size = self.memSize(Type.anyerror),
21165 .size = self.memSize(.anyerror),
1356721166 .disp = err_off,
1356821167 } },
1356921168 },
13570 Immediate.u(0),
21169 .u(0),
1357121170 );
1357221171
1357321172 if (maybe_inst) |inst| self.eflags_inst = inst;
1357421173 return MCValue{ .eflags = .a };
1357521174}
1357621175
13577fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
21176fn isNonErr(self: *CodeGen, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
1357821177 const is_err_res = try self.isErr(inst, eu_ty, eu_mcv);
1357921178 switch (is_err_res) {
1358021179 .eflags => |cc| {
......@@ -13589,7 +21188,7 @@ fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MC
1358921188 }
1359021189}
1359121190
13592fn isNonErrPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
21191fn isNonErrPtr(self: *CodeGen, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
1359321192 const is_err_res = try self.isErrPtr(inst, ptr_ty, ptr_mcv);
1359421193 switch (is_err_res) {
1359521194 .eflags => |cc| {
......@@ -13604,7 +21203,7 @@ fn isNonErrPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue
1360421203 }
1360521204}
1360621205
13607fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
21206fn airIsNull(self: *CodeGen, inst: Air.Inst.Index) !void {
1360821207 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1360921208 const operand = try self.resolveInst(un_op);
1361021209 const ty = self.typeOf(un_op);
......@@ -13612,7 +21211,7 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
1361221211 return self.finishAir(inst, result, .{ un_op, .none, .none });
1361321212}
1361421213
13615fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
21214fn airIsNullPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1361621215 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1361721216 const operand = try self.resolveInst(un_op);
1361821217 const ty = self.typeOf(un_op);
......@@ -13620,7 +21219,7 @@ fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
1362021219 return self.finishAir(inst, result, .{ un_op, .none, .none });
1362121220}
1362221221
13623fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
21222fn airIsNonNull(self: *CodeGen, inst: Air.Inst.Index) !void {
1362421223 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1362521224 const operand = try self.resolveInst(un_op);
1362621225 const ty = self.typeOf(un_op);
......@@ -13632,7 +21231,7 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
1363221231 return self.finishAir(inst, result, .{ un_op, .none, .none });
1363321232}
1363421233
13635fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
21234fn airIsNonNullPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1363621235 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1363721236 const operand = try self.resolveInst(un_op);
1363821237 const ty = self.typeOf(un_op);
......@@ -13643,7 +21242,7 @@ fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
1364321242 return self.finishAir(inst, result, .{ un_op, .none, .none });
1364421243}
1364521244
13646fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
21245fn airIsErr(self: *CodeGen, inst: Air.Inst.Index) !void {
1364721246 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1364821247 const operand = try self.resolveInst(un_op);
1364921248 const ty = self.typeOf(un_op);
......@@ -13651,7 +21250,7 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
1365121250 return self.finishAir(inst, result, .{ un_op, .none, .none });
1365221251}
1365321252
13654fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
21253fn airIsErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1365521254 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1365621255 const operand = try self.resolveInst(un_op);
1365721256 const ty = self.typeOf(un_op);
......@@ -13659,7 +21258,7 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
1365921258 return self.finishAir(inst, result, .{ un_op, .none, .none });
1366021259}
1366121260
13662fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
21261fn airIsNonErr(self: *CodeGen, inst: Air.Inst.Index) !void {
1366321262 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1366421263 const operand = try self.resolveInst(un_op);
1366521264 const ty = self.typeOf(un_op);
......@@ -13667,7 +21266,7 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
1366721266 return self.finishAir(inst, result, .{ un_op, .none, .none });
1366821267}
1366921268
13670fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
21269fn airIsNonErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1367121270 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1367221271 const operand = try self.resolveInst(un_op);
1367321272 const ty = self.typeOf(un_op);
......@@ -13675,7 +21274,7 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
1367521274 return self.finishAir(inst, result, .{ un_op, .none, .none });
1367621275}
1367721276
13678fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
21277fn airLoop(self: *CodeGen, inst: Air.Inst.Index) !void {
1367921278 // A loop is a setup to be able to jump back to the beginning.
1368021279 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1368121280 const loop = self.air.extraData(Air.Block, ty_pl.payload);
......@@ -13686,15 +21285,14 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
1368621285
1368721286 try self.loops.putNoClobber(self.gpa, inst, .{
1368821287 .state = state,
13689 .jmp_target = @intCast(self.mir_instructions.len),
21288 .target = @intCast(self.mir_instructions.len),
1369021289 });
1369121290 defer assert(self.loops.remove(inst));
1369221291
1369321292 try self.genBodyBlock(body);
13694 self.finishAirBookkeeping();
1369521293}
1369621294
13697fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
21295fn airBlock(self: *CodeGen, inst: Air.Inst.Index) !void {
1369821296 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1369921297 const extra = self.air.extraData(Air.Block, ty_pl.payload);
1370021298 try self.asmPseudo(.pseudo_dbg_enter_block_none);
......@@ -13702,10 +21300,10 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
1370221300 try self.asmPseudo(.pseudo_dbg_leave_block_none);
1370321301}
1370421302
13705fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
21303fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
1370621304 // A block is a setup to be able to jump to the end.
1370721305 const inst_tracking_i = self.inst_tracking.count();
13708 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));
21306 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(.unreach));
1370921307
1371021308 self.scope_generation += 1;
1371121309 try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() });
......@@ -13729,28 +21327,197 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
1372921327 const tracking = &self.inst_tracking.values()[inst_tracking_i];
1373021328 if (self.liveness.isUnused(inst)) try tracking.die(self, inst);
1373121329 self.getValueIfFree(tracking.short, inst);
13732 self.finishAirBookkeeping();
1373321330}
1373421331
13735fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwitch, condition: MCValue) !void {
21332fn lowerSwitchBr(
21333 self: *CodeGen,
21334 inst: Air.Inst.Index,
21335 switch_br: Air.UnwrappedSwitch,
21336 condition: MCValue,
21337 condition_dies: bool,
21338 is_loop: bool,
21339) !void {
1373621340 const zcu = self.pt.zcu;
1373721341 const condition_ty = self.typeOf(switch_br.operand);
13738 const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.cases_len + 1);
13739 defer self.gpa.free(liveness.deaths);
1374021342
13741 const signedness = switch (condition_ty.zigTypeTag(zcu)) {
13742 .bool, .pointer => .unsigned,
13743 .int, .@"enum", .error_set => condition_ty.intInfo(zcu).signedness,
13744 else => unreachable,
21343 const ExpectedContents = extern struct {
21344 liveness_deaths: [1 << 8 | 1]Air.Inst.Index,
21345 bigint_limbs: [std.math.big.int.calcTwosCompLimbCount(1 << 8)]std.math.big.Limb,
21346 relocs: [1 << 6]Mir.Inst.Index,
1374521347 };
21348 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
21349 std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
21350 const allocator = stack.get();
1374621351
1374721352 self.scope_generation += 1;
1374821353 const state = try self.saveState();
1374921354
13750 var it = switch_br.iterateCases();
13751 while (it.next()) |case| {
13752 var relocs = try self.gpa.alloc(Mir.Inst.Index, case.items.len + case.ranges.len);
13753 defer self.gpa.free(relocs);
21355 const liveness = try self.liveness.getSwitchBr(allocator, inst, switch_br.cases_len + 1);
21356 defer allocator.free(liveness.deaths);
21357
21358 if (!self.mod.pic and self.target.ofmt == .elf) table: {
21359 var prong_items: u32 = 0;
21360 var min: ?Value = null;
21361 var max: ?Value = null;
21362 {
21363 var cases_it = switch_br.iterateCases();
21364 while (cases_it.next()) |case| {
21365 prong_items += @intCast(case.items.len + case.ranges.len);
21366 for (case.items) |item| {
21367 const val = Value.fromInterned(item.toInterned().?);
21368 if (min == null or val.compareHetero(.lt, min.?, zcu)) min = val;
21369 if (max == null or val.compareHetero(.gt, max.?, zcu)) max = val;
21370 }
21371 for (case.ranges) |range| {
21372 const low = Value.fromInterned(range[0].toInterned().?);
21373 if (min == null or low.compareHetero(.lt, min.?, zcu)) min = low;
21374 const high = Value.fromInterned(range[1].toInterned().?);
21375 if (max == null or high.compareHetero(.gt, max.?, zcu)) max = high;
21376 }
21377 }
21378 }
21379 // This condition also triggers for switches with no non-else prongs and switches on bool.
21380 if (prong_items < 1 << 2 or prong_items > 1 << 8) break :table;
21381
21382 var min_space: Value.BigIntSpace = undefined;
21383 const min_bigint = min.?.toBigInt(&min_space, zcu);
21384 var max_space: Value.BigIntSpace = undefined;
21385 const max_bigint = max.?.toBigInt(&max_space, zcu);
21386 const limbs = try allocator.alloc(
21387 std.math.big.Limb,
21388 @max(min_bigint.limbs.len, max_bigint.limbs.len) + 1,
21389 );
21390 defer allocator.free(limbs);
21391 const table_len = table_len: {
21392 var table_len_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
21393 table_len_bigint.sub(max_bigint, min_bigint);
21394 assert(table_len_bigint.positive); // min <= max
21395 break :table_len @as(u11, table_len_bigint.toConst().to(u10) catch break :table) + 1; // no more than a 1024 entry table
21396 };
21397 assert(prong_items <= table_len); // each prong item introduces at least one unique integer to the range
21398 if (prong_items < table_len >> 2) break :table; // no more than 75% waste
21399
21400 const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: {
21401 const condition_index = try self.allocTempRegOrMem(condition_ty, true);
21402 try self.genCopy(condition_ty, condition_index, condition, .{});
21403 break :condition_index condition_index;
21404 };
21405 try self.spillEflagsIfOccupied();
21406 if (min.?.orderAgainstZero(zcu).compare(.neq)) try self.genBinOpMir(
21407 .{ ._, .sub },
21408 condition_ty,
21409 condition_index,
21410 .{ .air_ref = Air.internedToRef(min.?.toIntern()) },
21411 );
21412 const else_reloc = if (switch_br.else_body_len > 0) else_reloc: {
21413 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_index, .{ .immediate = table_len - 1 });
21414 break :else_reloc try self.asmJccReloc(.a, undefined);
21415 } else undefined;
21416 const table_start: u31 = @intCast(self.mir_table.items.len);
21417 {
21418 const condition_index_reg = if (condition_index.isRegister())
21419 condition_index.getReg().?
21420 else
21421 try self.copyToTmpRegister(.usize, condition_index);
21422 const condition_index_lock = self.register_manager.lockReg(condition_index_reg);
21423 defer if (condition_index_lock) |lock| self.register_manager.unlockReg(lock);
21424 try self.truncateRegister(condition_ty, condition_index_reg);
21425 const ptr_size = @divExact(self.target.ptrBitWidth(), 8);
21426 try self.asmMemory(.{ ._, .jmp }, .{
21427 .base = .table,
21428 .mod = .{ .rm = .{
21429 .size = .ptr,
21430 .index = registerAlias(condition_index_reg, ptr_size),
21431 .scale = .fromFactor(@intCast(ptr_size)),
21432 .disp = table_start * ptr_size,
21433 } },
21434 });
21435 }
21436 const else_reloc_marker: u32 = 0;
21437 assert(self.mir_instructions.len > else_reloc_marker);
21438 try self.mir_table.appendNTimes(self.gpa, else_reloc_marker, table_len);
21439 if (is_loop) try self.loop_switches.putNoClobber(self.gpa, inst, .{
21440 .start = table_start,
21441 .len = table_len,
21442 .min = min.?,
21443 .else_relocs = if (switch_br.else_body_len > 0) .{ .forward = .empty } else .@"unreachable",
21444 });
21445 defer if (is_loop) {
21446 var loop_switch_data = self.loop_switches.fetchRemove(inst).?.value;
21447 switch (loop_switch_data.else_relocs) {
21448 .@"unreachable", .backward => {},
21449 .forward => |*else_relocs| else_relocs.deinit(self.gpa),
21450 }
21451 };
21452 var cases_it = switch_br.iterateCases();
21453 while (cases_it.next()) |case| {
21454 {
21455 const table = self.mir_table.items[table_start..][0..table_len];
21456 for (case.items) |item| {
21457 const val = Value.fromInterned(item.toInterned().?);
21458 var val_space: Value.BigIntSpace = undefined;
21459 const val_bigint = val.toBigInt(&val_space, zcu);
21460 var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
21461 index_bigint.sub(val_bigint, min_bigint);
21462 table[index_bigint.toConst().to(u10) catch unreachable] = @intCast(self.mir_instructions.len);
21463 }
21464 for (case.ranges) |range| {
21465 var low_space: Value.BigIntSpace = undefined;
21466 const low_bigint = Value.fromInterned(range[0].toInterned().?).toBigInt(&low_space, zcu);
21467 var high_space: Value.BigIntSpace = undefined;
21468 const high_bigint = Value.fromInterned(range[1].toInterned().?).toBigInt(&high_space, zcu);
21469 var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
21470 index_bigint.sub(low_bigint, min_bigint);
21471 const start = index_bigint.toConst().to(u10) catch unreachable;
21472 index_bigint.sub(high_bigint, min_bigint);
21473 const end = @as(u11, index_bigint.toConst().to(u10) catch unreachable) + 1;
21474 @memset(table[start..end], @intCast(self.mir_instructions.len));
21475 }
21476 }
21477
21478 for (liveness.deaths[case.idx]) |operand| try self.processDeath(operand);
21479
21480 try self.genBodyBlock(case.body);
21481 try self.restoreState(state, &.{}, .{
21482 .emit_instructions = false,
21483 .update_tracking = true,
21484 .resurrect = true,
21485 .close_scope = true,
21486 });
21487 }
21488 if (switch_br.else_body_len > 0) {
21489 const else_body = cases_it.elseBody();
21490
21491 const else_deaths = liveness.deaths.len - 1;
21492 for (liveness.deaths[else_deaths]) |operand| try self.processDeath(operand);
21493
21494 self.performReloc(else_reloc);
21495 if (is_loop) {
21496 const loop_switch_data = self.loop_switches.getPtr(inst).?;
21497 for (loop_switch_data.else_relocs.forward.items) |reloc| self.performReloc(reloc);
21498 loop_switch_data.else_relocs.forward.deinit(self.gpa);
21499 loop_switch_data.else_relocs = .{ .backward = @intCast(self.mir_instructions.len) };
21500 }
21501 for (self.mir_table.items[table_start..][0..table_len]) |*entry| if (entry.* == else_reloc_marker) {
21502 entry.* = @intCast(self.mir_instructions.len);
21503 };
21504
21505 try self.genBodyBlock(else_body);
21506 try self.restoreState(state, &.{}, .{
21507 .emit_instructions = false,
21508 .update_tracking = true,
21509 .resurrect = true,
21510 .close_scope = true,
21511 });
21512 }
21513 return;
21514 }
21515
21516 const signedness = if (condition_ty.isAbiInt(zcu)) condition_ty.intInfo(zcu).signedness else .unsigned;
21517 var cases_it = switch_br.iterateCases();
21518 while (cases_it.next()) |case| {
21519 var relocs = try allocator.alloc(Mir.Inst.Index, case.items.len + case.ranges.len);
21520 defer allocator.free(relocs);
1375421521
1375521522 try self.spillEflagsIfOccupied();
1375621523 for (case.items, relocs[0..case.items.len]) |item, *reloc| {
......@@ -13833,9 +21600,8 @@ fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwit
1383321600 // Relocate the "skip" branch to fall through to the next case.
1383421601 self.performReloc(skip_case_reloc);
1383521602 }
13836
1383721603 if (switch_br.else_body_len > 0) {
13838 const else_body = it.elseBody();
21604 const else_body = cases_it.elseBody();
1383921605
1384021606 const else_deaths = liveness.deaths.len - 1;
1384121607 for (liveness.deaths[else_deaths]) |operand| try self.processDeath(operand);
......@@ -13850,24 +21616,23 @@ fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwit
1385021616 }
1385121617}
1385221618
13853fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
21619fn airSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void {
1385421620 const switch_br = self.air.unwrapSwitch(inst);
1385521621 const condition = try self.resolveInst(switch_br.operand);
1385621622
1385721623 // If the condition dies here in this switch instruction, process
1385821624 // that death now instead of later as this has an effect on
1385921625 // whether it needs to be spilled in the branches
13860 if (self.liveness.operandDies(inst, 0)) {
21626 const condition_dies = self.liveness.operandDies(inst, 0);
21627 if (condition_dies) {
1386121628 if (switch_br.operand.toIndex()) |op_inst| try self.processDeath(op_inst);
1386221629 }
13863
13864 try self.lowerSwitchBr(inst, switch_br, condition);
21630 try self.lowerSwitchBr(inst, switch_br, condition, condition_dies, false);
1386521631
1386621632 // We already took care of pl_op.operand earlier, so there's nothing left to do
13867 self.finishAirBookkeeping();
1386821633}
1386921634
13870fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
21635fn airLoopSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void {
1387121636 const switch_br = self.air.unwrapSwitch(inst);
1387221637 const condition = try self.resolveInst(switch_br.operand);
1387321638
......@@ -13879,7 +21644,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
1387921644 try self.genCopy(self.typeOf(switch_br.operand), mat_cond, condition, .{});
1388021645 break :mat_cond mat_cond;
1388121646 };
13882 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(mat_cond));
21647 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(mat_cond));
1388321648
1388421649 // If the condition dies here in this switch instruction, process
1388521650 // that death now instead of later as this has an effect on
......@@ -13893,25 +21658,83 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
1389321658
1389421659 try self.loops.putNoClobber(self.gpa, inst, .{
1389521660 .state = state,
13896 .jmp_target = @intCast(self.mir_instructions.len),
21661 .target = @intCast(self.mir_instructions.len),
1389721662 });
1389821663 defer assert(self.loops.remove(inst));
1389921664
1390021665 // Stop tracking block result without forgetting tracking info
1390121666 try self.freeValue(mat_cond);
1390221667
13903 try self.lowerSwitchBr(inst, switch_br, mat_cond);
21668 try self.lowerSwitchBr(inst, switch_br, mat_cond, true, true);
1390421669
1390521670 try self.processDeath(inst);
13906 self.finishAirBookkeeping();
1390721671}
1390821672
13909fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {
21673fn airSwitchDispatch(self: *CodeGen, inst: Air.Inst.Index) !void {
1391021674 const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
1391121675
1391221676 const block_ty = self.typeOfIndex(br.block_inst);
13913 const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
1391421677 const loop_data = self.loops.getPtr(br.block_inst).?;
21678 if (self.loop_switches.getPtr(br.block_inst)) |table| {
21679 // Process operand death so that it is properly accounted for in the State below.
21680 const condition_dies = self.liveness.operandDies(inst, 0);
21681
21682 try self.restoreState(loop_data.state, &.{}, .{
21683 .emit_instructions = true,
21684 .update_tracking = false,
21685 .resurrect = false,
21686 .close_scope = false,
21687 });
21688
21689 const condition_ty = self.typeOf(br.operand);
21690 const condition = try self.resolveInst(br.operand);
21691 const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: {
21692 const condition_index = try self.allocTempRegOrMem(condition_ty, true);
21693 try self.genCopy(condition_ty, condition_index, condition, .{});
21694 break :condition_index condition_index;
21695 };
21696 try self.spillEflagsIfOccupied();
21697 if (table.min.orderAgainstZero(self.pt.zcu).compare(.neq)) try self.genBinOpMir(
21698 .{ ._, .sub },
21699 condition_ty,
21700 condition_index,
21701 .{ .air_ref = Air.internedToRef(table.min.toIntern()) },
21702 );
21703 switch (table.else_relocs) {
21704 .@"unreachable" => {},
21705 .forward => |*else_relocs| {
21706 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_index, .{ .immediate = table.len - 1 });
21707 try else_relocs.append(self.gpa, try self.asmJccReloc(.a, undefined));
21708 },
21709 .backward => |else_reloc| {
21710 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_index, .{ .immediate = table.len - 1 });
21711 _ = try self.asmJccReloc(.a, else_reloc);
21712 },
21713 }
21714 {
21715 const condition_index_reg = if (condition_index.isRegister())
21716 condition_index.getReg().?
21717 else
21718 try self.copyToTmpRegister(.usize, condition_index);
21719 const condition_index_lock = self.register_manager.lockReg(condition_index_reg);
21720 defer if (condition_index_lock) |lock| self.register_manager.unlockReg(lock);
21721 try self.truncateRegister(condition_ty, condition_index_reg);
21722 const ptr_size = @divExact(self.target.ptrBitWidth(), 8);
21723 try self.asmMemory(.{ ._, .jmp }, .{
21724 .base = .table,
21725 .mod = .{ .rm = .{
21726 .size = .ptr,
21727 .index = registerAlias(condition_index_reg, ptr_size),
21728 .scale = .fromFactor(@intCast(ptr_size)),
21729 .disp = @intCast(table.start * ptr_size),
21730 } },
21731 });
21732 }
21733
21734 return self.finishAir(inst, .none, .{ br.operand, .none, .none });
21735 }
21736
21737 const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
1391521738 done: {
1391621739 try self.getValue(block_tracking.short, null);
1391721740 const src_mcv = try self.resolveInst(br.operand);
......@@ -13945,15 +21768,13 @@ fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {
1394521768
1394621769 // Emit a jump with a relocation. It will be patched up after the block ends.
1394721770 // Leave the jump offset undefined
13948 _ = try self.asmJmpReloc(loop_data.jmp_target);
21771 _ = try self.asmJmpReloc(loop_data.target);
1394921772
1395021773 // Stop tracking block result without forgetting tracking info
1395121774 try self.freeValue(block_tracking.short);
13952
13953 self.finishAirBookkeeping();
1395421775}
1395521776
13956fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {
21777fn performReloc(self: *CodeGen, reloc: Mir.Inst.Index) void {
1395721778 const next_inst: u32 = @intCast(self.mir_instructions.len);
1395821779 switch (self.mir_instructions.items(.tag)[reloc]) {
1395921780 .j, .jmp => {},
......@@ -13966,7 +21787,7 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {
1396621787 self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;
1396721788}
1396821789
13969fn airBr(self: *Self, inst: Air.Inst.Index) !void {
21790fn airBr(self: *CodeGen, inst: Air.Inst.Index) !void {
1397021791 const zcu = self.pt.zcu;
1397121792 const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
1397221793
......@@ -14007,7 +21828,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
1400721828 }
1400821829
1400921830 if (first_br) {
14010 block_tracking.* = InstTracking.init(block_result);
21831 block_tracking.* = .init(block_result);
1401121832 try self.saveRetroactiveState(&block_data.state);
1401221833 } else try self.restoreState(block_data.state, &.{}, .{
1401321834 .emit_instructions = true,
......@@ -14023,11 +21844,9 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
1402321844
1402421845 // Stop tracking block result without forgetting tracking info
1402521846 try self.freeValue(block_tracking.short);
14026
14027 self.finishAirBookkeeping();
1402821847}
1402921848
14030fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {
21849fn airRepeat(self: *CodeGen, inst: Air.Inst.Index) !void {
1403121850 const loop_inst = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat.loop_inst;
1403221851 const repeat_info = self.loops.get(loop_inst).?;
1403321852 try self.restoreState(repeat_info.state, &.{}, .{
......@@ -14036,11 +21855,10 @@ fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {
1403621855 .resurrect = false,
1403721856 .close_scope = true,
1403821857 });
14039 _ = try self.asmJmpReloc(repeat_info.jmp_target);
14040 self.finishAirBookkeeping();
21858 _ = try self.asmJmpReloc(repeat_info.target);
1404121859}
1404221860
14043fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
21861fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
1404421862 const pt = self.pt;
1404521863 const zcu = pt.zcu;
1404621864 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -14054,7 +21872,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1405421872 extra_i += inputs.len;
1405521873
1405621874 var result: MCValue = .none;
14057 var args = std.ArrayList(MCValue).init(self.gpa);
21875 var args: std.ArrayList(MCValue) = .init(self.gpa);
1405821876 try args.ensureTotalCapacity(outputs.len + inputs.len);
1405921877 defer {
1406021878 for (args.items) |arg| if (arg.getReg()) |reg| self.register_manager.unlockReg(.{
......@@ -14062,15 +21880,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1406221880 });
1406321881 args.deinit();
1406421882 }
14065 var arg_map = std.StringHashMap(u8).init(self.gpa);
21883 var arg_map: std.StringHashMap(u8) = .init(self.gpa);
1406621884 try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len));
1406721885 defer arg_map.deinit();
1406821886
1406921887 var outputs_extra_i = extra_i;
1407021888 for (outputs) |output| {
14071 const extra_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]);
14072 const constraint = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
14073 const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
21889 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
21890 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
21891 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
1407421892 // This equation accounts for the fact that even if we have exactly 4 bytes
1407521893 // for the string, we still use the next u32 for the null terminator.
1407621894 extra_i += (constraint.len + name.len + (2 + 3)) / 4;
......@@ -14097,8 +21915,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1409721915 const is_early_clobber = constraint[1] == '&';
1409821916 const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..];
1409921917 const arg_mcv: MCValue = arg_mcv: {
14100 const arg_maybe_reg: ?Register = if (mem.eql(u8, rest, "r") or
14101 mem.eql(u8, rest, "f") or mem.eql(u8, rest, "x"))
21918 const arg_maybe_reg: ?Register = if (std.mem.eql(u8, rest, "r") or
21919 std.mem.eql(u8, rest, "f") or std.mem.eql(u8, rest, "x"))
1410221920 registerAlias(
1410321921 self.register_manager.tryAllocReg(maybe_inst, switch (rest[0]) {
1410421922 'r' => abi.RegisterClass.gp,
......@@ -14108,20 +21926,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1410821926 }) orelse return self.fail("ran out of registers lowering inline asm", .{}),
1410921927 @intCast(ty.abiSize(zcu)),
1411021928 )
14111 else if (mem.eql(u8, rest, "m"))
21929 else if (std.mem.eql(u8, rest, "m"))
1411221930 if (output != .none) null else return self.fail(
1411321931 "memory constraint unsupported for asm result: '{s}'",
1411421932 .{constraint},
1411521933 )
14116 else if (mem.eql(u8, rest, "g") or
14117 mem.eql(u8, rest, "rm") or mem.eql(u8, rest, "mr") or
14118 mem.eql(u8, rest, "r,m") or mem.eql(u8, rest, "m,r"))
21934 else if (std.mem.eql(u8, rest, "g") or
21935 std.mem.eql(u8, rest, "rm") or std.mem.eql(u8, rest, "mr") or
21936 std.mem.eql(u8, rest, "r,m") or std.mem.eql(u8, rest, "m,r"))
1411921937 self.register_manager.tryAllocReg(maybe_inst, abi.RegisterClass.gp) orelse
1412021938 if (output != .none)
1412121939 null
1412221940 else
1412321941 return self.fail("ran out of registers lowering inline asm", .{})
14124 else if (mem.startsWith(u8, rest, "{") and mem.endsWith(u8, rest, "}"))
21942 else if (std.mem.startsWith(u8, rest, "{") and std.mem.endsWith(u8, rest, "}"))
1412521943 parseRegName(rest["{".len .. rest.len - "}".len]) orelse
1412621944 return self.fail("invalid register constraint: '{s}'", .{constraint})
1412721945 else if (rest.len == 1 and std.ascii.isDigit(rest[0])) {
......@@ -14134,18 +21952,19 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1413421952 break :arg_mcv if (arg_maybe_reg) |reg| .{ .register = reg } else arg: {
1413521953 const ptr_mcv = try self.resolveInst(output);
1413621954 switch (ptr_mcv) {
14137 .immediate => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_|
21955 .immediate => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_|
1413821956 break :arg ptr_mcv.deref(),
1413921957 .register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(),
1414021958 else => {},
1414121959 }
14142 break :arg .{ .indirect = .{ .reg = try self.copyToTmpRegister(Type.usize, ptr_mcv) } };
21960 break :arg .{ .indirect = .{ .reg = try self.copyToTmpRegister(.usize, ptr_mcv) } };
1414321961 };
1414421962 };
14145 if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| {
14146 _ = self.register_manager.lockReg(reg);
21963 if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |tracked_index| {
21964 try self.register_manager.getRegIndex(tracked_index, if (output == .none) inst else null);
21965 _ = self.register_manager.lockRegIndexAssumeUnused(tracked_index);
1414721966 };
14148 if (!mem.eql(u8, name, "_"))
21967 if (!std.mem.eql(u8, name, "_"))
1414921968 arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
1415021969 args.appendAssumeCapacity(arg_mcv);
1415121970 if (output == .none) result = arg_mcv;
......@@ -14153,17 +21972,17 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1415321972 }
1415421973
1415521974 for (inputs) |input| {
14156 const input_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]);
14157 const constraint = mem.sliceTo(input_bytes, 0);
14158 const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
21975 const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
21976 const constraint = std.mem.sliceTo(input_bytes, 0);
21977 const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
1415921978 // This equation accounts for the fact that even if we have exactly 4 bytes
1416021979 // for the string, we still use the next u32 for the null terminator.
1416121980 extra_i += (constraint.len + name.len + (2 + 3)) / 4;
1416221981
1416321982 const ty = self.typeOf(input);
1416421983 const input_mcv = try self.resolveInst(input);
14165 const arg_mcv: MCValue = if (mem.eql(u8, constraint, "r") or
14166 mem.eql(u8, constraint, "f") or mem.eql(u8, constraint, "x"))
21984 const arg_mcv: MCValue = if (std.mem.eql(u8, constraint, "r") or
21985 std.mem.eql(u8, constraint, "f") or std.mem.eql(u8, constraint, "x"))
1416721986 arg: {
1416821987 const rc = switch (constraint[0]) {
1416921988 'r' => abi.RegisterClass.gp,
......@@ -14177,16 +21996,16 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1417721996 const reg = try self.register_manager.allocReg(null, rc);
1417821997 try self.genSetReg(reg, ty, input_mcv, .{});
1417921998 break :arg .{ .register = registerAlias(reg, @intCast(ty.abiSize(zcu))) };
14180 } else if (mem.eql(u8, constraint, "i") or mem.eql(u8, constraint, "n"))
21999 } else if (std.mem.eql(u8, constraint, "i") or std.mem.eql(u8, constraint, "n"))
1418122000 switch (input_mcv) {
1418222001 .immediate => |imm| .{ .immediate = imm },
1418322002 else => return self.fail("immediate operand requires comptime value: '{s}'", .{
1418422003 constraint,
1418522004 }),
1418622005 }
14187 else if (mem.eql(u8, constraint, "m")) arg: {
22006 else if (std.mem.eql(u8, constraint, "m")) arg: {
1418822007 switch (input_mcv) {
14189 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_|
22008 .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_|
1419022009 break :arg input_mcv,
1419122010 .indirect, .load_frame => break :arg input_mcv,
1419222011 .load_symbol, .load_direct, .load_got, .load_tlv => {},
......@@ -14201,24 +22020,24 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1420122020 try self.genCopy(ty, temp_mcv, input_mcv, .{});
1420222021 break :arg temp_mcv;
1420322022 };
14204 try self.genSetReg(addr_reg, Type.usize, input_mcv.address(), .{});
22023 try self.genSetReg(addr_reg, .usize, input_mcv.address(), .{});
1420522024 break :arg .{ .indirect = .{ .reg = addr_reg } };
14206 } else if (mem.eql(u8, constraint, "g") or
14207 mem.eql(u8, constraint, "rm") or mem.eql(u8, constraint, "mr") or
14208 mem.eql(u8, constraint, "r,m") or mem.eql(u8, constraint, "m,r"))
22025 } else if (std.mem.eql(u8, constraint, "g") or
22026 std.mem.eql(u8, constraint, "rm") or std.mem.eql(u8, constraint, "mr") or
22027 std.mem.eql(u8, constraint, "r,m") or std.mem.eql(u8, constraint, "m,r"))
1420922028 arg: {
1421022029 switch (input_mcv) {
1421122030 .register, .indirect, .load_frame => break :arg input_mcv,
14212 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_|
22031 .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_|
1421322032 break :arg input_mcv,
1421422033 else => {},
1421522034 }
1421622035 const temp_mcv = try self.allocTempRegOrMem(ty, true);
1421722036 try self.genCopy(ty, temp_mcv, input_mcv, .{});
1421822037 break :arg temp_mcv;
14219 } else if (mem.eql(u8, constraint, "X"))
22038 } else if (std.mem.eql(u8, constraint, "X"))
1422022039 input_mcv
14221 else if (mem.startsWith(u8, constraint, "{") and mem.endsWith(u8, constraint, "}")) arg: {
22040 else if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) arg: {
1422222041 const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse
1422322042 return self.fail("invalid register constraint: '{s}'", .{constraint});
1422422043 try self.register_manager.getReg(reg, null);
......@@ -14233,7 +22052,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1423322052 if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| {
1423422053 _ = self.register_manager.lockReg(reg);
1423522054 };
14236 if (!mem.eql(u8, name, "_"))
22055 if (!std.mem.eql(u8, name, "_"))
1423722056 arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
1423822057 args.appendAssumeCapacity(arg_mcv);
1423922058 }
......@@ -14241,7 +22060,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1424122060 {
1424222061 var clobber_i: u32 = 0;
1424322062 while (clobber_i < clobbers_len) : (clobber_i += 1) {
14244 const clobber = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
22063 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
1424522064 // This equation accounts for the fact that even if we have exactly 4 bytes
1424622065 // for the string, we still use the next u32 for the null terminator.
1424722066 extra_i += clobber.len / 4 + 1;
......@@ -14294,20 +22113,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1429422113 labels.deinit(self.gpa);
1429522114 }
1429622115
14297 const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
14298 var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");
22116 const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
22117 var line_it = std.mem.tokenizeAny(u8, asm_source, "\n\r;");
1429922118 next_line: while (line_it.next()) |line| {
14300 var mnem_it = mem.tokenizeAny(u8, line, " \t");
14301 var prefix: Instruction.Prefix = .none;
22119 var mnem_it = std.mem.tokenizeAny(u8, line, " \t");
22120 var prefix: encoder.Instruction.Prefix = .none;
1430222121 const mnem_str = while (mnem_it.next()) |mnem_str| {
1430322122 if (mnem_str[0] == '#') continue :next_line;
14304 if (mem.startsWith(u8, mnem_str, "//")) continue :next_line;
14305 if (std.meta.stringToEnum(Instruction.Prefix, mnem_str)) |pre| {
22123 if (std.mem.startsWith(u8, mnem_str, "//")) continue :next_line;
22124 if (std.meta.stringToEnum(encoder.Instruction.Prefix, mnem_str)) |pre| {
1430622125 if (prefix != .none) return self.fail("extra prefix: '{s}'", .{mnem_str});
1430722126 prefix = pre;
1430822127 continue;
1430922128 }
14310 if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str;
22129 if (!std.mem.endsWith(u8, mnem_str, ":")) break mnem_str;
1431122130 const label_name = mnem_str[0 .. mnem_str.len - ":".len];
1431222131 if (!Label.isValid(.definition, label_name))
1431322132 return self.fail("invalid label: '{s}'", .{label_name});
......@@ -14332,21 +22151,21 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1433222151
1433322152 var mnem_size: ?Memory.Size = if (prefix == .directive)
1433422153 null
14335 else if (mem.endsWith(u8, mnem_str, "b"))
22154 else if (std.mem.endsWith(u8, mnem_str, "b"))
1433622155 .byte
14337 else if (mem.endsWith(u8, mnem_str, "w"))
22156 else if (std.mem.endsWith(u8, mnem_str, "w"))
1433822157 .word
14339 else if (mem.endsWith(u8, mnem_str, "l"))
22158 else if (std.mem.endsWith(u8, mnem_str, "l"))
1434022159 .dword
14341 else if (mem.endsWith(u8, mnem_str, "q") and
14342 (std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !mem.endsWith(u8, mnem_str, "dq")))
22160 else if (std.mem.endsWith(u8, mnem_str, "q") and
22161 (std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !std.mem.endsWith(u8, mnem_str, "dq")))
1434322162 .qword
14344 else if (mem.endsWith(u8, mnem_str, "t"))
22163 else if (std.mem.endsWith(u8, mnem_str, "t"))
1434522164 .tbyte
1434622165 else
1434722166 null;
1434822167 const mnem_tag = while (true) break std.meta.stringToEnum(
14349 Instruction.Mnemonic,
22168 encoder.Instruction.Mnemonic,
1435022169 mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size != null)],
1435122170 ) orelse if (mnem_size) |_| {
1435222171 mnem_size = null;
......@@ -14367,18 +22186,18 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1436722186 .{ ._, .pseudo }
1436822187 else for (std.enums.values(Mir.Inst.Fixes)) |fixes| {
1436922188 const fixes_name = @tagName(fixes);
14370 const space_i = mem.indexOfScalar(u8, fixes_name, ' ');
22189 const space_i = std.mem.indexOfScalar(u8, fixes_name, ' ');
1437122190 const fixes_prefix = if (space_i) |i|
14372 std.meta.stringToEnum(Instruction.Prefix, fixes_name[0..i]).?
22191 std.meta.stringToEnum(encoder.Instruction.Prefix, fixes_name[0..i]).?
1437322192 else
1437422193 .none;
1437522194 if (fixes_prefix != prefix) continue;
1437622195 const pattern = fixes_name[if (space_i) |i| i + " ".len else 0..];
14377 const wildcard_i = mem.indexOfScalar(u8, pattern, '_').?;
22196 const wildcard_i = std.mem.indexOfScalar(u8, pattern, '_').?;
1437822197 const mnem_prefix = pattern[0..wildcard_i];
1437922198 const mnem_suffix = pattern[wildcard_i + "_".len ..];
14380 if (!mem.startsWith(u8, mnem_name, mnem_prefix)) continue;
14381 if (!mem.endsWith(u8, mnem_name, mnem_suffix)) continue;
22199 if (!std.mem.startsWith(u8, mnem_name, mnem_prefix)) continue;
22200 if (!std.mem.endsWith(u8, mnem_name, mnem_suffix)) continue;
1438222201 break .{ fixes, std.meta.stringToEnum(
1438322202 Mir.Inst.Tag,
1438422203 mnem_name[mnem_prefix.len .. mnem_name.len - mnem_suffix.len],
......@@ -14390,31 +22209,25 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1439022209 });
1439122210 };
1439222211
14393 const Operand = union(enum) {
14394 none,
14395 reg: Register,
14396 mem: Memory,
14397 imm: Immediate,
14398 inst: Mir.Inst.Index,
14399 };
14400 var ops: [4]Operand = .{.none} ** 4;
22212 var ops: [4]Operand = @splat(.none);
22213 var ops_len: usize = 0;
1440122214
1440222215 var last_op = false;
14403 var op_it = mem.splitScalar(u8, mnem_it.rest(), ',');
22216 var op_it = std.mem.splitScalar(u8, mnem_it.rest(), ',');
1440422217 next_op: for (&ops) |*op| {
1440522218 const op_str = while (!last_op) {
1440622219 const full_str = op_it.next() orelse break :next_op;
14407 const code_str = if (mem.indexOfScalar(u8, full_str, '#') orelse
14408 mem.indexOf(u8, full_str, "//")) |comment|
22220 const code_str = if (std.mem.indexOfScalar(u8, full_str, '#') orelse
22221 std.mem.indexOf(u8, full_str, "//")) |comment|
1440922222 code: {
1441022223 last_op = true;
1441122224 break :code full_str[0..comment];
1441222225 } else full_str;
14413 const trim_str = mem.trim(u8, code_str, " \t*");
22226 const trim_str = std.mem.trim(u8, code_str, " \t*");
1441422227 if (trim_str.len > 0) break trim_str;
1441522228 } else break;
14416 if (mem.startsWith(u8, op_str, "%%")) {
14417 const colon = mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');
22229 if (std.mem.startsWith(u8, op_str, "%%")) {
22230 const colon = std.mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');
1441822231 const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse
1441922232 return self.fail("invalid register: '{s}'", .{op_str});
1442022233 if (colon) |colon_pos| {
......@@ -14428,12 +22241,12 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1442822241 } },
1442922242 } };
1443022243 } else {
14431 if (mnem_size) |size| if (reg.bitSize() != size.bitSize())
22244 if (mnem_size) |size| if (reg.bitSize() != size.bitSize(self.target))
1443222245 return self.fail("invalid register size: '{s}'", .{op_str});
1443322246 op.* = .{ .reg = reg };
1443422247 }
14435 } else if (mem.startsWith(u8, op_str, "%[") and mem.endsWith(u8, op_str, "]")) {
14436 const colon = mem.indexOfScalarPos(u8, op_str, "%[".len, ':');
22248 } else if (std.mem.startsWith(u8, op_str, "%[") and std.mem.endsWith(u8, op_str, "]")) {
22249 const colon = std.mem.indexOfScalarPos(u8, op_str, "%[".len, ':');
1443722250 const modifier = if (colon) |colon_pos|
1443822251 op_str[colon_pos + ":".len .. op_str.len - "]".len]
1443922252 else
......@@ -14442,15 +22255,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1444222255 arg_map.get(op_str["%[".len .. colon orelse op_str.len - "]".len]) orelse
1444322256 return self.fail("no matching constraint: '{s}'", .{op_str})
1444422257 ]) {
14445 .immediate => |imm| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "c"))
14446 .{ .imm = Immediate.u(imm) }
22258 .immediate => |imm| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "c"))
22259 .{ .imm = .u(imm) }
1444722260 else
1444822261 return self.fail("invalid modifier: '{s}'", .{modifier}),
14449 .register => |reg| if (mem.eql(u8, modifier, ""))
22262 .register => |reg| if (std.mem.eql(u8, modifier, ""))
1445022263 .{ .reg = reg }
1445122264 else
1445222265 return self.fail("invalid modifier: '{s}'", .{modifier}),
14453 .memory => |addr| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "P"))
22266 .memory => |addr| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "P"))
1445422267 .{ .mem = .{
1445522268 .base = .{ .reg = .ds },
1445622269 .mod = .{ .rm = .{
......@@ -14461,7 +22274,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1446122274 } }
1446222275 else
1446322276 return self.fail("invalid modifier: '{s}'", .{modifier}),
14464 .indirect => |reg_off| if (mem.eql(u8, modifier, ""))
22277 .indirect => |reg_off| if (std.mem.eql(u8, modifier, ""))
1446522278 .{ .mem = .{
1446622279 .base = .{ .reg = reg_off.reg },
1446722280 .mod = .{ .rm = .{
......@@ -14472,7 +22285,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1447222285 } }
1447322286 else
1447422287 return self.fail("invalid modifier: '{s}'", .{modifier}),
14475 .load_frame => |frame_addr| if (mem.eql(u8, modifier, ""))
22288 .load_frame => |frame_addr| if (std.mem.eql(u8, modifier, ""))
1447622289 .{ .mem = .{
1447722290 .base = .{ .frame = frame_addr.index },
1447822291 .mod = .{ .rm = .{
......@@ -14483,42 +22296,42 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1448322296 } }
1448422297 else
1448522298 return self.fail("invalid modifier: '{s}'", .{modifier}),
14486 .lea_got => |sym_index| if (mem.eql(u8, modifier, "P"))
14487 .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_got = sym_index }) }
22299 .lea_got => |sym_index| if (std.mem.eql(u8, modifier, "P"))
22300 .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_got = sym_index }) }
1448822301 else
1448922302 return self.fail("invalid modifier: '{s}'", .{modifier}),
14490 .lea_symbol => |sym_off| if (mem.eql(u8, modifier, "P"))
14491 .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_symbol = sym_off }) }
22303 .lea_symbol => |sym_off| if (std.mem.eql(u8, modifier, "P"))
22304 .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_symbol = sym_off }) }
1449222305 else
1449322306 return self.fail("invalid modifier: '{s}'", .{modifier}),
1449422307 else => return self.fail("invalid constraint: '{s}'", .{op_str}),
1449522308 };
14496 } else if (mem.startsWith(u8, op_str, "$")) {
22309 } else if (std.mem.startsWith(u8, op_str, "$")) {
1449722310 if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| {
1449822311 if (mnem_size) |size| {
14499 const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - (size.bitSize() - 1));
22312 const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - (size.bitSize(self.target) - 1));
1450022313 if ((if (s < 0) ~s else s) > max)
1450122314 return self.fail("invalid immediate size: '{s}'", .{op_str});
1450222315 }
14503 op.* = .{ .imm = Immediate.s(s) };
22316 op.* = .{ .imm = .s(s) };
1450422317 } else |_| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| {
1450522318 if (mnem_size) |size| {
14506 const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - size.bitSize());
22319 const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - size.bitSize(self.target));
1450722320 if (u > max)
1450822321 return self.fail("invalid immediate size: '{s}'", .{op_str});
1450922322 }
14510 op.* = .{ .imm = Immediate.u(u) };
22323 op.* = .{ .imm = .u(u) };
1451122324 } else |_| return self.fail("invalid immediate: '{s}'", .{op_str});
14512 } else if (mem.endsWith(u8, op_str, ")")) {
14513 const open = mem.indexOfScalar(u8, op_str, '(') orelse
22325 } else if (std.mem.endsWith(u8, op_str, ")")) {
22326 const open = std.mem.indexOfScalar(u8, op_str, '(') orelse
1451422327 return self.fail("invalid operand: '{s}'", .{op_str});
14515 var sib_it = mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ',');
22328 var sib_it = std.mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ',');
1451622329 const base_str = sib_it.next() orelse
1451722330 return self.fail("invalid memory operand: '{s}'", .{op_str});
14518 if (base_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))
22331 if (base_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%"))
1451922332 return self.fail("invalid memory operand: '{s}'", .{op_str});
1452022333 const index_str = sib_it.next() orelse "";
14521 if (index_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))
22334 if (index_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%"))
1452222335 return self.fail("invalid memory operand: '{s}'", .{op_str});
1452322336 const scale_str = sib_it.next() orelse "";
1452422337 if (index_str.len == 0 and scale_str.len > 0)
......@@ -14550,10 +22363,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1455022363 else
1455122364 .none,
1455222365 .scale = scale,
14553 .disp = if (mem.startsWith(u8, op_str[0..open], "%[") and
14554 mem.endsWith(u8, op_str[0..open], "]"))
22366 .disp = if (std.mem.startsWith(u8, op_str[0..open], "%[") and
22367 std.mem.endsWith(u8, op_str[0..open], "]"))
1455522368 disp: {
14556 const colon = mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');
22369 const colon = std.mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');
1455722370 const modifier = if (colon) |colon_pos|
1455822371 op_str[colon_pos + ":".len .. open - "]".len]
1455922372 else
......@@ -14562,9 +22375,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1456222375 arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse
1456322376 return self.fail("no matching constraint: '{s}'", .{op_str})
1456422377 ]) {
14565 .immediate => |imm| if (mem.eql(u8, modifier, "") or
14566 mem.eql(u8, modifier, "c"))
14567 math.cast(i32, @as(i64, @bitCast(imm))) orelse
22378 .immediate => |imm| if (std.mem.eql(u8, modifier, "") or
22379 std.mem.eql(u8, modifier, "c"))
22380 std.math.cast(i32, @as(i64, @bitCast(imm))) orelse
1456822381 return self.fail("invalid displacement: '{s}'", .{op_str})
1456922382 else
1457022383 return self.fail("invalid modifier: '{s}'", .{modifier}),
......@@ -14592,8 +22405,12 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1459222405 try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len));
1459322406 op.* = .{ .inst = label_gop.value_ptr.target };
1459422407 } else return self.fail("invalid operand: '{s}'", .{op_str});
22408 ops_len += 1;
1459522409 } else if (op_it.next()) |op_str| return self.fail("extra operand: '{s}'", .{op_str});
1459622410
22411 // convert from att syntax to intel syntax
22412 std.mem.reverse(Operand, ops[0..ops_len]);
22413
1459722414 (if (prefix == .directive) switch (mnem_tag) {
1459822415 .@".cfi_def_cfa" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none)
1459922416 self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, ops[0].reg, ops[1].imm)
......@@ -14649,68 +22466,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1464922466 error.InvalidInstruction,
1465022467 .@".cfi_escape" => error.InvalidInstruction,
1465122468 else => unreachable,
14652 } else switch (ops[0]) {
14653 .none => self.asmOpOnly(mnem_fixed_tag),
14654 .reg => |reg0| switch (ops[1]) {
14655 .none => self.asmRegister(mnem_fixed_tag, reg0),
14656 .reg => |reg1| switch (ops[2]) {
14657 .none => self.asmRegisterRegister(mnem_fixed_tag, reg1, reg0),
14658 .reg => |reg2| switch (ops[3]) {
14659 .none => self.asmRegisterRegisterRegister(mnem_fixed_tag, reg2, reg1, reg0),
14660 else => error.InvalidInstruction,
14661 },
14662 .mem => |mem2| switch (ops[3]) {
14663 .none => self.asmMemoryRegisterRegister(mnem_fixed_tag, mem2, reg1, reg0),
14664 else => error.InvalidInstruction,
14665 },
14666 else => error.InvalidInstruction,
14667 },
14668 .mem => |mem1| switch (ops[2]) {
14669 .none => self.asmMemoryRegister(mnem_fixed_tag, mem1, reg0),
14670 else => error.InvalidInstruction,
14671 },
14672 else => error.InvalidInstruction,
14673 },
14674 .mem => |mem0| switch (ops[1]) {
14675 .none => self.asmMemory(mnem_fixed_tag, mem0),
14676 .reg => |reg1| switch (ops[2]) {
14677 .none => self.asmRegisterMemory(mnem_fixed_tag, reg1, mem0),
14678 else => error.InvalidInstruction,
14679 },
14680 else => error.InvalidInstruction,
14681 },
14682 .imm => |imm0| switch (ops[1]) {
14683 .none => self.asmImmediate(mnem_fixed_tag, imm0),
14684 .reg => |reg1| switch (ops[2]) {
14685 .none => self.asmRegisterImmediate(mnem_fixed_tag, reg1, imm0),
14686 .reg => |reg2| switch (ops[3]) {
14687 .none => self.asmRegisterRegisterImmediate(mnem_fixed_tag, reg2, reg1, imm0),
14688 .reg => |reg3| self.asmRegisterRegisterRegisterImmediate(
14689 mnem_fixed_tag,
14690 reg3,
14691 reg2,
14692 reg1,
14693 imm0,
14694 ),
14695 else => error.InvalidInstruction,
14696 },
14697 .mem => |mem2| switch (ops[3]) {
14698 .none => self.asmMemoryRegisterImmediate(mnem_fixed_tag, mem2, reg1, imm0),
14699 else => error.InvalidInstruction,
14700 },
14701 else => error.InvalidInstruction,
14702 },
14703 .mem => |mem1| switch (ops[2]) {
14704 .none => self.asmMemoryImmediate(mnem_fixed_tag, mem1, imm0),
14705 else => error.InvalidInstruction,
14706 },
14707 else => error.InvalidInstruction,
14708 },
14709 .inst => |inst0| switch (ops[1]) {
14710 .none => self.asmReloc(mnem_fixed_tag, inst0),
14711 else => error.InvalidInstruction,
14712 },
14713 }) catch |err| switch (err) {
22469 } else self.asmOps(mnem_fixed_tag, ops)) catch |err| switch (err) {
1471422470 error.InvalidInstruction => return self.fail(
1471522471 "invalid instruction: '{s} {s} {s} {s} {s}'",
1471622472 .{
......@@ -14730,10 +22486,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1473022486 return self.fail("undefined label: '{s}'", .{label.key_ptr.*});
1473122487
1473222488 for (outputs, args.items[0..outputs.len]) |output, arg_mcv| {
14733 const extra_bytes = mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);
22489 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);
1473422490 const constraint =
14735 mem.sliceTo(mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);
14736 const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
22491 std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);
22492 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
1473722493 // This equation accounts for the fact that even if we have exactly 4 bytes
1473822494 // for the string, we still use the next u32 for the null terminator.
1473922495 outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
......@@ -14745,7 +22501,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1474522501 }
1474622502
1474722503 simple: {
14748 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
22504 var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
1474922505 var buf_index: usize = 0;
1475022506 for (outputs) |output| {
1475122507 if (output == .none) continue;
......@@ -14775,9 +22531,12 @@ const MoveStrategy = union(enum) {
1477522531 extract: Mir.Inst.FixedTag,
1477622532 };
1477722533
14778 pub fn read(strat: MoveStrategy, self: *Self, dst_reg: Register, src_mem: Memory) !void {
22534 pub fn read(strat: MoveStrategy, self: *CodeGen, dst_reg: Register, src_mem: Memory) !void {
1477922535 switch (strat) {
14780 .move => |tag| try self.asmRegisterMemory(tag, dst_reg, src_mem),
22536 .move => |tag| try self.asmRegisterMemory(tag, switch (tag[1]) {
22537 else => dst_reg,
22538 .lea => if (dst_reg.bitSize() >= 32) dst_reg else dst_reg.to32(),
22539 }, src_mem),
1478122540 .x87_load_store => {
1478222541 try self.asmMemory(.{ .f_, .ld }, src_mem);
1478322542 assert(dst_reg != .st7);
......@@ -14787,34 +22546,50 @@ const MoveStrategy = union(enum) {
1478722546 ie.insert,
1478822547 dst_reg,
1478922548 src_mem,
14790 Immediate.u(0),
22549 .u(0),
1479122550 ),
1479222551 .vex_insert_extract => |ie| try self.asmRegisterRegisterMemoryImmediate(
1479322552 ie.insert,
1479422553 dst_reg,
1479522554 dst_reg,
1479622555 src_mem,
14797 Immediate.u(0),
22556 .u(0),
1479822557 ),
1479922558 }
1480022559 }
14801 pub fn write(strat: MoveStrategy, self: *Self, dst_mem: Memory, src_reg: Register) !void {
22560 pub fn write(strat: MoveStrategy, self: *CodeGen, dst_mem: Memory, src_reg: Register) !void {
1480222561 switch (strat) {
1480322562 .move => |tag| try self.asmMemoryRegister(tag, dst_mem, src_reg),
1480422563 .x87_load_store => {
1480522564 try self.asmRegister(.{ .f_, .ld }, src_reg);
1480622565 try self.asmMemory(.{ .f_p, .st }, dst_mem);
1480722566 },
14808 .insert_extract, .vex_insert_extract => |ie| try self.asmMemoryRegisterImmediate(
14809 ie.extract,
14810 dst_mem,
14811 src_reg,
14812 Immediate.u(0),
14813 ),
22567 .insert_extract, .vex_insert_extract => |ie| if (ie.extract[0] != .p_w or self.hasFeature(.sse4_1))
22568 try self.asmMemoryRegisterImmediate(ie.extract, dst_mem, src_reg, .u(0))
22569 else if (self.hasFeature(.sse2)) {
22570 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
22571 try self.asmRegisterRegisterImmediate(ie.extract, tmp_reg.to32(), src_reg.to128(), .u(0));
22572 try self.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.to16());
22573 } else {
22574 const tmp_frame_index = try self.allocFrameIndex(.init(.{
22575 .size = 16,
22576 .alignment = .@"16",
22577 }));
22578 try self.asmMemoryRegister(.{ ._ps, .mova }, .{
22579 .base = .{ .frame = tmp_frame_index },
22580 .mod = .{ .rm = .{ .size = .xword } },
22581 }, src_reg.to128());
22582 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
22583 try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg.to16(), .{
22584 .base = .{ .frame = tmp_frame_index },
22585 .mod = .{ .rm = .{ .size = .word } },
22586 });
22587 try self.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.to16());
22588 },
1481422589 }
1481522590 }
1481622591};
14817fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !MoveStrategy {
22592fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) !MoveStrategy {
1481822593 const pt = self.pt;
1481922594 const zcu = pt.zcu;
1482022595 switch (class) {
......@@ -14823,7 +22598,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
1482322598 .mmx => {},
1482422599 .sse => switch (ty.zigTypeTag(zcu)) {
1482522600 else => {
14826 const classes = mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
22601 const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
1482722602 assert(std.mem.indexOfNone(abi.Class, classes, &.{
1482822603 .integer, .sse, .sseup, .memory, .float, .float_combine,
1482922604 }) == null);
......@@ -14853,10 +22628,13 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
1485322628 else
1485422629 .{ ._q, .mov } },
1485522630 9...16 => return .{ .move = if (self.hasFeature(.avx))
14856 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14857 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
22631 .{ if (aligned) .v_dqa else .v_dqu, .mov }
22632 else if (self.hasFeature(.sse2))
22633 .{ if (aligned) ._dqa else ._dqu, .mov }
22634 else
22635 .{ ._ps, if (aligned) .mova else .movu } },
1485822636 17...32 => if (self.hasFeature(.avx))
14859 return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
22637 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1486022638 else => {},
1486122639 } else switch (abi_size) {
1486222640 4 => return .{ .move = if (self.hasFeature(.avx))
......@@ -14865,15 +22643,18 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
1486522643 .{ ._ss, .mov } },
1486622644 5...8 => return .{ .move = if (self.hasFeature(.avx))
1486722645 .{ .v_sd, .mov }
22646 else if (self.hasFeature(.sse2))
22647 .{ ._sd, .mov }
1486822648 else
14869 .{ ._sd, .mov } },
22649 .{ ._ps, .movl } },
1487022650 9...16 => return .{ .move = if (self.hasFeature(.avx))
14871 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
14872 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
14873 17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned)
14874 .{ .v_pd, .mova }
22651 .{ .v_pd, if (aligned) .mova else .movu }
22652 else if (self.hasFeature(.sse2))
22653 .{ ._pd, if (aligned) .mova else .movu }
1487522654 else
14876 .{ .v_pd, .movu } },
22655 .{ ._ps, if (aligned) .mova else .movu } },
22656 17...32 => if (self.hasFeature(.avx))
22657 return .{ .move = .{ .v_pd, if (aligned) .mova else .movu } },
1487722658 else => {},
1487822659 }
1487922660 },
......@@ -14891,11 +22672,16 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
1489122672 .{ ._ss, .mov } },
1489222673 64 => return .{ .move = if (self.hasFeature(.avx))
1489322674 .{ .v_sd, .mov }
22675 else if (self.hasFeature(.sse2))
22676 .{ ._sd, .mov }
1489422677 else
14895 .{ ._sd, .mov } },
22678 .{ ._ps, .movl } },
1489622679 128 => return .{ .move = if (self.hasFeature(.avx))
14897 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14898 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
22680 .{ if (aligned) .v_dqa else .v_dqu, .mov }
22681 else if (self.hasFeature(.sse2))
22682 .{ if (aligned) ._dqa else ._dqu, .mov }
22683 else
22684 .{ ._ps, if (aligned) .mova else .movu } },
1489922685 else => {},
1490022686 },
1490122687 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
......@@ -14909,65 +22695,62 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
1490922695 .int => switch (ty.childType(zcu).intInfo(zcu).bits) {
1491022696 1...8 => switch (ty.vectorLen(zcu)) {
1491122697 1...16 => return .{ .move = if (self.hasFeature(.avx))
14912 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14913 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
22698 .{ if (aligned) .v_dqa else .v_dqu, .mov }
22699 else if (self.hasFeature(.sse2))
22700 .{ if (aligned) ._dqa else ._dqu, .mov }
22701 else
22702 .{ ._ps, if (aligned) .mova else .movu } },
1491422703 17...32 => if (self.hasFeature(.avx))
14915 return .{ .move = if (aligned)
14916 .{ .v_, .movdqa }
14917 else
14918 .{ .v_, .movdqu } },
22704 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1491922705 else => {},
1492022706 },
1492122707 9...16 => switch (ty.vectorLen(zcu)) {
1492222708 1...8 => return .{ .move = if (self.hasFeature(.avx))
14923 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14924 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
22709 .{ if (aligned) .v_dqa else .v_dqu, .mov }
22710 else if (self.hasFeature(.sse2))
22711 .{ if (aligned) ._dqa else ._dqu, .mov }
22712 else
22713 .{ ._ps, if (aligned) .mova else .movu } },
1492522714 9...16 => if (self.hasFeature(.avx))
14926 return .{ .move = if (aligned)
14927 .{ .v_, .movdqa }
14928 else
14929 .{ .v_, .movdqu } },
22715 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1493022716 else => {},
1493122717 },
1493222718 17...32 => switch (ty.vectorLen(zcu)) {
1493322719 1...4 => return .{ .move = if (self.hasFeature(.avx))
14934 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14935 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
22720 .{ if (aligned) .v_dqa else .v_dqu, .mov }
22721 else if (self.hasFeature(.sse2))
22722 .{ if (aligned) ._dqa else ._dqu, .mov }
22723 else
22724 .{ ._ps, if (aligned) .mova else .movu } },
1493622725 5...8 => if (self.hasFeature(.avx))
14937 return .{ .move = if (aligned)
14938 .{ .v_, .movdqa }
14939 else
14940 .{ .v_, .movdqu } },
22726 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1494122727 else => {},
1494222728 },
1494322729 33...64 => switch (ty.vectorLen(zcu)) {
1494422730 1...2 => return .{ .move = if (self.hasFeature(.avx))
14945 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14946 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
22731 .{ if (aligned) .v_dqa else .v_dqu, .mov }
22732 else if (self.hasFeature(.sse2))
22733 .{ if (aligned) ._dqa else ._dqu, .mov }
22734 else
22735 .{ ._ps, if (aligned) .mova else .movu } },
1494722736 3...4 => if (self.hasFeature(.avx))
14948 return .{ .move = if (aligned)
14949 .{ .v_, .movdqa }
14950 else
14951 .{ .v_, .movdqu } },
22737 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1495222738 else => {},
1495322739 },
1495422740 65...128 => switch (ty.vectorLen(zcu)) {
1495522741 1 => return .{ .move = if (self.hasFeature(.avx))
14956 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14957 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
22742 .{ if (aligned) .v_dqa else .v_dqu, .mov }
22743 else if (self.hasFeature(.sse2))
22744 .{ if (aligned) ._dqa else ._dqu, .mov }
22745 else
22746 .{ ._ps, if (aligned) .mova else .movu } },
1495822747 2 => if (self.hasFeature(.avx))
14959 return .{ .move = if (aligned)
14960 .{ .v_, .movdqa }
14961 else
14962 .{ .v_, .movdqu } },
22748 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1496322749 else => {},
1496422750 },
1496522751 129...256 => switch (ty.vectorLen(zcu)) {
1496622752 1 => if (self.hasFeature(.avx))
14967 return .{ .move = if (aligned)
14968 .{ .v_, .movdqa }
14969 else
14970 .{ .v_, .movdqu } },
22753 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1497122754 else => {},
1497222755 },
1497322756 else => {},
......@@ -14975,13 +22758,13 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
1497522758 .pointer, .optional => if (ty.childType(zcu).isPtrAtRuntime(zcu))
1497622759 switch (ty.vectorLen(zcu)) {
1497722760 1...2 => return .{ .move = if (self.hasFeature(.avx))
14978 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14979 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
22761 .{ if (aligned) .v_dqa else .v_dqu, .mov }
22762 else if (self.hasFeature(.sse2))
22763 .{ if (aligned) ._dqa else ._dqu, .mov }
22764 else
22765 .{ ._ps, if (aligned) .mova else .movu } },
1498022766 3...4 => if (self.hasFeature(.avx))
14981 return .{ .move = if (aligned)
14982 .{ .v_, .movdqa }
14983 else
14984 .{ .v_, .movdqu } },
22767 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1498522768 else => {},
1498622769 }
1498722770 else
......@@ -14989,46 +22772,42 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
1498922772 .float => switch (ty.childType(zcu).floatBits(self.target.*)) {
1499022773 16 => switch (ty.vectorLen(zcu)) {
1499122774 1...8 => return .{ .move = if (self.hasFeature(.avx))
14992 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14993 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
22775 .{ if (aligned) .v_dqa else .v_dqu, .mov }
22776 else if (self.hasFeature(.sse2))
22777 .{ if (aligned) ._dqa else ._dqu, .mov }
22778 else
22779 .{ ._ps, if (aligned) .mova else .movu } },
1499422780 9...16 => if (self.hasFeature(.avx))
14995 return .{ .move = if (aligned)
14996 .{ .v_, .movdqa }
14997 else
14998 .{ .v_, .movdqu } },
22781 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1499922782 else => {},
1500022783 },
1500122784 32 => switch (ty.vectorLen(zcu)) {
1500222785 1...4 => return .{ .move = if (self.hasFeature(.avx))
15003 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
15004 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
22786 .{ .v_ps, if (aligned) .mova else .movu }
22787 else
22788 .{ ._ps, if (aligned) .mova else .movu } },
1500522789 5...8 => if (self.hasFeature(.avx))
15006 return .{ .move = if (aligned)
15007 .{ .v_ps, .mova }
15008 else
15009 .{ .v_ps, .movu } },
22790 return .{ .move = .{ .v_ps, if (aligned) .mova else .movu } },
1501022791 else => {},
1501122792 },
1501222793 64 => switch (ty.vectorLen(zcu)) {
1501322794 1...2 => return .{ .move = if (self.hasFeature(.avx))
15014 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
15015 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
22795 .{ .v_pd, if (aligned) .mova else .movu }
22796 else
22797 .{ ._pd, if (aligned) .mova else .movu } },
1501622798 3...4 => if (self.hasFeature(.avx))
15017 return .{ .move = if (aligned)
15018 .{ .v_pd, .mova }
15019 else
15020 .{ .v_pd, .movu } },
22799 return .{ .move = .{ .v_pd, if (aligned) .mova else .movu } },
1502122800 else => {},
1502222801 },
1502322802 128 => switch (ty.vectorLen(zcu)) {
1502422803 1 => return .{ .move = if (self.hasFeature(.avx))
15025 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
15026 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
22804 .{ if (aligned) .v_dqa else .v_dqu, .mov }
22805 else if (self.hasFeature(.sse2))
22806 .{ if (aligned) ._dqa else ._dqu, .mov }
22807 else
22808 .{ ._ps, if (aligned) .mova else .movu } },
1502722809 2 => if (self.hasFeature(.avx))
15028 return .{ .move = if (aligned)
15029 .{ .v_, .movdqa }
15030 else
15031 .{ .v_, .movdqu } },
22810 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1503222811 else => {},
1503322812 },
1503422813 else => {},
......@@ -15045,7 +22824,7 @@ const CopyOptions = struct {
1504522824 safety: bool = false,
1504622825};
1504722826
15048fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) InnerError!void {
22827fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) InnerError!void {
1504922828 const pt = self.pt;
1505022829
1505122830 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
......@@ -15059,6 +22838,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
1505922838 .immediate,
1506022839 .eflags,
1506122840 .register_overflow,
22841 .register_mask,
1506222842 .lea_direct,
1506322843 .lea_got,
1506422844 .lea_tlv,
......@@ -15088,42 +22868,93 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
1508822868 .off = -dst_reg_off.off,
1508922869 } },
1509022870 }, opts),
15091 .register_pair => |dst_regs| {
15092 const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = switch (src_mcv) {
15093 .register_pair, .memory, .indirect, .load_frame => null,
15094 .load_symbol, .load_direct, .load_got, .load_tlv => src: {
22871 inline .register_pair, .register_triple, .register_quadruple => |dst_regs, dst_tag| {
22872 const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = src_info: switch (src_mcv) {
22873 .undef, .memory, .indirect, .load_frame => null,
22874 .register => |src_reg| switch (dst_regs[0].class()) {
22875 .general_purpose => switch (src_reg.class()) {
22876 else => unreachable,
22877 .sse => if (ty.abiSize(pt.zcu) <= 16) {
22878 if (self.hasFeature(.avx)) {
22879 try self.asmRegisterRegister(.{ .v_q, .mov }, dst_regs[0].to64(), src_reg.to128());
22880 try self.asmRegisterRegisterImmediate(.{ .vp_q, .extr }, dst_regs[1].to64(), src_reg.to128(), .u(1));
22881 } else if (self.hasFeature(.sse4_1)) {
22882 try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128());
22883 try self.asmRegisterRegisterImmediate(.{ .p_q, .extr }, dst_regs[1].to64(), src_reg.to128(), .u(1));
22884 } else {
22885 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
22886 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
22887 defer self.register_manager.unlockReg(tmp_lock);
22888
22889 try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128());
22890 try self.asmRegisterRegister(.{ ._ps, .movhl }, tmp_reg.to128(), src_reg.to128());
22891 try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[1].to64(), tmp_reg.to128());
22892 }
22893 return;
22894 } else unreachable,
22895 },
22896 else => unreachable,
22897 },
22898 dst_tag => |src_regs| {
22899 var hazard_regs = src_regs;
22900 for (dst_regs, &hazard_regs, 1..) |dst_reg, src_reg, hazard_index| {
22901 const dst_id = dst_reg.id();
22902 if (dst_id == src_reg.id()) continue;
22903 var mir_tag: Mir.Inst.Tag = .mov;
22904 for (hazard_regs[hazard_index..]) |*hazard_reg| {
22905 if (dst_id != hazard_reg.id()) continue;
22906 mir_tag = .xchg;
22907 hazard_reg.* = src_reg;
22908 }
22909 try self.asmRegisterRegister(.{ ._, mir_tag }, dst_reg.to64(), src_reg.to64());
22910 }
22911 return;
22912 },
22913 .load_symbol, .load_direct, .load_got, .load_tlv => {
1509522914 const src_addr_reg =
1509622915 (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64();
1509722916 const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
1509822917 errdefer self.register_manager.unlockReg(src_addr_lock);
1509922918
15100 try self.genSetReg(src_addr_reg, Type.usize, src_mcv.address(), opts);
15101 break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
22919 try self.genSetReg(src_addr_reg, .usize, src_mcv.address(), opts);
22920 break :src_info .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
1510222921 },
15103 .air_ref => |src_ref| return self.genCopy(
15104 ty,
15105 dst_mcv,
15106 try self.resolveInst(src_ref),
15107 opts,
15108 ),
22922 .air_ref => |src_ref| return self.genCopy(ty, dst_mcv, try self.resolveInst(src_ref), opts),
1510922923 else => return self.fail("TODO implement genCopy for {s} of {}", .{
1511022924 @tagName(src_mcv), ty.fmt(pt),
1511122925 }),
1511222926 };
1511322927 defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock);
1511422928
15115 var part_disp: i32 = 0;
15116 for (dst_regs, try self.splitType(ty), 0..) |dst_reg, dst_ty, part_i| {
15117 try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) {
15118 .register_pair => |src_regs| .{ .register = src_regs[part_i] },
15119 .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
15120 .load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{
15121 .reg = src_info.?.addr_reg,
15122 .off = part_disp,
15123 } },
22929 for ([_]bool{ false, true }) |emit_hazard| {
22930 var hazard_count: u3 = 0;
22931 var part_disp: i32 = 0;
22932 for (dst_regs, try self.splitType(dst_regs.len, ty), 0..) |dst_reg, dst_ty, part_i| {
22933 defer part_disp += @intCast(dst_ty.abiSize(pt.zcu));
22934 const is_hazard = if (src_mcv.getReg()) |src_reg|
22935 dst_reg.id() == src_reg.id()
22936 else if (src_info) |info|
22937 dst_reg.id() == info.addr_reg.id()
22938 else
22939 false;
22940 if (is_hazard) hazard_count += 1;
22941 if (is_hazard != emit_hazard) continue;
22942 try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) {
22943 .undef => if (opts.safety and part_i > 0) .{ .register = dst_regs[0] } else .undef,
22944 dst_tag => |src_regs| .{ .register = src_regs[part_i] },
22945 .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
22946 .load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{
22947 .reg = src_info.?.addr_reg,
22948 .off = part_disp,
22949 } },
22950 else => unreachable,
22951 }, opts);
22952 }
22953 switch (hazard_count) {
22954 0 => break,
22955 1 => continue,
1512422956 else => unreachable,
15125 }, opts);
15126 part_disp += @intCast(dst_ty.abiSize(pt.zcu));
22957 }
1512722958 }
1512822959 },
1512922960 .indirect => |reg_off| try self.genSetMem(
......@@ -15135,13 +22966,13 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
1513522966 ),
1513622967 .memory, .load_symbol, .load_direct, .load_got, .load_tlv => {
1513722968 switch (dst_mcv) {
15138 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr|
22969 .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr|
1513922970 return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv, opts),
1514022971 .load_symbol, .load_direct, .load_got, .load_tlv => {},
1514122972 else => unreachable,
1514222973 }
1514322974
15144 const addr_reg = try self.copyToTmpRegister(Type.usize, dst_mcv.address());
22975 const addr_reg = try self.copyToTmpRegister(.usize, dst_mcv.address());
1514522976 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
1514622977 defer self.register_manager.unlockReg(addr_lock);
1514722978
......@@ -15158,7 +22989,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
1515822989}
1515922990
1516022991fn genSetReg(
15161 self: *Self,
22992 self: *CodeGen,
1516222993 dst_reg: Register,
1516322994 ty: Type,
1516422995 src_mcv: MCValue,
......@@ -15179,17 +23010,17 @@ fn genSetReg(
1517923010 => unreachable,
1518023011 .undef => if (opts.safety) switch (dst_reg.class()) {
1518123012 .general_purpose => switch (abi_size) {
15182 1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), Immediate.u(0xAA)),
15183 2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), Immediate.u(0xAAAA)),
23013 1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), .u(0xAA)),
23014 2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), .u(0xAAAA)),
1518423015 3...4 => try self.asmRegisterImmediate(
1518523016 .{ ._, .mov },
1518623017 dst_reg.to32(),
15187 Immediate.s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))),
23018 .s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))),
1518823019 ),
1518923020 5...8 => try self.asmRegisterImmediate(
1519023021 .{ ._, .mov },
1519123022 dst_reg.to64(),
15192 Immediate.u(0xAAAAAAAAAAAAAAAA),
23023 .u(0xAAAAAAAAAAAAAAAA),
1519323024 ),
1519423025 else => unreachable,
1519523026 },
......@@ -15203,20 +23034,20 @@ fn genSetReg(
1520323034 // register is the fastest way to zero a register.
1520423035 try self.spillEflagsIfOccupied();
1520523036 try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
15206 } else if (abi_size > 4 and math.cast(u32, imm) != null) {
23037 } else if (abi_size > 4 and std.math.cast(u32, imm) != null) {
1520723038 // 32-bit moves zero-extend to 64-bit.
15208 try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(imm));
23039 try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(imm));
1520923040 } else if (abi_size <= 4 and @as(i64, @bitCast(imm)) < 0) {
1521023041 try self.asmRegisterImmediate(
1521123042 .{ ._, .mov },
1521223043 registerAlias(dst_reg, abi_size),
15213 Immediate.s(@intCast(@as(i64, @bitCast(imm)))),
23044 .s(@intCast(@as(i64, @bitCast(imm)))),
1521423045 );
1521523046 } else {
1521623047 try self.asmRegisterImmediate(
1521723048 .{ ._, .mov },
1521823049 registerAlias(dst_reg, abi_size),
15219 Immediate.u(imm),
23050 .u(imm),
1522023051 );
1522123052 }
1522223053 },
......@@ -15233,7 +23064,7 @@ fn genSetReg(
1523323064 src_reg,
1523423065 ),
1523523066 .x87, .mmx, .ip => unreachable,
15236 .sse => try self.asmRegisterRegister(
23067 .sse => if (self.hasFeature(.sse2)) try self.asmRegisterRegister(
1523723068 switch (abi_size) {
1523823069 1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov },
1523923070 5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov },
......@@ -15241,7 +23072,20 @@ fn genSetReg(
1524123072 },
1524223073 registerAlias(dst_reg, @max(abi_size, 4)),
1524323074 src_reg.to128(),
15244 ),
23075 ) else {
23076 const frame_index = try self.allocFrameIndex(.init(.{
23077 .size = 4,
23078 .alignment = .@"4",
23079 }));
23080 try self.asmMemoryRegister(.{ ._ss, .mov }, .{
23081 .base = .{ .frame = frame_index },
23082 .mod = .{ .rm = .{ .size = .dword } },
23083 }, src_reg.to128());
23084 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(dst_reg, abi_size), .{
23085 .base = .{ .frame = frame_index },
23086 .mod = .{ .rm = .{ .size = .fromSize(abi_size) } },
23087 });
23088 },
1524523089 },
1524623090 .segment => try self.asmRegisterRegister(
1524723091 .{ ._, .mov },
......@@ -15286,17 +23130,17 @@ fn genSetReg(
1528623130 .sse => try self.asmRegisterRegister(
1528723131 @as(?Mir.Inst.FixedTag, switch (ty.scalarType(zcu).zigTypeTag(zcu)) {
1528823132 else => switch (abi_size) {
15289 1...16 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else .{ ._, .movdqa },
15290 17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null,
23133 1...16 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else .{ ._dqa, .mov },
23134 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null,
1529123135 else => null,
1529223136 },
1529323137 .float => switch (ty.scalarType(zcu).floatBits(self.target.*)) {
1529423138 16, 128 => switch (abi_size) {
1529523139 2...16 => if (self.hasFeature(.avx))
15296 .{ .v_, .movdqa }
23140 .{ .v_dqa, .mov }
1529723141 else
15298 .{ ._, .movdqa },
15299 17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null,
23142 .{ ._dqa, .mov },
23143 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null,
1530023144 else => null,
1530123145 },
1530223146 32 => if (self.hasFeature(.avx)) .{ .v_ps, .mova } else .{ ._ps, .mova },
......@@ -15311,7 +23155,42 @@ fn genSetReg(
1531123155 },
1531223156 .ip => unreachable,
1531323157 },
15314 .register_pair => |src_regs| try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts),
23158 inline .register_pair,
23159 .register_triple,
23160 .register_quadruple,
23161 => |src_regs| switch (dst_reg.class()) {
23162 .general_purpose => switch (src_regs[0].class()) {
23163 .general_purpose => try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts),
23164 else => unreachable,
23165 },
23166 .sse => switch (src_regs[0].class()) {
23167 .general_purpose => if (abi_size <= 16) {
23168 if (self.hasFeature(.avx)) {
23169 try self.asmRegisterRegister(.{ .v_q, .mov }, dst_reg.to128(), src_regs[0].to64());
23170 try self.asmRegisterRegisterRegisterImmediate(
23171 .{ .vp_q, .insr },
23172 dst_reg.to128(),
23173 dst_reg.to128(),
23174 src_regs[1].to64(),
23175 .u(1),
23176 );
23177 } else if (self.hasFeature(.sse4_1)) {
23178 try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64());
23179 try self.asmRegisterRegisterImmediate(.{ .p_q, .insr }, dst_reg.to128(), src_regs[1].to64(), .u(1));
23180 } else {
23181 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
23182 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
23183 defer self.register_manager.unlockReg(tmp_lock);
23184
23185 try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64());
23186 try self.asmRegisterRegister(.{ ._q, .mov }, tmp_reg.to128(), src_regs[1].to64());
23187 try self.asmRegisterRegister(.{ ._ps, .movlh }, dst_reg.to128(), tmp_reg.to128());
23188 }
23189 } else unreachable,
23190 else => unreachable,
23191 },
23192 else => unreachable,
23193 },
1531523194 .register_offset,
1531623195 .indirect,
1531723196 .load_frame,
......@@ -15325,15 +23204,15 @@ fn genSetReg(
1532523204 .load_frame => |frame_addr| try self.moveStrategy(
1532623205 ty,
1532723206 dst_reg.class(),
15328 self.getFrameAddrAlignment(frame_addr).compare(.gte, Alignment.fromLog2Units(
15329 math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
23207 self.getFrameAddrAlignment(frame_addr).compare(.gte, .fromLog2Units(
23208 std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
1533023209 )),
1533123210 ),
1533223211 .lea_frame => .{ .move = .{ ._, .lea } },
1533323212 else => unreachable,
1533423213 }).read(self, registerAlias(dst_reg, abi_size), switch (src_mcv) {
1533523214 .register_offset, .indirect => |reg_off| .{
15336 .base = .{ .reg = reg_off.reg },
23215 .base = .{ .reg = reg_off.reg.to64() },
1533723216 .mod = .{ .rm = .{
1533823217 .size = self.memSize(ty),
1533923218 .disp = reg_off.off,
......@@ -15348,9 +23227,50 @@ fn genSetReg(
1534823227 },
1534923228 else => unreachable,
1535023229 }),
23230 .register_mask => |src_reg_mask| {
23231 assert(src_reg_mask.reg.class() == .sse);
23232 const has_avx = self.hasFeature(.avx);
23233 const bits_reg = switch (dst_reg.class()) {
23234 .general_purpose => dst_reg,
23235 else => try self.register_manager.allocReg(null, abi.RegisterClass.gp),
23236 };
23237 const bits_lock = self.register_manager.lockReg(bits_reg);
23238 defer if (bits_lock) |lock| self.register_manager.unlockReg(lock);
23239
23240 const pack_reg = switch (src_reg_mask.info.scalar) {
23241 else => src_reg_mask.reg,
23242 .word => try self.register_manager.allocReg(null, abi.RegisterClass.sse),
23243 };
23244 const pack_lock = self.register_manager.lockReg(pack_reg);
23245 defer if (pack_lock) |lock| self.register_manager.unlockReg(lock);
23246
23247 var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.info.scalar.bitSize(self.target), 8));
23248 switch (src_reg_mask.info.scalar) {
23249 else => {},
23250 .word => {
23251 const src_alias = registerAlias(src_reg_mask.reg, mask_size);
23252 const pack_alias = registerAlias(pack_reg, mask_size);
23253 if (has_avx) {
23254 try self.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, src_alias, src_alias);
23255 } else {
23256 try self.asmRegisterRegister(.{ ._dqa, .mov }, pack_alias, src_alias);
23257 try self.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias);
23258 }
23259 mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable;
23260 },
23261 }
23262 try self.asmRegisterRegister(.{ switch (src_reg_mask.info.scalar) {
23263 .byte, .word => if (has_avx) .vp_b else .p_b,
23264 .dword => if (has_avx) .v_ps else ._ps,
23265 .qword => if (has_avx) .v_pd else ._pd,
23266 else => unreachable,
23267 }, .movmsk }, bits_reg.to32(), registerAlias(pack_reg, mask_size));
23268 if (src_reg_mask.info.inverted) try self.asmRegister(.{ ._, .not }, registerAlias(bits_reg, abi_size));
23269 try self.genSetReg(dst_reg, ty, .{ .register = bits_reg }, .{});
23270 },
1535123271 .memory, .load_symbol, .load_direct, .load_got, .load_tlv => {
1535223272 switch (src_mcv) {
15353 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr|
23273 .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr|
1535423274 return (try self.moveStrategy(
1535523275 ty,
1535623276 dst_reg.class(),
......@@ -15396,18 +23316,14 @@ fn genSetReg(
1539623316 else => unreachable,
1539723317 }
1539823318
15399 const addr_reg = try self.copyToTmpRegister(Type.usize, src_mcv.address());
23319 const addr_reg = try self.copyToTmpRegister(.usize, src_mcv.address());
1540023320 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
1540123321 defer self.register_manager.unlockReg(addr_lock);
1540223322
15403 try (try self.moveStrategy(ty, dst_reg.class(), false)).read(
15404 self,
15405 registerAlias(dst_reg, abi_size),
15406 .{
15407 .base = .{ .reg = addr_reg },
15408 .mod = .{ .rm = .{ .size = self.memSize(ty) } },
15409 },
15410 );
23323 try (try self.moveStrategy(ty, dst_reg.class(), false)).read(self, registerAlias(dst_reg, abi_size), .{
23324 .base = .{ .reg = addr_reg.to64() },
23325 .mod = .{ .rm = .{ .size = self.memSize(ty) } },
23326 });
1541123327 },
1541223328 .lea_symbol => |sym_off| switch (self.bin_file.tag) {
1541323329 .elf, .macho => try self.asmRegisterMemory(
......@@ -15447,7 +23363,7 @@ fn genSetReg(
1544723363}
1544823364
1544923365fn genSetMem(
15450 self: *Self,
23366 self: *CodeGen,
1545123367 base: Memory.Base,
1545223368 disp: i32,
1545323369 ty: Type,
......@@ -15461,6 +23377,7 @@ fn genSetMem(
1546123377 .none => .{ .immediate = @bitCast(@as(i64, disp)) },
1546223378 .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
1546323379 .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
23380 .table => unreachable,
1546423381 .reloc => |sym_index| .{ .lea_symbol = .{ .sym_index = sym_index, .off = disp } },
1546523382 };
1546623383 switch (src_mcv) {
......@@ -15478,31 +23395,31 @@ fn genSetMem(
1547823395 ),
1547923396 .immediate => |imm| switch (abi_size) {
1548023397 1, 2, 4 => {
15481 const immediate = switch (if (ty.isAbiInt(zcu))
23398 const immediate: Immediate = switch (if (ty.isAbiInt(zcu))
1548223399 ty.intInfo(zcu).signedness
1548323400 else
1548423401 .unsigned) {
15485 .signed => Immediate.s(@truncate(@as(i64, @bitCast(imm)))),
15486 .unsigned => Immediate.u(@as(u32, @intCast(imm))),
23402 .signed => .s(@truncate(@as(i64, @bitCast(imm)))),
23403 .unsigned => .u(@as(u32, @intCast(imm))),
1548723404 };
1548823405 try self.asmMemoryImmediate(
1548923406 .{ ._, .mov },
1549023407 .{ .base = base, .mod = .{ .rm = .{
15491 .size = Memory.Size.fromSize(abi_size),
23408 .size = .fromSize(abi_size),
1549223409 .disp = disp,
1549323410 } } },
1549423411 immediate,
1549523412 );
1549623413 },
1549723414 3, 5...7 => unreachable,
15498 else => if (math.cast(i32, @as(i64, @bitCast(imm)))) |small| {
23415 else => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| {
1549923416 try self.asmMemoryImmediate(
1550023417 .{ ._, .mov },
1550123418 .{ .base = base, .mod = .{ .rm = .{
15502 .size = Memory.Size.fromSize(abi_size),
23419 .size = .fromSize(abi_size),
1550323420 .disp = disp,
1550423421 } } },
15505 Immediate.s(small),
23422 .s(small),
1550623423 );
1550723424 } else {
1550823425 var offset: i32 = 0;
......@@ -15512,10 +23429,10 @@ fn genSetMem(
1551223429 .size = .dword,
1551323430 .disp = disp + offset,
1551423431 } } },
15515 if (ty.isSignedInt(zcu)) Immediate.s(
15516 @truncate(@as(i64, @bitCast(imm)) >> (math.cast(u6, offset * 8) orelse 63)),
15517 ) else Immediate.u(
15518 @as(u32, @truncate(if (math.cast(u6, offset * 8)) |shift| imm >> shift else 0)),
23432 if (ty.isSignedInt(zcu)) .s(
23433 @truncate(@as(i64, @bitCast(imm)) >> (std.math.cast(u6, offset * 8) orelse 63)),
23434 ) else .u(
23435 @as(u32, @truncate(if (std.math.cast(u6, offset * 8)) |shift| imm >> shift else 0)),
1551923436 ),
1552023437 );
1552123438 },
......@@ -15542,9 +23459,11 @@ fn genSetMem(
1554223459 .general_purpose, .segment, .x87, .ip => @divExact(src_alias.bitSize(), 8),
1554323460 .mmx, .sse => abi_size,
1554423461 });
15545 const src_align = Alignment.fromNonzeroByteUnits(math.ceilPowerOfTwoAssert(u32, src_size));
23462 const src_align: InternPool.Alignment = .fromNonzeroByteUnits(
23463 std.math.ceilPowerOfTwoAssert(u32, src_size),
23464 );
1554623465 if (src_size > mem_size) {
15547 const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
23466 const frame_index = try self.allocFrameIndex(.init(.{
1554823467 .size = src_size,
1554923468 .alignment = src_align,
1555023469 }));
......@@ -15552,7 +23471,7 @@ fn genSetMem(
1555223471 try (try self.moveStrategy(ty, src_alias.class(), true)).write(
1555323472 self,
1555423473 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{
15555 .size = Memory.Size.fromSize(src_size),
23474 .size = .fromSize(src_size),
1555623475 } } },
1555723476 src_alias,
1555823477 );
......@@ -15568,19 +23487,23 @@ fn genSetMem(
1556823487 .index = frame_index,
1556923488 .off = disp,
1557023489 }).compare(.gte, src_align),
23490 .table => unreachable,
1557123491 .reloc => false,
1557223492 })).write(
1557323493 self,
1557423494 .{ .base = base, .mod = .{ .rm = .{
15575 .size = Memory.Size.fromBitSize(@min(self.memSize(ty).bitSize(), src_alias.bitSize())),
23495 .size = .fromBitSize(@min(
23496 self.memSize(ty).bitSize(self.target),
23497 src_alias.bitSize(),
23498 )),
1557623499 .disp = disp,
1557723500 } } },
1557823501 src_alias,
1557923502 );
1558023503 },
15581 .register_pair => |src_regs| {
23504 inline .register_pair, .register_triple, .register_quadruple => |src_regs| {
1558223505 var part_disp: i32 = disp;
15583 for (try self.splitType(ty), src_regs) |src_ty, src_reg| {
23506 for (try self.splitType(src_regs.len, ty), src_regs) |src_ty, src_reg| {
1558423507 try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg }, opts);
1558523508 part_disp += @intCast(src_ty.abiSize(zcu));
1558623509 }
......@@ -15609,7 +23532,7 @@ fn genSetMem(
1560923532 try self.genSetMem(
1561023533 base,
1561123534 disp + @as(i32, @intCast(child_ty.abiSize(zcu))),
15612 Type.bool,
23535 .bool,
1561323536 .{ .eflags = ro.eflags },
1561423537 opts,
1561523538 );
......@@ -15645,6 +23568,13 @@ fn genSetMem(
1564523568
1564623569 try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts);
1564723570 },
23571 .register_mask => {
23572 const src_reg = try self.copyToTmpRegister(ty, src_mcv);
23573 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
23574 defer self.register_manager.unlockReg(src_lock);
23575
23576 try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts);
23577 },
1564823578 .memory,
1564923579 .indirect,
1565023580 .load_direct,
......@@ -15666,51 +23596,104 @@ fn genSetMem(
1566623596
1566723597 try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts);
1566823598 },
15669 else => try self.genInlineMemcpy(
15670 dst_ptr_mcv,
15671 src_mcv.address(),
15672 .{ .immediate = abi_size },
15673 ),
23599 else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }, .{ .no_alias = true }),
1567423600 },
1567523601 .air_ref => |src_ref| try self.genSetMem(base, disp, ty, try self.resolveInst(src_ref), opts),
1567623602 }
1567723603}
1567823604
15679fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {
23605fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue, opts: struct {
23606 no_alias: bool,
23607}) InnerError!void {
23608 if (opts.no_alias and dst_ptr.isAddress() and src_ptr.isAddress()) switch (len) {
23609 else => {},
23610 .immediate => |len_imm| switch (len_imm) {
23611 else => {},
23612 1 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {
23613 try self.asmRegisterMemory(.{ ._, .mov }, reg.to8(), try src_ptr.deref().mem(self, .{ .size = .byte }));
23614 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .byte }), reg.to8());
23615 return;
23616 },
23617 2 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {
23618 try self.asmRegisterMemory(.{ ._, .mov }, reg.to16(), try src_ptr.deref().mem(self, .{ .size = .word }));
23619 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .word }), reg.to16());
23620 return;
23621 },
23622 4 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {
23623 try self.asmRegisterMemory(.{ ._, .mov }, reg.to32(), try src_ptr.deref().mem(self, .{ .size = .dword }));
23624 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .dword }), reg.to32());
23625 return;
23626 },
23627 8 => if (self.target.cpu.arch == .x86_64) {
23628 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {
23629 try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), try src_ptr.deref().mem(self, .{ .size = .qword }));
23630 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .qword }), reg.to64());
23631 return;
23632 }
23633 },
23634 16 => if (self.hasFeature(.avx)) {
23635 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
23636 try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
23637 try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
23638 return;
23639 }
23640 } else if (self.hasFeature(.sse2)) {
23641 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
23642 try self.asmRegisterMemory(.{ ._dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
23643 try self.asmMemoryRegister(.{ ._dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
23644 return;
23645 }
23646 } else if (self.hasFeature(.sse)) {
23647 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
23648 try self.asmRegisterMemory(.{ ._ps, .movu }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
23649 try self.asmMemoryRegister(.{ ._ps, .movu }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
23650 return;
23651 }
23652 },
23653 32 => if (self.hasFeature(.avx)) {
23654 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
23655 try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to256(), try src_ptr.deref().mem(self, .{ .size = .yword }));
23656 try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .yword }), reg.to256());
23657 return;
23658 }
23659 },
23660 },
23661 };
1568023662 try self.spillRegisters(&.{ .rsi, .rdi, .rcx });
15681 try self.genSetReg(.rsi, Type.usize, src_ptr, .{});
15682 try self.genSetReg(.rdi, Type.usize, dst_ptr, .{});
15683 try self.genSetReg(.rcx, Type.usize, len, .{});
23663 try self.genSetReg(.rsi, .usize, src_ptr, .{});
23664 try self.genSetReg(.rdi, .usize, dst_ptr, .{});
23665 try self.genSetReg(.rcx, .usize, len, .{});
1568423666 try self.asmOpOnly(.{ .@"rep _sb", .mov });
1568523667}
1568623668
1568723669fn genInlineMemset(
15688 self: *Self,
23670 self: *CodeGen,
1568923671 dst_ptr: MCValue,
1569023672 value: MCValue,
1569123673 len: MCValue,
1569223674 opts: CopyOptions,
1569323675) InnerError!void {
1569423676 try self.spillRegisters(&.{ .rdi, .al, .rcx });
15695 try self.genSetReg(.rdi, Type.usize, dst_ptr, .{});
15696 try self.genSetReg(.al, Type.u8, value, opts);
15697 try self.genSetReg(.rcx, Type.usize, len, .{});
23677 try self.genSetReg(.rdi, .usize, dst_ptr, .{});
23678 try self.genSetReg(.al, .u8, value, opts);
23679 try self.genSetReg(.rcx, .usize, len, .{});
1569823680 try self.asmOpOnly(.{ .@"rep _sb", .sto });
1569923681}
1570023682
1570123683fn genExternSymbolRef(
15702 self: *Self,
23684 self: *CodeGen,
1570323685 comptime tag: Mir.Inst.Tag,
1570423686 lib: ?[]const u8,
1570523687 callee: []const u8,
1570623688) InnerError!void {
1570723689 if (self.bin_file.cast(.coff)) |coff_file| {
1570823690 const global_index = try coff_file.getGlobalSymbol(callee, lib);
23691 const scratch_reg = abi.getCAbiLinkerScratchReg(self.fn_type.fnCallingConvention(self.pt.zcu));
1570923692 _ = try self.addInst(.{
1571023693 .tag = .mov,
1571123694 .ops = .import_reloc,
1571223695 .data = .{ .rx = .{
15713 .r1 = .rax,
23696 .r1 = scratch_reg,
1571423697 .payload = try self.addExtra(bits.SymbolOffset{
1571523698 .sym_index = link.File.Coff.global_symbol_bit | global_index,
1571623699 }),
......@@ -15718,14 +23701,14 @@ fn genExternSymbolRef(
1571823701 });
1571923702 switch (tag) {
1572023703 .mov => {},
15721 .call => try self.asmRegister(.{ ._, .call }, .rax),
23704 .call => try self.asmRegister(.{ ._, .call }, scratch_reg),
1572223705 else => unreachable,
1572323706 }
1572423707 } else return self.fail("TODO implement calling extern functions", .{});
1572523708}
1572623709
1572723710fn genLazySymbolRef(
15728 self: *Self,
23711 self: *CodeGen,
1572923712 comptime tag: Mir.Inst.Tag,
1573023713 reg: Register,
1573123714 lazy_sym: link.File.LazySymbol,
......@@ -15737,10 +23720,10 @@ fn genLazySymbolRef(
1573723720 return self.fail("{s} creating lazy symbol", .{@errorName(err)});
1573823721 if (self.mod.pic) {
1573923722 switch (tag) {
15740 .lea, .call => try self.genSetReg(reg, Type.usize, .{
23723 .lea, .call => try self.genSetReg(reg, .usize, .{
1574123724 .lea_symbol = .{ .sym_index = sym_index },
1574223725 }, .{}),
15743 .mov => try self.genSetReg(reg, Type.usize, .{
23726 .mov => try self.genSetReg(reg, .usize, .{
1574423727 .load_symbol = .{ .sym_index = sym_index },
1574523728 }, .{}),
1574623729 else => unreachable,
......@@ -15755,7 +23738,7 @@ fn genLazySymbolRef(
1575523738 .base = .{ .reloc = sym_index },
1575623739 .mod = .{ .rm = .{ .size = .qword } },
1575723740 }),
15758 .call => try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym_index })),
23741 .call => try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym_index })),
1575923742 else => unreachable,
1576023743 }
1576123744 } else if (self.bin_file.cast(.plan9)) |p9_file| {
......@@ -15781,7 +23764,7 @@ fn genLazySymbolRef(
1578123764 .mov => try self.asmRegisterMemory(
1578223765 .{ ._, tag },
1578323766 reg.to64(),
15784 Memory.initSib(.qword, .{ .base = .{ .reg = reg.to64() } }),
23767 .initSib(.qword, .{ .base = .{ .reg = reg.to64() } }),
1578523768 ),
1578623769 else => unreachable,
1578723770 }
......@@ -15790,8 +23773,8 @@ fn genLazySymbolRef(
1579023773 return self.fail("{s} creating lazy symbol", .{@errorName(err)});
1579123774 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
1579223775 switch (tag) {
15793 .lea, .call => try self.genSetReg(reg, Type.usize, .{ .lea_got = sym_index }, .{}),
15794 .mov => try self.genSetReg(reg, Type.usize, .{ .load_got = sym_index }, .{}),
23776 .lea, .call => try self.genSetReg(reg, .usize, .{ .lea_got = sym_index }, .{}),
23777 .mov => try self.genSetReg(reg, .usize, .{ .load_got = sym_index }, .{}),
1579523778 else => unreachable,
1579623779 }
1579723780 switch (tag) {
......@@ -15805,10 +23788,10 @@ fn genLazySymbolRef(
1580523788 return self.fail("{s} creating lazy symbol", .{@errorName(err)});
1580623789 const sym = zo.symbols.items[sym_index];
1580723790 switch (tag) {
15808 .lea, .call => try self.genSetReg(reg, Type.usize, .{
23791 .lea, .call => try self.genSetReg(reg, .usize, .{
1580923792 .lea_symbol = .{ .sym_index = sym.nlist_idx },
1581023793 }, .{}),
15811 .mov => try self.genSetReg(reg, Type.usize, .{
23794 .mov => try self.genSetReg(reg, .usize, .{
1581223795 .load_symbol = .{ .sym_index = sym.nlist_idx },
1581323796 }, .{}),
1581423797 else => unreachable,
......@@ -15823,7 +23806,7 @@ fn genLazySymbolRef(
1582323806 }
1582423807}
1582523808
15826fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {
23809fn airIntFromPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1582723810 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1582823811 const result = result: {
1582923812 // TODO: handle case where the operand is a slice not a raw pointer
......@@ -15838,7 +23821,7 @@ fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {
1583823821 return self.finishAir(inst, result, .{ un_op, .none, .none });
1583923822}
1584023823
15841fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
23824fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
1584223825 const pt = self.pt;
1584323826 const zcu = pt.zcu;
1584423827 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -15852,18 +23835,20 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
1585223835 else => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv,
1585323836 };
1585423837
15855 const dst_rc = self.regClassForType(dst_ty);
15856 const src_rc = self.regClassForType(src_ty);
23838 const dst_rc = self.regSetForType(dst_ty);
23839 const src_rc = self.regSetForType(src_ty);
1585723840
15858 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
23841 const src_lock = if (src_mcv.getReg()) |src_reg| self.register_manager.lockReg(src_reg) else null;
1585923842 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
1586023843
15861 const dst_mcv = if (dst_rc.supersetOf(src_rc) and dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and
23844 const dst_mcv = if ((if (src_mcv.getReg()) |src_reg| src_reg.class() == .general_purpose else true) and
23845 dst_rc.supersetOf(src_rc) and dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and
23846 dst_ty.abiAlignment(zcu).order(src_ty.abiAlignment(zcu)).compare(.lte) and
1586223847 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
1586323848 const dst_mcv = try self.allocRegOrMem(inst, true);
15864 try self.genCopy(switch (math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) {
23849 try self.genCopy(switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) {
1586523850 .lt => dst_ty,
15866 .eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
23851 .eq => if (!dst_mcv.isBase() or src_mcv.isBase()) dst_ty else src_ty,
1586723852 .gt => src_ty,
1586823853 }, dst_mcv, src_mcv, .{});
1586923854 break :dst dst_mcv;
......@@ -15878,7 +23863,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
1587823863 const bit_size = dst_ty.bitSize(zcu);
1587923864 if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;
1588023865
15881 const dst_limbs_len = math.divCeil(i32, @intCast(bit_size), 64) catch unreachable;
23866 const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable;
1588223867 const high_mcv: MCValue = switch (dst_mcv) {
1588323868 .register => |dst_reg| .{ .register = dst_reg },
1588423869 .register_pair => |dst_regs| .{ .register = dst_regs[1] },
......@@ -15887,23 +23872,49 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
1588723872 const high_reg = if (high_mcv.isRegister())
1588823873 high_mcv.getReg().?
1588923874 else
15890 try self.copyToTmpRegister(Type.usize, high_mcv);
23875 try self.copyToTmpRegister(.usize, high_mcv);
1589123876 const high_lock = self.register_manager.lockReg(high_reg);
1589223877 defer if (high_lock) |lock| self.register_manager.unlockReg(lock);
15893
1589423878 try self.truncateRegister(dst_ty, high_reg);
1589523879 if (!high_mcv.isRegister()) try self.genCopy(
15896 if (abi_size <= 8) dst_ty else Type.usize,
23880 if (abi_size <= 8) dst_ty else .usize,
1589723881 high_mcv,
1589823882 .{ .register = high_reg },
1589923883 .{},
1590023884 );
23885 var offset = dst_limbs_len * 8;
23886 if (offset < abi_size) {
23887 const dst_signedness: std.builtin.Signedness = if (dst_ty.isAbiInt(zcu))
23888 dst_ty.intInfo(zcu).signedness
23889 else
23890 .unsigned;
23891 const ext_mcv: MCValue = ext_mcv: switch (dst_signedness) {
23892 .signed => {
23893 try self.asmRegisterImmediate(.{ ._r, .sa }, high_reg, .u(63));
23894 break :ext_mcv .{ .register = high_reg };
23895 },
23896 .unsigned => .{ .immediate = 0 },
23897 };
23898 while (offset < abi_size) : (offset += 8) {
23899 const limb_mcv: MCValue = switch (dst_mcv) {
23900 .register => |dst_reg| .{ .register = dst_reg },
23901 .register_pair => |dst_regs| .{ .register = dst_regs[@divExact(offset, 8)] },
23902 else => dst_mcv.address().offset(offset).deref(),
23903 };
23904 const limb_lock = if (limb_mcv.isRegister())
23905 self.register_manager.lockReg(limb_mcv.getReg().?)
23906 else
23907 null;
23908 defer if (limb_lock) |lock| self.register_manager.unlockReg(lock);
23909 try self.genCopy(.usize, limb_mcv, ext_mcv, .{});
23910 }
23911 }
1590123912 break :result dst_mcv;
1590223913 };
1590323914 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1590423915}
1590523916
15906fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
23917fn airArrayToSlice(self: *CodeGen, inst: Air.Inst.Index) !void {
1590723918 const pt = self.pt;
1590823919 const zcu = pt.zcu;
1590923920 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -15914,12 +23925,12 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
1591423925 const array_ty = ptr_ty.childType(zcu);
1591523926 const array_len = array_ty.arrayLen(zcu);
1591623927
15917 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));
23928 const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu));
1591823929 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr, .{});
1591923930 try self.genSetMem(
1592023931 .{ .frame = frame_index },
1592123932 @intCast(ptr_ty.abiSize(zcu)),
15922 Type.usize,
23933 .usize,
1592323934 .{ .immediate = array_len },
1592423935 .{},
1592523936 );
......@@ -15928,7 +23939,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
1592823939 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1592923940}
1593023941
15931fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
23942fn airFloatFromInt(self: *CodeGen, inst: Air.Inst.Index) !void {
1593223943 const pt = self.pt;
1593323944 const zcu = pt.zcu;
1593423945 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -15940,7 +23951,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
1594023951 const src_bits: u32 = @intCast(src_ty.bitSize(zcu));
1594123952 const src_signedness =
1594223953 if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned;
15943 const src_size = math.divCeil(u32, @max(switch (src_signedness) {
23954 const src_size = std.math.divCeil(u32, @max(switch (src_signedness) {
1594423955 .signed => src_bits,
1594523956 .unsigned => src_bits + 1,
1594623957 }, 32), 8) catch unreachable;
......@@ -15967,7 +23978,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
1596723978 intCompilerRtAbiName(src_bits),
1596823979 floatCompilerRtAbiName(dst_bits),
1596923980 }) catch unreachable,
15970 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
23981 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
1597123982 }
1597223983
1597323984 const src_mcv = try self.resolveInst(ty_op.operand);
......@@ -15980,7 +23991,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
1598023991
1598123992 if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg);
1598223993
15983 const dst_reg = try self.register_manager.allocReg(inst, self.regClassForType(dst_ty));
23994 const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty));
1598423995 const dst_mcv = MCValue{ .register = dst_reg };
1598523996 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1598623997 defer self.register_manager.unlockReg(dst_lock);
......@@ -16008,7 +24019,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
1600824019 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1600924020}
1601024021
16011fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
24022fn airIntFromFloat(self: *CodeGen, inst: Air.Inst.Index) !void {
1601224023 const pt = self.pt;
1601324024 const zcu = pt.zcu;
1601424025 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -16017,7 +24028,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
1601724028 const dst_bits: u32 = @intCast(dst_ty.bitSize(zcu));
1601824029 const dst_signedness =
1601924030 if (dst_ty.isAbiInt(zcu)) dst_ty.intInfo(zcu).signedness else .unsigned;
16020 const dst_size = math.divCeil(u32, @max(switch (dst_signedness) {
24031 const dst_size = std.math.divCeil(u32, @max(switch (dst_signedness) {
1602124032 .signed => dst_bits,
1602224033 .unsigned => dst_bits + 1,
1602324034 }, 32), 8) catch unreachable;
......@@ -16047,7 +24058,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
1604724058 floatCompilerRtAbiName(src_bits),
1604824059 intCompilerRtAbiName(dst_bits),
1604924060 }) catch unreachable,
16050 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
24061 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
1605124062 }
1605224063
1605324064 const src_mcv = try self.resolveInst(ty_op.operand);
......@@ -16058,7 +24069,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
1605824069 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
1605924070 defer self.register_manager.unlockReg(src_lock);
1606024071
16061 const dst_reg = try self.register_manager.allocReg(inst, self.regClassForType(dst_ty));
24072 const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty));
1606224073 const dst_mcv = MCValue{ .register = dst_reg };
1606324074 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1606424075 defer self.register_manager.unlockReg(dst_lock);
......@@ -16080,7 +24091,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
1608024091 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1608124092}
1608224093
16083fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
24094fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void {
1608424095 const pt = self.pt;
1608524096 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1608624097 const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
......@@ -16097,37 +24108,37 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1609724108 if (val_abi_size > 8) {
1609824109 const exp_addr_mcv: MCValue = switch (exp_mcv) {
1609924110 .memory, .indirect, .load_frame => exp_mcv.address(),
16100 else => .{ .register = try self.copyToTmpRegister(Type.usize, exp_mcv.address()) },
24111 else => .{ .register = try self.copyToTmpRegister(.usize, exp_mcv.address()) },
1610124112 };
1610224113 const exp_addr_lock =
1610324114 if (exp_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
1610424115 defer if (exp_addr_lock) |lock| self.register_manager.unlockReg(lock);
1610524116
16106 try self.genSetReg(.rax, Type.usize, exp_addr_mcv.deref(), .{});
16107 try self.genSetReg(.rdx, Type.usize, exp_addr_mcv.offset(8).deref(), .{});
24117 try self.genSetReg(.rax, .usize, exp_addr_mcv.deref(), .{});
24118 try self.genSetReg(.rdx, .usize, exp_addr_mcv.offset(8).deref(), .{});
1610824119 } else try self.genSetReg(.rax, val_ty, exp_mcv, .{});
1610924120
1611024121 const new_mcv = try self.resolveInst(extra.new_value);
1611124122 const new_reg = if (val_abi_size > 8) new: {
1611224123 const new_addr_mcv: MCValue = switch (new_mcv) {
1611324124 .memory, .indirect, .load_frame => new_mcv.address(),
16114 else => .{ .register = try self.copyToTmpRegister(Type.usize, new_mcv.address()) },
24125 else => .{ .register = try self.copyToTmpRegister(.usize, new_mcv.address()) },
1611524126 };
1611624127 const new_addr_lock =
1611724128 if (new_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
1611824129 defer if (new_addr_lock) |lock| self.register_manager.unlockReg(lock);
1611924130
16120 try self.genSetReg(.rbx, Type.usize, new_addr_mcv.deref(), .{});
16121 try self.genSetReg(.rcx, Type.usize, new_addr_mcv.offset(8).deref(), .{});
24131 try self.genSetReg(.rbx, .usize, new_addr_mcv.deref(), .{});
24132 try self.genSetReg(.rcx, .usize, new_addr_mcv.offset(8).deref(), .{});
1612224133 break :new null;
1612324134 } else try self.copyToTmpRegister(val_ty, new_mcv);
1612424135 const new_lock = if (new_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;
1612524136 defer if (new_lock) |lock| self.register_manager.unlockReg(lock);
1612624137
1612724138 const ptr_mcv = try self.resolveInst(extra.ptr);
16128 const mem_size = Memory.Size.fromSize(val_abi_size);
24139 const mem_size: Memory.Size = .fromSize(val_abi_size);
1612924140 const ptr_mem: Memory = switch (ptr_mcv) {
16130 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
24141 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }),
1613124142 else => .{
1613224143 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
1613324144 .mod = .{ .rm = .{ .size = mem_size } },
......@@ -16140,6 +24151,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1614024151 const ptr_lock = switch (ptr_mem.base) {
1614124152 .none, .frame, .reloc => null,
1614224153 .reg => |reg| self.register_manager.lockReg(reg),
24154 .table => unreachable,
1614324155 };
1614424156 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
1614524157
......@@ -16159,16 +24171,16 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1615924171 }
1616024172
1616124173 const dst_mcv = try self.allocRegOrMem(inst, false);
16162 try self.genCopy(Type.usize, dst_mcv, .{ .register = .rax }, .{});
16163 try self.genCopy(Type.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx }, .{});
16164 try self.genCopy(Type.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne }, .{});
24174 try self.genCopy(.usize, dst_mcv, .{ .register = .rax }, .{});
24175 try self.genCopy(.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx }, .{});
24176 try self.genCopy(.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne }, .{});
1616524177 break :result dst_mcv;
1616624178 };
1616724179 return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
1616824180}
1616924181
1617024182fn atomicOp(
16171 self: *Self,
24183 self: *CodeGen,
1617224184 ptr_mcv: MCValue,
1617324185 val_mcv: MCValue,
1617424186 ptr_ty: Type,
......@@ -16192,9 +24204,9 @@ fn atomicOp(
1619224204 defer if (val_lock) |lock| self.register_manager.unlockReg(lock);
1619324205
1619424206 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));
16195 const mem_size = Memory.Size.fromSize(val_abi_size);
24207 const mem_size: Memory.Size = .fromSize(val_abi_size);
1619624208 const ptr_mem: Memory = switch (ptr_mcv) {
16197 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
24209 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }),
1619824210 else => .{
1619924211 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
1620024212 .mod = .{ .rm = .{ .size = mem_size } },
......@@ -16207,6 +24219,7 @@ fn atomicOp(
1620724219 const mem_lock = switch (ptr_mem.base) {
1620824220 .none, .frame, .reloc => null,
1620924221 .reg => |reg| self.register_manager.lockReg(reg),
24222 .table => unreachable,
1621024223 };
1621124224 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
1621224225
......@@ -16309,11 +24322,11 @@ fn atomicOp(
1630924322 });
1631024323 try self.genSetReg(sse_reg, val_ty, .{ .register = .rax }, .{});
1631124324 switch (mir_tag[0]) {
16312 .v_ss, .v_sd => if (val_mcv.isMemory()) try self.asmRegisterRegisterMemory(
24325 .v_ss, .v_sd => if (val_mcv.isBase()) try self.asmRegisterRegisterMemory(
1631324326 mir_tag,
1631424327 sse_reg.to128(),
1631524328 sse_reg.to128(),
16316 try val_mcv.mem(self, self.memSize(val_ty)),
24329 try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }),
1631724330 ) else try self.asmRegisterRegisterRegister(
1631824331 mir_tag,
1631924332 sse_reg.to128(),
......@@ -16323,10 +24336,10 @@ fn atomicOp(
1632324336 else
1632424337 try self.copyToTmpRegister(val_ty, val_mcv)).to128(),
1632524338 ),
16326 ._ss, ._sd => if (val_mcv.isMemory()) try self.asmRegisterMemory(
24339 ._ss, ._sd => if (val_mcv.isBase()) try self.asmRegisterMemory(
1632724340 mir_tag,
1632824341 sse_reg.to128(),
16329 try val_mcv.mem(self, self.memSize(val_ty)),
24342 try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }),
1633024343 ) else try self.asmRegisterRegister(
1633124344 mir_tag,
1633224345 sse_reg.to128(),
......@@ -16381,7 +24394,7 @@ fn atomicOp(
1638124394 try self.asmCmovccRegisterMemory(
1638224395 cc,
1638324396 registerAlias(tmp_reg, cmov_abi_size),
16384 try val_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
24397 try val_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }),
1638524398 );
1638624399 },
1638724400 else => {
......@@ -16434,11 +24447,11 @@ fn atomicOp(
1643424447 const val_mem_mcv: MCValue = switch (val_mcv) {
1643524448 .memory, .indirect, .load_frame => val_mcv,
1643624449 else => .{ .indirect = .{
16437 .reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),
24450 .reg = try self.copyToTmpRegister(.usize, val_mcv.address()),
1643824451 } },
1643924452 };
16440 const val_lo_mem = try val_mem_mcv.mem(self, .qword);
16441 const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .qword);
24453 const val_lo_mem = try val_mem_mcv.mem(self, .{ .size = .qword });
24454 const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .{ .size = .qword });
1644224455 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
1644324456 try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
1644424457 try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
......@@ -16491,7 +24504,7 @@ fn atomicOp(
1649124504 },
1649224505 };
1649324506
16494 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .register = .rcx });
24507 const tmp_reg = try self.copyToTmpRegister(.usize, .{ .register = .rcx });
1649524508 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
1649624509 defer self.register_manager.unlockReg(tmp_lock);
1649724510
......@@ -16526,7 +24539,7 @@ fn atomicOp(
1652624539 }
1652724540}
1652824541
16529fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
24542fn airAtomicRmw(self: *CodeGen, inst: Air.Inst.Index) !void {
1653024543 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
1653124544 const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
1653224545
......@@ -16547,7 +24560,7 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
1654724560 return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
1654824561}
1654924562
16550fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
24563fn airAtomicLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
1655124564 const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
1655224565
1655324566 const ptr_ty = self.typeOf(atomic_load.ptr);
......@@ -16568,7 +24581,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
1656824581 return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });
1656924582}
1657024583
16571fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
24584fn airAtomicStore(self: *CodeGen, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
1657224585 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1657324586
1657424587 const ptr_ty = self.typeOf(bin_op.lhs);
......@@ -16581,7 +24594,7 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr
1658124594 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1658224595}
1658324596
16584fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
24597fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
1658524598 const pt = self.pt;
1658624599 const zcu = pt.zcu;
1658724600 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -16593,13 +24606,17 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1659324606 const reg_locks = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdi, .rsi, .rcx });
1659424607 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
1659524608
16596 const dst_ptr = try self.resolveInst(bin_op.lhs);
16597 const dst_ptr_ty = self.typeOf(bin_op.lhs);
16598 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
16599 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
16600 else => null,
24609 const dst = try self.resolveInst(bin_op.lhs);
24610 const dst_ty = self.typeOf(bin_op.lhs);
24611 const dst_locks: [2]?RegisterLock = switch (dst) {
24612 .register => |dst_reg| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null },
24613 .register_pair => |dst_regs| .{
24614 self.register_manager.lockRegAssumeUnused(dst_regs[0]),
24615 self.register_manager.lockRegAssumeUnused(dst_regs[1]),
24616 },
24617 else => @splat(null),
1660124618 };
16602 defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock);
24619 for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);
1660324620
1660424621 const src_val = try self.resolveInst(bin_op.rhs);
1660524622 const elem_ty = self.typeOf(bin_op.rhs);
......@@ -16612,16 +24629,20 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1661224629 const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu));
1661324630
1661424631 if (elem_abi_size == 1) {
16615 const ptr: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {
16616 // TODO: this only handles slices stored in the stack
16617 .slice => dst_ptr,
16618 .one => dst_ptr,
24632 const dst_ptr: MCValue = switch (dst_ty.ptrSize(zcu)) {
24633 .slice => switch (dst) {
24634 .register_pair => |dst_regs| .{ .register = dst_regs[0] },
24635 else => dst,
24636 },
24637 .one => dst,
1661924638 .c, .many => unreachable,
1662024639 };
16621 const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {
16622 // TODO: this only handles slices stored in the stack
16623 .slice => dst_ptr.address().offset(8).deref(),
16624 .one => .{ .immediate = dst_ptr_ty.childType(zcu).arrayLen(zcu) },
24640 const len: MCValue = switch (dst_ty.ptrSize(zcu)) {
24641 .slice => switch (dst) {
24642 .register_pair => |dst_regs| .{ .register = dst_regs[1] },
24643 else => dst.address().offset(8).deref(),
24644 },
24645 .one => .{ .immediate = dst_ty.childType(zcu).arrayLen(zcu) },
1662524646 .c, .many => unreachable,
1662624647 };
1662724648 const len_lock: ?RegisterLock = switch (len) {
......@@ -16630,20 +24651,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1663024651 };
1663124652 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
1663224653
16633 try self.genInlineMemset(ptr, src_val, len, .{ .safety = safety });
24654 try self.genInlineMemset(dst_ptr, src_val, len, .{ .safety = safety });
1663424655 break :result;
1663524656 }
1663624657
1663724658 // Store the first element, and then rely on memcpy copying forwards.
1663824659 // Length zero requires a runtime check - so we handle arrays specially
1663924660 // here to elide it.
16640 switch (dst_ptr_ty.ptrSize(zcu)) {
24661 switch (dst_ty.ptrSize(zcu)) {
1664124662 .slice => {
16642 const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(zcu);
24663 const slice_ptr_ty = dst_ty.slicePtrFieldType(zcu);
1664324664
16644 // TODO: this only handles slices stored in the stack
16645 const ptr = dst_ptr;
16646 const len = dst_ptr.address().offset(8).deref();
24665 const dst_ptr: MCValue = switch (dst) {
24666 .register_pair => |dst_regs| .{ .register = dst_regs[0] },
24667 else => dst,
24668 };
24669 const len: MCValue = switch (dst) {
24670 .register_pair => |dst_regs| .{ .register = dst_regs[1] },
24671 else => dst.address().offset(8).deref(),
24672 };
1664724673
1664824674 // Used to store the number of elements for comparison.
1664924675 // After comparison, updated to store number of bytes needed to copy.
......@@ -16652,11 +24678,11 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1665224678 const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
1665324679 defer self.register_manager.unlockReg(len_lock);
1665424680
16655 try self.genSetReg(len_reg, Type.usize, len, .{});
24681 try self.genSetReg(len_reg, .usize, len, .{});
1665624682 try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg);
1665724683
1665824684 const skip_reloc = try self.asmJccReloc(.z, undefined);
16659 try self.store(slice_ptr_ty, ptr, src_val, .{ .safety = safety });
24685 try self.store(slice_ptr_ty, dst_ptr, src_val, .{ .safety = safety });
1666024686
1666124687 const second_elem_ptr_reg =
1666224688 try self.register_manager.allocReg(null, abi.RegisterClass.gp);
......@@ -16665,29 +24691,29 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1666524691 self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
1666624692 defer self.register_manager.unlockReg(second_elem_ptr_lock);
1666724693
16668 try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
16669 .reg = try self.copyToTmpRegister(Type.usize, ptr),
24694 try self.genSetReg(second_elem_ptr_reg, .usize, .{ .register_offset = .{
24695 .reg = try self.copyToTmpRegister(.usize, dst_ptr),
1667024696 .off = elem_abi_size,
1667124697 } }, .{});
1667224698
16673 try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 });
24699 try self.genBinOpMir(.{ ._, .sub }, .usize, len_mcv, .{ .immediate = 1 });
1667424700 try self.asmRegisterRegisterImmediate(
1667524701 .{ .i_, .mul },
1667624702 len_reg,
1667724703 len_reg,
16678 Immediate.s(elem_abi_size),
24704 .s(elem_abi_size),
1667924705 );
16680 try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);
24706 try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, len_mcv, .{ .no_alias = false });
1668124707
1668224708 self.performReloc(skip_reloc);
1668324709 },
1668424710 .one => {
1668524711 const elem_ptr_ty = try pt.singleMutPtrType(elem_ty);
1668624712
16687 const len = dst_ptr_ty.childType(zcu).arrayLen(zcu);
24713 const len = dst_ty.childType(zcu).arrayLen(zcu);
1668824714
1668924715 assert(len != 0); // prevented by Sema
16690 try self.store(elem_ptr_ty, dst_ptr, src_val, .{ .safety = safety });
24716 try self.store(elem_ptr_ty, dst, src_val, .{ .safety = safety });
1669124717
1669224718 const second_elem_ptr_reg =
1669324719 try self.register_manager.allocReg(null, abi.RegisterClass.gp);
......@@ -16696,13 +24722,13 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1669624722 self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
1669724723 defer self.register_manager.unlockReg(second_elem_ptr_lock);
1669824724
16699 try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
16700 .reg = try self.copyToTmpRegister(Type.usize, dst_ptr),
24725 try self.genSetReg(second_elem_ptr_reg, .usize, .{ .register_offset = .{
24726 .reg = try self.copyToTmpRegister(.usize, dst),
1670124727 .off = elem_abi_size,
1670224728 } }, .{});
1670324729
1670424730 const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
16705 try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy);
24731 try self.genInlineMemcpy(second_elem_ptr_mcv, dst, bytes_to_copy, .{ .no_alias = false });
1670624732 },
1670724733 .c, .many => unreachable,
1670824734 }
......@@ -16710,7 +24736,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1671024736 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
1671124737}
1671224738
16713fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
24739fn airMemcpy(self: *CodeGen, inst: Air.Inst.Index) !void {
1671424740 const pt = self.pt;
1671524741 const zcu = pt.zcu;
1671624742 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -16719,64 +24745,87 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
1671924745 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
1672024746 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
1672124747
16722 const dst_ptr = try self.resolveInst(bin_op.lhs);
16723 const dst_ptr_ty = self.typeOf(bin_op.lhs);
16724 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
16725 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
16726 else => null,
24748 const dst = try self.resolveInst(bin_op.lhs);
24749 const dst_ty = self.typeOf(bin_op.lhs);
24750 const dst_locks: [2]?RegisterLock = switch (dst) {
24751 .register => |dst_reg| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null },
24752 .register_pair => |dst_regs| .{
24753 self.register_manager.lockRegAssumeUnused(dst_regs[0]),
24754 self.register_manager.lockReg(dst_regs[1]),
24755 },
24756 else => @splat(null),
1672724757 };
16728 defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock);
24758 for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);
1672924759
16730 const src_ptr = try self.resolveInst(bin_op.rhs);
16731 const src_ptr_lock: ?RegisterLock = switch (src_ptr) {
16732 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
16733 else => null,
24760 const src = try self.resolveInst(bin_op.rhs);
24761 const src_locks: [2]?RegisterLock = switch (src) {
24762 .register => |src_reg| .{ self.register_manager.lockReg(src_reg), null },
24763 .register_pair => |src_regs| .{
24764 self.register_manager.lockRegAssumeUnused(src_regs[0]),
24765 self.register_manager.lockRegAssumeUnused(src_regs[1]),
24766 },
24767 else => @splat(null),
1673424768 };
16735 defer if (src_ptr_lock) |lock| self.register_manager.unlockReg(lock);
24769 for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock);
1673624770
16737 const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {
24771 const len: MCValue = switch (dst_ty.ptrSize(zcu)) {
1673824772 .slice => len: {
1673924773 const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
1674024774 const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
1674124775 defer self.register_manager.unlockReg(len_lock);
1674224776
16743 try self.asmRegisterMemoryImmediate(
16744 .{ .i_, .mul },
16745 len_reg,
16746 try dst_ptr.address().offset(8).deref().mem(self, .qword),
16747 Immediate.s(@intCast(dst_ptr_ty.childType(zcu).abiSize(zcu))),
16748 );
24777 switch (dst) {
24778 .register_pair => |dst_regs| try self.asmRegisterRegisterImmediate(
24779 .{ .i_, .mul },
24780 len_reg,
24781 dst_regs[1],
24782 .s(@intCast(dst_ty.childType(zcu).abiSize(zcu))),
24783 ),
24784 else => try self.asmRegisterMemoryImmediate(
24785 .{ .i_, .mul },
24786 len_reg,
24787 try dst.address().offset(8).deref().mem(self, .{ .size = .qword }),
24788 .s(@intCast(dst_ty.childType(zcu).abiSize(zcu))),
24789 ),
24790 }
1674924791 break :len .{ .register = len_reg };
1675024792 },
1675124793 .one => len: {
16752 const array_ty = dst_ptr_ty.childType(zcu);
24794 const array_ty = dst_ty.childType(zcu);
1675324795 break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(zcu) };
1675424796 },
1675524797 .c, .many => unreachable,
1675624798 };
1675724799 const len_lock: ?RegisterLock = switch (len) {
16758 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
24800 .register => |reg| self.register_manager.lockReg(reg),
1675924801 else => null,
1676024802 };
1676124803 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
1676224804
16763 // TODO: dst_ptr and src_ptr could be slices rather than raw pointers
16764 try self.genInlineMemcpy(dst_ptr, src_ptr, len);
24805 const dst_ptr: MCValue = switch (dst) {
24806 .register_pair => |dst_regs| .{ .register = dst_regs[0] },
24807 else => dst,
24808 };
24809 const src_ptr: MCValue = switch (src) {
24810 .register_pair => |src_regs| .{ .register = src_regs[0] },
24811 else => src,
24812 };
24813
24814 try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{ .no_alias = true });
1676524815
1676624816 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
1676724817}
1676824818
16769fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
24819fn airTagName(self: *CodeGen, inst: Air.Inst.Index) !void {
1677024820 const pt = self.pt;
1677124821 const zcu = pt.zcu;
1677224822 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1677324823 const inst_ty = self.typeOfIndex(inst);
1677424824 const enum_ty = self.typeOf(un_op);
16775 const resolved_cc = abi.resolveCallingConvention(.auto, self.target.*);
1677624825
1677724826 // We need a properly aligned and sized call frame to be able to call this function.
1677824827 {
16779 const needed_call_frame = FrameAlloc.init(.{
24828 const needed_call_frame: FrameAlloc = .init(.{
1678024829 .size = inst_ty.abiSize(zcu),
1678124830 .alignment = inst_ty.abiAlignment(zcu),
1678224831 });
......@@ -16790,23 +24839,23 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
1679024839 }
1679124840
1679224841 try self.spillEflagsIfOccupied();
16793 try self.spillCallerPreservedRegs(resolved_cc);
24842 try self.spillCallerPreservedRegs(.auto);
1679424843
16795 const param_regs = abi.getCAbiIntParamRegs(resolved_cc);
24844 const param_regs = abi.getCAbiIntParamRegs(.auto);
1679624845
1679724846 const dst_mcv = try self.allocRegOrMem(inst, false);
16798 try self.genSetReg(param_regs[0], Type.usize, dst_mcv.address(), .{});
24847 try self.genSetReg(param_regs[0], .usize, dst_mcv.address(), .{});
1679924848
1680024849 const operand = try self.resolveInst(un_op);
1680124850 try self.genSetReg(param_regs[1], enum_ty, operand, .{});
1680224851
1680324852 const enum_lazy_sym: link.File.LazySymbol = .{ .kind = .code, .ty = enum_ty.toIntern() };
16804 try self.genLazySymbolRef(.call, .rax, enum_lazy_sym);
24853 try self.genLazySymbolRef(.call, abi.getCAbiLinkerScratchReg(self.fn_type.fnCallingConvention(zcu)), enum_lazy_sym);
1680524854
1680624855 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
1680724856}
1680824857
16809fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
24858fn airErrorName(self: *CodeGen, inst: Air.Inst.Index) !void {
1681024859 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1681124860
1681224861 const err_ty = self.typeOf(un_op);
......@@ -16908,13 +24957,13 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
1690824957 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
1690924958}
1691024959
16911fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
24960fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void {
1691224961 const pt = self.pt;
1691324962 const zcu = pt.zcu;
1691424963 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1691524964 const vector_ty = self.typeOfIndex(inst);
1691624965 const vector_len = vector_ty.vectorLen(zcu);
16917 const dst_rc = self.regClassForType(vector_ty);
24966 const dst_rc = self.regSetForType(vector_ty);
1691824967 const scalar_ty = self.typeOf(ty_op.operand);
1691924968
1692024969 const result: MCValue = result: {
......@@ -16930,27 +24979,23 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1693024979 try self.genSetReg(
1693124980 regs[1],
1693224981 vector_ty,
16933 .{ .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - vector_len) },
24982 .{ .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - vector_len) },
1693424983 .{},
1693524984 );
1693624985 const src_mcv = try self.resolveInst(ty_op.operand);
16937 const abi_size = @max(math.divCeil(u32, vector_len, 8) catch unreachable, 4);
24986 const abi_size = @max(std.math.divCeil(u32, vector_len, 8) catch unreachable, 4);
1693824987 try self.asmCmovccRegisterRegister(
1693924988 switch (src_mcv) {
1694024989 .eflags => |cc| cc,
1694124990 .register => |src_reg| cc: {
16942 try self.asmRegisterImmediate(
16943 .{ ._, .@"test" },
16944 src_reg.to8(),
16945 Immediate.u(1),
16946 );
24991 try self.asmRegisterImmediate(.{ ._, .@"test" }, src_reg.to8(), .u(1));
1694724992 break :cc .nz;
1694824993 },
1694924994 else => cc: {
1695024995 try self.asmMemoryImmediate(
1695124996 .{ ._, .@"test" },
16952 try src_mcv.mem(self, .byte),
16953 Immediate.u(1),
24997 try src_mcv.mem(self, .{ .size = .byte }),
24998 .u(1),
1695424999 );
1695525000 break :cc .nz;
1695625001 },
......@@ -16990,10 +25035,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1699025035 defer self.register_manager.unlockReg(dst_lock);
1699125036
1699225037 const src_mcv = try self.resolveInst(ty_op.operand);
16993 if (src_mcv.isMemory()) try self.asmRegisterMemory(
25038 if (src_mcv.isBase()) try self.asmRegisterMemory(
1699425039 mir_tag,
1699525040 registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))),
16996 try src_mcv.mem(self, self.memSize(scalar_ty)),
25041 try src_mcv.mem(self, .{ .size = self.memSize(scalar_ty) }),
1699725042 ) else {
1699825043 if (mir_tag[0] == .v_i128) break :avx2;
1699925044 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
......@@ -17037,7 +25082,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1703725082 .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
1703825083 dst_alias,
1703925084 dst_alias,
17040 Immediate.u(0b00_00_00_00),
25085 .u(0b00_00_00_00),
1704125086 );
1704225087 if (switch (scalar_bits) {
1704325088 1...8 => vector_len > 4,
......@@ -17049,7 +25094,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1704925094 .{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf },
1705025095 dst_alias,
1705125096 dst_alias,
17052 Immediate.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),
25097 .u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),
1705325098 );
1705425099 break :result .{ .register = dst_reg };
1705525100 },
......@@ -17066,10 +25111,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1706625111 const src_mcv = try self.resolveInst(ty_op.operand);
1706725112 if (self.hasFeature(.avx)) {
1706825113 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
17069 if (src_mcv.isMemory()) try self.asmRegisterMemory(
25114 if (src_mcv.isBase()) try self.asmRegisterMemory(
1707025115 .{ .v_ss, .broadcast },
1707125116 dst_reg.to128(),
17072 try src_mcv.mem(self, .dword),
25117 try src_mcv.mem(self, .{ .size = .dword }),
1707325118 ) else {
1707425119 const src_reg = if (src_mcv.isRegister())
1707525120 src_mcv.getReg().?
......@@ -17080,7 +25125,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1708025125 dst_reg.to128(),
1708125126 src_reg.to128(),
1708225127 src_reg.to128(),
17083 Immediate.u(0),
25128 .u(0),
1708425129 );
1708525130 }
1708625131 break :result .{ .register = dst_reg };
......@@ -17095,7 +25140,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1709525140 .{ ._ps, .shuf },
1709625141 dst_reg.to128(),
1709725142 dst_reg.to128(),
17098 Immediate.u(0),
25143 .u(0),
1709925144 );
1710025145 break :result dst_mcv;
1710125146 }
......@@ -17103,10 +25148,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1710325148 5...8 => if (self.hasFeature(.avx)) {
1710425149 const src_mcv = try self.resolveInst(ty_op.operand);
1710525150 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
17106 if (src_mcv.isMemory()) try self.asmRegisterMemory(
25151 if (src_mcv.isBase()) try self.asmRegisterMemory(
1710725152 .{ .v_ss, .broadcast },
1710825153 dst_reg.to256(),
17109 try src_mcv.mem(self, .dword),
25154 try src_mcv.mem(self, .{ .size = .dword }),
1711025155 ) else {
1711125156 const src_reg = if (src_mcv.isRegister())
1711225157 src_mcv.getReg().?
......@@ -17122,14 +25167,14 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1712225167 dst_reg.to128(),
1712325168 src_reg.to128(),
1712425169 src_reg.to128(),
17125 Immediate.u(0),
25170 .u(0),
1712625171 );
1712725172 try self.asmRegisterRegisterRegisterImmediate(
1712825173 .{ .v_f128, .insert },
1712925174 dst_reg.to256(),
1713025175 dst_reg.to256(),
1713125176 dst_reg.to128(),
17132 Immediate.u(1),
25177 .u(1),
1713325178 );
1713425179 }
1713525180 }
......@@ -17149,10 +25194,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1714925194 const src_mcv = try self.resolveInst(ty_op.operand);
1715025195 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
1715125196 if (self.hasFeature(.sse3)) {
17152 if (src_mcv.isMemory()) try self.asmRegisterMemory(
25197 if (src_mcv.isBase()) try self.asmRegisterMemory(
1715325198 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
1715425199 dst_reg.to128(),
17155 try src_mcv.mem(self, .qword),
25200 try src_mcv.mem(self, .{ .size = .qword }),
1715625201 ) else try self.asmRegisterRegister(
1715725202 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
1715825203 dst_reg.to128(),
......@@ -17174,10 +25219,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1717425219 3...4 => if (self.hasFeature(.avx)) {
1717525220 const src_mcv = try self.resolveInst(ty_op.operand);
1717625221 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
17177 if (src_mcv.isMemory()) try self.asmRegisterMemory(
25222 if (src_mcv.isBase()) try self.asmRegisterMemory(
1717825223 .{ .v_sd, .broadcast },
1717925224 dst_reg.to256(),
17180 try src_mcv.mem(self, .qword),
25225 try src_mcv.mem(self, .{ .size = .qword }),
1718125226 ) else {
1718225227 const src_reg = if (src_mcv.isRegister())
1718325228 src_mcv.getReg().?
......@@ -17198,7 +25243,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1719825243 dst_reg.to256(),
1719925244 dst_reg.to256(),
1720025245 dst_reg.to128(),
17201 Immediate.u(1),
25246 .u(1),
1720225247 );
1720325248 }
1720425249 }
......@@ -17217,10 +25262,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1721725262 2 => if (self.hasFeature(.avx)) {
1721825263 const src_mcv = try self.resolveInst(ty_op.operand);
1721925264 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
17220 if (src_mcv.isMemory()) try self.asmRegisterMemory(
25265 if (src_mcv.isBase()) try self.asmRegisterMemory(
1722125266 .{ .v_f128, .broadcast },
1722225267 dst_reg.to256(),
17223 try src_mcv.mem(self, .xword),
25268 try src_mcv.mem(self, .{ .size = .xword }),
1722425269 ) else {
1722525270 const src_reg = if (src_mcv.isRegister())
1722625271 src_mcv.getReg().?
......@@ -17231,7 +25276,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1723125276 dst_reg.to256(),
1723225277 src_reg.to256(),
1723325278 src_reg.to128(),
17234 Immediate.u(1),
25279 .u(1),
1723525280 );
1723625281 }
1723725282 break :result .{ .register = dst_reg };
......@@ -17247,7 +25292,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1724725292 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1724825293}
1724925294
17250fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
25295fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
1725125296 const pt = self.pt;
1725225297 const zcu = pt.zcu;
1725325298 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
......@@ -17308,7 +25353,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1730825353 mask_alias,
1730925354 mask_alias,
1731025355 mask_reg.to128(),
17311 Immediate.u(1),
25356 .u(1),
1731225357 );
1731325358 break :broadcast;
1731425359 },
......@@ -17326,9 +25371,9 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1732625371 else => unreachable,
1732725372 }, .broadcast },
1732825373 mask_alias,
17329 if (pred_mcv.isMemory()) try pred_mcv.mem(self, .byte) else .{
25374 if (pred_mcv.isBase()) try pred_mcv.mem(self, .{ .size = .byte }) else .{
1733025375 .base = .{ .reg = (try self.copyToTmpRegister(
17331 Type.usize,
25376 .usize,
1733225377 pred_mcv.address(),
1733325378 )).to64() },
1733425379 .mod = .{ .rm = .{ .size = .byte } },
......@@ -17362,7 +25407,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1736225407 .{ if (has_avx) .vp_w else .p_w, .shufl },
1736325408 mask_alias,
1736425409 mask_alias,
17365 Immediate.u(0b00_00_00_00),
25410 .u(0b00_00_00_00),
1736625411 );
1736725412 if (abi_size <= 8) break :broadcast;
1736825413 }
......@@ -17370,7 +25415,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1737025415 .{ if (has_avx) .vp_d else .p_d, .shuf },
1737125416 mask_alias,
1737225417 mask_alias,
17373 Immediate.u(switch (elem_abi_size) {
25418 .u(switch (elem_abi_size) {
1737425419 1...2, 5...8 => 0b01_00_01_00,
1737525420 3...4 => 0b00_00_00_00,
1737625421 else => unreachable,
......@@ -17386,12 +25431,12 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1738625431 .ty = mask_elem_ty.toIntern(),
1738725432 .storage = .{ .u64 = bit / elem_bits },
1738825433 } });
17389 const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
25434 const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
1739025435 .ty = mask_ty.toIntern(),
1739125436 .storage = .{ .elems = mask_elems[0..vec_len] },
1739225437 } })));
1739325438 const mask_mem: Memory = .{
17394 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
25439 .base = .{ .reg = try self.copyToTmpRegister(.usize, mask_mcv.address()) },
1739525440 .mod = .{ .rm = .{ .size = self.memSize(ty) } },
1739625441 };
1739725442 if (has_avx) try self.asmRegisterRegisterMemory(
......@@ -17411,12 +25456,12 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1741125456 .ty = mask_elem_ty.toIntern(),
1741225457 .storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) },
1741325458 } });
17414 const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
25459 const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
1741525460 .ty = mask_ty.toIntern(),
1741625461 .storage = .{ .elems = mask_elems[0..vec_len] },
1741725462 } })));
1741825463 const mask_mem: Memory = .{
17419 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
25464 .base = .{ .reg = try self.copyToTmpRegister(.usize, mask_mcv.address()) },
1742025465 .mod = .{ .rm = .{ .size = self.memSize(ty) } },
1742125466 };
1742225467 if (has_avx) {
......@@ -17520,11 +25565,11 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1752025565 try self.genSetReg(dst_reg, ty, rhs_mcv, .{});
1752125566 break :rhs dst_alias;
1752225567 };
17523 if (lhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(
25568 if (lhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister(
1752425569 mir_tag,
1752525570 dst_alias,
1752625571 rhs_alias,
17527 try lhs_mcv.mem(self, self.memSize(ty)),
25572 try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }),
1752825573 mask_alias,
1752925574 ) else try self.asmRegisterRegisterRegisterRegister(
1753025575 mir_tag,
......@@ -17536,10 +25581,10 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1753625581 try self.copyToTmpRegister(ty, lhs_mcv), abi_size),
1753725582 mask_alias,
1753825583 );
17539 } else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
25584 } else if (has_blend) if (lhs_mcv.isBase()) try self.asmRegisterMemoryRegister(
1754025585 mir_tag,
1754125586 dst_alias,
17542 try lhs_mcv.mem(self, self.memSize(ty)),
25587 try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }),
1754325588 mask_alias,
1754425589 ) else try self.asmRegisterRegisterRegister(
1754525590 mir_tag,
......@@ -17561,10 +25606,10 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1756125606 },
1756225607 }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(pt)});
1756325608 try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
17564 if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
25609 if (rhs_mcv.isBase()) try self.asmRegisterMemory(
1756525610 .{ mir_fixes, .andn },
1756625611 mask_alias,
17567 try rhs_mcv.mem(self, Memory.Size.fromSize(abi_size)),
25612 try rhs_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
1756825613 ) else try self.asmRegisterRegister(
1756925614 .{ mir_fixes, .andn },
1757025615 mask_alias,
......@@ -17580,7 +25625,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1758025625 return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
1758125626}
1758225627
17583fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
25628fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void {
1758425629 const pt = self.pt;
1758525630 const zcu = pt.zcu;
1758625631 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -17640,16 +25685,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1764025685
1764125686 for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) |variant| unpck: {
1764225687 if (elem_abi_size > 8) break :unpck;
17643 if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)
17644 has_avx
17645 else
17646 self.hasFeature(.avx2)) 32 else 16)) break :unpck;
25688 if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :unpck;
1764725689
17648 var sources = [1]?u1{null} ** 2;
25690 var sources: [2]?u1 = @splat(null);
1764925691 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
1765025692 const mask_elem = maybe_mask_elem orelse continue;
1765125693 const mask_elem_index =
17652 math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
25694 std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
1765325695 const elem_byte = (elem_index >> 1) * elem_abi_size;
1765425696 if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) | @as(u4, switch (variant) {
1765525697 .unpckl => 0b0000,
......@@ -17701,11 +25743,11 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1770125743 },
1770225744 else => unreachable,
1770325745 } };
17704 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemory(
25746 if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemory(
1770525747 mir_tag,
1770625748 dst_alias,
1770725749 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17708 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
25750 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
1770925751 ) else try self.asmRegisterRegisterRegister(
1771025752 mir_tag,
1771125753 dst_alias,
......@@ -17714,10 +25756,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1771425756 rhs_mcv.getReg().?
1771525757 else
1771625758 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17717 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
25759 ) else if (rhs_mcv.isBase()) try self.asmRegisterMemory(
1771825760 mir_tag,
1771925761 dst_alias,
17720 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
25762 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
1772125763 ) else try self.asmRegisterRegister(
1772225764 mir_tag,
1772325765 dst_alias,
......@@ -17731,10 +25773,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1773125773
1773225774 pshufd: {
1773325775 if (elem_abi_size != 4) break :pshufd;
17734 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd;
25776 if (max_abi_size > self.vectorSize(.float)) break :pshufd;
1773525777
1773625778 var control: u8 = 0b00_00_00_00;
17737 var sources = [1]?u1{null} ** 1;
25779 var sources: [1]?u1 = @splat(null);
1773825780 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
1773925781 const mask_elem = maybe_mask_elem orelse continue;
1774025782 const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
......@@ -17746,10 +25788,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1774625788 } else sources[(elem_index & 0b010) >> 1] = source;
1774725789
1774825790 const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
17749 const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
25791 const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
1775025792 if (elem_index & 0b100 == 0)
17751 control |= select
17752 else if (control & @as(u8, 0b11) << select_bit != select) break :pshufd;
25793 control |= select_mask
25794 else if (control & @as(u8, 0b11) << select_bit != select_mask) break :pshufd;
1775325795 }
1775425796
1775525797 const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
......@@ -17763,11 +25805,11 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1776325805 try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
1776425806 const dst_alias = registerAlias(dst_reg, max_abi_size);
1776525807
17766 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
25808 if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate(
1776725809 .{ if (has_avx) .vp_d else .p_d, .shuf },
1776825810 dst_alias,
17769 try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17770 Immediate.u(control),
25811 try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
25812 .u(control),
1777125813 ) else try self.asmRegisterRegisterImmediate(
1777225814 .{ if (has_avx) .vp_d else .p_d, .shuf },
1777325815 dst_alias,
......@@ -17775,17 +25817,17 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1777525817 src_mcv.getReg().?
1777625818 else
1777725819 try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),
17778 Immediate.u(control),
25820 .u(control),
1777925821 );
1778025822 break :result .{ .register = dst_reg };
1778125823 }
1778225824
1778325825 shufps: {
1778425826 if (elem_abi_size != 4) break :shufps;
17785 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps;
25827 if (max_abi_size > self.vectorSize(.float)) break :shufps;
1778625828
1778725829 var control: u8 = 0b00_00_00_00;
17788 var sources = [1]?u1{null} ** 2;
25830 var sources: [2]?u1 = @splat(null);
1778925831 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
1779025832 const mask_elem = maybe_mask_elem orelse continue;
1779125833 const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
......@@ -17797,10 +25839,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1779725839 } else sources[(elem_index & 0b010) >> 1] = source;
1779825840
1779925841 const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
17800 const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
25842 const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
1780125843 if (elem_index & 0b100 == 0)
17802 control |= select
17803 else if (control & @as(u8, 0b11) << select_bit != select) break :shufps;
25844 control |= select_mask
25845 else if (control & @as(u8, 0b11) << select_bit != select_mask) break :shufps;
1780425846 }
1780525847 if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;
1780625848
......@@ -17819,12 +25861,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1781925861 const dst_reg = dst_mcv.getReg().?;
1782025862 const dst_alias = registerAlias(dst_reg, max_abi_size);
1782125863
17822 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
25864 if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
1782325865 .{ .v_ps, .shuf },
1782425866 dst_alias,
1782525867 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17826 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17827 Immediate.u(control),
25868 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
25869 .u(control),
1782825870 ) else try self.asmRegisterRegisterRegisterImmediate(
1782925871 .{ .v_ps, .shuf },
1783025872 dst_alias,
......@@ -17833,12 +25875,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1783325875 rhs_mcv.getReg().?
1783425876 else
1783525877 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17836 Immediate.u(control),
17837 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
25878 .u(control),
25879 ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
1783825880 .{ ._ps, .shuf },
1783925881 dst_alias,
17840 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17841 Immediate.u(control),
25882 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
25883 .u(control),
1784225884 ) else try self.asmRegisterRegisterImmediate(
1784325885 .{ ._ps, .shuf },
1784425886 dst_alias,
......@@ -17846,17 +25888,17 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1784625888 rhs_mcv.getReg().?
1784725889 else
1784825890 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17849 Immediate.u(control),
25891 .u(control),
1785025892 );
1785125893 break :result dst_mcv;
1785225894 }
1785325895
1785425896 shufpd: {
1785525897 if (elem_abi_size != 8) break :shufpd;
17856 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd;
25898 if (max_abi_size > self.vectorSize(.float)) break :shufpd;
1785725899
1785825900 var control: u4 = 0b0_0_0_0;
17859 var sources = [1]?u1{null} ** 2;
25901 var sources: [2]?u1 = @splat(null);
1786025902 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
1786125903 const mask_elem = maybe_mask_elem orelse continue;
1786225904 const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
......@@ -17886,12 +25928,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1788625928 const dst_reg = dst_mcv.getReg().?;
1788725929 const dst_alias = registerAlias(dst_reg, max_abi_size);
1788825930
17889 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
25931 if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
1789025932 .{ .v_pd, .shuf },
1789125933 dst_alias,
1789225934 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17893 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17894 Immediate.u(control),
25935 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
25936 .u(control),
1789525937 ) else try self.asmRegisterRegisterRegisterImmediate(
1789625938 .{ .v_pd, .shuf },
1789725939 dst_alias,
......@@ -17900,12 +25942,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1790025942 rhs_mcv.getReg().?
1790125943 else
1790225944 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17903 Immediate.u(control),
17904 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
25945 .u(control),
25946 ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
1790525947 .{ ._pd, .shuf },
1790625948 dst_alias,
17907 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17908 Immediate.u(control),
25949 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
25950 .u(control),
1790925951 ) else try self.asmRegisterRegisterImmediate(
1791025952 .{ ._pd, .shuf },
1791125953 dst_alias,
......@@ -17913,27 +25955,27 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1791325955 rhs_mcv.getReg().?
1791425956 else
1791525957 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17916 Immediate.u(control),
25958 .u(control),
1791725959 );
1791825960 break :result dst_mcv;
1791925961 }
1792025962
1792125963 blend: {
1792225964 if (elem_abi_size < 2) break :blend;
17923 if (dst_abi_size > @as(u32, if (has_avx) 32 else 16)) break :blend;
25965 if (dst_abi_size > self.vectorSize(.float)) break :blend;
1792425966 if (!self.hasFeature(.sse4_1)) break :blend;
1792525967
1792625968 var control: u8 = 0b0_0_0_0_0_0_0_0;
1792725969 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
1792825970 const mask_elem = maybe_mask_elem orelse continue;
1792925971 const mask_elem_index =
17930 math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
25972 std.math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
1793125973 if (mask_elem_index != elem_index) break :blend;
1793225974
17933 const select = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
25975 const select_mask = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
1793425976 if (elem_index & 0b1000 == 0)
17935 control |= select
17936 else if (control & @as(u8, 0b1) << @truncate(elem_index) != select) break :blend;
25977 control |= select_mask
25978 else if (control & @as(u8, 0b1) << @truncate(elem_index) != select_mask) break :blend;
1793725979 }
1793825980
1793925981 if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {
......@@ -17956,12 +25998,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1795625998
1795725999 const rhs_mcv = try self.resolveInst(extra.b);
1795826000 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
17959 if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
26001 if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
1796026002 .{ .vp_d, .blend },
1796126003 registerAlias(dst_reg, dst_abi_size),
1796226004 registerAlias(lhs_reg, dst_abi_size),
17963 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
17964 Immediate.u(expanded_control),
26005 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
26006 .u(expanded_control),
1796526007 ) else try self.asmRegisterRegisterRegisterImmediate(
1796626008 .{ .vp_d, .blend },
1796726009 registerAlias(dst_reg, dst_abi_size),
......@@ -17970,7 +26012,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1797026012 rhs_mcv.getReg().?
1797126013 else
1797226014 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
17973 Immediate.u(expanded_control),
26015 .u(expanded_control),
1797426016 );
1797526017 break :result .{ .register = dst_reg };
1797626018 }
......@@ -18008,1810 +26050,4076 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1800826050 try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
1800926051 const dst_reg = dst_mcv.getReg().?;
1801026052
18011 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
26053 if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
1801226054 .{ .vp_w, .blend },
1801326055 registerAlias(dst_reg, dst_abi_size),
1801426056 registerAlias(if (lhs_mcv.isRegister())
1801526057 lhs_mcv.getReg().?
1801626058 else
1801726059 dst_reg, dst_abi_size),
18018 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18019 Immediate.u(expanded_control),
26060 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
26061 .u(expanded_control),
1802026062 ) else try self.asmRegisterRegisterRegisterImmediate(
1802126063 .{ .vp_w, .blend },
1802226064 registerAlias(dst_reg, dst_abi_size),
1802326065 registerAlias(if (lhs_mcv.isRegister())
1802426066 lhs_mcv.getReg().?
1802526067 else
18026 dst_reg, dst_abi_size),
26068 dst_reg, dst_abi_size),
26069 registerAlias(if (rhs_mcv.isRegister())
26070 rhs_mcv.getReg().?
26071 else
26072 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
26073 .u(expanded_control),
26074 ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
26075 .{ .p_w, .blend },
26076 registerAlias(dst_reg, dst_abi_size),
26077 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
26078 .u(expanded_control),
26079 ) else try self.asmRegisterRegisterImmediate(
26080 .{ .p_w, .blend },
26081 registerAlias(dst_reg, dst_abi_size),
26082 registerAlias(if (rhs_mcv.isRegister())
26083 rhs_mcv.getReg().?
26084 else
26085 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
26086 .u(expanded_control),
26087 );
26088 break :result .{ .register = dst_reg };
26089 }
26090
26091 const expanded_control = switch (elem_abi_size) {
26092 4, 8 => control,
26093 16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) |
26094 @as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00),
26095 else => unreachable,
26096 };
26097
26098 const lhs_mcv = try self.resolveInst(extra.a);
26099 const rhs_mcv = try self.resolveInst(extra.b);
26100
26101 const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
26102 self.reuseOperand(inst, extra.a, 0, lhs_mcv))
26103 lhs_mcv
26104 else if (has_avx and lhs_mcv.isRegister())
26105 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
26106 else
26107 try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
26108 const dst_reg = dst_mcv.getReg().?;
26109
26110 if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
26111 switch (elem_abi_size) {
26112 4 => .{ .v_ps, .blend },
26113 8, 16 => .{ .v_pd, .blend },
26114 else => unreachable,
26115 },
26116 registerAlias(dst_reg, dst_abi_size),
26117 registerAlias(if (lhs_mcv.isRegister())
26118 lhs_mcv.getReg().?
26119 else
26120 dst_reg, dst_abi_size),
26121 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
26122 .u(expanded_control),
26123 ) else try self.asmRegisterRegisterRegisterImmediate(
26124 switch (elem_abi_size) {
26125 4 => .{ .v_ps, .blend },
26126 8, 16 => .{ .v_pd, .blend },
26127 else => unreachable,
26128 },
26129 registerAlias(dst_reg, dst_abi_size),
26130 registerAlias(if (lhs_mcv.isRegister())
26131 lhs_mcv.getReg().?
26132 else
26133 dst_reg, dst_abi_size),
26134 registerAlias(if (rhs_mcv.isRegister())
26135 rhs_mcv.getReg().?
26136 else
26137 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
26138 .u(expanded_control),
26139 ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
26140 switch (elem_abi_size) {
26141 4 => .{ ._ps, .blend },
26142 8, 16 => .{ ._pd, .blend },
26143 else => unreachable,
26144 },
26145 registerAlias(dst_reg, dst_abi_size),
26146 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
26147 .u(expanded_control),
26148 ) else try self.asmRegisterRegisterImmediate(
26149 switch (elem_abi_size) {
26150 4 => .{ ._ps, .blend },
26151 8, 16 => .{ ._pd, .blend },
26152 else => unreachable,
26153 },
26154 registerAlias(dst_reg, dst_abi_size),
26155 registerAlias(if (rhs_mcv.isRegister())
26156 rhs_mcv.getReg().?
26157 else
26158 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
26159 .u(expanded_control),
26160 );
26161 break :result .{ .register = dst_reg };
26162 }
26163
26164 blendv: {
26165 if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :blendv;
26166
26167 const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8);
26168 const select_mask_ty = try pt.vectorType(.{
26169 .len = @intCast(mask_elems.len),
26170 .child = select_mask_elem_ty.toIntern(),
26171 });
26172 var select_mask_elems: [32]InternPool.Index = undefined;
26173 for (
26174 select_mask_elems[0..mask_elems.len],
26175 mask_elems,
26176 0..,
26177 ) |*select_mask_elem, maybe_mask_elem, elem_index| {
26178 const mask_elem = maybe_mask_elem orelse continue;
26179 const mask_elem_index =
26180 std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
26181 if (mask_elem_index != elem_index) break :blendv;
26182
26183 select_mask_elem.* = (if (mask_elem < 0)
26184 try select_mask_elem_ty.maxIntScalar(pt, select_mask_elem_ty)
26185 else
26186 try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern();
26187 }
26188 const select_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
26189 .ty = select_mask_ty.toIntern(),
26190 .storage = .{ .elems = select_mask_elems[0..mask_elems.len] },
26191 } })));
26192
26193 if (self.hasFeature(.sse4_1)) {
26194 const mir_tag: Mir.Inst.FixedTag = .{
26195 if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
26196 (dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) {
26197 4 => if (has_avx) .v_ps else ._ps,
26198 8 => if (has_avx) .v_pd else ._pd,
26199 else => unreachable,
26200 } else if (has_avx) .vp_b else .p_b,
26201 .blendv,
26202 };
26203
26204 const select_mask_reg = if (!has_avx) reg: {
26205 try self.register_manager.getKnownReg(.xmm0, null);
26206 try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{});
26207 break :reg .xmm0;
26208 } else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
26209 const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size);
26210 const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg);
26211 defer self.register_manager.unlockReg(select_mask_lock);
26212
26213 const lhs_mcv = try self.resolveInst(extra.a);
26214 const rhs_mcv = try self.resolveInst(extra.b);
26215
26216 const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
26217 self.reuseOperand(inst, extra.a, 0, lhs_mcv))
26218 lhs_mcv
26219 else if (has_avx and lhs_mcv.isRegister())
26220 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
26221 else
26222 try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
26223 const dst_reg = dst_mcv.getReg().?;
26224 const dst_alias = registerAlias(dst_reg, dst_abi_size);
26225
26226 if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister(
26227 mir_tag,
26228 dst_alias,
26229 if (lhs_mcv.isRegister())
26230 registerAlias(lhs_mcv.getReg().?, dst_abi_size)
26231 else
26232 dst_alias,
26233 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
26234 select_mask_alias,
26235 ) else try self.asmRegisterRegisterRegisterRegister(
26236 mir_tag,
26237 dst_alias,
26238 if (lhs_mcv.isRegister())
26239 registerAlias(lhs_mcv.getReg().?, dst_abi_size)
26240 else
26241 dst_alias,
1802726242 registerAlias(if (rhs_mcv.isRegister())
1802826243 rhs_mcv.getReg().?
1802926244 else
1803026245 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18031 Immediate.u(expanded_control),
18032 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18033 .{ .p_w, .blend },
18034 registerAlias(dst_reg, dst_abi_size),
18035 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18036 Immediate.u(expanded_control),
18037 ) else try self.asmRegisterRegisterImmediate(
18038 .{ .p_w, .blend },
18039 registerAlias(dst_reg, dst_abi_size),
26246 select_mask_alias,
26247 ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryRegister(
26248 mir_tag,
26249 dst_alias,
26250 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
26251 select_mask_alias,
26252 ) else try self.asmRegisterRegisterRegister(
26253 mir_tag,
26254 dst_alias,
1804026255 registerAlias(if (rhs_mcv.isRegister())
1804126256 rhs_mcv.getReg().?
1804226257 else
1804326258 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18044 Immediate.u(expanded_control),
26259 select_mask_alias,
1804526260 );
18046 break :result .{ .register = dst_reg };
26261 break :result dst_mcv;
1804726262 }
1804826263
18049 const expanded_control = switch (elem_abi_size) {
18050 4, 8 => control,
18051 16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) |
18052 @as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00),
18053 else => unreachable,
18054 };
18055
1805626264 const lhs_mcv = try self.resolveInst(extra.a);
1805726265 const rhs_mcv = try self.resolveInst(extra.b);
1805826266
18059 const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
18060 self.reuseOperand(inst, extra.a, 0, lhs_mcv))
18061 lhs_mcv
18062 else if (has_avx and lhs_mcv.isRegister())
18063 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
26267 const dst_mcv: MCValue = if (rhs_mcv.isRegister() and
26268 self.reuseOperand(inst, extra.b, 1, rhs_mcv))
26269 rhs_mcv
1806426270 else
18065 try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
26271 try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv);
1806626272 const dst_reg = dst_mcv.getReg().?;
26273 const dst_alias = registerAlias(dst_reg, dst_abi_size);
1806726274
18068 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
18069 switch (elem_abi_size) {
18070 4 => .{ .v_ps, .blend },
18071 8, 16 => .{ .v_pd, .blend },
18072 else => unreachable,
18073 },
18074 registerAlias(dst_reg, dst_abi_size),
18075 registerAlias(if (lhs_mcv.isRegister())
18076 lhs_mcv.getReg().?
18077 else
18078 dst_reg, dst_abi_size),
18079 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18080 Immediate.u(expanded_control),
18081 ) else try self.asmRegisterRegisterRegisterImmediate(
18082 switch (elem_abi_size) {
18083 4 => .{ .v_ps, .blend },
18084 8, 16 => .{ .v_pd, .blend },
26275 const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
26276 const mask_alias = registerAlias(mask_reg, dst_abi_size);
26277 const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
26278 defer self.register_manager.unlockReg(mask_lock);
26279
26280 const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat())
26281 switch (elem_ty.floatBits(self.target.*)) {
26282 16, 80, 128 => .p_,
26283 32 => ._ps,
26284 64 => ._pd,
1808526285 else => unreachable,
18086 },
18087 registerAlias(dst_reg, dst_abi_size),
18088 registerAlias(if (lhs_mcv.isRegister())
26286 }
26287 else
26288 .p_;
26289 try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
26290 if (lhs_mcv.isBase()) try self.asmRegisterMemory(
26291 .{ mir_fixes, .andn },
26292 mask_alias,
26293 try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
26294 ) else try self.asmRegisterRegister(
26295 .{ mir_fixes, .andn },
26296 mask_alias,
26297 if (lhs_mcv.isRegister())
1808926298 lhs_mcv.getReg().?
1809026299 else
18091 dst_reg, dst_abi_size),
18092 registerAlias(if (rhs_mcv.isRegister())
18093 rhs_mcv.getReg().?
18094 else
18095 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18096 Immediate.u(expanded_control),
18097 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18098 switch (elem_abi_size) {
18099 4 => .{ ._ps, .blend },
18100 8, 16 => .{ ._pd, .blend },
26300 try self.copyToTmpRegister(dst_ty, lhs_mcv),
26301 );
26302 try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
26303 break :result dst_mcv;
26304 }
26305
26306 pshufb: {
26307 if (max_abi_size > 16) break :pshufb;
26308 if (!self.hasFeature(.ssse3)) break :pshufb;
26309
26310 const temp_regs =
26311 try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);
26312 const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);
26313 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
26314
26315 const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);
26316 try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});
26317
26318 const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);
26319 try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});
26320
26321 var lhs_mask_elems: [16]InternPool.Index = undefined;
26322 for (lhs_mask_elems[0..max_abi_size], 0..) |*lhs_mask_elem, byte_index| {
26323 const elem_index = byte_index / elem_abi_size;
26324 lhs_mask_elem.* = try pt.intern(.{ .int = .{
26325 .ty = .u8_type,
26326 .storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
26327 const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
26328 if (mask_elem < 0) break :elem 0b1_00_00000;
26329 const mask_elem_index: u31 = @intCast(mask_elem);
26330 const byte_off: u32 = @intCast(byte_index % elem_abi_size);
26331 break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
26332 } },
26333 } });
26334 }
26335 const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
26336 const lhs_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
26337 .ty = lhs_mask_ty.toIntern(),
26338 .storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
26339 } })));
26340 const lhs_mask_mem: Memory = .{
26341 .base = .{ .reg = try self.copyToTmpRegister(.usize, lhs_mask_mcv.address()) },
26342 .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
26343 };
26344 if (has_avx) try self.asmRegisterRegisterMemory(
26345 .{ .vp_b, .shuf },
26346 lhs_temp_alias,
26347 lhs_temp_alias,
26348 lhs_mask_mem,
26349 ) else try self.asmRegisterMemory(
26350 .{ .p_b, .shuf },
26351 lhs_temp_alias,
26352 lhs_mask_mem,
26353 );
26354
26355 var rhs_mask_elems: [16]InternPool.Index = undefined;
26356 for (rhs_mask_elems[0..max_abi_size], 0..) |*rhs_mask_elem, byte_index| {
26357 const elem_index = byte_index / elem_abi_size;
26358 rhs_mask_elem.* = try pt.intern(.{ .int = .{
26359 .ty = .u8_type,
26360 .storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
26361 const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
26362 if (mask_elem >= 0) break :elem 0b1_00_00000;
26363 const mask_elem_index: u31 = @intCast(~mask_elem);
26364 const byte_off: u32 = @intCast(byte_index % elem_abi_size);
26365 break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
26366 } },
26367 } });
26368 }
26369 const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
26370 const rhs_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
26371 .ty = rhs_mask_ty.toIntern(),
26372 .storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },
26373 } })));
26374 const rhs_mask_mem: Memory = .{
26375 .base = .{ .reg = try self.copyToTmpRegister(.usize, rhs_mask_mcv.address()) },
26376 .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
26377 };
26378 if (has_avx) try self.asmRegisterRegisterMemory(
26379 .{ .vp_b, .shuf },
26380 rhs_temp_alias,
26381 rhs_temp_alias,
26382 rhs_mask_mem,
26383 ) else try self.asmRegisterMemory(
26384 .{ .p_b, .shuf },
26385 rhs_temp_alias,
26386 rhs_mask_mem,
26387 );
26388
26389 if (has_avx) try self.asmRegisterRegisterRegister(
26390 .{ switch (elem_ty.zigTypeTag(zcu)) {
26391 else => break :result null,
26392 .int => .vp_,
26393 .float => switch (elem_ty.floatBits(self.target.*)) {
26394 32 => .v_ps,
26395 64 => .v_pd,
26396 16, 80, 128 => break :result null,
26397 else => unreachable,
26398 },
26399 }, .@"or" },
26400 lhs_temp_alias,
26401 lhs_temp_alias,
26402 rhs_temp_alias,
26403 ) else try self.asmRegisterRegister(
26404 .{ switch (elem_ty.zigTypeTag(zcu)) {
26405 else => break :result null,
26406 .int => .p_,
26407 .float => switch (elem_ty.floatBits(self.target.*)) {
26408 32 => ._ps,
26409 64 => ._pd,
26410 16, 80, 128 => break :result null,
26411 else => unreachable,
26412 },
26413 }, .@"or" },
26414 lhs_temp_alias,
26415 rhs_temp_alias,
26416 );
26417 break :result .{ .register = temp_regs[0] };
26418 }
26419
26420 break :result null;
26421 }) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{
26422 lhs_ty.fmt(pt),
26423 rhs_ty.fmt(pt),
26424 dst_ty.fmt(pt),
26425 Value.fromInterned(extra.mask).fmtValue(pt),
26426 });
26427 return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
26428}
26429
26430fn airReduce(self: *CodeGen, inst: Air.Inst.Index) !void {
26431 const pt = self.pt;
26432 const zcu = pt.zcu;
26433 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
26434
26435 const result: MCValue = result: {
26436 const operand_ty = self.typeOf(reduce.operand);
26437 if (operand_ty.isVector(zcu) and operand_ty.childType(zcu).toIntern() == .bool_type) {
26438 try self.spillEflagsIfOccupied();
26439
26440 const abi_size: u32 = @intCast(operand_ty.abiSize(zcu));
26441 const operand_mcv = try self.resolveInst(reduce.operand);
26442 const mask_len = operand_ty.vectorLen(zcu);
26443 const mask_len_minus_one = (std.math.cast(u6, mask_len - 1) orelse {
26444 const acc_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
26445 const acc_lock = self.register_manager.lockRegAssumeUnused(acc_reg);
26446 defer self.register_manager.unlockReg(acc_lock);
26447 var limb_offset: i31 = 0;
26448 while (limb_offset < abi_size) : (limb_offset += 8) {
26449 try self.asmRegisterMemory(
26450 .{ ._, if (limb_offset == 0) .mov else switch (reduce.operation) {
26451 .Or => .@"or",
26452 .And => .@"and",
26453 else => return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}),
26454 } },
26455 acc_reg.to64(),
26456 try operand_mcv.mem(self, .{
26457 .size = .qword,
26458 .disp = limb_offset,
26459 }),
26460 );
26461 }
26462 switch (reduce.operation) {
26463 .Or => {
26464 try self.asmRegisterRegister(.{ ._, .@"test" }, acc_reg.to64(), acc_reg.to64());
26465 break :result .{ .eflags = .nz };
26466 },
26467 .And => {
26468 try self.asmRegisterImmediate(.{ ._, .cmp }, acc_reg.to64(), .s(-1));
26469 break :result .{ .eflags = .z };
26470 },
1810126471 else => unreachable,
26472 }
26473 });
26474 const mask = @as(u64, std.math.maxInt(u64)) >> ~mask_len_minus_one;
26475 switch (reduce.operation) {
26476 .Or => {
26477 if (operand_mcv.isBase()) try self.asmMemoryImmediate(
26478 .{ ._, .@"test" },
26479 try operand_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
26480 if (mask_len < abi_size * 8)
26481 .u(mask)
26482 else
26483 .s(-1),
26484 ) else {
26485 const operand_reg = registerAlias(operand_reg: {
26486 if (operand_mcv.isRegister()) {
26487 const operand_reg = operand_mcv.getReg().?;
26488 if (operand_reg.class() == .general_purpose) break :operand_reg operand_reg;
26489 }
26490 break :operand_reg try self.copyToTmpRegister(operand_ty, operand_mcv);
26491 }, abi_size);
26492 const operand_lock = self.register_manager.lockReg(operand_reg);
26493 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
26494
26495 if (mask_len < abi_size * 8) try self.asmRegisterImmediate(
26496 .{ ._, .@"test" },
26497 operand_reg,
26498 .u(mask),
26499 ) else try self.asmRegisterRegister(
26500 .{ ._, .@"test" },
26501 operand_reg,
26502 operand_reg,
26503 );
26504 }
26505 break :result .{ .eflags = .nz };
1810226506 },
18103 registerAlias(dst_reg, dst_abi_size),
18104 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18105 Immediate.u(expanded_control),
18106 ) else try self.asmRegisterRegisterImmediate(
18107 switch (elem_abi_size) {
18108 4 => .{ ._ps, .blend },
18109 8, 16 => .{ ._pd, .blend },
18110 else => unreachable,
26507 .And => {
26508 const tmp_reg = registerAlias(
26509 try self.copyToTmpRegister(operand_ty, operand_mcv),
26510 abi_size,
26511 );
26512 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
26513 defer self.register_manager.unlockReg(tmp_lock);
26514
26515 try self.asmRegister(.{ ._, .not }, tmp_reg);
26516 if (mask_len < abi_size * 8)
26517 try self.asmRegisterImmediate(.{ ._, .@"test" }, tmp_reg, .u(mask))
26518 else
26519 try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg);
26520 break :result .{ .eflags = .z };
1811126521 },
18112 registerAlias(dst_reg, dst_abi_size),
18113 registerAlias(if (rhs_mcv.isRegister())
18114 rhs_mcv.getReg().?
18115 else
18116 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18117 Immediate.u(expanded_control),
18118 );
18119 break :result .{ .register = dst_reg };
26522 else => return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}),
26523 }
1812026524 }
26525 return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)});
26526 };
26527 return self.finishAir(inst, result, .{ reduce.operand, .none, .none });
26528}
1812126529
18122 blendv: {
18123 if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)
18124 has_avx
18125 else
18126 self.hasFeature(.avx2)) 32 else 16)) break :blendv;
26530fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
26531 const pt = self.pt;
26532 const zcu = pt.zcu;
26533 const result_ty = self.typeOfIndex(inst);
26534 const len: usize = @intCast(result_ty.arrayLen(zcu));
26535 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
26536 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);
26537 const result: MCValue = result: {
26538 switch (result_ty.zigTypeTag(zcu)) {
26539 .@"struct" => {
26540 const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
26541 if (result_ty.containerLayout(zcu) == .@"packed") {
26542 const loaded_struct = zcu.intern_pool.loadStructType(result_ty.toIntern());
26543 try self.genInlineMemset(
26544 .{ .lea_frame = .{ .index = frame_index } },
26545 .{ .immediate = 0 },
26546 .{ .immediate = result_ty.abiSize(zcu) },
26547 .{},
26548 );
26549 for (elements, 0..) |elem, elem_i_usize| {
26550 const elem_i: u32 = @intCast(elem_i_usize);
26551 if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
1812726552
18128 const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8);
18129 const select_mask_ty = try pt.vectorType(.{
18130 .len = @intCast(mask_elems.len),
18131 .child = select_mask_elem_ty.toIntern(),
18132 });
18133 var select_mask_elems: [32]InternPool.Index = undefined;
18134 for (
18135 select_mask_elems[0..mask_elems.len],
18136 mask_elems,
18137 0..,
18138 ) |*select_mask_elem, maybe_mask_elem, elem_index| {
18139 const mask_elem = maybe_mask_elem orelse continue;
18140 const mask_elem_index =
18141 math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
18142 if (mask_elem_index != elem_index) break :blendv;
26553 const elem_ty = result_ty.fieldType(elem_i, zcu);
26554 const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu));
26555 if (elem_bit_size > 64) {
26556 return self.fail(
26557 "TODO airAggregateInit implement packed structs with large fields",
26558 .{},
26559 );
26560 }
26561 const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu));
26562 const elem_abi_bits = elem_abi_size * 8;
26563 const elem_off = pt.structPackedFieldBitOffset(loaded_struct, elem_i);
26564 const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
26565 const elem_bit_off = elem_off % elem_abi_bits;
26566 const elem_mcv = try self.resolveInst(elem);
26567 const mat_elem_mcv = switch (elem_mcv) {
26568 .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index },
26569 else => elem_mcv,
26570 };
26571 const elem_lock = switch (mat_elem_mcv) {
26572 .register => |reg| self.register_manager.lockReg(reg),
26573 .immediate => |imm| lock: {
26574 if (imm == 0) continue;
26575 break :lock null;
26576 },
26577 else => null,
26578 };
26579 defer if (elem_lock) |lock| self.register_manager.unlockReg(lock);
1814326580
18144 select_mask_elem.* = (if (mask_elem < 0)
18145 try select_mask_elem_ty.maxIntScalar(pt, select_mask_elem_ty)
18146 else
18147 try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern();
18148 }
18149 const select_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
18150 .ty = select_mask_ty.toIntern(),
18151 .storage = .{ .elems = select_mask_elems[0..mask_elems.len] },
18152 } })));
26581 const elem_extra_bits = self.regExtraBits(elem_ty);
26582 {
26583 const temp_reg = try self.copyToTmpRegister(elem_ty, mat_elem_mcv);
26584 const temp_alias = registerAlias(temp_reg, elem_abi_size);
26585 const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
26586 defer self.register_manager.unlockReg(temp_lock);
1815326587
18154 if (self.hasFeature(.sse4_1)) {
18155 const mir_tag: Mir.Inst.FixedTag = .{
18156 if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
18157 (dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) {
18158 4 => if (has_avx) .v_ps else ._ps,
18159 8 => if (has_avx) .v_pd else ._pd,
18160 else => unreachable,
18161 } else if (has_avx) .vp_b else .p_b,
18162 .blendv,
18163 };
26588 if (elem_bit_off < elem_extra_bits) {
26589 try self.truncateRegister(elem_ty, temp_alias);
26590 }
26591 if (elem_bit_off > 0) try self.genShiftBinOpMir(
26592 .{ ._l, .sh },
26593 elem_ty,
26594 .{ .register = temp_alias },
26595 .u8,
26596 .{ .immediate = elem_bit_off },
26597 );
26598 try self.genBinOpMir(
26599 .{ ._, .@"or" },
26600 elem_ty,
26601 .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
26602 .{ .register = temp_alias },
26603 );
26604 }
26605 if (elem_bit_off > elem_extra_bits) {
26606 const temp_reg = try self.copyToTmpRegister(elem_ty, mat_elem_mcv);
26607 const temp_alias = registerAlias(temp_reg, elem_abi_size);
26608 const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
26609 defer self.register_manager.unlockReg(temp_lock);
1816426610
18165 const select_mask_reg = if (!has_avx) reg: {
18166 try self.register_manager.getKnownReg(.xmm0, null);
18167 try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{});
18168 break :reg .xmm0;
18169 } else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
18170 const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size);
18171 const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg);
18172 defer self.register_manager.unlockReg(select_mask_lock);
26611 if (elem_extra_bits > 0) {
26612 try self.truncateRegister(elem_ty, temp_alias);
26613 }
26614 try self.genShiftBinOpMir(
26615 .{ ._r, .sh },
26616 elem_ty,
26617 .{ .register = temp_reg },
26618 .u8,
26619 .{ .immediate = elem_abi_bits - elem_bit_off },
26620 );
26621 try self.genBinOpMir(
26622 .{ ._, .@"or" },
26623 elem_ty,
26624 .{ .load_frame = .{
26625 .index = frame_index,
26626 .off = elem_byte_off + @as(i32, @intCast(elem_abi_size)),
26627 } },
26628 .{ .register = temp_alias },
26629 );
26630 }
26631 }
26632 } else for (elements, 0..) |elem, elem_i| {
26633 if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
1817326634
18174 const lhs_mcv = try self.resolveInst(extra.a);
18175 const rhs_mcv = try self.resolveInst(extra.b);
26635 const elem_ty = result_ty.fieldType(elem_i, zcu);
26636 const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, zcu));
26637 const elem_mcv = try self.resolveInst(elem);
26638 const mat_elem_mcv = switch (elem_mcv) {
26639 .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index },
26640 else => elem_mcv,
26641 };
26642 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv, .{});
26643 }
26644 break :result .{ .load_frame = .{ .index = frame_index } };
26645 },
26646 .array, .vector => {
26647 const elem_ty = result_ty.childType(zcu);
26648 if (result_ty.isVector(zcu) and elem_ty.toIntern() == .bool_type) {
26649 const result_size: u32 = @intCast(result_ty.abiSize(zcu));
26650 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
26651 try self.asmRegisterRegister(
26652 .{ ._, .xor },
26653 registerAlias(dst_reg, @min(result_size, 4)),
26654 registerAlias(dst_reg, @min(result_size, 4)),
26655 );
1817626656
18177 const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
18178 self.reuseOperand(inst, extra.a, 0, lhs_mcv))
18179 lhs_mcv
18180 else if (has_avx and lhs_mcv.isRegister())
18181 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
18182 else
18183 try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
18184 const dst_reg = dst_mcv.getReg().?;
18185 const dst_alias = registerAlias(dst_reg, dst_abi_size);
26657 for (elements, 0..) |elem, elem_i| {
26658 const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem });
26659 const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg);
26660 defer self.register_manager.unlockReg(elem_lock);
1818626661
18187 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(
18188 mir_tag,
18189 dst_alias,
18190 if (lhs_mcv.isRegister())
18191 registerAlias(lhs_mcv.getReg().?, dst_abi_size)
18192 else
18193 dst_alias,
18194 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18195 select_mask_alias,
18196 ) else try self.asmRegisterRegisterRegisterRegister(
18197 mir_tag,
18198 dst_alias,
18199 if (lhs_mcv.isRegister())
18200 registerAlias(lhs_mcv.getReg().?, dst_abi_size)
18201 else
18202 dst_alias,
18203 registerAlias(if (rhs_mcv.isRegister())
18204 rhs_mcv.getReg().?
18205 else
18206 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18207 select_mask_alias,
18208 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
18209 mir_tag,
18210 dst_alias,
18211 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18212 select_mask_alias,
18213 ) else try self.asmRegisterRegisterRegister(
18214 mir_tag,
18215 dst_alias,
18216 registerAlias(if (rhs_mcv.isRegister())
18217 rhs_mcv.getReg().?
18218 else
18219 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18220 select_mask_alias,
18221 );
18222 break :result dst_mcv;
18223 }
26662 try self.asmRegisterImmediate(
26663 .{ ._, .@"and" },
26664 registerAlias(elem_reg, @min(result_size, 4)),
26665 .u(1),
26666 );
26667 if (elem_i > 0) try self.asmRegisterImmediate(
26668 .{ ._l, .sh },
26669 registerAlias(elem_reg, result_size),
26670 .u(@intCast(elem_i)),
26671 );
26672 try self.asmRegisterRegister(
26673 .{ ._, .@"or" },
26674 registerAlias(dst_reg, result_size),
26675 registerAlias(elem_reg, result_size),
26676 );
26677 }
26678 break :result .{ .register = dst_reg };
26679 } else {
26680 const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
26681 const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
26682
26683 for (elements, 0..) |elem, elem_i| {
26684 const elem_mcv = try self.resolveInst(elem);
26685 const mat_elem_mcv = switch (elem_mcv) {
26686 .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index },
26687 else => elem_mcv,
26688 };
26689 const elem_off: i32 = @intCast(elem_size * elem_i);
26690 try self.genSetMem(
26691 .{ .frame = frame_index },
26692 elem_off,
26693 elem_ty,
26694 mat_elem_mcv,
26695 .{},
26696 );
26697 }
26698 if (result_ty.sentinel(zcu)) |sentinel| try self.genSetMem(
26699 .{ .frame = frame_index },
26700 @intCast(elem_size * elements.len),
26701 elem_ty,
26702 try self.genTypedValue(sentinel),
26703 .{},
26704 );
26705 break :result .{ .load_frame = .{ .index = frame_index } };
26706 }
26707 },
26708 else => unreachable,
26709 }
26710 };
1822426711
18225 const lhs_mcv = try self.resolveInst(extra.a);
18226 const rhs_mcv = try self.resolveInst(extra.b);
26712 if (elements.len <= Liveness.bpi - 1) {
26713 var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
26714 @memcpy(buf[0..elements.len], elements);
26715 return self.finishAir(inst, result, buf);
26716 }
26717 var bt = self.liveness.iterateBigTomb(inst);
26718 for (elements) |elem| try self.feed(&bt, elem);
26719 return self.finishAirResult(inst, result);
26720}
1822726721
18228 const dst_mcv: MCValue = if (rhs_mcv.isRegister() and
18229 self.reuseOperand(inst, extra.b, 1, rhs_mcv))
18230 rhs_mcv
18231 else
18232 try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv);
18233 const dst_reg = dst_mcv.getReg().?;
18234 const dst_alias = registerAlias(dst_reg, dst_abi_size);
26722fn airUnionInit(self: *CodeGen, inst: Air.Inst.Index) !void {
26723 const pt = self.pt;
26724 const zcu = pt.zcu;
26725 const ip = &zcu.intern_pool;
26726 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
26727 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
26728 const result: MCValue = result: {
26729 const union_ty = self.typeOfIndex(inst);
26730 const layout = union_ty.unionGetLayout(zcu);
1823526731
18236 const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
18237 const mask_alias = registerAlias(mask_reg, dst_abi_size);
18238 const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
18239 defer self.register_manager.unlockReg(mask_lock);
26732 const src_ty = self.typeOf(extra.init);
26733 const src_mcv = try self.resolveInst(extra.init);
26734 if (layout.tag_size == 0) {
26735 if (layout.abi_size <= src_ty.abiSize(zcu) and
26736 self.reuseOperand(inst, extra.init, 0, src_mcv)) break :result src_mcv;
1824026737
18241 const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat())
18242 switch (elem_ty.floatBits(self.target.*)) {
18243 16, 80, 128 => .p_,
18244 32 => ._ps,
18245 64 => ._pd,
18246 else => unreachable,
18247 }
18248 else
18249 .p_;
18250 try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
18251 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
18252 .{ mir_fixes, .andn },
18253 mask_alias,
18254 try lhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18255 ) else try self.asmRegisterRegister(
18256 .{ mir_fixes, .andn },
18257 mask_alias,
18258 if (lhs_mcv.isRegister())
18259 lhs_mcv.getReg().?
18260 else
18261 try self.copyToTmpRegister(dst_ty, lhs_mcv),
18262 );
18263 try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
26738 const dst_mcv = try self.allocRegOrMem(inst, true);
26739 try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
1826426740 break :result dst_mcv;
1826526741 }
1826626742
18267 pshufb: {
18268 if (max_abi_size > 16) break :pshufb;
18269 if (!self.hasFeature(.ssse3)) break :pshufb;
26743 const dst_mcv = try self.allocRegOrMem(inst, false);
1827026744
18271 const temp_regs =
18272 try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);
18273 const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);
18274 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
26745 const loaded_union = zcu.typeToUnion(union_ty).?;
26746 const field_name = loaded_union.loadTagType(ip).names.get(ip)[extra.field_index];
26747 const tag_ty: Type = .fromInterned(loaded_union.enum_tag_ty);
26748 const field_index = tag_ty.enumFieldIndex(field_name, zcu).?;
26749 const tag_val = try pt.enumValueFieldIndex(tag_ty, field_index);
26750 const tag_int_val = try tag_val.intFromEnum(tag_ty, pt);
26751 const tag_int = tag_int_val.toUnsignedInt(zcu);
26752 const tag_off: i32 = @intCast(layout.tagOffset());
26753 try self.genCopy(
26754 tag_ty,
26755 dst_mcv.address().offset(tag_off).deref(),
26756 .{ .immediate = tag_int },
26757 .{},
26758 );
1827526759
18276 const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);
18277 try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});
26760 const pl_off: i32 = @intCast(layout.payloadOffset());
26761 try self.genCopy(src_ty, dst_mcv.address().offset(pl_off).deref(), src_mcv, .{});
1827826762
18279 const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);
18280 try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});
26763 break :result dst_mcv;
26764 };
26765 return self.finishAir(inst, result, .{ extra.init, .none, .none });
26766}
1828126767
18282 var lhs_mask_elems: [16]InternPool.Index = undefined;
18283 for (lhs_mask_elems[0..max_abi_size], 0..) |*lhs_mask_elem, byte_index| {
18284 const elem_index = byte_index / elem_abi_size;
18285 lhs_mask_elem.* = try pt.intern(.{ .int = .{
18286 .ty = .u8_type,
18287 .storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
18288 const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
18289 if (mask_elem < 0) break :elem 0b1_00_00000;
18290 const mask_elem_index: u31 = @intCast(mask_elem);
18291 const byte_off: u32 = @intCast(byte_index % elem_abi_size);
18292 break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
18293 } },
18294 } });
18295 }
18296 const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
18297 const lhs_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
18298 .ty = lhs_mask_ty.toIntern(),
18299 .storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
18300 } })));
18301 const lhs_mask_mem: Memory = .{
18302 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },
18303 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
18304 };
18305 if (has_avx) try self.asmRegisterRegisterMemory(
18306 .{ .vp_b, .shuf },
18307 lhs_temp_alias,
18308 lhs_temp_alias,
18309 lhs_mask_mem,
18310 ) else try self.asmRegisterMemory(
18311 .{ .p_b, .shuf },
18312 lhs_temp_alias,
18313 lhs_mask_mem,
18314 );
26768fn airPrefetch(self: *CodeGen, inst: Air.Inst.Index) !void {
26769 const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
26770 return self.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });
26771}
1831526772
18316 var rhs_mask_elems: [16]InternPool.Index = undefined;
18317 for (rhs_mask_elems[0..max_abi_size], 0..) |*rhs_mask_elem, byte_index| {
18318 const elem_index = byte_index / elem_abi_size;
18319 rhs_mask_elem.* = try pt.intern(.{ .int = .{
18320 .ty = .u8_type,
18321 .storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
18322 const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
18323 if (mask_elem >= 0) break :elem 0b1_00_00000;
18324 const mask_elem_index: u31 = @intCast(~mask_elem);
18325 const byte_off: u32 = @intCast(byte_index % elem_abi_size);
18326 break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
18327 } },
18328 } });
26773fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void {
26774 const pt = self.pt;
26775 const zcu = pt.zcu;
26776 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
26777 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
26778 const ty = self.typeOfIndex(inst);
26779
26780 const ops = [3]Air.Inst.Ref{ extra.lhs, extra.rhs, pl_op.operand };
26781 const result = result: {
26782 if (switch (ty.scalarType(zcu).floatBits(self.target.*)) {
26783 16, 80, 128 => true,
26784 32, 64 => !self.hasFeature(.fma),
26785 else => unreachable,
26786 }) {
26787 if (ty.zigTypeTag(zcu) != .float) return self.fail("TODO implement airMulAdd for {}", .{
26788 ty.fmt(pt),
26789 });
26790
26791 var callee_buf: ["__fma?".len]u8 = undefined;
26792 break :result try self.genCall(.{ .lib = .{
26793 .return_type = ty.toIntern(),
26794 .param_types = &.{ ty.toIntern(), ty.toIntern(), ty.toIntern() },
26795 .callee = std.fmt.bufPrint(&callee_buf, "{s}fma{s}", .{
26796 floatLibcAbiPrefix(ty),
26797 floatLibcAbiSuffix(ty),
26798 }) catch unreachable,
26799 } }, &.{ ty, ty, ty }, &.{
26800 .{ .air_ref = extra.lhs }, .{ .air_ref = extra.rhs }, .{ .air_ref = pl_op.operand },
26801 }, .{});
26802 }
26803
26804 var mcvs: [3]MCValue = undefined;
26805 var locks: [3]?RegisterManager.RegisterLock = @splat(null);
26806 defer for (locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
26807 var order: [3]u2 = @splat(0);
26808 var unused: std.StaticBitSet(3) = .initFull();
26809 for (ops, &mcvs, &locks, 0..) |op, *mcv, *lock, op_i| {
26810 const op_index: u2 = @intCast(op_i);
26811 mcv.* = try self.resolveInst(op);
26812 if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) {
26813 order[op_index] = 1;
26814 unused.unset(0);
26815 } else if (unused.isSet(2) and mcv.isBase()) {
26816 order[op_index] = 3;
26817 unused.unset(2);
1832926818 }
18330 const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
18331 const rhs_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
18332 .ty = rhs_mask_ty.toIntern(),
18333 .storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },
18334 } })));
18335 const rhs_mask_mem: Memory = .{
18336 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },
18337 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
18338 };
18339 if (has_avx) try self.asmRegisterRegisterMemory(
18340 .{ .vp_b, .shuf },
18341 rhs_temp_alias,
18342 rhs_temp_alias,
18343 rhs_mask_mem,
18344 ) else try self.asmRegisterMemory(
18345 .{ .p_b, .shuf },
18346 rhs_temp_alias,
18347 rhs_mask_mem,
18348 );
26819 switch (mcv.*) {
26820 .register => |reg| lock.* = self.register_manager.lockReg(reg),
26821 else => {},
26822 }
26823 }
26824 for (&order, &mcvs, &locks) |*mop_index, *mcv, *lock| {
26825 if (mop_index.* != 0) continue;
26826 mop_index.* = 1 + @as(u2, @intCast(unused.toggleFirstSet().?));
26827 if (mop_index.* > 1 and mcv.isRegister()) continue;
26828 const reg = try self.copyToTmpRegister(ty, mcv.*);
26829 mcv.* = .{ .register = reg };
26830 if (lock.*) |old_lock| self.register_manager.unlockReg(old_lock);
26831 lock.* = self.register_manager.lockRegAssumeUnused(reg);
26832 }
1834926833
18350 if (has_avx) try self.asmRegisterRegisterRegister(
18351 .{ switch (elem_ty.zigTypeTag(zcu)) {
18352 else => break :result null,
18353 .int => .vp_,
18354 .float => switch (elem_ty.floatBits(self.target.*)) {
18355 32 => .v_ps,
18356 64 => .v_pd,
18357 16, 80, 128 => break :result null,
26834 const mir_tag = @as(?Mir.Inst.FixedTag, if (std.mem.eql(u2, &order, &.{ 1, 3, 2 }) or
26835 std.mem.eql(u2, &order, &.{ 3, 1, 2 }))
26836 switch (ty.zigTypeTag(zcu)) {
26837 .float => switch (ty.floatBits(self.target.*)) {
26838 32 => .{ .v_ss, .fmadd132 },
26839 64 => .{ .v_sd, .fmadd132 },
26840 16, 80, 128 => null,
26841 else => unreachable,
26842 },
26843 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
26844 .float => switch (ty.childType(zcu).floatBits(self.target.*)) {
26845 32 => switch (ty.vectorLen(zcu)) {
26846 1 => .{ .v_ss, .fmadd132 },
26847 2...8 => .{ .v_ps, .fmadd132 },
26848 else => null,
26849 },
26850 64 => switch (ty.vectorLen(zcu)) {
26851 1 => .{ .v_sd, .fmadd132 },
26852 2...4 => .{ .v_pd, .fmadd132 },
26853 else => null,
26854 },
26855 16, 80, 128 => null,
26856 else => unreachable,
26857 },
26858 else => unreachable,
26859 },
26860 else => unreachable,
26861 }
26862 else if (std.mem.eql(u2, &order, &.{ 2, 1, 3 }) or std.mem.eql(u2, &order, &.{ 1, 2, 3 }))
26863 switch (ty.zigTypeTag(zcu)) {
26864 .float => switch (ty.floatBits(self.target.*)) {
26865 32 => .{ .v_ss, .fmadd213 },
26866 64 => .{ .v_sd, .fmadd213 },
26867 16, 80, 128 => null,
26868 else => unreachable,
26869 },
26870 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
26871 .float => switch (ty.childType(zcu).floatBits(self.target.*)) {
26872 32 => switch (ty.vectorLen(zcu)) {
26873 1 => .{ .v_ss, .fmadd213 },
26874 2...8 => .{ .v_ps, .fmadd213 },
26875 else => null,
26876 },
26877 64 => switch (ty.vectorLen(zcu)) {
26878 1 => .{ .v_sd, .fmadd213 },
26879 2...4 => .{ .v_pd, .fmadd213 },
26880 else => null,
26881 },
26882 16, 80, 128 => null,
1835826883 else => unreachable,
1835926884 },
18360 }, .@"or" },
18361 lhs_temp_alias,
18362 lhs_temp_alias,
18363 rhs_temp_alias,
18364 ) else try self.asmRegisterRegister(
18365 .{ switch (elem_ty.zigTypeTag(zcu)) {
18366 else => break :result null,
18367 .int => .p_,
18368 .float => switch (elem_ty.floatBits(self.target.*)) {
18369 32 => ._ps,
18370 64 => ._pd,
18371 16, 80, 128 => break :result null,
26885 else => unreachable,
26886 },
26887 else => unreachable,
26888 }
26889 else if (std.mem.eql(u2, &order, &.{ 2, 3, 1 }) or std.mem.eql(u2, &order, &.{ 3, 2, 1 }))
26890 switch (ty.zigTypeTag(zcu)) {
26891 .float => switch (ty.floatBits(self.target.*)) {
26892 32 => .{ .v_ss, .fmadd231 },
26893 64 => .{ .v_sd, .fmadd231 },
26894 16, 80, 128 => null,
26895 else => unreachable,
26896 },
26897 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
26898 .float => switch (ty.childType(zcu).floatBits(self.target.*)) {
26899 32 => switch (ty.vectorLen(zcu)) {
26900 1 => .{ .v_ss, .fmadd231 },
26901 2...8 => .{ .v_ps, .fmadd231 },
26902 else => null,
26903 },
26904 64 => switch (ty.vectorLen(zcu)) {
26905 1 => .{ .v_sd, .fmadd231 },
26906 2...4 => .{ .v_pd, .fmadd231 },
26907 else => null,
26908 },
26909 16, 80, 128 => null,
1837226910 else => unreachable,
1837326911 },
18374 }, .@"or" },
18375 lhs_temp_alias,
18376 rhs_temp_alias,
18377 );
18378 break :result .{ .register = temp_regs[0] };
18379 }
26912 else => unreachable,
26913 },
26914 else => unreachable,
26915 }
26916 else
26917 unreachable) orelse return self.fail("TODO implement airMulAdd for {}", .{ty.fmt(pt)});
1838026918
18381 break :result null;
18382 }) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{
18383 lhs_ty.fmt(pt), rhs_ty.fmt(pt), dst_ty.fmt(pt),
18384 Value.fromInterned(extra.mask).fmtValue(pt),
18385 });
18386 return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
26919 var mops: [3]MCValue = undefined;
26920 for (order, mcvs) |mop_index, mcv| mops[mop_index - 1] = mcv;
26921
26922 const abi_size: u32 = @intCast(ty.abiSize(zcu));
26923 const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);
26924 const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
26925 if (mops[2].isRegister()) try self.asmRegisterRegisterRegister(
26926 mir_tag,
26927 mop1_reg,
26928 mop2_reg,
26929 registerAlias(mops[2].getReg().?, abi_size),
26930 ) else try self.asmRegisterRegisterMemory(
26931 mir_tag,
26932 mop1_reg,
26933 mop2_reg,
26934 try mops[2].mem(self, .{ .size = .fromSize(abi_size) }),
26935 );
26936 break :result mops[0];
26937 };
26938 return self.finishAir(inst, result, ops);
1838726939}
1838826940
18389fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
26941fn airVaStart(self: *CodeGen, inst: Air.Inst.Index) !void {
1839026942 const pt = self.pt;
1839126943 const zcu = pt.zcu;
18392 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
26944 const va_list_ty = self.air.instructions.items(.data)[@intFromEnum(inst)].ty;
26945 const ptr_anyopaque_ty = try pt.singleMutPtrType(.anyopaque);
1839326946
18394 const result: MCValue = result: {
18395 const operand_ty = self.typeOf(reduce.operand);
18396 if (operand_ty.isVector(zcu) and operand_ty.childType(zcu).toIntern() == .bool_type) {
26947 const result: MCValue = switch (self.fn_type.fnCallingConvention(zcu)) {
26948 .x86_64_sysv => result: {
26949 const info = self.va_info.sysv;
26950 const dst_fi = try self.allocFrameIndex(.initSpill(va_list_ty, zcu));
26951 var field_off: u31 = 0;
26952 // gp_offset: c_uint,
26953 try self.genSetMem(
26954 .{ .frame = dst_fi },
26955 field_off,
26956 .c_uint,
26957 .{ .immediate = info.gp_count * 8 },
26958 .{},
26959 );
26960 field_off += @intCast(Type.c_uint.abiSize(zcu));
26961 // fp_offset: c_uint,
26962 try self.genSetMem(
26963 .{ .frame = dst_fi },
26964 field_off,
26965 .c_uint,
26966 .{ .immediate = abi.SysV.c_abi_int_param_regs.len * 8 + info.fp_count * 16 },
26967 .{},
26968 );
26969 field_off += @intCast(Type.c_uint.abiSize(zcu));
26970 // overflow_arg_area: *anyopaque,
26971 try self.genSetMem(
26972 .{ .frame = dst_fi },
26973 field_off,
26974 ptr_anyopaque_ty,
26975 .{ .lea_frame = info.overflow_arg_area },
26976 .{},
26977 );
26978 field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu));
26979 // reg_save_area: *anyopaque,
26980 try self.genSetMem(
26981 .{ .frame = dst_fi },
26982 field_off,
26983 ptr_anyopaque_ty,
26984 .{ .lea_frame = info.reg_save_area },
26985 .{},
26986 );
26987 field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu));
26988 break :result .{ .load_frame = .{ .index = dst_fi } };
26989 },
26990 .x86_64_win => return self.fail("TODO implement c_va_start for Win64", .{}),
26991 else => |cc| return self.fail("{s} does not support var args", .{@tagName(cc)}),
26992 };
26993 return self.finishAir(inst, result, .{ .none, .none, .none });
26994}
26995
26996fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void {
26997 const pt = self.pt;
26998 const zcu = pt.zcu;
26999 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
27000 const ty = self.typeOfIndex(inst);
27001 const promote_ty = self.promoteVarArg(ty);
27002 const ptr_anyopaque_ty = try pt.singleMutPtrType(.anyopaque);
27003 const unused = self.liveness.isUnused(inst);
27004
27005 const result: MCValue = switch (self.fn_type.fnCallingConvention(zcu)) {
27006 .x86_64_sysv => result: {
1839727007 try self.spillEflagsIfOccupied();
1839827008
18399 const operand_mcv = try self.resolveInst(reduce.operand);
18400 const mask_len = (math.cast(u6, operand_ty.vectorLen(zcu)) orelse
18401 return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}));
18402 const mask = (@as(u64, 1) << mask_len) - 1;
18403 const abi_size: u32 = @intCast(operand_ty.abiSize(zcu));
18404 switch (reduce.operation) {
18405 .Or => {
18406 if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
18407 .{ ._, .@"test" },
18408 try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)),
18409 Immediate.u(mask),
18410 ) else {
18411 const operand_reg = registerAlias(if (operand_mcv.isRegister())
18412 operand_mcv.getReg().?
18413 else
18414 try self.copyToTmpRegister(operand_ty, operand_mcv), abi_size);
18415 if (mask_len < abi_size * 8) try self.asmRegisterImmediate(
18416 .{ ._, .@"test" },
18417 operand_reg,
18418 Immediate.u(mask),
18419 ) else try self.asmRegisterRegister(
18420 .{ ._, .@"test" },
18421 operand_reg,
18422 operand_reg,
18423 );
18424 }
18425 break :result .{ .eflags = .nz };
27009 const tmp_regs =
27010 try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
27011 const offset_reg = tmp_regs[0].to32();
27012 const addr_reg = tmp_regs[1].to64();
27013 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
27014 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
27015
27016 const promote_mcv = try self.allocTempRegOrMem(promote_ty, true);
27017 const promote_lock = switch (promote_mcv) {
27018 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
27019 else => null,
27020 };
27021 defer if (promote_lock) |lock| self.register_manager.unlockReg(lock);
27022
27023 const ptr_arg_list_reg =
27024 try self.copyToTmpRegister(self.typeOf(ty_op.operand), .{ .air_ref = ty_op.operand });
27025 const ptr_arg_list_lock = self.register_manager.lockRegAssumeUnused(ptr_arg_list_reg);
27026 defer self.register_manager.unlockReg(ptr_arg_list_lock);
27027
27028 const gp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 0 } };
27029 const fp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 4 } };
27030 const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } };
27031 const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } };
27032
27033 const classes = std.mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none);
27034 switch (classes[0]) {
27035 .integer => {
27036 assert(classes.len == 1);
27037
27038 try self.genSetReg(offset_reg, .c_uint, gp_offset, .{});
27039 try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u(
27040 abi.SysV.c_abi_int_param_regs.len * 8,
27041 ));
27042 const mem_reloc = try self.asmJccReloc(.ae, undefined);
27043
27044 try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{});
27045 if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
27046 .base = .{ .reg = addr_reg },
27047 .mod = .{ .rm = .{
27048 .size = .qword,
27049 .index = offset_reg.to64(),
27050 } },
27051 });
27052 try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
27053 .base = .{ .reg = offset_reg.to64() },
27054 .mod = .{ .rm = .{
27055 .size = .qword,
27056 .disp = 8,
27057 } },
27058 });
27059 try self.genCopy(.c_uint, gp_offset, .{ .register = offset_reg }, .{});
27060 const done_reloc = try self.asmJmpReloc(undefined);
27061
27062 self.performReloc(mem_reloc);
27063 try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{});
27064 try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
27065 .base = .{ .reg = addr_reg },
27066 .mod = .{ .rm = .{
27067 .size = .qword,
27068 .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)),
27069 } },
27070 });
27071 try self.genCopy(
27072 ptr_anyopaque_ty,
27073 overflow_arg_area,
27074 .{ .register = offset_reg.to64() },
27075 .{},
27076 );
27077
27078 self.performReloc(done_reloc);
27079 if (!unused) try self.genCopy(promote_ty, promote_mcv, .{
27080 .indirect = .{ .reg = addr_reg },
27081 }, .{});
1842627082 },
18427 .And => {
18428 const tmp_reg = try self.copyToTmpRegister(operand_ty, operand_mcv);
18429 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
18430 defer self.register_manager.unlockReg(tmp_lock);
27083 .sse => {
27084 assert(classes.len == 1);
27085
27086 try self.genSetReg(offset_reg, .c_uint, fp_offset, .{});
27087 try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u(
27088 abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16,
27089 ));
27090 const mem_reloc = try self.asmJccReloc(.ae, undefined);
27091
27092 try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{});
27093 if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
27094 .base = .{ .reg = addr_reg },
27095 .mod = .{ .rm = .{
27096 .size = .qword,
27097 .index = offset_reg.to64(),
27098 } },
27099 });
27100 try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
27101 .base = .{ .reg = offset_reg.to64() },
27102 .mod = .{ .rm = .{
27103 .size = .qword,
27104 .disp = 16,
27105 } },
27106 });
27107 try self.genCopy(.c_uint, fp_offset, .{ .register = offset_reg }, .{});
27108 const done_reloc = try self.asmJmpReloc(undefined);
1843127109
18432 try self.asmRegister(.{ ._, .not }, tmp_reg);
18433 if (mask_len < abi_size * 8)
18434 try self.asmRegisterImmediate(.{ ._, .@"test" }, tmp_reg, Immediate.u(mask))
18435 else
18436 try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg);
18437 break :result .{ .eflags = .z };
27110 self.performReloc(mem_reloc);
27111 try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{});
27112 try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
27113 .base = .{ .reg = addr_reg },
27114 .mod = .{ .rm = .{
27115 .size = .qword,
27116 .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)),
27117 } },
27118 });
27119 try self.genCopy(
27120 ptr_anyopaque_ty,
27121 overflow_arg_area,
27122 .{ .register = offset_reg.to64() },
27123 .{},
27124 );
27125
27126 self.performReloc(done_reloc);
27127 if (!unused) try self.genCopy(promote_ty, promote_mcv, .{
27128 .indirect = .{ .reg = addr_reg },
27129 }, .{});
1843827130 },
18439 else => return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}),
27131 .memory => {
27132 assert(classes.len == 1);
27133 unreachable;
27134 },
27135 else => return self.fail("TODO implement c_va_arg for {} on SysV", .{promote_ty.fmt(pt)}),
1844027136 }
18441 }
18442 return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)});
18443 };
18444 return self.finishAir(inst, result, .{ reduce.operand, .none, .none });
18445}
1844627137
18447fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
18448 const pt = self.pt;
18449 const zcu = pt.zcu;
18450 const result_ty = self.typeOfIndex(inst);
18451 const len: usize = @intCast(result_ty.arrayLen(zcu));
18452 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
18453 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);
18454 const result: MCValue = result: {
18455 switch (result_ty.zigTypeTag(zcu)) {
18456 .@"struct" => {
18457 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));
18458 if (result_ty.containerLayout(zcu) == .@"packed") {
18459 const struct_obj = zcu.typeToStruct(result_ty).?;
18460 try self.genInlineMemset(
18461 .{ .lea_frame = .{ .index = frame_index } },
18462 .{ .immediate = 0 },
18463 .{ .immediate = result_ty.abiSize(zcu) },
18464 .{},
18465 );
18466 for (elements, 0..) |elem, elem_i_usize| {
18467 const elem_i: u32 = @intCast(elem_i_usize);
18468 if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
27138 if (unused) break :result .unreach;
27139 if (ty.toIntern() == promote_ty.toIntern()) break :result promote_mcv;
1846927140
18470 const elem_ty = result_ty.fieldType(elem_i, zcu);
18471 const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu));
18472 if (elem_bit_size > 64) {
18473 return self.fail(
18474 "TODO airAggregateInit implement packed structs with large fields",
18475 .{},
18476 );
18477 }
18478 const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu));
18479 const elem_abi_bits = elem_abi_size * 8;
18480 const elem_off = pt.structPackedFieldBitOffset(struct_obj, elem_i);
18481 const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
18482 const elem_bit_off = elem_off % elem_abi_bits;
18483 const elem_mcv = try self.resolveInst(elem);
18484 const mat_elem_mcv = switch (elem_mcv) {
18485 .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index },
18486 else => elem_mcv,
18487 };
18488 const elem_lock = switch (mat_elem_mcv) {
18489 .register => |reg| self.register_manager.lockReg(reg),
18490 .immediate => |imm| lock: {
18491 if (imm == 0) continue;
18492 break :lock null;
18493 },
18494 else => null,
18495 };
18496 defer if (elem_lock) |lock| self.register_manager.unlockReg(lock);
27141 if (!promote_ty.isRuntimeFloat()) {
27142 const dst_mcv = try self.allocRegOrMem(inst, true);
27143 try self.genCopy(ty, dst_mcv, promote_mcv, .{});
27144 break :result dst_mcv;
27145 }
1849727146
18498 const elem_extra_bits = self.regExtraBits(elem_ty);
18499 {
18500 const temp_reg = try self.copyToTmpRegister(elem_ty, mat_elem_mcv);
18501 const temp_alias = registerAlias(temp_reg, elem_abi_size);
18502 const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
18503 defer self.register_manager.unlockReg(temp_lock);
27147 assert(ty.toIntern() == .f32_type and promote_ty.toIntern() == .f64_type);
27148 const dst_mcv = if (promote_mcv.isRegister())
27149 promote_mcv
27150 else
27151 try self.copyToRegisterWithInstTracking(inst, ty, promote_mcv);
27152 const dst_reg = dst_mcv.getReg().?.to128();
27153 const dst_lock = self.register_manager.lockReg(dst_reg);
27154 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
1850427155
18505 if (elem_bit_off < elem_extra_bits) {
18506 try self.truncateRegister(elem_ty, temp_alias);
18507 }
18508 if (elem_bit_off > 0) try self.genShiftBinOpMir(
18509 .{ ._l, .sh },
18510 elem_ty,
18511 .{ .register = temp_alias },
18512 Type.u8,
18513 .{ .immediate = elem_bit_off },
18514 );
18515 try self.genBinOpMir(
18516 .{ ._, .@"or" },
18517 elem_ty,
18518 .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
18519 .{ .register = temp_alias },
18520 );
18521 }
18522 if (elem_bit_off > elem_extra_bits) {
18523 const temp_reg = try self.copyToTmpRegister(elem_ty, mat_elem_mcv);
18524 const temp_alias = registerAlias(temp_reg, elem_abi_size);
18525 const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
18526 defer self.register_manager.unlockReg(temp_lock);
27156 if (self.hasFeature(.avx)) if (promote_mcv.isBase()) try self.asmRegisterRegisterMemory(
27157 .{ .v_ss, .cvtsd2 },
27158 dst_reg,
27159 dst_reg,
27160 try promote_mcv.mem(self, .{ .size = .qword }),
27161 ) else try self.asmRegisterRegisterRegister(
27162 .{ .v_ss, .cvtsd2 },
27163 dst_reg,
27164 dst_reg,
27165 (if (promote_mcv.isRegister())
27166 promote_mcv.getReg().?
27167 else
27168 try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(),
27169 ) else if (promote_mcv.isBase()) try self.asmRegisterMemory(
27170 .{ ._ss, .cvtsd2 },
27171 dst_reg,
27172 try promote_mcv.mem(self, .{ .size = .qword }),
27173 ) else try self.asmRegisterRegister(
27174 .{ ._ss, .cvtsd2 },
27175 dst_reg,
27176 (if (promote_mcv.isRegister())
27177 promote_mcv.getReg().?
27178 else
27179 try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(),
27180 );
27181 break :result promote_mcv;
27182 },
27183 .x86_64_win => return self.fail("TODO implement c_va_arg for Win64", .{}),
27184 else => |cc| return self.fail("{s} does not support var args", .{@tagName(cc)}),
27185 };
27186 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
27187}
1852727188
18528 if (elem_extra_bits > 0) {
18529 try self.truncateRegister(elem_ty, temp_alias);
18530 }
18531 try self.genShiftBinOpMir(
18532 .{ ._r, .sh },
18533 elem_ty,
18534 .{ .register = temp_reg },
18535 Type.u8,
18536 .{ .immediate = elem_abi_bits - elem_bit_off },
18537 );
18538 try self.genBinOpMir(
18539 .{ ._, .@"or" },
18540 elem_ty,
18541 .{ .load_frame = .{
18542 .index = frame_index,
18543 .off = elem_byte_off + @as(i32, @intCast(elem_abi_size)),
18544 } },
18545 .{ .register = temp_alias },
18546 );
18547 }
18548 }
18549 } else for (elements, 0..) |elem, elem_i| {
18550 if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
27189fn airVaCopy(self: *CodeGen, inst: Air.Inst.Index) !void {
27190 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
27191 const ptr_va_list_ty = self.typeOf(ty_op.operand);
1855127192
18552 const elem_ty = result_ty.fieldType(elem_i, zcu);
18553 const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, zcu));
18554 const elem_mcv = try self.resolveInst(elem);
18555 const mat_elem_mcv = switch (elem_mcv) {
18556 .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index },
18557 else => elem_mcv,
18558 };
18559 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv, .{});
18560 }
18561 break :result .{ .load_frame = .{ .index = frame_index } };
18562 },
18563 .array, .vector => {
18564 const elem_ty = result_ty.childType(zcu);
18565 if (result_ty.isVector(zcu) and elem_ty.toIntern() == .bool_type) {
18566 const result_size: u32 = @intCast(result_ty.abiSize(zcu));
18567 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
18568 try self.asmRegisterRegister(
18569 .{ ._, .xor },
18570 registerAlias(dst_reg, @min(result_size, 4)),
18571 registerAlias(dst_reg, @min(result_size, 4)),
18572 );
27193 const dst_mcv = try self.allocRegOrMem(inst, true);
27194 try self.load(dst_mcv, ptr_va_list_ty, .{ .air_ref = ty_op.operand });
27195 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
27196}
1857327197
18574 for (elements, 0..) |elem, elem_i| {
18575 const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem });
18576 const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg);
18577 defer self.register_manager.unlockReg(elem_lock);
27198fn airVaEnd(self: *CodeGen, inst: Air.Inst.Index) !void {
27199 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
27200 return self.finishAir(inst, .unreach, .{ un_op, .none, .none });
27201}
1857827202
18579 try self.asmRegisterImmediate(
18580 .{ ._, .@"and" },
18581 registerAlias(elem_reg, @min(result_size, 4)),
18582 Immediate.u(1),
18583 );
18584 if (elem_i > 0) try self.asmRegisterImmediate(
18585 .{ ._l, .sh },
18586 registerAlias(elem_reg, result_size),
18587 Immediate.u(@intCast(elem_i)),
18588 );
18589 try self.asmRegisterRegister(
18590 .{ ._, .@"or" },
18591 registerAlias(dst_reg, result_size),
18592 registerAlias(elem_reg, result_size),
18593 );
18594 }
18595 break :result .{ .register = dst_reg };
18596 } else {
18597 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));
18598 const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
27203fn resolveInst(self: *CodeGen, ref: Air.Inst.Ref) InnerError!MCValue {
27204 const zcu = self.pt.zcu;
27205 const ty = self.typeOf(ref);
1859927206
18600 for (elements, 0..) |elem, elem_i| {
18601 const elem_mcv = try self.resolveInst(elem);
18602 const mat_elem_mcv = switch (elem_mcv) {
18603 .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index },
18604 else => elem_mcv,
18605 };
18606 const elem_off: i32 = @intCast(elem_size * elem_i);
18607 try self.genSetMem(
18608 .{ .frame = frame_index },
18609 elem_off,
18610 elem_ty,
18611 mat_elem_mcv,
27207 // If the type has no codegen bits, no need to store it.
27208 if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return .none;
27209
27210 const mcv = if (ref.toIndex()) |inst| mcv: {
27211 break :mcv self.inst_tracking.getPtr(inst).?.short;
27212 } else mcv: {
27213 const ip_index = ref.toInterned().?;
27214 const gop = try self.const_tracking.getOrPut(self.gpa, ip_index);
27215 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
27216 const const_mcv = try self.genTypedValue(.fromInterned(ip_index));
27217 switch (const_mcv) {
27218 .lea_tlv => |tlv_sym| switch (self.bin_file.tag) {
27219 .elf, .macho => {
27220 if (self.mod.pic) {
27221 try self.spillRegisters(&.{ .rdi, .rax });
27222 } else {
27223 try self.spillRegisters(&.{.rax});
27224 }
27225 const frame_index = try self.allocFrameIndex(.init(.{
27226 .size = 8,
27227 .alignment = .@"8",
27228 }));
27229 try self.genSetMem(
27230 .{ .frame = frame_index },
27231 0,
27232 .usize,
27233 .{ .lea_symbol = .{ .sym_index = tlv_sym } },
1861227234 .{},
1861327235 );
18614 }
18615 if (result_ty.sentinel(zcu)) |sentinel| try self.genSetMem(
18616 .{ .frame = frame_index },
18617 @intCast(elem_size * elements.len),
18618 elem_ty,
18619 try self.genTypedValue(sentinel),
18620 .{},
18621 );
18622 break :result .{ .load_frame = .{ .index = frame_index } };
18623 }
18624 },
18625 else => unreachable,
18626 }
27236 break :init .{ .load_frame = .{ .index = frame_index } };
27237 },
27238 else => break :init const_mcv,
27239 },
27240 else => break :init const_mcv,
27241 }
27242 });
27243 break :mcv gop.value_ptr.short;
1862727244 };
1862827245
18629 if (elements.len <= Liveness.bpi - 1) {
18630 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
18631 @memcpy(buf[0..elements.len], elements);
18632 return self.finishAir(inst, result, buf);
27246 switch (mcv) {
27247 .none, .unreach, .dead => unreachable,
27248 else => return mcv,
1863327249 }
18634 var bt = self.liveness.iterateBigTomb(inst);
18635 for (elements) |elem| try self.feed(&bt, elem);
18636 return self.finishAirResult(inst, result);
1863727250}
1863827251
18639fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
18640 const pt = self.pt;
18641 const zcu = pt.zcu;
18642 const ip = &zcu.intern_pool;
18643 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
18644 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
18645 const result: MCValue = result: {
18646 const union_ty = self.typeOfIndex(inst);
18647 const layout = union_ty.unionGetLayout(zcu);
18648
18649 const src_ty = self.typeOf(extra.init);
18650 const src_mcv = try self.resolveInst(extra.init);
18651 if (layout.tag_size == 0) {
18652 if (layout.abi_size <= src_ty.abiSize(zcu) and
18653 self.reuseOperand(inst, extra.init, 0, src_mcv)) break :result src_mcv;
18654
18655 const dst_mcv = try self.allocRegOrMem(inst, true);
18656 try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
18657 break :result dst_mcv;
18658 }
18659
18660 const dst_mcv = try self.allocRegOrMem(inst, false);
18661
18662 const union_obj = zcu.typeToUnion(union_ty).?;
18663 const field_name = union_obj.loadTagType(ip).names.get(ip)[extra.field_index];
18664 const tag_ty = Type.fromInterned(union_obj.enum_tag_ty);
18665 const field_index = tag_ty.enumFieldIndex(field_name, zcu).?;
18666 const tag_val = try pt.enumValueFieldIndex(tag_ty, field_index);
18667 const tag_int_val = try tag_val.intFromEnum(tag_ty, pt);
18668 const tag_int = tag_int_val.toUnsignedInt(zcu);
18669 const tag_off: i32 = @intCast(layout.tagOffset());
18670 try self.genCopy(
18671 tag_ty,
18672 dst_mcv.address().offset(tag_off).deref(),
18673 .{ .immediate = tag_int },
18674 .{},
18675 );
27252fn getResolvedInstValue(self: *CodeGen, inst: Air.Inst.Index) *InstTracking {
27253 const tracking = self.inst_tracking.getPtr(inst).?;
27254 return switch (tracking.short) {
27255 .none, .unreach, .dead => unreachable,
27256 else => tracking,
27257 };
27258}
1867627259
18677 const pl_off: i32 = @intCast(layout.payloadOffset());
18678 try self.genCopy(src_ty, dst_mcv.address().offset(pl_off).deref(), src_mcv, .{});
27260/// If the MCValue is an immediate, and it does not fit within this type,
27261/// we put it in a register.
27262/// A potential opportunity for future optimization here would be keeping track
27263/// of the fact that the instruction is available both as an immediate
27264/// and as a register.
27265fn limitImmediateType(self: *CodeGen, operand: Air.Inst.Ref, comptime T: type) !MCValue {
27266 const mcv = try self.resolveInst(operand);
27267 const ti = @typeInfo(T).int;
27268 switch (mcv) {
27269 .immediate => |imm| {
27270 // This immediate is unsigned.
27271 const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed));
27272 if (imm >= std.math.maxInt(U)) {
27273 return MCValue{ .register = try self.copyToTmpRegister(.usize, mcv) };
27274 }
27275 },
27276 else => {},
27277 }
27278 return mcv;
27279}
1867927280
18680 break :result dst_mcv;
27281fn genTypedValue(self: *CodeGen, val: Value) InnerError!MCValue {
27282 const pt = self.pt;
27283 return switch (try codegen.genTypedValue(self.bin_file, pt, self.src_loc, val, self.target.*)) {
27284 .mcv => |mcv| switch (mcv) {
27285 .none => .none,
27286 .undef => .undef,
27287 .immediate => |imm| .{ .immediate = imm },
27288 .memory => |addr| .{ .memory = addr },
27289 .load_symbol => |sym_index| .{ .load_symbol = .{ .sym_index = sym_index } },
27290 .lea_symbol => |sym_index| .{ .lea_symbol = .{ .sym_index = sym_index } },
27291 .load_direct => |sym_index| .{ .load_direct = sym_index },
27292 .lea_direct => |sym_index| .{ .lea_direct = sym_index },
27293 .load_got => |sym_index| .{ .lea_got = sym_index },
27294 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
27295 },
27296 .fail => |msg| return self.failMsg(msg),
1868127297 };
18682 return self.finishAir(inst, result, .{ extra.init, .none, .none });
1868327298}
1868427299
18685fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
18686 const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
18687 return self.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });
18688}
27300const CallMCValues = struct {
27301 args: []MCValue,
27302 return_value: InstTracking,
27303 stack_byte_count: u31,
27304 stack_align: InternPool.Alignment,
27305 gp_count: u32,
27306 fp_count: u32,
27307
27308 fn deinit(self: *CallMCValues, func: *CodeGen) void {
27309 func.gpa.free(self.args);
27310 self.* = undefined;
27311 }
27312};
1868927313
18690fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
27314/// Caller must call `CallMCValues.deinit`.
27315fn resolveCallingConventionValues(
27316 self: *CodeGen,
27317 fn_info: InternPool.Key.FuncType,
27318 var_args: []const Type,
27319 stack_frame_base: FrameIndex,
27320) !CallMCValues {
1869127321 const pt = self.pt;
1869227322 const zcu = pt.zcu;
18693 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
18694 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
18695 const ty = self.typeOfIndex(inst);
27323 const ip = &zcu.intern_pool;
27324 const cc = fn_info.cc;
27325 const param_types = try self.gpa.alloc(Type, fn_info.param_types.len + var_args.len);
27326 defer self.gpa.free(param_types);
1869627327
18697 const ops = [3]Air.Inst.Ref{ extra.lhs, extra.rhs, pl_op.operand };
18698 const result = result: {
18699 if (switch (ty.scalarType(zcu).floatBits(self.target.*)) {
18700 16, 80, 128 => true,
18701 32, 64 => !self.hasFeature(.fma),
18702 else => unreachable,
18703 }) {
18704 if (ty.zigTypeTag(zcu) != .float) return self.fail("TODO implement airMulAdd for {}", .{
18705 ty.fmt(pt),
18706 });
27328 for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*dest, src|
27329 dest.* = .fromInterned(src);
27330 for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty|
27331 param_ty.* = self.promoteVarArg(arg_ty);
1870727332
18708 var callee_buf: ["__fma?".len]u8 = undefined;
18709 break :result try self.genCall(.{ .lib = .{
18710 .return_type = ty.toIntern(),
18711 .param_types = &.{ ty.toIntern(), ty.toIntern(), ty.toIntern() },
18712 .callee = std.fmt.bufPrint(&callee_buf, "{s}fma{s}", .{
18713 floatLibcAbiPrefix(ty),
18714 floatLibcAbiSuffix(ty),
18715 }) catch unreachable,
18716 } }, &.{ ty, ty, ty }, &.{
18717 .{ .air_ref = extra.lhs }, .{ .air_ref = extra.rhs }, .{ .air_ref = pl_op.operand },
18718 });
18719 }
27333 var result: CallMCValues = .{
27334 .args = try self.gpa.alloc(MCValue, param_types.len),
27335 // These undefined values must be populated before returning from this function.
27336 .return_value = undefined,
27337 .stack_byte_count = 0,
27338 .stack_align = undefined,
27339 .gp_count = 0,
27340 .fp_count = 0,
27341 };
27342 errdefer self.gpa.free(result.args);
1872027343
18721 var mcvs: [3]MCValue = undefined;
18722 var locks = [1]?RegisterManager.RegisterLock{null} ** 3;
18723 defer for (locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
18724 var order = [1]u2{0} ** 3;
18725 var unused = std.StaticBitSet(3).initFull();
18726 for (ops, &mcvs, &locks, 0..) |op, *mcv, *lock, op_i| {
18727 const op_index: u2 = @intCast(op_i);
18728 mcv.* = try self.resolveInst(op);
18729 if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) {
18730 order[op_index] = 1;
18731 unused.unset(0);
18732 } else if (unused.isSet(2) and mcv.isMemory()) {
18733 order[op_index] = 3;
18734 unused.unset(2);
18735 }
18736 switch (mcv.*) {
18737 .register => |reg| lock.* = self.register_manager.lockReg(reg),
18738 else => {},
27344 const ret_ty: Type = .fromInterned(fn_info.return_type);
27345 switch (cc) {
27346 .naked => {
27347 assert(result.args.len == 0);
27348 result.return_value = .init(.unreach);
27349 result.stack_align = switch (self.target.cpu.arch) {
27350 else => unreachable,
27351 .x86 => .@"4",
27352 .x86_64 => .@"8",
27353 };
27354 },
27355 .x86_64_sysv, .x86_64_win => |cc_opts| {
27356 var ret_int_reg_i: u32 = 0;
27357 var ret_sse_reg_i: u32 = 0;
27358 var param_int_reg_i: u32 = 0;
27359 var param_sse_reg_i: u32 = 0;
27360 result.stack_align = .fromByteUnits(cc_opts.incoming_stack_alignment orelse 16);
27361
27362 switch (cc) {
27363 .x86_64_sysv => {},
27364 .x86_64_win => result.stack_byte_count += @intCast(4 * 8),
27365 else => unreachable,
1873927366 }
18740 }
18741 for (&order, &mcvs, &locks) |*mop_index, *mcv, *lock| {
18742 if (mop_index.* != 0) continue;
18743 mop_index.* = 1 + @as(u2, @intCast(unused.toggleFirstSet().?));
18744 if (mop_index.* > 1 and mcv.isRegister()) continue;
18745 const reg = try self.copyToTmpRegister(ty, mcv.*);
18746 mcv.* = .{ .register = reg };
18747 if (lock.*) |old_lock| self.register_manager.unlockReg(old_lock);
18748 lock.* = self.register_manager.lockRegAssumeUnused(reg);
18749 }
1875027367
18751 const mir_tag = @as(?Mir.Inst.FixedTag, if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or
18752 mem.eql(u2, &order, &.{ 3, 1, 2 }))
18753 switch (ty.zigTypeTag(zcu)) {
18754 .float => switch (ty.floatBits(self.target.*)) {
18755 32 => .{ .v_ss, .fmadd132 },
18756 64 => .{ .v_sd, .fmadd132 },
18757 16, 80, 128 => null,
27368 // Return values
27369 if (ret_ty.isNoReturn(zcu)) {
27370 result.return_value = .init(.unreach);
27371 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
27372 // TODO: is this even possible for C calling convention?
27373 result.return_value = .init(.none);
27374 } else {
27375 var ret_tracking: [4]InstTracking = undefined;
27376 var ret_tracking_i: usize = 0;
27377
27378 const classes = switch (cc) {
27379 .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none),
27380 .x86_64_win => &.{abi.classifyWindows(ret_ty, zcu)},
1875827381 else => unreachable,
18759 },
18760 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
18761 .float => switch (ty.childType(zcu).floatBits(self.target.*)) {
18762 32 => switch (ty.vectorLen(zcu)) {
18763 1 => .{ .v_ss, .fmadd132 },
18764 2...8 => .{ .v_ps, .fmadd132 },
18765 else => null,
18766 },
18767 64 => switch (ty.vectorLen(zcu)) {
18768 1 => .{ .v_sd, .fmadd132 },
18769 2...4 => .{ .v_pd, .fmadd132 },
18770 else => null,
18771 },
18772 16, 80, 128 => null,
18773 else => unreachable,
27382 };
27383 for (classes) |class| switch (class) {
27384 .integer => {
27385 const ret_int_reg = registerAlias(
27386 abi.getCAbiIntReturnRegs(cc)[ret_int_reg_i],
27387 @intCast(@min(ret_ty.abiSize(zcu), 8)),
27388 );
27389 ret_int_reg_i += 1;
27390
27391 ret_tracking[ret_tracking_i] = .init(.{ .register = ret_int_reg });
27392 ret_tracking_i += 1;
27393 },
27394 .sse, .float, .float_combine, .win_i128 => {
27395 const ret_sse_regs = abi.getCAbiSseReturnRegs(cc);
27396 const abi_size: u32 = @intCast(ret_ty.abiSize(zcu));
27397 const reg_size = @min(abi_size, self.vectorSize(.float));
27398 var byte_offset: u32 = 0;
27399 while (byte_offset < abi_size) : (byte_offset += reg_size) {
27400 const ret_sse_reg = registerAlias(ret_sse_regs[ret_sse_reg_i], reg_size);
27401 ret_sse_reg_i += 1;
27402
27403 ret_tracking[ret_tracking_i] = .init(.{ .register = ret_sse_reg });
27404 ret_tracking_i += 1;
27405 }
27406 },
27407 .sseup => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .sse),
27408 .x87 => {
27409 ret_tracking[ret_tracking_i] = .init(.{ .register = abi.getCAbiX87ReturnRegs(cc)[0] });
27410 ret_tracking_i += 1;
27411 },
27412 .x87up => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .x87),
27413 .complex_x87 => {
27414 ret_tracking[ret_tracking_i] = .init(.{ .register_pair = abi.getCAbiX87ReturnRegs(cc)[0..2].* });
27415 ret_tracking_i += 1;
1877427416 },
27417 .memory => {
27418 const ret_int_reg = abi.getCAbiIntReturnRegs(cc)[ret_int_reg_i].to64();
27419 ret_int_reg_i += 1;
27420 const ret_indirect_reg = abi.getCAbiIntParamRegs(cc)[param_int_reg_i];
27421 param_int_reg_i += 1;
27422
27423 ret_tracking[ret_tracking_i] = .{
27424 .short = .{ .indirect = .{ .reg = ret_int_reg } },
27425 .long = .{ .indirect = .{ .reg = ret_indirect_reg } },
27426 };
27427 ret_tracking_i += 1;
27428 },
27429 .none, .integer_per_element => unreachable,
27430 };
27431 result.return_value = switch (ret_tracking_i) {
1877527432 else => unreachable,
18776 },
18777 else => unreachable,
27433 1 => ret_tracking[0],
27434 2 => .init(.{ .register_pair = .{
27435 ret_tracking[0].short.register,
27436 ret_tracking[1].short.register,
27437 } }),
27438 3 => .init(.{ .register_triple = .{
27439 ret_tracking[0].short.register,
27440 ret_tracking[1].short.register,
27441 ret_tracking[2].short.register,
27442 } }),
27443 4 => .init(.{ .register_quadruple = .{
27444 ret_tracking[0].short.register,
27445 ret_tracking[1].short.register,
27446 ret_tracking[2].short.register,
27447 ret_tracking[3].short.register,
27448 } }),
27449 };
1877827450 }
18779 else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
18780 switch (ty.zigTypeTag(zcu)) {
18781 .float => switch (ty.floatBits(self.target.*)) {
18782 32 => .{ .v_ss, .fmadd213 },
18783 64 => .{ .v_sd, .fmadd213 },
18784 16, 80, 128 => null,
27451
27452 // Input params
27453 for (param_types, result.args) |ty, *arg| {
27454 assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
27455 switch (cc) {
27456 .x86_64_sysv => {},
27457 .x86_64_win => {
27458 param_int_reg_i = @max(param_int_reg_i, param_sse_reg_i);
27459 param_sse_reg_i = param_int_reg_i;
27460 },
1878527461 else => unreachable,
18786 },
18787 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
18788 .float => switch (ty.childType(zcu).floatBits(self.target.*)) {
18789 32 => switch (ty.vectorLen(zcu)) {
18790 1 => .{ .v_ss, .fmadd213 },
18791 2...8 => .{ .v_ps, .fmadd213 },
18792 else => null,
18793 },
18794 64 => switch (ty.vectorLen(zcu)) {
18795 1 => .{ .v_sd, .fmadd213 },
18796 2...4 => .{ .v_pd, .fmadd213 },
18797 else => null,
27462 }
27463
27464 var arg_mcv: [4]MCValue = undefined;
27465 var arg_mcv_i: usize = 0;
27466
27467 const classes = switch (cc) {
27468 .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none),
27469 .x86_64_win => &.{abi.classifyWindows(ty, zcu)},
27470 else => unreachable,
27471 };
27472 classes: for (classes) |class| switch (class) {
27473 .integer => {
27474 const param_int_regs = abi.getCAbiIntParamRegs(cc);
27475 if (param_int_reg_i >= param_int_regs.len) break;
27476
27477 const param_int_reg =
27478 registerAlias(param_int_regs[param_int_reg_i], @intCast(@min(ty.abiSize(zcu), 8)));
27479 param_int_reg_i += 1;
27480
27481 arg_mcv[arg_mcv_i] = .{ .register = param_int_reg };
27482 arg_mcv_i += 1;
27483 },
27484 .sse, .float, .float_combine => {
27485 const param_sse_regs = abi.getCAbiSseParamRegs(cc);
27486 const abi_size: u32 = @intCast(ty.abiSize(zcu));
27487 const reg_size = @min(abi_size, self.vectorSize(.float));
27488 var byte_offset: u32 = 0;
27489 while (byte_offset < abi_size) : (byte_offset += reg_size) {
27490 if (param_sse_reg_i >= param_sse_regs.len) break :classes;
27491
27492 const param_sse_reg = registerAlias(param_sse_regs[param_sse_reg_i], reg_size);
27493 param_sse_reg_i += 1;
27494
27495 arg_mcv[arg_mcv_i] = .{ .register = param_sse_reg };
27496 arg_mcv_i += 1;
27497 }
27498 },
27499 .sseup => assert(arg_mcv[arg_mcv_i - 1].register.class() == .sse),
27500 .x87, .x87up, .complex_x87, .memory, .win_i128 => switch (cc) {
27501 .x86_64_sysv => switch (class) {
27502 .x87, .x87up, .complex_x87, .memory => break,
27503 else => unreachable,
1879827504 },
18799 16, 80, 128 => null,
27505 .x86_64_win => if (ty.abiSize(zcu) > 8) {
27506 const param_int_reg = abi.getCAbiIntParamRegs(cc)[param_int_reg_i].to64();
27507 param_int_reg_i += 1;
27508
27509 arg_mcv[arg_mcv_i] = .{ .indirect = .{ .reg = param_int_reg } };
27510 arg_mcv_i += 1;
27511 } else break,
1880027512 else => unreachable,
1880127513 },
18802 else => unreachable,
18803 },
18804 else => unreachable,
27514 .none => unreachable,
27515 .integer_per_element => {
27516 const param_int_regs_len: u32 =
27517 @intCast(abi.getCAbiIntParamRegs(cc).len);
27518 const remaining_param_int_regs: u3 =
27519 @intCast(param_int_regs_len - param_int_reg_i);
27520 param_int_reg_i = param_int_regs_len;
27521
27522 const frame_elem_align = 8;
27523 const frame_elems_len = ty.vectorLen(zcu) - remaining_param_int_regs;
27524 const frame_elem_size = std.mem.alignForward(
27525 u64,
27526 ty.childType(zcu).abiSize(zcu),
27527 frame_elem_align,
27528 );
27529 const frame_size: u31 = @intCast(frame_elems_len * frame_elem_size);
27530
27531 result.stack_byte_count =
27532 std.mem.alignForward(u31, result.stack_byte_count, frame_elem_align);
27533 arg_mcv[arg_mcv_i] = .{ .elementwise_regs_then_frame = .{
27534 .regs = remaining_param_int_regs,
27535 .frame_off = @intCast(result.stack_byte_count),
27536 .frame_index = stack_frame_base,
27537 } };
27538 arg_mcv_i += 1;
27539 result.stack_byte_count += frame_size;
27540 },
27541 } else {
27542 arg.* = switch (arg_mcv_i) {
27543 else => unreachable,
27544 1 => arg_mcv[0],
27545 2 => .{ .register_pair = .{
27546 arg_mcv[0].register,
27547 arg_mcv[1].register,
27548 } },
27549 3 => .{ .register_triple = .{
27550 arg_mcv[0].register,
27551 arg_mcv[1].register,
27552 arg_mcv[2].register,
27553 } },
27554 4 => .{ .register_quadruple = .{
27555 arg_mcv[0].register,
27556 arg_mcv[1].register,
27557 arg_mcv[2].register,
27558 arg_mcv[3].register,
27559 } },
27560 };
27561 continue;
27562 }
27563
27564 const param_align = ty.abiAlignment(zcu).max(.@"8");
27565 result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count));
27566 result.stack_align = result.stack_align.max(param_align);
27567 arg.* = .{ .load_frame = .{
27568 .index = stack_frame_base,
27569 .off = result.stack_byte_count,
27570 } };
27571 result.stack_byte_count += @intCast(ty.abiSize(zcu));
1880527572 }
18806 else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
18807 switch (ty.zigTypeTag(zcu)) {
18808 .float => switch (ty.floatBits(self.target.*)) {
18809 32 => .{ .v_ss, .fmadd231 },
18810 64 => .{ .v_sd, .fmadd231 },
18811 16, 80, 128 => null,
18812 else => unreachable,
18813 },
18814 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
18815 .float => switch (ty.childType(zcu).floatBits(self.target.*)) {
18816 32 => switch (ty.vectorLen(zcu)) {
18817 1 => .{ .v_ss, .fmadd231 },
18818 2...8 => .{ .v_ps, .fmadd231 },
18819 else => null,
18820 },
18821 64 => switch (ty.vectorLen(zcu)) {
18822 1 => .{ .v_sd, .fmadd231 },
18823 2...4 => .{ .v_pd, .fmadd231 },
18824 else => null,
18825 },
18826 16, 80, 128 => null,
27573 assert(param_int_reg_i <= 6);
27574 result.gp_count = param_int_reg_i;
27575 assert(param_sse_reg_i <= 16);
27576 result.fp_count = param_sse_reg_i;
27577 },
27578 .auto => {
27579 result.stack_align = abi.zigcc.stack_align orelse .fromByteUnits(self.vectorSize(.float));
27580
27581 var param_gpr = abi.getCAbiIntParamRegs(cc);
27582 var param_x87 = abi.getCAbiX87ParamRegs(cc);
27583 var param_sse = abi.getCAbiSseParamRegs(cc);
27584
27585 // Return values
27586 result.return_value = if (ret_ty.isNoReturn(zcu))
27587 .init(.unreach)
27588 else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu))
27589 .init(.none)
27590 else return_value: {
27591 const ret_gpr = abi.getCAbiIntReturnRegs(cc);
27592 const ret_size: u31 = @intCast(ret_ty.abiSize(zcu));
27593 if (abi.zigcc.return_in_regs) switch (self.regClassForType(ret_ty)) {
27594 .general_purpose => if (ret_size <= @as(u4, switch (self.target.cpu.arch) {
27595 else => unreachable,
27596 .x86 => 4,
27597 .x86_64 => 8,
27598 }))
27599 break :return_value .init(.{ .register = registerAlias(ret_gpr[0], ret_size) })
27600 else if (ret_gpr.len >= 2 and ret_ty.isSliceAtRuntime(zcu))
27601 break :return_value .init(.{ .register_pair = ret_gpr[0..2].* }),
27602 .segment, .mmx, .ip => unreachable,
27603 .x87 => break :return_value .init(.{ .register = .st0 }),
27604 .sse => if (ret_size <= self.vectorSize(.float)) break :return_value .init(.{
27605 .register = registerAlias(abi.getCAbiSseReturnRegs(cc)[0], @max(ret_size, 16)),
27606 }),
27607 };
27608 const ret_indirect_reg = param_gpr[0];
27609 param_gpr = param_gpr[1..];
27610 break :return_value .{
27611 .short = .{ .indirect = .{ .reg = ret_gpr[0] } },
27612 .long = .{ .indirect = .{ .reg = ret_indirect_reg } },
27613 };
27614 };
27615
27616 // Input params
27617 for (param_types, result.args) |param_ty, *arg| {
27618 if (!param_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
27619 arg.* = .none;
27620 continue;
27621 }
27622 const param_size: u31 = @intCast(param_ty.abiSize(zcu));
27623 if (abi.zigcc.params_in_regs) switch (self.regClassForType(param_ty)) {
27624 .general_purpose => if (param_gpr.len >= 1 and param_size <= @as(u4, switch (self.target.cpu.arch) {
1882727625 else => unreachable,
27626 .x86 => 4,
27627 .x86_64 => 8,
27628 })) {
27629 arg.* = .{ .register = registerAlias(param_gpr[0], param_size) };
27630 param_gpr = param_gpr[1..];
27631 continue;
27632 } else if (param_gpr.len >= 2 and param_ty.isSliceAtRuntime(zcu)) {
27633 arg.* = .{ .register_pair = param_gpr[0..2].* };
27634 param_gpr = param_gpr[2..];
27635 continue;
1882827636 },
18829 else => unreachable,
18830 },
18831 else => unreachable,
27637 .segment, .mmx, .ip => unreachable,
27638 .x87 => if (param_x87.len >= 1) {
27639 arg.* = .{ .register = param_x87[0] };
27640 param_x87 = param_x87[1..];
27641 continue;
27642 },
27643 .sse => if (param_sse.len >= 1 and param_size <= self.vectorSize(.float)) {
27644 arg.* = .{
27645 .register = registerAlias(param_sse[0], @max(param_size, 16)),
27646 };
27647 param_sse = param_sse[1..];
27648 continue;
27649 },
27650 };
27651 const param_align = param_ty.abiAlignment(zcu);
27652 result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count));
27653 result.stack_align = result.stack_align.max(param_align);
27654 arg.* = .{ .load_frame = .{
27655 .index = stack_frame_base,
27656 .off = result.stack_byte_count,
27657 } };
27658 result.stack_byte_count += param_size;
1883227659 }
18833 else
18834 unreachable) orelse return self.fail("TODO implement airMulAdd for {}", .{ty.fmt(pt)});
27660 },
27661 else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),
27662 }
1883527663
18836 var mops: [3]MCValue = undefined;
18837 for (order, mcvs) |mop_index, mcv| mops[mop_index - 1] = mcv;
27664 result.stack_byte_count = @intCast(result.stack_align.forward(result.stack_byte_count));
27665 return result;
27666}
1883827667
18839 const abi_size: u32 = @intCast(ty.abiSize(zcu));
18840 const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);
18841 const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
18842 if (mops[2].isRegister()) try self.asmRegisterRegisterRegister(
18843 mir_tag,
18844 mop1_reg,
18845 mop2_reg,
18846 registerAlias(mops[2].getReg().?, abi_size),
18847 ) else try self.asmRegisterRegisterMemory(
18848 mir_tag,
18849 mop1_reg,
18850 mop2_reg,
18851 try mops[2].mem(self, Memory.Size.fromSize(abi_size)),
18852 );
18853 break :result mops[0];
27668fn fail(self: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
27669 @branchHint(.cold);
27670 const zcu = self.pt.zcu;
27671 switch (self.owner) {
27672 .nav_index => |i| return zcu.codegenFail(i, format, args),
27673 .lazy_sym => |s| return zcu.codegenFailType(s.ty, format, args),
27674 }
27675 return error.CodegenFail;
27676}
27677
27678fn failMsg(self: *CodeGen, msg: *Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } {
27679 @branchHint(.cold);
27680 const zcu = self.pt.zcu;
27681 switch (self.owner) {
27682 .nav_index => |i| return zcu.codegenFailMsg(i, msg),
27683 .lazy_sym => |s| return zcu.codegenFailTypeMsg(s.ty, msg),
27684 }
27685 return error.CodegenFail;
27686}
27687
27688fn parseRegName(name: []const u8) ?Register {
27689 if (@hasDecl(Register, "parseRegName")) {
27690 return Register.parseRegName(name);
27691 }
27692 return std.meta.stringToEnum(Register, name);
27693}
27694
27695/// Returns register wide enough to hold at least `size_bytes`.
27696fn registerAlias(reg: Register, size_bytes: u32) Register {
27697 return switch (reg.class()) {
27698 .general_purpose => if (size_bytes == 0)
27699 unreachable // should be comptime-known
27700 else if (size_bytes <= 1)
27701 reg.to8()
27702 else if (size_bytes <= 2)
27703 reg.to16()
27704 else if (size_bytes <= 4)
27705 reg.to32()
27706 else if (size_bytes <= 8)
27707 reg.to64()
27708 else
27709 unreachable,
27710 .segment => if (size_bytes <= 2)
27711 reg
27712 else
27713 unreachable,
27714 .x87 => if (size_bytes >= 10 and size_bytes <= 16)
27715 reg
27716 else
27717 unreachable,
27718 .mmx => if (size_bytes <= 8)
27719 reg
27720 else
27721 unreachable,
27722 .sse => if (size_bytes <= 16)
27723 reg.to128()
27724 else if (size_bytes <= 32)
27725 reg.to256()
27726 else
27727 unreachable,
27728 .ip => if (size_bytes <= 2)
27729 .ip
27730 else if (size_bytes <= 4)
27731 .eip
27732 else if (size_bytes <= 8)
27733 .rip
27734 else
27735 unreachable,
27736 };
27737}
27738
27739fn memSize(self: *CodeGen, ty: Type) Memory.Size {
27740 const zcu = self.pt.zcu;
27741 return switch (ty.zigTypeTag(zcu)) {
27742 .float => .fromBitSize(ty.floatBits(self.target.*)),
27743 else => .fromSize(@intCast(ty.abiSize(zcu))),
1885427744 };
18855 return self.finishAir(inst, result, ops);
1885627745}
1885727746
18858fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
27747fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Type {
1885927748 const pt = self.pt;
1886027749 const zcu = pt.zcu;
18861 const va_list_ty = self.air.instructions.items(.data)[@intFromEnum(inst)].ty;
18862 const ptr_anyopaque_ty = try pt.singleMutPtrType(Type.anyopaque);
18863
18864 const result: MCValue = switch (abi.resolveCallingConvention(
18865 self.fn_type.fnCallingConvention(zcu),
18866 self.target.*,
18867 )) {
18868 .x86_64_sysv => result: {
18869 const info = self.va_info.sysv;
18870 const dst_fi = try self.allocFrameIndex(FrameAlloc.initSpill(va_list_ty, zcu));
18871 var field_off: u31 = 0;
18872 // gp_offset: c_uint,
18873 try self.genSetMem(
18874 .{ .frame = dst_fi },
18875 field_off,
18876 Type.c_uint,
18877 .{ .immediate = info.gp_count * 8 },
18878 .{},
18879 );
18880 field_off += @intCast(Type.c_uint.abiSize(zcu));
18881 // fp_offset: c_uint,
18882 try self.genSetMem(
18883 .{ .frame = dst_fi },
18884 field_off,
18885 Type.c_uint,
18886 .{ .immediate = abi.SysV.c_abi_int_param_regs.len * 8 + info.fp_count * 16 },
18887 .{},
18888 );
18889 field_off += @intCast(Type.c_uint.abiSize(zcu));
18890 // overflow_arg_area: *anyopaque,
18891 try self.genSetMem(
18892 .{ .frame = dst_fi },
18893 field_off,
18894 ptr_anyopaque_ty,
18895 .{ .lea_frame = info.overflow_arg_area },
18896 .{},
18897 );
18898 field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu));
18899 // reg_save_area: *anyopaque,
18900 try self.genSetMem(
18901 .{ .frame = dst_fi },
18902 field_off,
18903 ptr_anyopaque_ty,
18904 .{ .lea_frame = info.reg_save_area },
18905 .{},
18906 );
18907 field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu));
18908 break :result .{ .load_frame = .{ .index = dst_fi } };
18909 },
18910 .x86_64_win => return self.fail("TODO implement c_va_start for Win64", .{}),
18911 else => unreachable,
27750 var parts: [parts_len]Type = undefined;
27751 if (ty.isVector(zcu)) if (std.math.divExact(u32, ty.vectorLen(zcu), parts_len)) |vec_len| return .{
27752 try pt.vectorType(.{ .len = vec_len, .child = ty.scalarType(zcu).toIntern() }),
27753 } ** parts_len else |err| switch (err) {
27754 error.DivisionByZero => unreachable,
27755 error.UnexpectedRemainder => {},
1891227756 };
18913 return self.finishAir(inst, result, .{ .none, .none, .none });
27757 const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
27758 if (classes.len == parts_len) for (&parts, classes, 0..) |*part, class, part_i| {
27759 part.* = switch (class) {
27760 .integer => if (part_i < parts_len - 1)
27761 .u64
27762 else part: {
27763 const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?;
27764 const elem_ty = try pt.intType(.unsigned, @intCast(elem_size * 8));
27765 break :part switch (@divExact(ty.abiSize(zcu) - part_i * 8, elem_size)) {
27766 1 => elem_ty,
27767 else => |array_len| try pt.arrayType(.{ .len = array_len, .child = elem_ty.toIntern() }),
27768 };
27769 },
27770 .float => .f32,
27771 .float_combine => try pt.arrayType(.{ .len = 2, .child = .f32_type }),
27772 .sse => .f64,
27773 else => break,
27774 };
27775 } else {
27776 var part_sizes: u64 = 0;
27777 for (parts) |part| part_sizes += part.abiSize(zcu);
27778 if (part_sizes == ty.abiSize(zcu)) return parts;
27779 };
27780 return self.fail("TODO implement splitType({d}, {})", .{ parts_len, ty.fmt(pt) });
1891427781}
1891527782
18916fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
27783/// Truncates the value in the register in place.
27784/// Clobbers any remaining bits.
27785fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {
1891727786 const pt = self.pt;
1891827787 const zcu = pt.zcu;
18919 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
18920 const ty = self.typeOfIndex(inst);
18921 const promote_ty = self.promoteVarArg(ty);
18922 const ptr_anyopaque_ty = try pt.singleMutPtrType(Type.anyopaque);
18923 const unused = self.liveness.isUnused(inst);
18924
18925 const result: MCValue = switch (abi.resolveCallingConvention(
18926 self.fn_type.fnCallingConvention(zcu),
18927 self.target.*,
18928 )) {
18929 .x86_64_sysv => result: {
18930 try self.spillEflagsIfOccupied();
18931
18932 const tmp_regs =
18933 try self.register_manager.allocRegs(2, .{null} ** 2, abi.RegisterClass.gp);
18934 const offset_reg = tmp_regs[0].to32();
18935 const addr_reg = tmp_regs[1].to64();
18936 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
18937 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
18938
18939 const promote_mcv = try self.allocTempRegOrMem(promote_ty, true);
18940 const promote_lock = switch (promote_mcv) {
18941 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
18942 else => null,
18943 };
18944 defer if (promote_lock) |lock| self.register_manager.unlockReg(lock);
18945
18946 const ptr_arg_list_reg =
18947 try self.copyToTmpRegister(self.typeOf(ty_op.operand), .{ .air_ref = ty_op.operand });
18948 const ptr_arg_list_lock = self.register_manager.lockRegAssumeUnused(ptr_arg_list_reg);
18949 defer self.register_manager.unlockReg(ptr_arg_list_lock);
18950
18951 const gp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 0 } };
18952 const fp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 4 } };
18953 const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } };
18954 const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } };
18955
18956 const classes = mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none);
18957 switch (classes[0]) {
18958 .integer => {
18959 assert(classes.len == 1);
18960
18961 try self.genSetReg(offset_reg, Type.c_uint, gp_offset, .{});
18962 try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(
18963 abi.SysV.c_abi_int_param_regs.len * 8,
18964 ));
18965 const mem_reloc = try self.asmJccReloc(.ae, undefined);
18966
18967 try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{});
18968 if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
18969 .base = .{ .reg = addr_reg },
18970 .mod = .{ .rm = .{
18971 .size = .qword,
18972 .index = offset_reg.to64(),
18973 } },
18974 });
18975 try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
18976 .base = .{ .reg = offset_reg.to64() },
18977 .mod = .{ .rm = .{
18978 .size = .qword,
18979 .disp = 8,
18980 } },
18981 });
18982 try self.genCopy(Type.c_uint, gp_offset, .{ .register = offset_reg }, .{});
18983 const done_reloc = try self.asmJmpReloc(undefined);
27788 const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{
27789 .signedness = .unsigned,
27790 .bits = @intCast(ty.bitSize(zcu)),
27791 };
27792 const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return;
27793 try self.spillEflagsIfOccupied();
27794 switch (int_info.signedness) {
27795 .signed => {
27796 try self.genShiftBinOpMir(.{ ._l, .sa }, .isize, .{ .register = reg }, .u8, .{ .immediate = shift });
27797 try self.genShiftBinOpMir(.{ ._r, .sa }, .isize, .{ .register = reg }, .u8, .{ .immediate = shift });
27798 },
27799 .unsigned => {
27800 const mask = ~@as(u64, 0) >> shift;
27801 if (int_info.bits <= 32) {
27802 try self.genBinOpMir(.{ ._, .@"and" }, .u32, .{ .register = reg }, .{ .immediate = mask });
27803 } else {
27804 const tmp_reg = try self.copyToTmpRegister(.usize, .{ .immediate = mask });
27805 try self.genBinOpMir(.{ ._, .@"and" }, .usize, .{ .register = reg }, .{ .register = tmp_reg });
27806 }
27807 },
27808 }
27809}
1898427810
18985 self.performReloc(mem_reloc);
18986 try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{});
18987 try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
18988 .base = .{ .reg = addr_reg },
18989 .mod = .{ .rm = .{
18990 .size = .qword,
18991 .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)),
18992 } },
18993 });
18994 try self.genCopy(
18995 ptr_anyopaque_ty,
18996 overflow_arg_area,
18997 .{ .register = offset_reg.to64() },
18998 .{},
18999 );
27811fn regBitSize(self: *CodeGen, ty: Type) u64 {
27812 const zcu = self.pt.zcu;
27813 const abi_size = ty.abiSize(zcu);
27814 return switch (ty.zigTypeTag(zcu)) {
27815 else => switch (abi_size) {
27816 1 => 8,
27817 2 => 16,
27818 3...4 => 32,
27819 5...8 => 64,
27820 else => unreachable,
27821 },
27822 .float => switch (abi_size) {
27823 1...16 => 128,
27824 17...32 => 256,
27825 else => unreachable,
27826 },
27827 };
27828}
1900027829
19001 self.performReloc(done_reloc);
19002 if (!unused) try self.genCopy(promote_ty, promote_mcv, .{
19003 .indirect = .{ .reg = addr_reg },
19004 }, .{});
19005 },
19006 .sse => {
19007 assert(classes.len == 1);
27830fn regExtraBits(self: *CodeGen, ty: Type) u64 {
27831 return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
27832}
1900827833
19009 try self.genSetReg(offset_reg, Type.c_uint, fp_offset, .{});
19010 try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(
19011 abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16,
19012 ));
19013 const mem_reloc = try self.asmJccReloc(.ae, undefined);
27834fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool {
27835 return switch (feature) {
27836 .@"64bit" => switch (cg.target.cpu.arch) {
27837 else => unreachable,
27838 .x86 => false,
27839 .x86_64 => true,
27840 },
27841 .false_deps_getmant,
27842 .false_deps_lzcnt_tzcnt,
27843 .false_deps_mulc,
27844 .false_deps_mullq,
27845 .false_deps_perm,
27846 .false_deps_popcnt,
27847 .false_deps_range,
27848 .slow_3ops_lea,
27849 .slow_incdec,
27850 .slow_lea,
27851 .slow_pmaddwd,
27852 .slow_pmulld,
27853 .slow_shld,
27854 .slow_two_mem_ops,
27855 .slow_unaligned_mem_16,
27856 .slow_unaligned_mem_32,
27857 => switch (cg.mod.optimize_mode) {
27858 .Debug, .ReleaseSafe, .ReleaseFast => null,
27859 .ReleaseSmall => false,
27860 },
27861 .fast_11bytenop,
27862 .fast_15bytenop,
27863 .fast_7bytenop,
27864 .fast_bextr,
27865 .fast_dpwssd,
27866 .fast_gather,
27867 .fast_hops,
27868 .fast_imm16,
27869 .fast_lzcnt,
27870 .fast_movbe,
27871 .fast_scalar_fsqrt,
27872 .fast_scalar_shift_masks,
27873 .fast_shld_rotate,
27874 .fast_variable_crosslane_shuffle,
27875 .fast_variable_perlane_shuffle,
27876 .fast_vector_fsqrt,
27877 .fast_vector_shift_masks,
27878 => switch (cg.mod.optimize_mode) {
27879 .Debug, .ReleaseSafe, .ReleaseFast => null,
27880 .ReleaseSmall => true,
27881 },
27882 .mmx => false,
27883 else => null,
27884 } orelse std.Target.x86.featureSetHas(cg.target.cpu.features, feature);
27885}
1901427886
19015 try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{});
19016 if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
19017 .base = .{ .reg = addr_reg },
19018 .mod = .{ .rm = .{
19019 .size = .qword,
19020 .index = offset_reg.to64(),
19021 } },
19022 });
19023 try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
19024 .base = .{ .reg = offset_reg.to64() },
19025 .mod = .{ .rm = .{
19026 .size = .qword,
19027 .disp = 16,
19028 } },
19029 });
19030 try self.genCopy(Type.c_uint, fp_offset, .{ .register = offset_reg }, .{});
19031 const done_reloc = try self.asmJmpReloc(undefined);
27887fn typeOf(self: *CodeGen, inst: Air.Inst.Ref) Type {
27888 const pt = self.pt;
27889 const zcu = pt.zcu;
27890 return self.air.typeOf(inst, &zcu.intern_pool);
27891}
1903227892
19033 self.performReloc(mem_reloc);
19034 try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{});
19035 try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
19036 .base = .{ .reg = addr_reg },
19037 .mod = .{ .rm = .{
19038 .size = .qword,
19039 .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)),
19040 } },
19041 });
19042 try self.genCopy(
19043 ptr_anyopaque_ty,
19044 overflow_arg_area,
19045 .{ .register = offset_reg.to64() },
19046 .{},
19047 );
27893fn typeOfIndex(self: *CodeGen, inst: Air.Inst.Index) Type {
27894 const pt = self.pt;
27895 const zcu = pt.zcu;
27896 const temp: Temp = .{ .index = inst };
27897 return switch (temp.unwrap(self)) {
27898 .ref => switch (self.air.instructions.items(.tag)[@intFromEnum(inst)]) {
27899 .loop_switch_br => self.typeOf(self.air.unwrapSwitch(inst).operand),
27900 else => self.air.typeOfIndex(inst, &zcu.intern_pool),
27901 },
27902 .temp => temp.typeOf(self),
27903 };
27904}
1904827905
19049 self.performReloc(done_reloc);
19050 if (!unused) try self.genCopy(promote_ty, promote_mcv, .{
19051 .indirect = .{ .reg = addr_reg },
19052 }, .{});
19053 },
19054 .memory => {
19055 assert(classes.len == 1);
19056 unreachable;
19057 },
19058 else => return self.fail("TODO implement c_va_arg for {} on SysV", .{
19059 promote_ty.fmt(pt),
19060 }),
19061 }
27906fn intCompilerRtAbiName(int_bits: u32) u8 {
27907 return switch (int_bits) {
27908 1...32 => 's',
27909 33...64 => 'd',
27910 65...128 => 't',
27911 else => unreachable,
27912 };
27913}
1906227914
19063 if (unused) break :result .unreach;
19064 if (ty.toIntern() == promote_ty.toIntern()) break :result promote_mcv;
27915fn floatCompilerRtAbiName(float_bits: u32) u8 {
27916 return switch (float_bits) {
27917 16 => 'h',
27918 32 => 's',
27919 64 => 'd',
27920 80 => 'x',
27921 128 => 't',
27922 else => unreachable,
27923 };
27924}
1906527925
19066 if (!promote_ty.isRuntimeFloat()) {
19067 const dst_mcv = try self.allocRegOrMem(inst, true);
19068 try self.genCopy(ty, dst_mcv, promote_mcv, .{});
19069 break :result dst_mcv;
19070 }
27926fn floatCompilerRtAbiType(self: *CodeGen, ty: Type, other_ty: Type) Type {
27927 if (ty.toIntern() == .f16_type and
27928 (other_ty.toIntern() == .f32_type or other_ty.toIntern() == .f64_type) and
27929 self.target.isDarwin()) return .u16;
27930 return ty;
27931}
1907127932
19072 assert(ty.toIntern() == .f32_type and promote_ty.toIntern() == .f64_type);
19073 const dst_mcv = if (promote_mcv.isRegister())
19074 promote_mcv
19075 else
19076 try self.copyToRegisterWithInstTracking(inst, ty, promote_mcv);
19077 const dst_reg = dst_mcv.getReg().?.to128();
19078 const dst_lock = self.register_manager.lockReg(dst_reg);
19079 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
27933fn floatLibcAbiPrefix(ty: Type) []const u8 {
27934 return switch (ty.toIntern()) {
27935 .f16_type, .f80_type => "__",
27936 .f32_type, .f64_type, .f128_type, .c_longdouble_type => "",
27937 else => unreachable,
27938 };
27939}
1908027940
19081 if (self.hasFeature(.avx)) if (promote_mcv.isMemory()) try self.asmRegisterRegisterMemory(
19082 .{ .v_ss, .cvtsd2 },
19083 dst_reg,
19084 dst_reg,
19085 try promote_mcv.mem(self, .qword),
19086 ) else try self.asmRegisterRegisterRegister(
19087 .{ .v_ss, .cvtsd2 },
19088 dst_reg,
19089 dst_reg,
19090 (if (promote_mcv.isRegister())
19091 promote_mcv.getReg().?
19092 else
19093 try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(),
19094 ) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(
19095 .{ ._ss, .cvtsd2 },
19096 dst_reg,
19097 try promote_mcv.mem(self, .qword),
19098 ) else try self.asmRegisterRegister(
19099 .{ ._ss, .cvtsd2 },
19100 dst_reg,
19101 (if (promote_mcv.isRegister())
19102 promote_mcv.getReg().?
19103 else
19104 try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(),
19105 );
19106 break :result promote_mcv;
19107 },
19108 .x86_64_win => return self.fail("TODO implement c_va_arg for Win64", .{}),
27941fn floatLibcAbiSuffix(ty: Type) []const u8 {
27942 return switch (ty.toIntern()) {
27943 .f16_type => "h",
27944 .f32_type => "f",
27945 .f64_type => "",
27946 .f80_type => "x",
27947 .f128_type => "q",
27948 .c_longdouble_type => "l",
1910927949 else => unreachable,
1911027950 };
19111 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1911227951}
1911327952
19114fn airVaCopy(self: *Self, inst: Air.Inst.Index) !void {
19115 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
19116 const ptr_va_list_ty = self.typeOf(ty_op.operand);
27953fn promoteInt(self: *CodeGen, ty: Type) Type {
27954 const pt = self.pt;
27955 const zcu = pt.zcu;
27956 const int_info: InternPool.Key.IntType = switch (ty.toIntern()) {
27957 .bool_type => .{ .signedness = .unsigned, .bits = 1 },
27958 else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty,
27959 };
27960 for ([_]Type{
27961 .c_int, .c_uint,
27962 .c_long, .c_ulong,
27963 .c_longlong, .c_ulonglong,
27964 }) |promote_ty| {
27965 const promote_info = promote_ty.intInfo(zcu);
27966 if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue;
27967 if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and
27968 promote_info.signedness == .signed) <= promote_info.bits) return promote_ty;
27969 }
27970 return ty;
27971}
1911727972
19118 const dst_mcv = try self.allocRegOrMem(inst, true);
19119 try self.load(dst_mcv, ptr_va_list_ty, .{ .air_ref = ty_op.operand });
19120 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
27973fn promoteVarArg(self: *CodeGen, ty: Type) Type {
27974 if (!ty.isRuntimeFloat()) return self.promoteInt(ty);
27975 switch (ty.floatBits(self.target.*)) {
27976 32, 64 => return .f64,
27977 else => |float_bits| {
27978 assert(float_bits == self.target.cTypeBitSize(.longdouble));
27979 return .c_longdouble;
27980 },
27981 }
1912127982}
1912227983
19123fn airVaEnd(self: *Self, inst: Air.Inst.Index) !void {
19124 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
19125 return self.finishAir(inst, .unreach, .{ un_op, .none, .none });
27984fn intInfo(ty: Type, cg: *CodeGen) ?std.builtin.Type.Int {
27985 const zcu = cg.pt.zcu;
27986 const ip = &zcu.intern_pool;
27987 var ty_index = ty.ip_index;
27988 while (true) switch (ip.indexToKey(ty_index)) {
27989 .int_type => |int_type| return int_type,
27990 .ptr_type => |ptr_type| return switch (ptr_type.flags.size) {
27991 .one, .many, .c => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
27992 .slice => null,
27993 },
27994 .opt_type => |opt_child| return if (!Type.fromInterned(opt_child).hasRuntimeBitsIgnoreComptime(zcu))
27995 .{ .signedness = .unsigned, .bits = 1 }
27996 else switch (ip.indexToKey(opt_child)) {
27997 .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
27998 .one, .many => switch (ptr_type.flags.is_allowzero) {
27999 false => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
28000 true => null,
28001 },
28002 .slice, .c => null,
28003 },
28004 else => null,
28005 },
28006 .error_union_type => |error_union_type| return if (!Type.fromInterned(error_union_type.payload_type)
28007 .hasRuntimeBitsIgnoreComptime(zcu)) .{ .signedness = .unsigned, .bits = zcu.errorSetBits() } else null,
28008 .simple_type => |simple_type| return switch (simple_type) {
28009 .bool => .{ .signedness = .unsigned, .bits = 1 },
28010 .anyerror => .{ .signedness = .unsigned, .bits = zcu.errorSetBits() },
28011 .isize => .{ .signedness = .signed, .bits = cg.target.ptrBitWidth() },
28012 .usize => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
28013 .c_char => .{ .signedness = cg.target.charSignedness(), .bits = cg.target.cTypeBitSize(.char) },
28014 .c_short => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.short) },
28015 .c_ushort => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.short) },
28016 .c_int => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.int) },
28017 .c_uint => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.int) },
28018 .c_long => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.long) },
28019 .c_ulong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.long) },
28020 .c_longlong => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.longlong) },
28021 .c_ulonglong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.longlong) },
28022 .f16, .f32, .f64, .f80, .f128, .c_longdouble => null,
28023 .anyopaque,
28024 .void,
28025 .type,
28026 .comptime_int,
28027 .comptime_float,
28028 .noreturn,
28029 .null,
28030 .undefined,
28031 .enum_literal,
28032 .adhoc_inferred_error_set,
28033 .generic_poison,
28034 => unreachable,
28035 },
28036 .struct_type => {
28037 const loaded_struct = ip.loadStructType(ty_index);
28038 switch (loaded_struct.layout) {
28039 .auto, .@"extern" => return null,
28040 .@"packed" => ty_index = loaded_struct.backingIntTypeUnordered(ip),
28041 }
28042 },
28043 .union_type => return switch (ip.loadUnionType(ty_index).flagsUnordered(ip).layout) {
28044 .auto, .@"extern" => null,
28045 .@"packed" => .{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) },
28046 },
28047 .enum_type => ty_index = ip.loadEnumType(ty_index).tag_ty,
28048 .error_set_type, .inferred_error_set_type => return .{ .signedness = .unsigned, .bits = zcu.errorSetBits() },
28049 else => return null,
28050 };
1912628051}
1912728052
19128fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
19129 const zcu = self.pt.zcu;
19130 const ty = self.typeOf(ref);
28053const Temp = struct {
28054 index: Air.Inst.Index,
1913128055
19132 // If the type has no codegen bits, no need to store it.
19133 if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return .none;
28056 fn unwrap(temp: Temp, cg: *CodeGen) union(enum) {
28057 ref: Air.Inst.Ref,
28058 temp: Index,
28059 } {
28060 switch (temp.index.unwrap()) {
28061 .ref => |ref| return .{ .ref = ref },
28062 .target => |target_index| {
28063 const temp_index: Index = @enumFromInt(target_index);
28064 assert(temp_index.isValid(cg));
28065 return .{ .temp = temp_index };
28066 },
28067 }
28068 }
1913428069
19135 const mcv = if (ref.toIndex()) |inst| mcv: {
19136 break :mcv self.inst_tracking.getPtr(inst).?.short;
19137 } else mcv: {
19138 const ip_index = ref.toInterned().?;
19139 const gop = try self.const_tracking.getOrPut(self.gpa, ip_index);
19140 if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(init: {
19141 const const_mcv = try self.genTypedValue(Value.fromInterned(ip_index));
19142 switch (const_mcv) {
19143 .lea_tlv => |tlv_sym| switch (self.bin_file.tag) {
19144 .elf, .macho => {
19145 if (self.mod.pic) {
19146 try self.spillRegisters(&.{ .rdi, .rax });
19147 } else {
19148 try self.spillRegisters(&.{.rax});
28070 fn typeOf(temp: Temp, cg: *CodeGen) Type {
28071 return switch (temp.unwrap(cg)) {
28072 .ref => |ref| cg.typeOf(ref),
28073 .temp => |temp_index| temp_index.typeOf(cg),
28074 };
28075 }
28076
28077 fn isMut(temp: Temp, cg: *CodeGen) bool {
28078 return switch (temp.unwrap(cg)) {
28079 .ref => false,
28080 .temp => |temp_index| switch (temp_index.tracking(cg).short) {
28081 .none,
28082 .unreach,
28083 .dead,
28084 .undef,
28085 .immediate,
28086 .eflags,
28087 .register_offset,
28088 .register_mask,
28089 .memory,
28090 .load_symbol,
28091 .lea_symbol,
28092 .indirect,
28093 .load_direct,
28094 .lea_direct,
28095 .load_got,
28096 .lea_got,
28097 .load_tlv,
28098 .lea_tlv,
28099 .lea_frame,
28100 .elementwise_regs_then_frame,
28101 .reserved_frame,
28102 .air_ref,
28103 => false,
28104 .register,
28105 .register_pair,
28106 .register_triple,
28107 .register_quadruple,
28108 .register_overflow,
28109 => true,
28110 .load_frame => |frame_addr| !frame_addr.index.isNamed(),
28111 },
28112 };
28113 }
28114
28115 fn tracking(temp: Temp, cg: *CodeGen) InstTracking {
28116 return cg.inst_tracking.get(temp.index).?;
28117 }
28118
28119 fn getOffset(temp: Temp, off: i32, cg: *CodeGen) !Temp {
28120 const new_temp_index = cg.next_temp_index;
28121 cg.temp_type[@intFromEnum(new_temp_index)] = .usize;
28122 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
28123 switch (temp.tracking(cg).short) {
28124 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28125 .register => |reg| {
28126 const new_reg =
28127 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
28128 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
28129 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
28130 .base = .{ .reg = reg.to64() },
28131 .mod = .{ .rm = .{
28132 .size = .qword,
28133 .disp = off,
28134 } },
28135 });
28136 },
28137 .register_offset => |reg_off| {
28138 const new_reg =
28139 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
28140 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
28141 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
28142 .base = .{ .reg = reg_off.reg.to64() },
28143 .mod = .{ .rm = .{
28144 .size = .qword,
28145 .disp = reg_off.off + off,
28146 } },
28147 });
28148 },
28149 .lea_symbol => |sym_off| new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = .{
28150 .sym_index = sym_off.sym_index,
28151 .off = sym_off.off + off,
28152 } }),
28153 .load_frame => |frame_addr| {
28154 const new_reg =
28155 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
28156 new_temp_index.tracking(cg).* = .init(.{ .register_offset = .{
28157 .reg = new_reg,
28158 .off = off,
28159 } });
28160 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
28161 .base = .{ .frame = frame_addr.index },
28162 .mod = .{ .rm = .{
28163 .size = .qword,
28164 .disp = frame_addr.off,
28165 } },
28166 });
28167 },
28168 .lea_frame => |frame_addr| new_temp_index.tracking(cg).* = .init(.{ .lea_frame = .{
28169 .index = frame_addr.index,
28170 .off = frame_addr.off + off,
28171 } }),
28172 }
28173 return .{ .index = new_temp_index.toIndex() };
28174 }
28175
28176 fn toOffset(temp: *Temp, off: i32, cg: *CodeGen) !void {
28177 if (off == 0) return;
28178 switch (temp.unwrap(cg)) {
28179 .ref => {},
28180 .temp => |temp_index| {
28181 const temp_tracking = temp_index.tracking(cg);
28182 switch (temp_tracking.short) {
28183 else => {},
28184 .register => |reg| {
28185 try cg.freeValue(temp_tracking.long);
28186 temp_tracking.* = .init(.{ .register_offset = .{
28187 .reg = reg,
28188 .off = off,
28189 } });
28190 return;
28191 },
28192 .register_offset => |reg_off| {
28193 try cg.freeValue(temp_tracking.long);
28194 temp_tracking.* = .init(.{ .register_offset = .{
28195 .reg = reg_off.reg,
28196 .off = reg_off.off + off,
28197 } });
28198 return;
28199 },
28200 .lea_symbol => |sym_off| {
28201 assert(std.meta.eql(temp_tracking.long.lea_symbol, sym_off));
28202 temp_tracking.* = .init(.{ .lea_symbol = .{
28203 .sym_index = sym_off.sym_index,
28204 .off = sym_off.off + off,
28205 } });
28206 return;
28207 },
28208 .lea_frame => |frame_addr| {
28209 assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr));
28210 temp_tracking.* = .init(.{ .lea_frame = .{
28211 .index = frame_addr.index,
28212 .off = frame_addr.off + off,
28213 } });
28214 return;
28215 },
28216 }
28217 },
28218 }
28219 const new_temp = try temp.getOffset(off, cg);
28220 try temp.die(cg);
28221 temp.* = new_temp;
28222 }
28223
28224 fn getLimb(temp: Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen) !Temp {
28225 const new_temp_index = cg.next_temp_index;
28226 cg.temp_type[@intFromEnum(new_temp_index)] = limb_ty;
28227 switch (temp.tracking(cg).short) {
28228 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28229 .immediate => |imm| {
28230 assert(limb_index == 0);
28231 new_temp_index.tracking(cg).* = .init(.{ .immediate = imm });
28232 },
28233 .register => |reg| {
28234 assert(limb_index == 0);
28235 const new_reg =
28236 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
28237 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
28238 try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());
28239 },
28240 .register_pair => |regs| {
28241 const new_reg =
28242 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
28243 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
28244 try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());
28245 },
28246 .register_offset => |reg_off| {
28247 assert(limb_index == 0);
28248 const new_reg =
28249 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
28250 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
28251 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
28252 .base = .{ .reg = reg_off.reg.to64() },
28253 .mod = .{ .rm = .{
28254 .size = .qword,
28255 .disp = reg_off.off + @as(u31, limb_index) * 8,
28256 } },
28257 });
28258 },
28259 .load_symbol => |sym_off| {
28260 const new_reg =
28261 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
28262 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
28263 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
28264 .base = .{ .reloc = sym_off.sym_index },
28265 .mod = .{ .rm = .{
28266 .size = .qword,
28267 .disp = sym_off.off + @as(u31, limb_index) * 8,
28268 } },
28269 });
28270 },
28271 .lea_symbol => |sym_off| {
28272 assert(limb_index == 0);
28273 new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = sym_off });
28274 },
28275 .load_frame => |frame_addr| {
28276 const new_reg =
28277 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
28278 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
28279 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
28280 .base = .{ .frame = frame_addr.index },
28281 .mod = .{ .rm = .{
28282 .size = .qword,
28283 .disp = frame_addr.off + @as(u31, limb_index) * 8,
28284 } },
28285 });
28286 },
28287 .lea_frame => |frame_addr| {
28288 assert(limb_index == 0);
28289 new_temp_index.tracking(cg).* = .init(.{ .lea_frame = frame_addr });
28290 },
28291 }
28292 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
28293 return .{ .index = new_temp_index.toIndex() };
28294 }
28295
28296 fn toLimb(temp: *Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen) !void {
28297 switch (temp.unwrap(cg)) {
28298 .ref => {},
28299 .temp => |temp_index| {
28300 const temp_tracking = temp_index.tracking(cg);
28301 switch (temp_tracking.short) {
28302 else => {},
28303 .register, .lea_symbol, .lea_frame => {
28304 assert(limb_index == 0);
28305 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
28306 return;
28307 },
28308 .register_pair => |regs| {
28309 switch (temp_tracking.long) {
28310 .none, .reserved_frame => {},
28311 else => temp_tracking.long =
28312 temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(),
1914928313 }
19150 const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
19151 .size = 8,
19152 .alignment = .@"8",
19153 }));
19154 try self.genSetMem(
19155 .{ .frame = frame_index },
19156 0,
19157 Type.usize,
19158 .{ .lea_symbol = .{ .sym_index = tlv_sym } },
19159 .{},
19160 );
19161 break :init .{ .load_frame = .{ .index = frame_index } };
28314 for (regs, 0..) |reg, reg_index| if (reg_index != limb_index)
28315 cg.register_manager.freeReg(reg);
28316 temp_tracking.* = .init(.{ .register = regs[limb_index] });
28317 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
28318 return;
1916228319 },
19163 else => break :init const_mcv,
19164 },
19165 else => break :init const_mcv,
19166 }
19167 });
19168 break :mcv gop.value_ptr.short;
19169 };
28320 .load_symbol => |sym_off| {
28321 assert(std.meta.eql(temp_tracking.long.load_symbol, sym_off));
28322 temp_tracking.* = .init(.{ .load_symbol = .{
28323 .sym_index = sym_off.sym_index,
28324 .off = sym_off.off + @as(u31, limb_index) * 8,
28325 } });
28326 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
28327 return;
28328 },
28329 .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) {
28330 assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr));
28331 temp_tracking.* = .init(.{ .load_frame = .{
28332 .index = frame_addr.index,
28333 .off = frame_addr.off + @as(u31, limb_index) * 8,
28334 } });
28335 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
28336 return;
28337 },
28338 }
28339 },
28340 }
28341 const new_temp = try temp.getLimb(limb_ty, limb_index, cg);
28342 try temp.die(cg);
28343 temp.* = new_temp;
28344 }
1917028345
19171 switch (mcv) {
19172 .none, .unreach, .dead => unreachable,
19173 else => return mcv,
28346 fn toSlicePtr(temp: *Temp, cg: *CodeGen) !void {
28347 const temp_ty = temp.typeOf(cg);
28348 if (temp_ty.isSlice(cg.pt.zcu)) try temp.toLimb(temp_ty.slicePtrFieldType(cg.pt.zcu), 0, cg);
28349 }
28350
28351 fn toSliceLen(temp: *Temp, cg: *CodeGen) !void {
28352 try temp.toLimb(.usize, 1, cg);
1917428353 }
19175}
1917628354
19177fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) *InstTracking {
19178 const tracking = self.inst_tracking.getPtr(inst).?;
19179 return switch (tracking.short) {
19180 .none, .unreach, .dead => unreachable,
19181 else => tracking,
19182 };
19183}
28355 fn toReg(temp: *Temp, new_reg: Register, cg: *CodeGen) !bool {
28356 const val, const ty = val_ty: switch (temp.unwrap(cg)) {
28357 .ref => |ref| .{ temp.tracking(cg).short, cg.typeOf(ref) },
28358 .temp => |temp_index| {
28359 const temp_tracking = temp_index.tracking(cg);
28360 if (temp_tracking.short == .register and
28361 temp_tracking.short.register == new_reg) return false;
28362 break :val_ty .{ temp_tracking.short, temp_index.typeOf(cg) };
28363 },
28364 };
28365 const new_temp_index = cg.next_temp_index;
28366 try cg.register_manager.getReg(new_reg, new_temp_index.toIndex());
28367 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
28368 try cg.genSetReg(new_reg, ty, val, .{});
28369 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
28370 try temp.die(cg);
28371 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
28372 temp.* = .{ .index = new_temp_index.toIndex() };
28373 return true;
28374 }
1918428375
19185/// If the MCValue is an immediate, and it does not fit within this type,
19186/// we put it in a register.
19187/// A potential opportunity for future optimization here would be keeping track
19188/// of the fact that the instruction is available both as an immediate
19189/// and as a register.
19190fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCValue {
19191 const mcv = try self.resolveInst(operand);
19192 const ti = @typeInfo(T).int;
19193 switch (mcv) {
19194 .immediate => |imm| {
19195 // This immediate is unsigned.
19196 const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed));
19197 if (imm >= math.maxInt(U)) {
19198 return MCValue{ .register = try self.copyToTmpRegister(Type.usize, mcv) };
19199 }
19200 },
19201 else => {},
28376 fn toRegClass(temp: *Temp, mut: bool, rc: Register.Class, cg: *CodeGen) !bool {
28377 const val = temp.tracking(cg).short;
28378 if (!mut or temp.isMut(cg)) switch (val) {
28379 else => {},
28380 .register => |reg| if (reg.class() == rc) return false,
28381 .register_offset => |reg_off| if (reg_off.reg.class() == rc and reg_off.off == 0) return false,
28382 };
28383 const ty = temp.typeOf(cg);
28384 const new_temp_index = cg.next_temp_index;
28385 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
28386 const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc));
28387 try cg.genSetReg(new_reg, ty, val, .{});
28388 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
28389 try temp.die(cg);
28390 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
28391 temp.* = .{ .index = new_temp_index.toIndex() };
28392 return true;
1920228393 }
19203 return mcv;
19204}
1920528394
19206fn genTypedValue(self: *Self, val: Value) InnerError!MCValue {
19207 const pt = self.pt;
19208 return switch (try codegen.genTypedValue(self.bin_file, pt, self.src_loc, val, self.target.*)) {
19209 .mcv => |mcv| switch (mcv) {
19210 .none => .none,
19211 .undef => .undef,
19212 .immediate => |imm| .{ .immediate = imm },
19213 .memory => |addr| .{ .memory = addr },
19214 .load_symbol => |sym_index| .{ .load_symbol = .{ .sym_index = sym_index } },
19215 .lea_symbol => |sym_index| .{ .lea_symbol = .{ .sym_index = sym_index } },
19216 .load_direct => |sym_index| .{ .load_direct = sym_index },
19217 .lea_direct => |sym_index| .{ .lea_direct = sym_index },
19218 .load_got => |sym_index| .{ .lea_got = sym_index },
19219 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
19220 },
19221 .fail => |msg| return self.failMsg(msg),
19222 };
19223}
28395 fn toPair(first_temp: *Temp, second_temp: *Temp, cg: *CodeGen) !void {
28396 while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| {
28397 if (try part_temp.toRegClass(true, .general_purpose, cg)) break;
28398 } else break;
28399 const first_temp_tracking = first_temp.unwrap(cg).temp.tracking(cg);
28400 const second_temp_tracking = second_temp.unwrap(cg).temp.tracking(cg);
28401 const result: MCValue = .{ .register_pair = .{
28402 first_temp_tracking.short.register,
28403 second_temp_tracking.short.register,
28404 } };
28405 const result_temp_index = cg.next_temp_index;
28406 const result_temp: Temp = .{ .index = result_temp_index.toIndex() };
28407 assert(cg.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
28408 assert(cg.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
28409 cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8;
28410 result_temp_index.tracking(cg).* = .init(result);
28411 first_temp.* = result_temp;
28412 }
1922428413
19225const CallMCValues = struct {
19226 args: []MCValue,
19227 return_value: InstTracking,
19228 stack_byte_count: u31,
19229 stack_align: Alignment,
19230 gp_count: u32,
19231 fp_count: u32,
28414 fn asMask(temp: Temp, info: MaskInfo, cg: *CodeGen) void {
28415 assert(info.scalar != .none);
28416 const mcv = &temp.unwrap(cg).temp.tracking(cg).short;
28417 const reg = mcv.register;
28418 mcv.* = .{ .register_mask = .{ .reg = reg, .info = info } };
28419 }
1923228420
19233 fn deinit(self: *CallMCValues, func: *Self) void {
19234 func.gpa.free(self.args);
19235 self.* = undefined;
28421 fn toLea(temp: *Temp, cg: *CodeGen) !bool {
28422 switch (temp.tracking(cg).short) {
28423 .none,
28424 .unreach,
28425 .dead,
28426 .undef,
28427 .eflags,
28428 .register_pair,
28429 .register_triple,
28430 .register_quadruple,
28431 .register_overflow,
28432 .register_mask,
28433 .elementwise_regs_then_frame,
28434 .reserved_frame,
28435 .air_ref,
28436 => unreachable, // not a valid pointer
28437 .immediate,
28438 .register,
28439 .register_offset,
28440 .lea_direct,
28441 .lea_got,
28442 .lea_tlv,
28443 .lea_frame,
28444 => return false,
28445 .memory,
28446 .indirect,
28447 .load_symbol,
28448 .load_direct,
28449 .load_got,
28450 .load_tlv,
28451 .load_frame,
28452 => return temp.toRegClass(true, .general_purpose, cg),
28453 .lea_symbol => |sym_off| {
28454 const off = sym_off.off;
28455 if (off == 0) return false;
28456 try temp.toOffset(-off, cg);
28457 while (try temp.toRegClass(true, .general_purpose, cg)) {}
28458 try temp.toOffset(off, cg);
28459 return true;
28460 },
28461 }
1923628462 }
19237};
1923828463
19239/// Caller must call `CallMCValues.deinit`.
19240fn resolveCallingConventionValues(
19241 self: *Self,
19242 fn_info: InternPool.Key.FuncType,
19243 var_args: []const Type,
19244 stack_frame_base: FrameIndex,
19245) !CallMCValues {
19246 const pt = self.pt;
19247 const zcu = pt.zcu;
19248 const ip = &zcu.intern_pool;
19249 const cc = fn_info.cc;
19250 const param_types = try self.gpa.alloc(Type, fn_info.param_types.len + var_args.len);
19251 defer self.gpa.free(param_types);
28464 fn toMemory(temp: *Temp, cg: *CodeGen) !bool {
28465 const temp_tracking = temp.tracking(cg);
28466 if (temp_tracking.short.isMemory()) return false;
28467 const new_temp_index = cg.next_temp_index;
28468 const ty = temp.typeOf(cg);
28469 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
28470 const new_frame_index = try cg.allocFrameIndex(.initSpill(ty, cg.pt.zcu));
28471 try cg.genSetMem(.{ .frame = new_frame_index }, 0, ty, temp_tracking.short, .{});
28472 new_temp_index.tracking(cg).* = .init(.{ .load_frame = .{ .index = new_frame_index } });
28473 try temp.die(cg);
28474 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
28475 temp.* = .{ .index = new_temp_index.toIndex() };
28476 return true;
28477 }
1925228478
19253 for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*dest, src| {
19254 dest.* = Type.fromInterned(src);
28479 // hack around linker relocation bugs
28480 fn toBase(temp: *Temp, cg: *CodeGen) !bool {
28481 const temp_tracking = temp.tracking(cg);
28482 if (temp_tracking.short.isBase()) return false;
28483 if (try temp.toMemory(cg)) return true;
28484 const new_temp_index = cg.next_temp_index;
28485 cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg);
28486 const new_reg =
28487 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
28488 try cg.genSetReg(new_reg, .usize, temp_tracking.short.address(), .{});
28489 new_temp_index.tracking(cg).* = .init(.{ .indirect = .{ .reg = new_reg } });
28490 try temp.die(cg);
28491 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
28492 temp.* = .{ .index = new_temp_index.toIndex() };
28493 return true;
1925528494 }
19256 for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty|
19257 param_ty.* = self.promoteVarArg(arg_ty);
1925828495
19259 var result: CallMCValues = .{
19260 .args = try self.gpa.alloc(MCValue, param_types.len),
19261 // These undefined values must be populated before returning from this function.
19262 .return_value = undefined,
19263 .stack_byte_count = 0,
19264 .stack_align = undefined,
19265 .gp_count = 0,
19266 .fp_count = 0,
28496 const AccessOptions = struct {
28497 disp: i32 = 0,
28498 safe: bool = false,
1926728499 };
19268 errdefer self.gpa.free(result.args);
19269
19270 const ret_ty = Type.fromInterned(fn_info.return_type);
1927128500
19272 const resolved_cc = abi.resolveCallingConvention(cc, self.target.*);
19273 switch (cc) {
19274 .naked => {
19275 assert(result.args.len == 0);
19276 result.return_value = InstTracking.init(.unreach);
19277 result.stack_align = .@"8";
19278 },
19279 .x86_64_sysv, .x86_64_win => |cc_opts| {
19280 var ret_int_reg_i: u32 = 0;
19281 var ret_sse_reg_i: u32 = 0;
19282 var param_int_reg_i: u32 = 0;
19283 var param_sse_reg_i: u32 = 0;
19284 result.stack_align = .fromByteUnits(cc_opts.incoming_stack_alignment orelse 16);
28501 fn load(ptr: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) !Temp {
28502 const val = try cg.tempAlloc(val_ty);
28503 try ptr.toOffset(opts.disp, cg);
28504 while (try ptr.toLea(cg)) {}
28505 const val_mcv = val.tracking(cg).short;
28506 switch (val_mcv) {
28507 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28508 .register => |val_reg| try ptr.loadReg(val_ty, registerAlias(
28509 val_reg,
28510 @intCast(val_ty.abiSize(cg.pt.zcu)),
28511 ), cg),
28512 inline .register_pair,
28513 .register_triple,
28514 .register_quadruple,
28515 => |val_regs| for (val_regs) |val_reg| {
28516 try ptr.loadReg(val_ty, val_reg, cg);
28517 try ptr.toOffset(@divExact(val_reg.bitSize(), 8), cg);
28518 while (try ptr.toLea(cg)) {}
28519 },
28520 .register_offset => |val_reg_off| switch (val_reg_off.off) {
28521 0 => try ptr.loadReg(val_ty, registerAlias(
28522 val_reg_off.reg,
28523 @intCast(val_ty.abiSize(cg.pt.zcu)),
28524 ), cg),
28525 else => unreachable,
28526 },
28527 .memory, .indirect, .load_frame, .load_symbol => {
28528 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
28529 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
28530 try val_ptr.memcpy(ptr, &len, cg);
28531 try val_ptr.die(cg);
28532 try len.die(cg);
28533 },
28534 }
28535 return val;
28536 }
1928528537
19286 switch (resolved_cc) {
19287 .x86_64_sysv => {},
19288 .x86_64_win => {
19289 // Align the stack to 16bytes before allocating shadow stack space (if any).
19290 result.stack_byte_count += @intCast(4 * Type.usize.abiSize(zcu));
28538 fn store(ptr: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) !void {
28539 const val_ty = val.typeOf(cg);
28540 try ptr.toOffset(opts.disp, cg);
28541 while (try ptr.toLea(cg)) {}
28542 val_to_gpr: while (true) : (while (try ptr.toLea(cg) or
28543 try val.toRegClass(false, .general_purpose, cg))
28544 {}) {
28545 const val_mcv = val.tracking(cg).short;
28546 switch (val_mcv) {
28547 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28548 .undef => if (opts.safe) {
28549 var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa });
28550 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
28551 try ptr.memset(&pat, &len, cg);
28552 try pat.die(cg);
28553 try len.die(cg);
28554 },
28555 .immediate => |val_imm| {
28556 const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31|
28557 .u(val_uimm31)
28558 else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32|
28559 .s(val_simm32)
28560 else
28561 continue :val_to_gpr;
28562 // hack around linker relocation bugs
28563 switch (ptr.tracking(cg).short) {
28564 else => {},
28565 .lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
28566 }
28567 try cg.asmMemoryImmediate(
28568 .{ ._, .mov },
28569 try ptr.tracking(cg).short.deref().mem(cg, .{
28570 .size = cg.memSize(val_ty),
28571 }),
28572 val_op,
28573 );
28574 },
28575 .eflags => |cc| {
28576 // hack around linker relocation bugs
28577 switch (ptr.tracking(cg).short) {
28578 else => {},
28579 .lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
28580 }
28581 try cg.asmSetccMemory(
28582 cc,
28583 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }),
28584 );
28585 },
28586 .register => |val_reg| try ptr.storeRegs(val_ty, &.{registerAlias(
28587 val_reg,
28588 @intCast(val_ty.abiSize(cg.pt.zcu)),
28589 )}, cg),
28590 inline .register_pair,
28591 .register_triple,
28592 .register_quadruple,
28593 => |val_regs| try ptr.storeRegs(val_ty, &val_regs, cg),
28594 .register_offset => |val_reg_off| switch (val_reg_off.off) {
28595 0 => try ptr.storeRegs(val_ty, &.{registerAlias(
28596 val_reg_off.reg,
28597 @intCast(val_ty.abiSize(cg.pt.zcu)),
28598 )}, cg),
28599 else => continue :val_to_gpr,
28600 },
28601 .register_overflow => |val_reg_ov| {
28602 const ip = &cg.pt.zcu.intern_pool;
28603 const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) {
28604 .tuple_type => |tuple_type| {
28605 const tuple_field_types = tuple_type.types.get(ip);
28606 assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type);
28607 break :first_ty tuple_field_types[0];
28608 },
28609 .opt_type => |opt_child| {
28610 assert(!val_ty.optionalReprIsPayload(cg.pt.zcu));
28611 break :first_ty opt_child;
28612 },
28613 else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }),
28614 });
28615 const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu));
28616 try ptr.storeRegs(first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg);
28617 try ptr.toOffset(first_size, cg);
28618 try cg.asmSetccMemory(
28619 val_reg_ov.eflags,
28620 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }),
28621 );
28622 },
28623 .lea_frame, .lea_symbol => continue :val_to_gpr,
28624 .memory, .indirect, .load_frame, .load_symbol => {
28625 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
28626 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
28627 try ptr.memcpy(&val_ptr, &len, cg);
28628 try val_ptr.die(cg);
28629 try len.die(cg);
1929128630 },
19292 else => unreachable,
1929328631 }
28632 break;
28633 }
28634 }
1929428635
19295 // Return values
19296 if (ret_ty.zigTypeTag(zcu) == .noreturn) {
19297 result.return_value = InstTracking.init(.unreach);
19298 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
19299 // TODO: is this even possible for C calling convention?
19300 result.return_value = InstTracking.init(.none);
19301 } else {
19302 var ret_tracking: [2]InstTracking = undefined;
19303 var ret_tracking_i: usize = 0;
28636 fn read(src: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) !Temp {
28637 var val = try cg.tempAlloc(val_ty);
28638 while (try src.toBase(cg)) {}
28639 const val_mcv = val.tracking(cg).short;
28640 switch (val_mcv) {
28641 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28642 .register => |val_reg| try src.readReg(opts.disp, val_ty, registerAlias(
28643 val_reg,
28644 @intCast(val_ty.abiSize(cg.pt.zcu)),
28645 ), cg),
28646 inline .register_pair, .register_triple, .register_quadruple => |val_regs| {
28647 var disp = opts.disp;
28648 for (val_regs) |val_reg| {
28649 try src.readReg(disp, val_ty, val_reg, cg);
28650 disp += @divExact(val_reg.bitSize(), 8);
28651 }
28652 },
28653 .register_offset => |val_reg_off| switch (val_reg_off.off) {
28654 0 => try src.readReg(opts.disp, val_ty, registerAlias(
28655 val_reg_off.reg,
28656 @intCast(val_ty.abiSize(cg.pt.zcu)),
28657 ), cg),
28658 else => unreachable,
28659 },
28660 .memory, .indirect, .load_frame, .load_symbol => {
28661 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
28662 var src_ptr =
28663 try cg.tempInit(.usize, src.tracking(cg).short.address().offset(opts.disp));
28664 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
28665 try val_ptr.memcpy(&src_ptr, &len, cg);
28666 try val_ptr.die(cg);
28667 try src_ptr.die(cg);
28668 try len.die(cg);
28669 },
28670 }
28671 return val;
28672 }
1930428673
19305 const classes = switch (resolved_cc) {
19306 .x86_64_sysv => mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none),
19307 .x86_64_win => &.{abi.classifyWindows(ret_ty, zcu)},
19308 else => unreachable,
19309 };
19310 for (classes) |class| switch (class) {
19311 .integer => {
19312 const ret_int_reg = registerAlias(
19313 abi.getCAbiIntReturnRegs(resolved_cc)[ret_int_reg_i],
19314 @intCast(@min(ret_ty.abiSize(zcu), 8)),
19315 );
19316 ret_int_reg_i += 1;
28674 fn write(dst: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) !void {
28675 const val_ty = val.typeOf(cg);
28676 while (try dst.toBase(cg)) {}
28677 val_to_gpr: while (true) : (while (try dst.toBase(cg) or
28678 try val.toRegClass(false, .general_purpose, cg))
28679 {}) {
28680 const val_mcv = val.tracking(cg).short;
28681 switch (val_mcv) {
28682 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28683 .undef => if (opts.safe) {
28684 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
28685 var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa });
28686 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
28687 try dst_ptr.memset(&pat, &len, cg);
28688 try dst_ptr.die(cg);
28689 try pat.die(cg);
28690 try len.die(cg);
28691 },
28692 .immediate => |val_imm| {
28693 const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31|
28694 .u(val_uimm31)
28695 else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32|
28696 .s(val_simm32)
28697 else
28698 continue :val_to_gpr;
28699 try cg.asmMemoryImmediate(
28700 .{ ._, .mov },
28701 try dst.tracking(cg).short.mem(cg, .{
28702 .size = cg.memSize(val_ty),
28703 .disp = opts.disp,
28704 }),
28705 val_op,
28706 );
28707 },
28708 .eflags => |cc| try cg.asmSetccMemory(
28709 cc,
28710 try dst.tracking(cg).short.mem(cg, .{
28711 .size = .byte,
28712 .disp = opts.disp,
28713 }),
28714 ),
28715 .register => |val_reg| try dst.writeRegs(opts.disp, val_ty, &.{registerAlias(
28716 val_reg,
28717 @intCast(val_ty.abiSize(cg.pt.zcu)),
28718 )}, cg),
28719 inline .register_pair,
28720 .register_triple,
28721 .register_quadruple,
28722 => |val_regs| try dst.writeRegs(opts.disp, val_ty, &val_regs, cg),
28723 .register_offset => |val_reg_off| switch (val_reg_off.off) {
28724 0 => try dst.writeRegs(opts.disp, val_ty, &.{registerAlias(
28725 val_reg_off.reg,
28726 @intCast(val_ty.abiSize(cg.pt.zcu)),
28727 )}, cg),
28728 else => continue :val_to_gpr,
28729 },
28730 .register_overflow => |val_reg_ov| {
28731 const ip = &cg.pt.zcu.intern_pool;
28732 const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) {
28733 .tuple_type => |tuple_type| {
28734 const tuple_field_types = tuple_type.types.get(ip);
28735 assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type);
28736 break :first_ty tuple_field_types[0];
28737 },
28738 .opt_type => |opt_child| {
28739 assert(!val_ty.optionalReprIsPayload(cg.pt.zcu));
28740 break :first_ty opt_child;
28741 },
28742 else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }),
28743 });
28744 const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu));
28745 try dst.writeRegs(opts.disp, first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg);
28746 try cg.asmSetccMemory(
28747 val_reg_ov.eflags,
28748 try dst.tracking(cg).short.mem(cg, .{
28749 .size = .byte,
28750 .disp = opts.disp + first_size,
28751 }),
28752 );
28753 },
28754 .lea_frame, .lea_symbol => continue :val_to_gpr,
28755 .memory, .indirect, .load_frame, .load_symbol => {
28756 var dst_ptr =
28757 try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
28758 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
28759 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
28760 try dst_ptr.memcpy(&val_ptr, &len, cg);
28761 try dst_ptr.die(cg);
28762 try val_ptr.die(cg);
28763 try len.die(cg);
28764 },
28765 }
28766 break;
28767 }
28768 }
1931728769
19318 ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_int_reg });
19319 ret_tracking_i += 1;
19320 },
19321 .sse, .float, .float_combine, .win_i128 => {
19322 const ret_sse_reg = registerAlias(
19323 abi.getCAbiSseReturnRegs(resolved_cc)[ret_sse_reg_i],
19324 @intCast(ret_ty.abiSize(zcu)),
19325 );
19326 ret_sse_reg_i += 1;
28770 fn loadReg(ptr: *Temp, dst_ty: Type, dst_reg: Register, cg: *CodeGen) !void {
28771 const dst_rc = dst_reg.class();
28772 const strat = try cg.moveStrategy(dst_ty, dst_rc, false);
28773 // hack around linker relocation bugs
28774 switch (ptr.tracking(cg).short) {
28775 else => {},
28776 .lea_symbol => |sym_off| if (dst_rc != .general_purpose or sym_off.off != 0)
28777 while (try ptr.toRegClass(false, .general_purpose, cg)) {},
28778 }
28779 try strat.read(cg, dst_reg, try ptr.tracking(cg).short.deref().mem(cg, .{
28780 .size = .fromBitSize(@min(8 * dst_ty.abiSize(cg.pt.zcu), dst_reg.bitSize())),
28781 }));
28782 }
1932728783
19328 ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_sse_reg });
19329 ret_tracking_i += 1;
19330 },
19331 .sseup => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .sse),
19332 .x87 => {
19333 ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = .st0 });
19334 ret_tracking_i += 1;
19335 },
19336 .x87up => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .x87),
19337 .complex_x87 => {
19338 ret_tracking[ret_tracking_i] =
19339 InstTracking.init(.{ .register_pair = .{ .st0, .st1 } });
19340 ret_tracking_i += 1;
19341 },
19342 .memory => {
19343 const ret_int_reg = abi.getCAbiIntReturnRegs(resolved_cc)[ret_int_reg_i].to64();
19344 ret_int_reg_i += 1;
19345 const ret_indirect_reg = abi.getCAbiIntParamRegs(resolved_cc)[param_int_reg_i];
19346 param_int_reg_i += 1;
28784 fn storeRegs(ptr: *Temp, src_ty: Type, src_regs: []const Register, cg: *CodeGen) !void {
28785 var part_disp: u31 = 0;
28786 var deferred_disp: u31 = 0;
28787 var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu));
28788 for (src_regs) |src_reg| {
28789 const src_rc = src_reg.class();
28790 const part_bit_size = @min(8 * src_abi_size, src_reg.bitSize());
28791 const part_size = @divExact(part_bit_size, 8);
28792 if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) {
28793 // hack around linker relocation bugs
28794 switch (ptr.tracking(cg).short) {
28795 else => {},
28796 .lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
28797 }
28798 const strat = try cg.moveStrategy(src_ty, src_rc, false);
28799 try strat.write(cg, try ptr.tracking(cg).short.deref().mem(cg, .{
28800 .size = .fromBitSize(part_bit_size),
28801 .disp = part_disp,
28802 }), registerAlias(src_reg, part_size));
28803 } else {
28804 const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size);
28805 const frame_index = try cg.allocFrameIndex(.init(.{
28806 .size = frame_size,
28807 .alignment = .fromNonzeroByteUnits(frame_size),
28808 }));
28809 const strat = try cg.moveStrategy(src_ty, src_rc, true);
28810 try strat.write(cg, .{
28811 .base = .{ .frame = frame_index },
28812 .mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
28813 }, registerAlias(src_reg, frame_size));
28814 try ptr.toOffset(deferred_disp, cg);
28815 deferred_disp = 0;
28816 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
28817 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
28818 try ptr.memcpy(&src_ptr, &len, cg);
28819 try src_ptr.die(cg);
28820 try len.die(cg);
28821 }
28822 part_disp += part_size;
28823 deferred_disp += part_size;
28824 src_abi_size -= part_size;
28825 }
28826 }
1934728827
19348 ret_tracking[ret_tracking_i] = .{
19349 .short = .{ .indirect = .{ .reg = ret_int_reg } },
19350 .long = .{ .indirect = .{ .reg = ret_indirect_reg } },
19351 };
19352 ret_tracking_i += 1;
19353 },
19354 .none, .integer_per_element => unreachable,
19355 };
19356 result.return_value = switch (ret_tracking_i) {
19357 else => unreachable,
19358 1 => ret_tracking[0],
19359 2 => InstTracking.init(.{ .register_pair = .{
19360 ret_tracking[0].short.register, ret_tracking[1].short.register,
19361 } }),
19362 };
28828 fn readReg(src: Temp, disp: i32, dst_ty: Type, dst_reg: Register, cg: *CodeGen) !void {
28829 const strat = try cg.moveStrategy(dst_ty, dst_reg.class(), false);
28830 try strat.read(cg, dst_reg, try src.tracking(cg).short.mem(cg, .{
28831 .size = .fromBitSize(@min(8 * dst_ty.abiSize(cg.pt.zcu), dst_reg.bitSize())),
28832 .disp = disp,
28833 }));
28834 }
28835
28836 fn writeRegs(dst: Temp, disp: i32, src_ty: Type, src_regs: []const Register, cg: *CodeGen) !void {
28837 var part_disp = disp;
28838 var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu));
28839 for (src_regs) |src_reg| {
28840 const src_rc = src_reg.class();
28841 const part_bit_size = @min(8 * src_abi_size, src_reg.bitSize());
28842 const part_size = @divExact(part_bit_size, 8);
28843 if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) {
28844 const strat = try cg.moveStrategy(src_ty, src_rc, false);
28845 try strat.write(cg, try dst.tracking(cg).short.mem(cg, .{
28846 .size = .fromBitSize(part_bit_size),
28847 .disp = part_disp,
28848 }), registerAlias(src_reg, part_size));
28849 } else {
28850 const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size);
28851 const frame_index = try cg.allocFrameIndex(.init(.{
28852 .size = frame_size,
28853 .alignment = .fromNonzeroByteUnits(frame_size),
28854 }));
28855 const strat = try cg.moveStrategy(src_ty, src_rc, true);
28856 try strat.write(cg, .{
28857 .base = .{ .frame = frame_index },
28858 .mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
28859 }, registerAlias(src_reg, frame_size));
28860 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address());
28861 try dst_ptr.toOffset(part_disp, cg);
28862 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
28863 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
28864 try dst_ptr.memcpy(&src_ptr, &len, cg);
28865 try dst_ptr.die(cg);
28866 try src_ptr.die(cg);
28867 try len.die(cg);
1936328868 }
28869 part_disp += part_size;
28870 src_abi_size -= part_size;
28871 }
28872 }
1936428873
19365 // Input params
19366 for (param_types, result.args) |ty, *arg| {
19367 assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
19368 switch (resolved_cc) {
19369 .x86_64_sysv => {},
19370 .x86_64_win => {
19371 param_int_reg_i = @max(param_int_reg_i, param_sse_reg_i);
19372 param_sse_reg_i = param_int_reg_i;
19373 },
19374 else => unreachable,
19375 }
28874 fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) !void {
28875 while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| {
28876 if (try temp.toReg(reg, cg)) break;
28877 } else break;
28878 try cg.asmOpOnly(.{ .@"rep _sb", .mov });
28879 }
1937628880
19377 var arg_mcv: [2]MCValue = undefined;
19378 var arg_mcv_i: usize = 0;
28881 fn memset(dst: *Temp, val: *Temp, len: *Temp, cg: *CodeGen) !void {
28882 while (true) for ([_]*Temp{ dst, val, len }, [_]Register{ .rdi, .rax, .rcx }) |temp, reg| {
28883 if (try temp.toReg(reg, cg)) break;
28884 } else break;
28885 try cg.asmOpOnly(.{ .@"rep _sb", .sto });
28886 }
1937928887
19380 const classes = switch (resolved_cc) {
19381 .x86_64_sysv => mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none),
19382 .x86_64_win => &.{abi.classifyWindows(ty, zcu)},
19383 else => unreachable,
19384 };
19385 for (classes) |class| switch (class) {
19386 .integer => {
19387 const param_int_regs = abi.getCAbiIntParamRegs(resolved_cc);
19388 if (param_int_reg_i >= param_int_regs.len) break;
28888 fn moveTo(temp: Temp, inst: Air.Inst.Index, cg: *CodeGen) !void {
28889 if (cg.liveness.isUnused(inst)) try temp.die(cg) else switch (temp.unwrap(cg)) {
28890 .ref => {
28891 const result = try cg.allocRegOrMem(inst, true);
28892 try cg.genCopy(cg.typeOfIndex(inst), result, temp.tracking(cg).short, .{});
28893 tracking_log.debug("{} => {} (birth)", .{ inst, result });
28894 cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));
28895 },
28896 .temp => |temp_index| {
28897 const temp_tracking = temp_index.tracking(cg);
28898 tracking_log.debug("{} => {} (birth)", .{ inst, temp_tracking.short });
28899 cg.inst_tracking.putAssumeCapacityNoClobber(inst, temp_tracking.*);
28900 assert(cg.reuseTemp(inst, temp_index.toIndex(), temp_tracking));
28901 },
28902 }
28903 }
1938928904
19390 const param_int_reg = registerAlias(
19391 abi.getCAbiIntParamRegs(resolved_cc)[param_int_reg_i],
19392 @intCast(@min(ty.abiSize(zcu), 8)),
19393 );
19394 param_int_reg_i += 1;
28905 fn die(temp: Temp, cg: *CodeGen) !void {
28906 switch (temp.unwrap(cg)) {
28907 .ref => {},
28908 .temp => |temp_index| try temp_index.tracking(cg).die(cg, temp_index.toIndex()),
28909 }
28910 }
1939528911
19396 arg_mcv[arg_mcv_i] = .{ .register = param_int_reg };
19397 arg_mcv_i += 1;
19398 },
19399 .sse, .float, .float_combine => {
19400 const param_sse_regs = abi.getCAbiSseParamRegs(resolved_cc);
19401 if (param_sse_reg_i >= param_sse_regs.len) break;
28912 const Index = enum(u4) {
28913 _,
1940228914
19403 const param_sse_reg = registerAlias(
19404 abi.getCAbiSseParamRegs(resolved_cc)[param_sse_reg_i],
19405 @intCast(ty.abiSize(zcu)),
19406 );
19407 param_sse_reg_i += 1;
28915 fn toIndex(index: Index) Air.Inst.Index {
28916 return .fromTargetIndex(@intFromEnum(index));
28917 }
1940828918
19409 arg_mcv[arg_mcv_i] = .{ .register = param_sse_reg };
19410 arg_mcv_i += 1;
19411 },
19412 .sseup => assert(arg_mcv[arg_mcv_i - 1].register.class() == .sse),
19413 .x87, .x87up, .complex_x87, .memory, .win_i128 => switch (resolved_cc) {
19414 .x86_64_sysv => switch (class) {
19415 .x87, .x87up, .complex_x87, .memory => break,
19416 else => unreachable,
19417 },
19418 .x86_64_win => if (ty.abiSize(zcu) > 8) {
19419 const param_int_reg =
19420 abi.getCAbiIntParamRegs(resolved_cc)[param_int_reg_i].to64();
19421 param_int_reg_i += 1;
28919 fn fromIndex(index: Air.Inst.Index) Index {
28920 return @enumFromInt(index.toTargetIndex());
28921 }
1942228922
19423 arg_mcv[arg_mcv_i] = .{ .indirect = .{ .reg = param_int_reg } };
19424 arg_mcv_i += 1;
19425 } else break,
19426 else => unreachable,
19427 },
19428 .none => unreachable,
19429 .integer_per_element => {
19430 const param_int_regs_len: u32 =
19431 @intCast(abi.getCAbiIntParamRegs(resolved_cc).len);
19432 const remaining_param_int_regs: u3 =
19433 @intCast(param_int_regs_len - param_int_reg_i);
19434 param_int_reg_i = param_int_regs_len;
28923 fn tracking(index: Index, cg: *CodeGen) *InstTracking {
28924 return &cg.inst_tracking.values()[@intFromEnum(index)];
28925 }
1943528926
19436 const frame_elem_align = 8;
19437 const frame_elems_len = ty.vectorLen(zcu) - remaining_param_int_regs;
19438 const frame_elem_size = mem.alignForward(
19439 u64,
19440 ty.childType(zcu).abiSize(zcu),
19441 frame_elem_align,
19442 );
19443 const frame_size: u31 = @intCast(frame_elems_len * frame_elem_size);
28927 fn isValid(index: Index, cg: *CodeGen) bool {
28928 return index.tracking(cg).short != .dead;
28929 }
1944428930
19445 result.stack_byte_count =
19446 mem.alignForward(u31, result.stack_byte_count, frame_elem_align);
19447 arg_mcv[arg_mcv_i] = .{ .elementwise_regs_then_frame = .{
19448 .regs = remaining_param_int_regs,
19449 .frame_off = @intCast(result.stack_byte_count),
19450 .frame_index = stack_frame_base,
19451 } };
19452 arg_mcv_i += 1;
19453 result.stack_byte_count += frame_size;
19454 },
19455 } else {
19456 arg.* = switch (arg_mcv_i) {
19457 else => unreachable,
19458 1 => arg_mcv[0],
19459 2 => .{ .register_pair = .{ arg_mcv[0].register, arg_mcv[1].register } },
19460 };
19461 continue;
19462 }
28931 fn typeOf(index: Index, cg: *CodeGen) Type {
28932 assert(index.isValid(cg));
28933 return cg.temp_type[@intFromEnum(index)];
28934 }
1946328935
19464 const param_size: u31 = @intCast(ty.abiSize(zcu));
19465 const param_align = ty.abiAlignment(zcu).max(.@"8");
19466 result.stack_byte_count = mem.alignForward(
19467 u31,
19468 result.stack_byte_count,
19469 @intCast(param_align.toByteUnits().?),
19470 );
19471 result.stack_align = result.stack_align.max(param_align);
19472 arg.* = .{ .load_frame = .{
19473 .index = stack_frame_base,
19474 .off = result.stack_byte_count,
19475 } };
19476 result.stack_byte_count += param_size;
28936 const max = std.math.maxInt(@typeInfo(Index).@"enum".tag_type);
28937 const Set = std.StaticBitSet(max);
28938 const SafetySet = if (std.debug.runtime_safety) Set else struct {
28939 inline fn initEmpty() @This() {
28940 return .{};
1947728941 }
19478 assert(param_int_reg_i <= 6);
19479 result.gp_count = param_int_reg_i;
19480 assert(param_sse_reg_i <= 16);
19481 result.fp_count = param_sse_reg_i;
19482 },
19483 .auto => {
19484 result.stack_align = .@"16";
1948528942
19486 // Return values
19487 if (ret_ty.zigTypeTag(zcu) == .noreturn) {
19488 result.return_value = InstTracking.init(.unreach);
19489 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
19490 result.return_value = InstTracking.init(.none);
19491 } else {
19492 const ret_reg = abi.getCAbiIntReturnRegs(resolved_cc)[0];
19493 const ret_ty_size: u31 = @intCast(ret_ty.abiSize(zcu));
19494 if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {
19495 const aliased_reg = registerAlias(ret_reg, ret_ty_size);
19496 result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
19497 } else {
19498 const ret_indirect_reg = abi.getCAbiIntParamRegs(resolved_cc)[0];
19499 result.return_value = .{
19500 .short = .{ .indirect = .{ .reg = ret_reg } },
19501 .long = .{ .indirect = .{ .reg = ret_indirect_reg } },
19502 };
19503 }
28943 inline fn isSet(_: @This(), index: usize) bool {
28944 assert(index < max);
28945 return true;
1950428946 }
1950528947
19506 // Input params
19507 for (param_types, result.args) |ty, *arg| {
19508 if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) {
19509 arg.* = .none;
19510 continue;
19511 }
19512 const param_size: u31 = @intCast(ty.abiSize(zcu));
19513 const param_align = ty.abiAlignment(zcu);
19514 result.stack_byte_count = mem.alignForward(
19515 u31,
19516 result.stack_byte_count,
19517 @intCast(param_align.toByteUnits().?),
19518 );
19519 result.stack_align = result.stack_align.max(param_align);
19520 arg.* = .{ .load_frame = .{
19521 .index = stack_frame_base,
19522 .off = result.stack_byte_count,
19523 } };
19524 result.stack_byte_count += param_size;
28948 inline fn set(_: @This(), index: usize) void {
28949 assert(index < max);
28950 }
28951
28952 inline fn eql(_: @This(), _: @This()) bool {
28953 return true;
28954 }
28955 };
28956 };
28957};
28958
28959fn resetTemps(cg: *CodeGen) void {
28960 for (0..@intFromEnum(cg.next_temp_index)) |temp_index| {
28961 const temp: Temp.Index = @enumFromInt(temp_index);
28962 assert(!temp.isValid(cg));
28963 cg.temp_type[temp_index] = undefined;
28964 }
28965 cg.next_temp_index = @enumFromInt(0);
28966}
28967
28968fn reuseTemp(
28969 cg: *CodeGen,
28970 new_inst: Air.Inst.Index,
28971 old_inst: Air.Inst.Index,
28972 tracking: *InstTracking,
28973) bool {
28974 switch (tracking.short) {
28975 .register,
28976 .register_pair,
28977 .register_offset,
28978 .register_overflow,
28979 .register_mask,
28980 .indirect,
28981 => for (tracking.short.getRegs()) |tracked_reg| {
28982 if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) |tracked_index| {
28983 cg.register_manager.registers[tracked_index] = new_inst;
1952528984 }
1952628985 },
19527 else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),
28986 .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,
28987 else => {},
28988 }
28989 switch (tracking.short) {
28990 .eflags, .register_overflow => cg.eflags_inst = new_inst,
28991 else => {},
1952828992 }
28993 tracking.reuse(cg, new_inst, old_inst);
28994 return true;
28995}
1952928996
19530 result.stack_byte_count = @intCast(result.stack_align.forward(result.stack_byte_count));
19531 return result;
28997fn tempAlloc(cg: *CodeGen, ty: Type) !Temp {
28998 const temp_index = cg.next_temp_index;
28999 temp_index.tracking(cg).* = .init(
29000 try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true),
29001 );
29002 cg.temp_type[@intFromEnum(temp_index)] = ty;
29003 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
29004 return .{ .index = temp_index.toIndex() };
1953229005}
1953329006
19534fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
19535 @branchHint(.cold);
19536 const zcu = self.pt.zcu;
19537 switch (self.owner) {
19538 .nav_index => |i| return zcu.codegenFail(i, format, args),
19539 .lazy_sym => |s| return zcu.codegenFailType(s.ty, format, args),
19540 }
19541 return error.CodegenFail;
29007fn tempAllocReg(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet) !Temp {
29008 const temp_index = cg.next_temp_index;
29009 temp_index.tracking(cg).* = .init(
29010 .{ .register = try cg.register_manager.allocReg(temp_index.toIndex(), rs) },
29011 );
29012 cg.temp_type[@intFromEnum(temp_index)] = ty;
29013 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
29014 return .{ .index = temp_index.toIndex() };
1954229015}
1954329016
19544fn failMsg(self: *Self, msg: *ErrorMsg) error{ OutOfMemory, CodegenFail } {
19545 @branchHint(.cold);
19546 const zcu = self.pt.zcu;
19547 switch (self.owner) {
19548 .nav_index => |i| return zcu.codegenFailMsg(i, msg),
19549 .lazy_sym => |s| return zcu.codegenFailTypeMsg(s.ty, msg),
19550 }
19551 return error.CodegenFail;
29017fn tempAllocRegPair(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet) !Temp {
29018 const temp_index = cg.next_temp_index;
29019 temp_index.tracking(cg).* = .init(
29020 .{ .register_pair = try cg.register_manager.allocRegs(2, temp_index.toIndex(), rs) },
29021 );
29022 cg.temp_type[@intFromEnum(temp_index)] = ty;
29023 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
29024 return .{ .index = temp_index.toIndex() };
1955229025}
1955329026
19554fn parseRegName(name: []const u8) ?Register {
19555 if (@hasDecl(Register, "parseRegName")) {
19556 return Register.parseRegName(name);
29027fn tempAllocMem(cg: *CodeGen, ty: Type) !Temp {
29028 const temp_index = cg.next_temp_index;
29029 temp_index.tracking(cg).* = .init(
29030 try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), false),
29031 );
29032 cg.temp_type[@intFromEnum(temp_index)] = ty;
29033 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
29034 return .{ .index = temp_index.toIndex() };
29035}
29036
29037fn tempInit(cg: *CodeGen, ty: Type, value: MCValue) !Temp {
29038 const temp_index = cg.next_temp_index;
29039 temp_index.tracking(cg).* = .init(value);
29040 cg.temp_type[@intFromEnum(temp_index)] = ty;
29041 try cg.getValue(value, temp_index.toIndex());
29042 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
29043 return .{ .index = temp_index.toIndex() };
29044}
29045
29046fn tempFromValue(cg: *CodeGen, value: Value) !Temp {
29047 return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.genTypedValue(value));
29048}
29049
29050fn tempFromOperand(
29051 cg: *CodeGen,
29052 inst: Air.Inst.Index,
29053 op_index: Liveness.OperandInt,
29054 op_ref: Air.Inst.Ref,
29055 ignore_death: bool,
29056) !Temp {
29057 const zcu = cg.pt.zcu;
29058 const ip = &zcu.intern_pool;
29059
29060 if (ignore_death or !cg.liveness.operandDies(inst, op_index)) {
29061 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };
29062 const val = op_ref.toInterned().?;
29063 const gop = try cg.const_tracking.getOrPut(cg.gpa, val);
29064 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
29065 const const_mcv = try cg.genTypedValue(.fromInterned(val));
29066 switch (const_mcv) {
29067 .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) {
29068 .elf, .macho => {
29069 if (cg.mod.pic) {
29070 try cg.spillRegisters(&.{ .rdi, .rax });
29071 } else {
29072 try cg.spillRegisters(&.{.rax});
29073 }
29074 const frame_index = try cg.allocFrameIndex(.init(.{
29075 .size = 8,
29076 .alignment = .@"8",
29077 }));
29078 try cg.genSetMem(
29079 .{ .frame = frame_index },
29080 0,
29081 .usize,
29082 .{ .lea_symbol = .{ .sym_index = tlv_sym } },
29083 .{},
29084 );
29085 break :init .{ .load_frame = .{ .index = frame_index } };
29086 },
29087 else => break :init const_mcv,
29088 },
29089 else => break :init const_mcv,
29090 }
29091 });
29092 return cg.tempInit(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
1955729093 }
19558 return std.meta.stringToEnum(Register, name);
29094
29095 const temp_index = cg.next_temp_index;
29096 const temp: Temp = .{ .index = temp_index.toIndex() };
29097 const op_inst = op_ref.toIndex().?;
29098 const tracking = cg.getResolvedInstValue(op_inst);
29099 temp_index.tracking(cg).* = tracking.*;
29100 if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
29101 cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref);
29102 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
29103 return temp;
29104}
29105
29106inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp {
29107 var temps: [op_refs.len]Temp = undefined;
29108 inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| {
29109 temp.* = try cg.tempFromOperand(inst, op_index, op_ref, inline for (0..op_index) |prev_op_index| {
29110 if (op_ref == op_refs[prev_op_index]) break true;
29111 } else false);
29112 }
29113 return temps;
1955929114}
1956029115
19561/// Returns register wide enough to hold at least `size_bytes`.
19562fn registerAlias(reg: Register, size_bytes: u32) Register {
19563 return switch (reg.class()) {
19564 .general_purpose => if (size_bytes == 0)
19565 unreachable // should be comptime-known
19566 else if (size_bytes <= 1)
19567 reg.to8()
19568 else if (size_bytes <= 2)
19569 reg.to16()
19570 else if (size_bytes <= 4)
19571 reg.to32()
19572 else if (size_bytes <= 8)
19573 reg.to64()
19574 else
19575 unreachable,
19576 .segment => if (size_bytes <= 2)
19577 reg
19578 else
19579 unreachable,
19580 .x87 => if (size_bytes == 16)
19581 reg
19582 else
19583 unreachable,
19584 .mmx => if (size_bytes <= 8)
19585 reg
19586 else
19587 unreachable,
19588 .sse => if (size_bytes <= 16)
19589 reg.to128()
19590 else if (size_bytes <= 32)
19591 reg.to256()
19592 else
19593 unreachable,
19594 .ip => if (size_bytes <= 2)
19595 .ip
19596 else if (size_bytes <= 4)
19597 .eip
19598 else if (size_bytes <= 8)
19599 .rip
19600 else
19601 unreachable,
29116const Operand = union(enum) {
29117 none,
29118 reg: Register,
29119 mem: Memory,
29120 imm: Immediate,
29121 inst: Mir.Inst.Index,
29122};
29123
29124const Select = struct {
29125 cg: *CodeGen,
29126 temps: [@intFromEnum(Select.Operand.Ref.none)]Temp,
29127 labels: [@intFromEnum(Label._)]struct {
29128 backward: ?Mir.Inst.Index,
29129 forward: [1]?Mir.Inst.Index,
29130 },
29131
29132 fn emitLabel(s: *Select, label_index: Label) void {
29133 if (label_index == ._) return;
29134 const label = &s.labels[@intFromEnum(label_index)];
29135 for (&label.forward) |*reloc| {
29136 if (reloc.*) |r| s.cg.performReloc(r);
29137 reloc.* = null;
29138 }
29139 label.backward = @intCast(s.cg.mir_instructions.len);
29140 }
29141
29142 fn emit(s: *Select, inst: Instruction) !void {
29143 s.emitLabel(inst[0]);
29144 const mir_tag: Mir.Inst.FixedTag = .{ inst[1], inst[2] };
29145 var mir_ops: [4]CodeGen.Operand = undefined;
29146 inline for (&mir_ops, 3..) |*mir_op, inst_index| mir_op.* = try inst[inst_index].lower(s);
29147 s.cg.asmOps(mir_tag, mir_ops) catch |err| switch (err) {
29148 error.InvalidInstruction => {
29149 const fixes = @tagName(mir_tag[0]);
29150 const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?;
29151 return s.cg.fail(
29152 "invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'",
29153 .{
29154 fixes[0..fixes_blank],
29155 @tagName(mir_tag[1]),
29156 fixes[fixes_blank + 1 ..],
29157 @tagName(mir_ops[0]),
29158 @tagName(mir_ops[1]),
29159 @tagName(mir_ops[2]),
29160 @tagName(mir_ops[3]),
29161 },
29162 );
29163 },
29164 else => |e| return e,
29165 };
29166 }
29167
29168 const Case = struct {
29169 required_features: [4]?std.Target.x86.Feature = @splat(null),
29170 dst_constraints: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]Constraint = @splat(.any),
29171 src_constraints: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Constraint = @splat(.any),
29172 patterns: []const Select.Pattern,
29173 extra_temps: [@intFromEnum(Select.Operand.Ref.dst0) - @intFromEnum(Select.Operand.Ref.tmp0)]TempSpec = @splat(.unused),
29174 dst_temps: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]TempSpec.Kind = @splat(.unused),
29175 clobbers: struct { eflags: bool = false } = .{},
29176 each: union(enum) {
29177 once: []const Instruction,
29178 },
1960229179 };
19603}
1960429180
19605fn memSize(self: *Self, ty: Type) Memory.Size {
19606 const pt = self.pt;
19607 const zcu = pt.zcu;
19608 return switch (ty.zigTypeTag(zcu)) {
19609 .float => Memory.Size.fromBitSize(ty.floatBits(self.target.*)),
19610 else => Memory.Size.fromSize(@intCast(ty.abiSize(zcu))),
29181 const Constraint = union(enum) {
29182 any,
29183 any_bool_vec,
29184 any_int,
29185 any_signed_int,
29186 any_float,
29187 po2_any,
29188 bool_vec: Memory.Size,
29189 vec: Memory.Size,
29190 signed_int_vec: Memory.Size,
29191 signed_int_or_full_vec: Memory.Size,
29192 unsigned_int_vec: Memory.Size,
29193 size: Memory.Size,
29194 multiple_size: Memory.Size,
29195 int: Memory.Size,
29196 scalar_int: Memory.Size,
29197 scalar_signed_int: Memory.Size,
29198 scalar_unsigned_int: Memory.Size,
29199 scalar_remainder_int: struct { of: Memory.Size, is: Memory.Size },
29200 exact_int: u16,
29201 exact_signed_int: u16,
29202 exact_unsigned_int: u16,
29203 signed_or_exact_int: Memory.Size,
29204 unsigned_or_exact_int: Memory.Size,
29205 po2_int: Memory.Size,
29206 signed_po2_int: Memory.Size,
29207 unsigned_po2_or_exact_int: Memory.Size,
29208 remainder_int: struct { of: Memory.Size, is: Memory.Size },
29209 exact_remainder_int: struct { of: Memory.Size, is: Memory.Size },
29210 signed_or_exact_remainder_int: struct { of: Memory.Size, is: Memory.Size },
29211 unsigned_or_exact_remainder_int: struct { of: Memory.Size, is: Memory.Size },
29212 signed_int: Memory.Size,
29213 unsigned_int: Memory.Size,
29214 elem_size_is: u8,
29215 po2_elem_size,
29216 elem_int: Memory.Size,
29217
29218 fn accepts(constraint: Constraint, ty: Type, cg: *CodeGen) bool {
29219 const zcu = cg.pt.zcu;
29220 return switch (constraint) {
29221 .any => true,
29222 .any_bool_vec => ty.isVector(zcu) and ty.childType(zcu).toIntern() == .bool_type,
29223 .any_int => intInfo(ty, cg) != null,
29224 .any_signed_int => if (intInfo(ty, cg)) |int_info| int_info.signedness == .signed else false,
29225 .any_float => ty.isRuntimeFloat(),
29226 .po2_any => std.math.isPowerOfTwo(ty.abiSize(zcu)),
29227 .bool_vec => |size| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and
29228 size.bitSize(cg.target) >= ty.vectorLen(zcu),
29229 .vec => |size| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() != .bool_type and
29230 size.bitSize(cg.target) >= ty.abiSize(zcu),
29231 .signed_int_vec => |size| ty.isVector(zcu) and size.bitSize(cg.target) >= 8 * ty.abiSize(zcu) and
29232 if (intInfo(ty.childType(zcu), cg)) |int_info| int_info.signedness == .signed else false,
29233 .signed_int_or_full_vec => |size| ty.isVector(zcu) and size.bitSize(cg.target) >= 8 * ty.abiSize(zcu) and
29234 if (intInfo(ty.childType(zcu), cg)) |int_info| switch (int_info.signedness) {
29235 .signed => true,
29236 .unsigned => int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits),
29237 } else false,
29238 .unsigned_int_vec => |size| ty.isVector(zcu) and size.bitSize(cg.target) >= 8 * ty.abiSize(zcu) and
29239 if (intInfo(ty.childType(zcu), cg)) |int_info| int_info.signedness == .unsigned else false,
29240 .size => |size| size.bitSize(cg.target) >= 8 * ty.abiSize(zcu),
29241 .multiple_size => |size| size.bitSize(cg.target) % 8 * ty.abiSize(zcu) == 0,
29242 .int => |size| if (intInfo(ty, cg)) |int_info| size.bitSize(cg.target) >= int_info.bits else false,
29243 .scalar_int => |size| if (intInfo(ty.scalarType(zcu), cg)) |int_info|
29244 size.bitSize(cg.target) >= int_info.bits
29245 else
29246 false,
29247 .scalar_signed_int => |size| if (intInfo(ty.scalarType(zcu), cg)) |int_info| switch (int_info.signedness) {
29248 .signed => size.bitSize(cg.target) >= int_info.bits,
29249 .unsigned => false,
29250 } else false,
29251 .scalar_unsigned_int => |size| if (intInfo(ty.scalarType(zcu), cg)) |int_info| switch (int_info.signedness) {
29252 .signed => false,
29253 .unsigned => size.bitSize(cg.target) >= int_info.bits,
29254 } else false,
29255 .scalar_remainder_int => |of_is| if (intInfo(ty.scalarType(zcu), cg)) |int_info|
29256 of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1
29257 else
29258 false,
29259 .exact_int => |bit_size| if (intInfo(ty, cg)) |int_info| bit_size == int_info.bits else false,
29260 .exact_signed_int => |bit_size| if (intInfo(ty, cg)) |int_info| switch (int_info.signedness) {
29261 .signed => bit_size == int_info.bits,
29262 .unsigned => false,
29263 } else false,
29264 .exact_unsigned_int => |bit_size| if (intInfo(ty, cg)) |int_info| switch (int_info.signedness) {
29265 .signed => false,
29266 .unsigned => bit_size == int_info.bits,
29267 } else false,
29268 .signed_or_exact_int => |size| if (intInfo(ty, cg)) |int_info| switch (int_info.signedness) {
29269 .signed => size.bitSize(cg.target) >= int_info.bits,
29270 .unsigned => size.bitSize(cg.target) == int_info.bits,
29271 } else false,
29272 .unsigned_or_exact_int => |size| if (intInfo(ty, cg)) |int_info| switch (int_info.signedness) {
29273 .signed => size.bitSize(cg.target) == int_info.bits,
29274 .unsigned => size.bitSize(cg.target) >= int_info.bits,
29275 } else false,
29276 .po2_int => |size| if (intInfo(ty, cg)) |int_info|
29277 std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits
29278 else
29279 false,
29280 .signed_po2_int => |size| if (intInfo(ty, cg)) |int_info| switch (int_info.signedness) {
29281 .signed => std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits,
29282 .unsigned => false,
29283 } else false,
29284 .unsigned_po2_or_exact_int => |size| if (intInfo(ty, cg)) |int_info| switch (int_info.signedness) {
29285 .signed => size.bitSize(cg.target) == int_info.bits,
29286 .unsigned => std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits,
29287 } else false,
29288 .remainder_int => |of_is| if (intInfo(ty, cg)) |int_info|
29289 of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1
29290 else
29291 false,
29292 .exact_remainder_int => |of_is| if (intInfo(ty, cg)) |int_info|
29293 of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1
29294 else
29295 false,
29296 .signed_or_exact_remainder_int => |of_is| if (intInfo(ty, cg)) |int_info| switch (int_info.signedness) {
29297 .signed => of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
29298 .unsigned => of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
29299 } else false,
29300 .unsigned_or_exact_remainder_int => |of_is| if (intInfo(ty, cg)) |int_info| switch (int_info.signedness) {
29301 .signed => of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
29302 .unsigned => of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
29303 } else false,
29304 .signed_int => |size| if (intInfo(ty, cg)) |int_info| switch (int_info.signedness) {
29305 .signed => size.bitSize(cg.target) >= int_info.bits,
29306 .unsigned => false,
29307 } else false,
29308 .unsigned_int => |size| if (intInfo(ty, cg)) |int_info| switch (int_info.signedness) {
29309 .signed => false,
29310 .unsigned => size.bitSize(cg.target) >= int_info.bits,
29311 } else false,
29312 .elem_size_is => |size| size == ty.elemType2(zcu).abiSize(zcu),
29313 .po2_elem_size => std.math.isPowerOfTwo(ty.elemType2(zcu).abiSize(zcu)),
29314 .elem_int => |size| if (intInfo(ty.elemType2(zcu), cg)) |elem_int_info|
29315 size.bitSize(cg.target) >= elem_int_info.bits
29316 else
29317 false,
29318 };
29319 }
29320 };
29321
29322 const Pattern = struct {
29323 src: [2]Src,
29324 commute: struct { u8, u8 } = .{ 0, 0 },
29325
29326 const Src = enum {
29327 none,
29328 any,
29329 imm8,
29330 imm16,
29331 imm32,
29332 simm32,
29333 mem,
29334 to_mem,
29335 mut_mem,
29336 to_mut_mem,
29337 gpr,
29338 to_gpr,
29339 mut_gpr,
29340 to_mut_gpr,
29341 mm,
29342 to_mm,
29343 mut_mm,
29344 to_mut_mm,
29345 xmm,
29346 to_xmm,
29347 mut_xmm,
29348 to_mut_xmm,
29349 ymm,
29350 to_ymm,
29351 mut_ymm,
29352 to_mut_ymm,
29353
29354 fn matches(src: Src, temp: Temp, cg: *CodeGen) bool {
29355 return switch (src) {
29356 .none => unreachable,
29357 .any => true,
29358 .imm8 => switch (temp.tracking(cg).short) {
29359 .immediate => |imm| std.math.cast(u8, imm) != null,
29360 else => false,
29361 },
29362 .imm16 => switch (temp.tracking(cg).short) {
29363 .immediate => |imm| std.math.cast(u16, imm) != null,
29364 else => false,
29365 },
29366 .imm32 => switch (temp.tracking(cg).short) {
29367 .immediate => |imm| std.math.cast(u32, imm) != null,
29368 else => false,
29369 },
29370 .simm32 => switch (temp.tracking(cg).short) {
29371 .immediate => |imm| std.math.cast(i32, @as(i64, @bitCast(imm))) != null,
29372 else => false,
29373 },
29374 .mem => temp.tracking(cg).short.isMemory(),
29375 .to_mem, .to_mut_mem => true,
29376 .mut_mem => temp.isMut(cg) and temp.tracking(cg).short.isMemory(),
29377 .gpr => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) {
29378 .register => |reg| reg.class() == .general_purpose,
29379 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and reg_off.off == 0,
29380 else => false,
29381 },
29382 .mut_gpr => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) {
29383 .register => |reg| reg.class() == .general_purpose,
29384 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and reg_off.off == 0,
29385 else => false,
29386 },
29387 .to_gpr, .to_mut_gpr => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8,
29388 .mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) {
29389 .register => |reg| reg.class() == .mmx,
29390 .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0,
29391 else => false,
29392 },
29393 .mut_mm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) {
29394 .register => |reg| reg.class() == .mmx,
29395 .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0,
29396 else => false,
29397 },
29398 .to_mm, .to_mut_mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8,
29399 .xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) {
29400 .register => |reg| reg.class() == .sse,
29401 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
29402 else => false,
29403 },
29404 .mut_xmm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) {
29405 .register => |reg| reg.class() == .sse,
29406 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
29407 else => false,
29408 },
29409 .to_xmm, .to_mut_xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16,
29410 .ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) {
29411 .register => |reg| reg.class() == .sse,
29412 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
29413 else => false,
29414 },
29415 .mut_ymm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) {
29416 .register => |reg| reg.class() == .sse,
29417 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
29418 else => false,
29419 },
29420 .to_ymm, .to_mut_ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32,
29421 };
29422 }
29423
29424 fn convert(src: Src, temp: *Temp, cg: *CodeGen) !bool {
29425 return switch (src) {
29426 .none => unreachable,
29427 .any, .imm8, .imm16, .imm32, .simm32 => false,
29428 .mem, .to_mem, .mut_mem, .to_mut_mem => try temp.toBase(cg),
29429 .gpr, .to_gpr => try temp.toRegClass(false, .general_purpose, cg),
29430 .mut_gpr, .to_mut_gpr => try temp.toRegClass(true, .general_purpose, cg),
29431 .mm, .to_mm => try temp.toRegClass(false, .mmx, cg),
29432 .mut_mm, .to_mut_mm => try temp.toRegClass(true, .mmx, cg),
29433 .xmm, .to_xmm, .ymm, .to_ymm => try temp.toRegClass(false, .sse, cg),
29434 .mut_xmm, .to_mut_xmm, .mut_ymm, .to_mut_ymm => try temp.toRegClass(true, .sse, cg),
29435 };
29436 }
29437 };
1961129438 };
19612}
1961329439
19614fn splitType(self: *Self, ty: Type) ![2]Type {
19615 const pt = self.pt;
19616 const zcu = pt.zcu;
19617 const classes = mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
19618 var parts: [2]Type = undefined;
19619 if (classes.len == 2) for (&parts, classes, 0..) |*part, class, part_i| {
19620 part.* = switch (class) {
19621 .integer => switch (part_i) {
19622 0 => Type.u64,
19623 1 => part: {
19624 const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?;
19625 const elem_ty = try pt.intType(.unsigned, @intCast(elem_size * 8));
19626 break :part switch (@divExact(ty.abiSize(zcu) - 8, elem_size)) {
19627 1 => elem_ty,
19628 else => |len| try pt.arrayType(.{ .len = len, .child = elem_ty.toIntern() }),
29440 const TempSpec = struct {
29441 type: Type = .noreturn,
29442 kind: Kind,
29443
29444 const unused: TempSpec = .{ .kind = .unused };
29445
29446 const Kind = union(enum) {
29447 unused,
29448 any,
29449 cc: Condition,
29450 reg: Register,
29451 rc: Register.Class,
29452 rc_mask: struct { rc: Register.Class, info: MaskInfo },
29453 mem,
29454 smin_mem: Select.Operand.Ref,
29455 smax_mem: Select.Operand.Ref,
29456 umin_mem: Select.Operand.Ref,
29457 umax_mem: Select.Operand.Ref,
29458 ref: Select.Operand.Ref,
29459 ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo },
29460
29461 fn finish(kind: Kind, temp: Temp, s: *const Select) void {
29462 switch (kind) {
29463 else => {},
29464 inline .rc_mask, .ref_mask => |mask| temp.asMask(mask.info, s.cg),
29465 }
29466 }
29467 };
29468
29469 fn create(spec: TempSpec, s: *Select) !?Temp {
29470 const cg = s.cg;
29471 return switch (spec.kind) {
29472 .unused => null,
29473 .any => try cg.tempAlloc(spec.type),
29474 .cc => |cc| try cg.tempInit(spec.type, .{ .eflags = cc }),
29475 .reg => |reg| try cg.tempInit(spec.type, .{ .register = reg }),
29476 .rc => |rc| try cg.tempAllocReg(spec.type, regSetForRegClass(rc)),
29477 .rc_mask => |rc_mask| try cg.tempAllocReg(spec.type, regSetForRegClass(rc_mask.rc)),
29478 .mem => try cg.tempAllocMem(spec.type),
29479 .smin_mem, .smax_mem, .umin_mem, .umax_mem => |ty_ref| {
29480 const pt = cg.pt;
29481 const zcu = pt.zcu;
29482 const ip = &zcu.intern_pool;
29483 const ty = ty_ref.deref(s).typeOf(s.cg);
29484 const vector_len, const scalar_ty: Type = switch (ip.indexToKey(ty.toIntern())) {
29485 else => .{ null, ty },
29486 .vector_type => |vector_type| .{ vector_type.len, .fromInterned(vector_type.child) },
1962929487 };
29488 const res_scalar_ty, const res_scalar_val: Value = res_scalar: switch (scalar_ty.toIntern()) {
29489 .bool_type => .{
29490 scalar_ty,
29491 .fromInterned(switch (spec.kind) {
29492 else => unreachable,
29493 .smin_mem, .umax_mem => .bool_true,
29494 .smax_mem, .umin_mem => .bool_false,
29495 }),
29496 },
29497 else => {
29498 const scalar_info = intInfo(scalar_ty, cg).?;
29499 const scalar_int_ty = try pt.intType(scalar_info.signedness, scalar_info.bits);
29500 if (scalar_info.bits <= 64) {
29501 const int_val: i64 = switch (spec.kind) {
29502 else => unreachable,
29503 .smin_mem => std.math.minInt(i64),
29504 .smax_mem => std.math.maxInt(i64),
29505 .umin_mem => 0,
29506 .umax_mem => -1,
29507 };
29508 const shift: u6 = @intCast(64 - scalar_info.bits);
29509 break :res_scalar .{ scalar_int_ty, switch (scalar_info.signedness) {
29510 .signed => try pt.intValue_i64(scalar_int_ty, int_val >> shift),
29511 .unsigned => try pt.intValue_u64(scalar_int_ty, @as(u64, @bitCast(int_val)) >> shift),
29512 } };
29513 }
29514 var big_int: std.math.big.int.Managed = try .init(cg.gpa);
29515 defer big_int.deinit();
29516 try big_int.setTwosCompIntLimit(switch (spec.kind) {
29517 else => unreachable,
29518 .smin_mem, .umin_mem => .min,
29519 .smax_mem, .umax_mem => .max,
29520 }, switch (spec.kind) {
29521 else => unreachable,
29522 .smin_mem, .smax_mem => .signed,
29523 .umin_mem, .umax_mem => .unsigned,
29524 }, scalar_info.bits);
29525 try big_int.truncate(&big_int, scalar_info.signedness, scalar_info.bits);
29526 break :res_scalar .{ scalar_int_ty, try pt.intValue_big(scalar_int_ty, big_int.toConst()) };
29527 },
29528 };
29529 const res_val: Value = if (vector_len) |len| .fromInterned(try pt.intern(.{ .aggregate = .{
29530 .ty = (try pt.vectorType(.{
29531 .len = len,
29532 .child = res_scalar_ty.toIntern(),
29533 })).toIntern(),
29534 .storage = .{ .repeated_elem = res_scalar_val.toIntern() },
29535 } })) else res_scalar_val;
29536 return try cg.tempFromValue(res_val);
1963029537 },
19631 else => unreachable,
29538 .ref => |ref| ref.deref(s),
29539 .ref_mask => |ref_mask| ref_mask.ref.deref(s),
29540 };
29541 }
29542 };
29543
29544 const Instruction = struct {
29545 Label,
29546 Mir.Inst.Fixes,
29547 Mir.Inst.Tag,
29548 Select.Operand,
29549 Select.Operand,
29550 Select.Operand,
29551 Select.Operand,
29552 };
29553 const Label = enum { @"0:", @"1:", @"_" };
29554 const Operand = struct {
29555 tag: Tag,
29556 base: Ref.Sized = .none,
29557 index: packed struct(u6) {
29558 ref: Ref,
29559 scale: Memory.Scale,
29560 } = .{ .ref = .none, .scale = .@"1" },
29561 adjust: Adjust = .none,
29562 imm: i32 = 0,
29563
29564 const Tag = enum {
29565 none,
29566 backward_label,
29567 forward_label,
29568 ref,
29569 simm,
29570 uimm,
29571 lea,
29572 mem,
29573 };
29574 const Adjust = packed struct(u8) {
29575 factor: i2,
29576 scale: Memory.Scale,
29577 amount: enum(u4) {
29578 none,
29579 ptr_size,
29580 ptr_bit_size,
29581 size,
29582 src0_size,
29583 bit_size,
29584 src0_bit_size,
29585 len,
29586 elem_limbs,
29587 src0_elem_size,
29588 src0_elem_size_times_src1,
29589 log2_src0_elem_size,
29590 smin,
29591 smax,
29592 umax,
1963229593 },
19633 .float => Type.f32,
19634 .float_combine => try pt.arrayType(.{ .len = 2, .child = .f32_type }),
19635 .sse => Type.f64,
19636 else => break,
29594
29595 const none: Adjust = .{ .factor = 0, .scale = .@"1", .amount = .none };
29596 const sub_ptr_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .ptr_size };
29597 const add_ptr_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .ptr_bit_size };
29598 const add_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .size };
29599 const sub_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .size };
29600 const add_src0_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_size };
29601 const sub_src0_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_size };
29602 const add_2_bit_size: Adjust = .{ .factor = 1, .scale = .@"2", .amount = .bit_size };
29603 const add_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .bit_size };
29604 const sub_bit_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .bit_size };
29605 const add_src0_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_bit_size };
29606 const sub_src0_bit_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_bit_size };
29607 const add_8_len: Adjust = .{ .factor = 1, .scale = .@"8", .amount = .len };
29608 const add_4_len: Adjust = .{ .factor = 1, .scale = .@"4", .amount = .len };
29609 const add_3_len: Adjust = .{ .factor = 1, .scale = .@"3", .amount = .len };
29610 const add_2_len: Adjust = .{ .factor = 1, .scale = .@"2", .amount = .len };
29611 const add_len: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .len };
29612 const sub_len: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .len };
29613 const add_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_elem_size };
29614 const add_2_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"2", .amount = .src0_elem_size };
29615 const add_4_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"4", .amount = .src0_elem_size };
29616 const add_8_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"8", .amount = .src0_elem_size };
29617 const sub_src0_elem_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_elem_size };
29618 const add_src0_elem_size_times_src1: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_elem_size_times_src1 };
29619 const sub_src0_elem_size_times_src1: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_elem_size_times_src1 };
29620 const add_log2_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .log2_src0_elem_size };
29621 const add_elem_limbs: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .elem_limbs };
29622 const add_umax: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .umax };
1963729623 };
19638 } else if (parts[0].abiSize(zcu) + parts[1].abiSize(zcu) == ty.abiSize(zcu)) return parts;
19639 return self.fail("TODO implement splitType for {}", .{ty.fmt(pt)});
19640}
29624 const Ref = enum(u4) {
29625 tmp0,
29626 tmp1,
29627 tmp2,
29628 tmp3,
29629 tmp4,
29630 tmp5,
29631 dst0,
29632 src0,
29633 src1,
29634 none,
1964129635
19642/// Truncates the value in the register in place.
19643/// Clobbers any remaining bits.
19644fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
19645 const pt = self.pt;
19646 const zcu = pt.zcu;
19647 const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.builtin.Type.Int{
19648 .signedness = .unsigned,
19649 .bits = @intCast(ty.bitSize(zcu)),
19650 };
19651 const shift = math.cast(u6, 64 - int_info.bits % 64) orelse return;
19652 try self.spillEflagsIfOccupied();
19653 switch (int_info.signedness) {
19654 .signed => {
19655 try self.genShiftBinOpMir(
19656 .{ ._l, .sa },
19657 Type.isize,
19658 .{ .register = reg },
19659 Type.u8,
19660 .{ .immediate = shift },
19661 );
19662 try self.genShiftBinOpMir(
19663 .{ ._r, .sa },
19664 Type.isize,
19665 .{ .register = reg },
19666 Type.u8,
19667 .{ .immediate = shift },
19668 );
19669 },
19670 .unsigned => {
19671 const mask = ~@as(u64, 0) >> shift;
19672 if (int_info.bits <= 32) {
19673 try self.genBinOpMir(
19674 .{ ._, .@"and" },
19675 Type.u32,
19676 .{ .register = reg },
19677 .{ .immediate = mask },
19678 );
19679 } else {
19680 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
19681 try self.genBinOpMir(
19682 .{ ._, .@"and" },
19683 Type.usize,
19684 .{ .register = reg },
19685 .{ .register = tmp_reg },
19686 );
19687 }
19688 },
19689 }
19690}
29636 const Sized = packed struct(u8) {
29637 ref: Ref,
29638 size: Memory.Size,
29639
29640 const none: Sized = .{ .ref = .none, .size = .none };
29641
29642 const tmp0: Sized = .{ .ref = .tmp0, .size = .none };
29643 const tmp0b: Sized = .{ .ref = .tmp0, .size = .byte };
29644 const tmp0w: Sized = .{ .ref = .tmp0, .size = .word };
29645 const tmp0d: Sized = .{ .ref = .tmp0, .size = .dword };
29646 const tmp0p: Sized = .{ .ref = .tmp0, .size = .ptr };
29647 const tmp0q: Sized = .{ .ref = .tmp0, .size = .qword };
29648 const tmp0x: Sized = .{ .ref = .tmp0, .size = .xword };
29649 const tmp0y: Sized = .{ .ref = .tmp0, .size = .yword };
29650
29651 const tmp1: Sized = .{ .ref = .tmp1, .size = .none };
29652 const tmp1b: Sized = .{ .ref = .tmp1, .size = .byte };
29653 const tmp1w: Sized = .{ .ref = .tmp1, .size = .word };
29654 const tmp1d: Sized = .{ .ref = .tmp1, .size = .dword };
29655 const tmp1p: Sized = .{ .ref = .tmp1, .size = .ptr };
29656 const tmp1q: Sized = .{ .ref = .tmp1, .size = .qword };
29657 const tmp1x: Sized = .{ .ref = .tmp1, .size = .xword };
29658 const tmp1y: Sized = .{ .ref = .tmp1, .size = .yword };
29659
29660 const tmp2: Sized = .{ .ref = .tmp2, .size = .none };
29661 const tmp2b: Sized = .{ .ref = .tmp2, .size = .byte };
29662 const tmp2w: Sized = .{ .ref = .tmp2, .size = .word };
29663 const tmp2d: Sized = .{ .ref = .tmp2, .size = .dword };
29664 const tmp2p: Sized = .{ .ref = .tmp2, .size = .ptr };
29665 const tmp2q: Sized = .{ .ref = .tmp2, .size = .qword };
29666 const tmp2x: Sized = .{ .ref = .tmp2, .size = .xword };
29667 const tmp2y: Sized = .{ .ref = .tmp2, .size = .yword };
29668
29669 const tmp3: Sized = .{ .ref = .tmp3, .size = .none };
29670 const tmp3b: Sized = .{ .ref = .tmp3, .size = .byte };
29671 const tmp3w: Sized = .{ .ref = .tmp3, .size = .word };
29672 const tmp3d: Sized = .{ .ref = .tmp3, .size = .dword };
29673 const tmp3p: Sized = .{ .ref = .tmp3, .size = .ptr };
29674 const tmp3q: Sized = .{ .ref = .tmp3, .size = .qword };
29675 const tmp3x: Sized = .{ .ref = .tmp3, .size = .xword };
29676 const tmp3y: Sized = .{ .ref = .tmp3, .size = .yword };
29677
29678 const tmp4: Sized = .{ .ref = .tmp4, .size = .none };
29679 const tmp4b: Sized = .{ .ref = .tmp4, .size = .byte };
29680 const tmp4w: Sized = .{ .ref = .tmp4, .size = .word };
29681 const tmp4d: Sized = .{ .ref = .tmp4, .size = .dword };
29682 const tmp4p: Sized = .{ .ref = .tmp4, .size = .ptr };
29683 const tmp4q: Sized = .{ .ref = .tmp4, .size = .qword };
29684 const tmp4x: Sized = .{ .ref = .tmp4, .size = .xword };
29685 const tmp4y: Sized = .{ .ref = .tmp4, .size = .yword };
29686
29687 const tmp5: Sized = .{ .ref = .tmp5, .size = .none };
29688 const tmp5b: Sized = .{ .ref = .tmp5, .size = .byte };
29689 const tmp5w: Sized = .{ .ref = .tmp5, .size = .word };
29690 const tmp5d: Sized = .{ .ref = .tmp5, .size = .dword };
29691 const tmp5p: Sized = .{ .ref = .tmp5, .size = .ptr };
29692 const tmp5q: Sized = .{ .ref = .tmp5, .size = .qword };
29693 const tmp5x: Sized = .{ .ref = .tmp5, .size = .xword };
29694 const tmp5y: Sized = .{ .ref = .tmp5, .size = .yword };
29695
29696 const dst0: Sized = .{ .ref = .dst0, .size = .none };
29697 const dst0b: Sized = .{ .ref = .dst0, .size = .byte };
29698 const dst0w: Sized = .{ .ref = .dst0, .size = .word };
29699 const dst0d: Sized = .{ .ref = .dst0, .size = .dword };
29700 const dst0p: Sized = .{ .ref = .dst0, .size = .ptr };
29701 const dst0q: Sized = .{ .ref = .dst0, .size = .qword };
29702 const dst0x: Sized = .{ .ref = .dst0, .size = .xword };
29703 const dst0y: Sized = .{ .ref = .dst0, .size = .yword };
29704
29705 const src0: Sized = .{ .ref = .src0, .size = .none };
29706 const src0b: Sized = .{ .ref = .src0, .size = .byte };
29707 const src0w: Sized = .{ .ref = .src0, .size = .word };
29708 const src0d: Sized = .{ .ref = .src0, .size = .dword };
29709 const src0p: Sized = .{ .ref = .src0, .size = .ptr };
29710 const src0q: Sized = .{ .ref = .src0, .size = .qword };
29711 const src0x: Sized = .{ .ref = .src0, .size = .xword };
29712 const src0y: Sized = .{ .ref = .src0, .size = .yword };
29713
29714 const src1: Sized = .{ .ref = .src1, .size = .none };
29715 const src1b: Sized = .{ .ref = .src1, .size = .byte };
29716 const src1w: Sized = .{ .ref = .src1, .size = .word };
29717 const src1d: Sized = .{ .ref = .src1, .size = .dword };
29718 const src1p: Sized = .{ .ref = .src1, .size = .ptr };
29719 const src1q: Sized = .{ .ref = .src1, .size = .qword };
29720 const src1x: Sized = .{ .ref = .src1, .size = .xword };
29721 const src1y: Sized = .{ .ref = .src1, .size = .yword };
29722 };
1969129723
19692fn regBitSize(self: *Self, ty: Type) u64 {
19693 const pt = self.pt;
19694 const zcu = pt.zcu;
19695 const abi_size = ty.abiSize(zcu);
19696 return switch (ty.zigTypeTag(zcu)) {
19697 else => switch (abi_size) {
19698 1 => 8,
19699 2 => 16,
19700 3...4 => 32,
19701 5...8 => 64,
19702 else => unreachable,
19703 },
19704 .float => switch (abi_size) {
19705 1...16 => 128,
19706 17...32 => 256,
19707 else => unreachable,
19708 },
19709 };
19710}
29724 fn deref(ref: Ref, s: *const Select) Temp {
29725 return s.temps[@intFromEnum(ref)];
29726 }
29727 };
1971129728
19712fn regExtraBits(self: *Self, ty: Type) u64 {
19713 return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
19714}
29729 const @"_": Select.Operand = .{ .tag = .none };
29730
29731 const @"0b": Select.Operand = .{ .tag = .backward_label, .base = .{ .ref = .tmp0, .size = .none } };
29732 const @"0f": Select.Operand = .{ .tag = .forward_label, .base = .{ .ref = .tmp0, .size = .none } };
29733 const @"1b": Select.Operand = .{ .tag = .backward_label, .base = .{ .ref = .tmp1, .size = .none } };
29734 const @"1f": Select.Operand = .{ .tag = .forward_label, .base = .{ .ref = .tmp1, .size = .none } };
29735
29736 const tmp0b: Select.Operand = .{ .tag = .ref, .base = .tmp0b };
29737 const tmp0w: Select.Operand = .{ .tag = .ref, .base = .tmp0w };
29738 const tmp0d: Select.Operand = .{ .tag = .ref, .base = .tmp0d };
29739 const tmp0p: Select.Operand = .{ .tag = .ref, .base = .tmp0p };
29740 const tmp0q: Select.Operand = .{ .tag = .ref, .base = .tmp0q };
29741 const tmp0x: Select.Operand = .{ .tag = .ref, .base = .tmp0x };
29742 const tmp0y: Select.Operand = .{ .tag = .ref, .base = .tmp0y };
29743
29744 const tmp1b: Select.Operand = .{ .tag = .ref, .base = .tmp1b };
29745 const tmp1w: Select.Operand = .{ .tag = .ref, .base = .tmp1w };
29746 const tmp1d: Select.Operand = .{ .tag = .ref, .base = .tmp1d };
29747 const tmp1p: Select.Operand = .{ .tag = .ref, .base = .tmp1p };
29748 const tmp1q: Select.Operand = .{ .tag = .ref, .base = .tmp1q };
29749 const tmp1x: Select.Operand = .{ .tag = .ref, .base = .tmp1x };
29750 const tmp1y: Select.Operand = .{ .tag = .ref, .base = .tmp1y };
29751
29752 const tmp2b: Select.Operand = .{ .tag = .ref, .base = .tmp2b };
29753 const tmp2w: Select.Operand = .{ .tag = .ref, .base = .tmp2w };
29754 const tmp2d: Select.Operand = .{ .tag = .ref, .base = .tmp2d };
29755 const tmp2p: Select.Operand = .{ .tag = .ref, .base = .tmp2p };
29756 const tmp2q: Select.Operand = .{ .tag = .ref, .base = .tmp2q };
29757 const tmp2x: Select.Operand = .{ .tag = .ref, .base = .tmp2x };
29758 const tmp2y: Select.Operand = .{ .tag = .ref, .base = .tmp2y };
29759
29760 const tmp3b: Select.Operand = .{ .tag = .ref, .base = .tmp3b };
29761 const tmp3w: Select.Operand = .{ .tag = .ref, .base = .tmp3w };
29762 const tmp3d: Select.Operand = .{ .tag = .ref, .base = .tmp3d };
29763 const tmp3p: Select.Operand = .{ .tag = .ref, .base = .tmp3p };
29764 const tmp3q: Select.Operand = .{ .tag = .ref, .base = .tmp3q };
29765 const tmp3x: Select.Operand = .{ .tag = .ref, .base = .tmp3x };
29766 const tmp3y: Select.Operand = .{ .tag = .ref, .base = .tmp3y };
29767
29768 const tmp4b: Select.Operand = .{ .tag = .ref, .base = .tmp4b };
29769 const tmp4w: Select.Operand = .{ .tag = .ref, .base = .tmp4w };
29770 const tmp4d: Select.Operand = .{ .tag = .ref, .base = .tmp4d };
29771 const tmp4p: Select.Operand = .{ .tag = .ref, .base = .tmp4p };
29772 const tmp4q: Select.Operand = .{ .tag = .ref, .base = .tmp4q };
29773 const tmp4x: Select.Operand = .{ .tag = .ref, .base = .tmp4x };
29774 const tmp4y: Select.Operand = .{ .tag = .ref, .base = .tmp4y };
29775
29776 const tmp5b: Select.Operand = .{ .tag = .ref, .base = .tmp5b };
29777 const tmp5w: Select.Operand = .{ .tag = .ref, .base = .tmp5w };
29778 const tmp5d: Select.Operand = .{ .tag = .ref, .base = .tmp5d };
29779 const tmp5p: Select.Operand = .{ .tag = .ref, .base = .tmp5p };
29780 const tmp5q: Select.Operand = .{ .tag = .ref, .base = .tmp5q };
29781 const tmp5x: Select.Operand = .{ .tag = .ref, .base = .tmp5x };
29782 const tmp5y: Select.Operand = .{ .tag = .ref, .base = .tmp5y };
29783
29784 const dst0b: Select.Operand = .{ .tag = .ref, .base = .dst0b };
29785 const dst0w: Select.Operand = .{ .tag = .ref, .base = .dst0w };
29786 const dst0d: Select.Operand = .{ .tag = .ref, .base = .dst0d };
29787 const dst0p: Select.Operand = .{ .tag = .ref, .base = .dst0p };
29788 const dst0q: Select.Operand = .{ .tag = .ref, .base = .dst0q };
29789 const dst0x: Select.Operand = .{ .tag = .ref, .base = .dst0x };
29790 const dst0y: Select.Operand = .{ .tag = .ref, .base = .dst0y };
29791
29792 const src0b: Select.Operand = .{ .tag = .ref, .base = .src0b };
29793 const src0w: Select.Operand = .{ .tag = .ref, .base = .src0w };
29794 const src0d: Select.Operand = .{ .tag = .ref, .base = .src0d };
29795 const src0p: Select.Operand = .{ .tag = .ref, .base = .src0p };
29796 const src0q: Select.Operand = .{ .tag = .ref, .base = .src0q };
29797 const src0x: Select.Operand = .{ .tag = .ref, .base = .src0x };
29798 const src0y: Select.Operand = .{ .tag = .ref, .base = .src0y };
29799
29800 const src1b: Select.Operand = .{ .tag = .ref, .base = .src1b };
29801 const src1w: Select.Operand = .{ .tag = .ref, .base = .src1w };
29802 const src1d: Select.Operand = .{ .tag = .ref, .base = .src1d };
29803 const src1p: Select.Operand = .{ .tag = .ref, .base = .src1p };
29804 const src1q: Select.Operand = .{ .tag = .ref, .base = .src1q };
29805 const src1x: Select.Operand = .{ .tag = .ref, .base = .src1x };
29806 const src1y: Select.Operand = .{ .tag = .ref, .base = .src1y };
29807
29808 fn si(imm: i32) Select.Operand {
29809 return .{ .tag = .simm, .imm = imm };
29810 }
29811 fn sa(base: Ref.Sized, adjust: Adjust) Select.Operand {
29812 return .{ .tag = .simm, .base = base, .adjust = adjust };
29813 }
29814 fn sia(imm: i32, base: Ref.Sized, adjust: Adjust) Select.Operand {
29815 return .{ .tag = .simm, .base = base, .adjust = adjust, .imm = imm };
29816 }
29817 fn ui(imm: i32) Select.Operand {
29818 return .{ .tag = .uimm, .imm = imm };
29819 }
29820 fn ua(base: Ref.Sized, adjust: Adjust) Select.Operand {
29821 return .{ .tag = .uimm, .base = base, .adjust = adjust };
29822 }
29823 fn uia(imm: i32, base: Ref.Sized, adjust: Adjust) Select.Operand {
29824 return .{ .tag = .uimm, .base = base, .adjust = adjust, .imm = imm };
29825 }
1971529826
19716fn hasFeature(self: *Self, feature: Target.x86.Feature) bool {
19717 return Target.x86.featureSetHas(self.target.cpu.features, feature);
19718}
19719fn hasAnyFeatures(self: *Self, features: anytype) bool {
19720 return Target.x86.featureSetHasAny(self.target.cpu.features, features);
19721}
19722fn hasAllFeatures(self: *Self, features: anytype) bool {
19723 return Target.x86.featureSetHasAll(self.target.cpu.features, features);
19724}
29827 fn lea(size: Memory.Size, base: Ref) Select.Operand {
29828 return .{
29829 .tag = .lea,
29830 .base = .{ .ref = base, .size = size },
29831 };
29832 }
29833 fn leaa(size: Memory.Size, base: Ref, adjust: Adjust) Select.Operand {
29834 return .{
29835 .tag = .lea,
29836 .base = .{ .ref = base, .size = size },
29837 .adjust = adjust,
29838 };
29839 }
29840 fn lead(size: Memory.Size, base: Ref, disp: i32) Select.Operand {
29841 return .{
29842 .tag = .lea,
29843 .base = .{ .ref = base, .size = size },
29844 .imm = disp,
29845 };
29846 }
29847 fn leai(size: Memory.Size, base: Ref, index: Ref) Select.Operand {
29848 return .{
29849 .tag = .lea,
29850 .base = .{ .ref = base, .size = size },
29851 .index = .{ .ref = index, .scale = .@"1" },
29852 };
29853 }
29854 fn leaia(size: Memory.Size, base: Ref, index: Ref, adjust: Adjust) Select.Operand {
29855 return .{
29856 .tag = .lea,
29857 .base = .{ .ref = base, .size = size },
29858 .index = .{ .ref = index, .scale = .@"1" },
29859 .adjust = adjust,
29860 };
29861 }
29862 fn leaid(size: Memory.Size, base: Ref, index: Ref, disp: i32) Select.Operand {
29863 return .{
29864 .tag = .lea,
29865 .base = .{ .ref = base, .size = size },
29866 .index = .{ .ref = index, .scale = .@"1" },
29867 .imm = disp,
29868 };
29869 }
29870 fn leasi(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref) Select.Operand {
29871 return .{
29872 .tag = .lea,
29873 .base = .{ .ref = base, .size = size },
29874 .index = .{ .ref = index, .scale = scale },
29875 };
29876 }
29877 fn leasid(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand {
29878 return .{
29879 .tag = .lea,
29880 .base = .{ .ref = base, .size = size },
29881 .index = .{ .ref = index, .scale = scale },
29882 .imm = disp,
29883 };
29884 }
29885 fn leasiad(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref, adjust: Adjust, disp: i32) Select.Operand {
29886 return .{
29887 .tag = .lea,
29888 .base = .{ .ref = base, .size = size },
29889 .index = .{ .ref = index, .scale = scale },
29890 .adjust = adjust,
29891 .imm = disp,
29892 };
29893 }
1972529894
19726fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {
19727 const pt = self.pt;
19728 const zcu = pt.zcu;
19729 return self.air.typeOf(inst, &zcu.intern_pool);
19730}
29895 fn mem(base: Ref.Sized) Select.Operand {
29896 return .{
29897 .tag = .mem,
29898 .base = base,
29899 };
29900 }
29901 fn memd(base: Ref.Sized, disp: i32) Select.Operand {
29902 return .{
29903 .tag = .mem,
29904 .base = base,
29905 .imm = disp,
29906 };
29907 }
29908 fn mema(base: Ref.Sized, adjust: Adjust) Select.Operand {
29909 return .{
29910 .tag = .mem,
29911 .base = base,
29912 .adjust = adjust,
29913 };
29914 }
29915 fn memad(base: Ref.Sized, adjust: Adjust, disp: i32) Select.Operand {
29916 return .{
29917 .tag = .mem,
29918 .base = base,
29919 .adjust = adjust,
29920 .imm = disp,
29921 };
29922 }
29923 fn memi(base: Ref.Sized, index: Ref) Select.Operand {
29924 return .{
29925 .tag = .mem,
29926 .base = base,
29927 .index = .{ .ref = index, .scale = .@"1" },
29928 };
29929 }
29930 fn memia(base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand {
29931 return .{
29932 .tag = .mem,
29933 .base = base,
29934 .index = .{ .ref = index, .scale = .@"1" },
29935 .adjust = adjust,
29936 };
29937 }
29938 fn memiad(base: Ref.Sized, index: Ref, adjust: Adjust, disp: i32) Select.Operand {
29939 return .{
29940 .tag = .mem,
29941 .base = base,
29942 .index = .{ .ref = index, .scale = .@"1" },
29943 .adjust = adjust,
29944 .imm = disp,
29945 };
29946 }
29947 fn memid(base: Ref.Sized, index: Ref, disp: i32) Select.Operand {
29948 return .{
29949 .tag = .mem,
29950 .base = base,
29951 .index = .{ .ref = index, .scale = .@"1" },
29952 .imm = disp,
29953 };
29954 }
29955 fn memsi(base: Ref.Sized, scale: Memory.Scale, index: Ref) Select.Operand {
29956 return .{
29957 .tag = .mem,
29958 .base = base,
29959 .index = .{ .ref = index, .scale = scale },
29960 };
29961 }
29962 fn memsia(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust) Select.Operand {
29963 return .{
29964 .tag = .mem,
29965 .base = base,
29966 .index = .{ .ref = index, .scale = scale },
29967 .adjust = adjust,
29968 };
29969 }
29970 fn memsid(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand {
29971 return .{
29972 .tag = .mem,
29973 .base = base,
29974 .index = .{ .ref = index, .scale = scale },
29975 .imm = disp,
29976 };
29977 }
29978 fn memsiad(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust, disp: i32) Select.Operand {
29979 return .{
29980 .tag = .mem,
29981 .base = base,
29982 .index = .{ .ref = index, .scale = scale },
29983 .adjust = adjust,
29984 .imm = disp,
29985 };
29986 }
1973129987
19732fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {
19733 const pt = self.pt;
19734 const zcu = pt.zcu;
19735 return switch (self.air.instructions.items(.tag)[@intFromEnum(inst)]) {
19736 .loop_switch_br => self.typeOf(self.air.unwrapSwitch(inst).operand),
19737 else => self.air.typeOfIndex(inst, &zcu.intern_pool),
19738 };
19739}
29988 fn adjustedImm(op: Select.Operand, comptime SignedImm: type, s: *const Select) SignedImm {
29989 const UnsignedImm = @Type(.{
29990 .int = .{ .signedness = .unsigned, .bits = @typeInfo(SignedImm).int.bits },
29991 });
29992 return @as(i5, op.adjust.factor) * op.adjust.scale.toFactor() * @as(SignedImm, switch (op.adjust.amount) {
29993 .none => 0,
29994 .ptr_size => @divExact(s.cg.target.ptrBitWidth(), 8),
29995 .ptr_bit_size => s.cg.target.ptrBitWidth(),
29996 .size => @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu)),
29997 .src0_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu)),
29998 .bit_size => @intCast(op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),
29999 .src0_bit_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),
30000 .len => @intCast(op.base.ref.deref(s).typeOf(s.cg).vectorLen(s.cg.pt.zcu)),
30001 .elem_limbs => @intCast(@divExact(
30002 op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
30003 @divExact(op.base.size.bitSize(s.cg.target), 8),
30004 )),
30005 .src0_elem_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)),
30006 .src0_elem_size_times_src1 => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) *
30007 Select.Operand.Ref.src1.deref(s).tracking(s.cg).short.immediate),
30008 .log2_src0_elem_size => @intCast(std.math.log2(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu))),
30009 .smin => @as(SignedImm, std.math.minInt(SignedImm)) >> @truncate(
30010 -%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
30011 ),
30012 .smax => @as(SignedImm, std.math.maxInt(SignedImm)) >> @truncate(
30013 -%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
30014 ),
30015 .umax => @bitCast(@as(UnsignedImm, std.math.maxInt(UnsignedImm)) >> @truncate(
30016 -%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
30017 )),
30018 }) + op.imm;
30019 }
1974030020
19741fn intCompilerRtAbiName(int_bits: u32) u8 {
19742 return switch (int_bits) {
19743 1...32 => 's',
19744 33...64 => 'd',
19745 65...128 => 't',
19746 else => unreachable,
30021 fn lower(op: Select.Operand, s: *Select) !CodeGen.Operand {
30022 return switch (op.tag) {
30023 .none => .none,
30024 .backward_label => .{ .inst = s.labels[@intFromEnum(op.base.ref)].backward.? },
30025 .forward_label => for (&s.labels[@intFromEnum(op.base.ref)].forward) |*label| {
30026 if (label.*) |_| continue;
30027 label.* = @intCast(s.cg.mir_instructions.len);
30028 break .{ .inst = undefined };
30029 } else unreachable,
30030 .ref => switch (op.base.ref.deref(s).tracking(s.cg).short) {
30031 .immediate => |imm| .{ .imm = switch (op.base.size) {
30032 .byte => if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u8, @intCast(imm))),
30033 .word => if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u16, @intCast(imm))),
30034 .dword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u32, @intCast(imm))),
30035 .qword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(imm),
30036 else => unreachable,
30037 } },
30038 else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = op.base.size }) },
30039 .register => |reg| .{ .reg = registerAlias(reg, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8))) },
30040 },
30041 .simm => .{ .imm = .s(op.adjustedImm(i32, s)) },
30042 .uimm => .{ .imm = .u(@bitCast(op.adjustedImm(i64, s))) },
30043 .lea => .{ .mem = .{
30044 .base = .{ .reg = registerAlias(op.base.ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)) },
30045 .mod = .{ .rm = .{
30046 .size = op.base.size,
30047 .index = switch (op.index.ref) {
30048 else => |ref| registerAlias(ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)),
30049 .none => .none,
30050 },
30051 .scale = op.index.scale,
30052 .disp = op.adjustedImm(i32, s),
30053 } },
30054 } },
30055 .mem => .{ .mem = try op.base.ref.deref(s).tracking(s.cg).short.mem(s.cg, .{
30056 .size = op.base.size,
30057 .index = switch (op.index.ref) {
30058 else => |ref| registerAlias(ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)),
30059 .none => .none,
30060 },
30061 .scale = op.index.scale,
30062 .disp = op.adjustedImm(i32, s),
30063 }) },
30064 };
30065 }
1974730066 };
19748}
30067};
30068fn select(
30069 cg: *CodeGen,
30070 dst_temps: []Temp,
30071 dst_tys: []const Type,
30072 src_temps: []Temp,
30073 cases: []const Select.Case,
30074) !void {
30075 cases: for (cases) |case| {
30076 for (case.required_features) |required_feature| if (required_feature) |feature| if (!cg.hasFeature(feature)) continue :cases;
30077 for (case.dst_constraints[0..dst_temps.len], dst_tys) |dst_constraint, dst_ty| if (!dst_constraint.accepts(dst_ty, cg)) continue :cases;
30078 for (case.src_constraints[0..src_temps.len], src_temps) |src_constraint, src_temp| if (!src_constraint.accepts(src_temp.typeOf(cg), cg)) continue :cases;
30079 if (std.debug.runtime_safety) {
30080 for (case.dst_constraints[dst_temps.len..]) |dst_constraint| assert(dst_constraint == .any);
30081 for (case.src_constraints[src_temps.len..]) |src_constraint| assert(src_constraint == .any);
30082 }
30083 patterns: for (case.patterns) |pattern| {
30084 for (pattern.src[0..src_temps.len], src_temps) |src_pattern, src_temp| if (!src_pattern.matches(src_temp, cg)) continue :patterns;
30085 if (std.debug.runtime_safety) for (pattern.src[src_temps.len..]) |src_pattern| assert(src_pattern == .none);
30086
30087 var s: Select = .{
30088 .cg = cg,
30089 .temps = undefined,
30090 .labels = @splat(.{ .forward = @splat(null), .backward = null }),
30091 };
30092 const tmp_slots = s.temps[@intFromEnum(Select.Operand.Ref.tmp0)..@intFromEnum(Select.Operand.Ref.dst0)];
30093 const dst_slots = s.temps[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)];
30094 const src_slots = s.temps[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)];
1974930095
19750fn floatCompilerRtAbiName(float_bits: u32) u8 {
19751 return switch (float_bits) {
19752 16 => 'h',
19753 32 => 's',
19754 64 => 'd',
19755 80 => 'x',
19756 128 => 't',
19757 else => unreachable,
19758 };
19759}
30096 @memcpy(src_slots[0..src_temps.len], src_temps);
30097 std.mem.swap(Temp, &src_slots[pattern.commute[0]], &src_slots[pattern.commute[1]]);
30098 for (tmp_slots, case.extra_temps) |*slot, spec| slot.* = try spec.create(&s) orelse continue;
1976030099
19761fn floatCompilerRtAbiType(self: *Self, ty: Type, other_ty: Type) Type {
19762 if (ty.toIntern() == .f16_type and
19763 (other_ty.toIntern() == .f32_type or other_ty.toIntern() == .f64_type) and
19764 self.target.isDarwin()) return Type.u16;
19765 return ty;
19766}
30100 while (true) for (pattern.src[0..src_temps.len], src_temps) |src_pattern, *src_temp| {
30101 if (try src_pattern.convert(src_temp, cg)) break;
30102 } else break;
30103 @memcpy(src_slots[0..src_temps.len], src_temps);
30104 std.mem.swap(Temp, &src_slots[pattern.commute[0]], &src_slots[pattern.commute[1]]);
1976730105
19768fn floatLibcAbiPrefix(ty: Type) []const u8 {
19769 return switch (ty.toIntern()) {
19770 .f16_type, .f80_type => "__",
19771 .f32_type, .f64_type, .f128_type, .c_longdouble_type => "",
19772 else => unreachable,
19773 };
19774}
30106 if (case.clobbers.eflags or case.each != .once) try cg.spillEflagsIfOccupied();
1977530107
19776fn floatLibcAbiSuffix(ty: Type) []const u8 {
19777 return switch (ty.toIntern()) {
19778 .f16_type => "h",
19779 .f32_type => "f",
19780 .f64_type => "",
19781 .f80_type => "x",
19782 .f128_type => "q",
19783 .c_longdouble_type => "l",
19784 else => unreachable,
19785 };
19786}
30108 for (dst_temps, dst_tys, case.dst_temps[0..dst_temps.len]) |*dst_temp, dst_ty, dst_kind|
30109 dst_temp.* = (try Select.TempSpec.create(.{ .type = dst_ty, .kind = dst_kind }, &s)).?;
30110 @memcpy(dst_slots[0..dst_temps.len], dst_temps);
1978730111
19788fn promoteInt(self: *Self, ty: Type) Type {
19789 const pt = self.pt;
19790 const zcu = pt.zcu;
19791 const int_info: InternPool.Key.IntType = switch (ty.toIntern()) {
19792 .bool_type => .{ .signedness = .unsigned, .bits = 1 },
19793 else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty,
19794 };
19795 for ([_]Type{
19796 Type.c_int, Type.c_uint,
19797 Type.c_long, Type.c_ulong,
19798 Type.c_longlong, Type.c_ulonglong,
19799 }) |promote_ty| {
19800 const promote_info = promote_ty.intInfo(zcu);
19801 if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue;
19802 if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and
19803 promote_info.signedness == .signed) <= promote_info.bits) return promote_ty;
19804 }
19805 return ty;
19806}
30112 switch (case.each) {
30113 .once => |body| {
30114 for (body) |inst| try s.emit(inst);
30115 s.emitLabel(.@"0:");
30116 },
30117 }
1980730118
19808fn promoteVarArg(self: *Self, ty: Type) Type {
19809 if (!ty.isRuntimeFloat()) return self.promoteInt(ty);
19810 switch (ty.floatBits(self.target.*)) {
19811 32, 64 => return Type.f64,
19812 else => |float_bits| {
19813 assert(float_bits == self.target.cTypeBitSize(.longdouble));
19814 return Type.c_longdouble;
19815 },
30119 for (dst_temps, case.dst_temps[0..dst_temps.len]) |dst_temp, dst_kind| dst_kind.finish(dst_temp, &s);
30120 for (case.extra_temps, tmp_slots) |spec, temp| if (spec.kind != .unused) try temp.die(cg);
30121 return;
30122 }
1981630123 }
30124 return error.SelectFailed;
1981730125}
src/arch/x86_64/Disassembler.zig+34-16
......@@ -38,28 +38,46 @@ pub fn next(dis: *Disassembler) Error!?Instruction {
3838
3939 const enc = try dis.parseEncoding(prefixes) orelse return error.UnknownOpcode;
4040 switch (enc.data.op_en) {
41 .zo => return inst(enc, .{}),
42 .d, .i => {
43 const imm = try dis.parseImm(enc.data.ops[0]);
41 .z => return inst(enc, .{}),
42 .o => {
43 const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
4444 return inst(enc, .{
45 .op1 = .{ .imm = imm },
45 .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
4646 });
4747 },
48 .zi => {
48 .zo => {
49 const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
50 return inst(enc, .{
51 .op1 = .{ .reg = enc.data.ops[0].toReg() },
52 .op2 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[1].regBitSize()) },
53 });
54 },
55 .oz => {
56 const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
57 return inst(enc, .{
58 .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
59 .op2 = .{ .reg = enc.data.ops[1].toReg() },
60 });
61 },
62 .oi => {
63 const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
4964 const imm = try dis.parseImm(enc.data.ops[1]);
5065 return inst(enc, .{
51 .op1 = .{ .reg = Register.rax.toBitSize(enc.data.ops[0].regBitSize()) },
66 .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
5267 .op2 = .{ .imm = imm },
5368 });
5469 },
55 .o, .oi => {
56 const reg_low_enc = @as(u3, @truncate(dis.code[dis.pos - 1]));
57 const op2: Instruction.Operand = if (enc.data.op_en == .oi) .{
58 .imm = try dis.parseImm(enc.data.ops[1]),
59 } else .none;
70 .i, .d => {
71 const imm = try dis.parseImm(enc.data.ops[0]);
6072 return inst(enc, .{
61 .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
62 .op2 = op2,
73 .op1 = .{ .imm = imm },
74 });
75 },
76 .zi => {
77 const imm = try dis.parseImm(enc.data.ops[1]);
78 return inst(enc, .{
79 .op1 = .{ .reg = enc.data.ops[0].toReg() },
80 .op2 = .{ .imm = imm },
6381 });
6482 },
6583 .m, .mi, .m1, .mc => {
......@@ -118,7 +136,7 @@ pub fn next(dis: *Disassembler) Error!?Instruction {
118136 const seg = segmentRegister(prefixes.legacy);
119137 const offset = try dis.parseOffset();
120138 return inst(enc, .{
121 .op1 = .{ .reg = Register.rax.toBitSize(enc.data.ops[0].regBitSize()) },
139 .op1 = .{ .reg = enc.data.ops[0].toReg() },
122140 .op2 = .{ .mem = Memory.initMoffs(seg, offset) },
123141 });
124142 },
......@@ -127,7 +145,7 @@ pub fn next(dis: *Disassembler) Error!?Instruction {
127145 const offset = try dis.parseOffset();
128146 return inst(enc, .{
129147 .op1 = .{ .mem = Memory.initMoffs(seg, offset) },
130 .op2 = .{ .reg = Register.rax.toBitSize(enc.data.ops[1].regBitSize()) },
148 .op2 = .{ .reg = enc.data.ops[1].toReg() },
131149 });
132150 },
133151 .mr, .mri, .mrc => {
......@@ -223,7 +241,7 @@ pub fn next(dis: *Disassembler) Error!?Instruction {
223241 .op3 = op3,
224242 });
225243 },
226 .rm0, .vmi, .rvm, .rvmr, .rvmi, .mvr => unreachable, // TODO
244 .rm0, .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => unreachable, // TODO
227245 }
228246}
229247
src/arch/x86_64/Emit.zig+58-34
......@@ -10,22 +10,21 @@ prev_di_loc: Loc,
1010/// Relative to the beginning of `code`.
1111prev_di_pc: usize,
1212
13code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .empty,
14relocs: std.ArrayListUnmanaged(Reloc) = .empty,
15
1613pub const Error = Lower.Error || error{
1714 EmitFail,
1815} || link.File.UpdateDebugInfoError;
1916
2017pub fn emitMir(emit: *Emit) Error!void {
2118 const gpa = emit.lower.bin_file.comp.gpa;
19 const code_offset_mapping = try emit.lower.allocator.alloc(u32, emit.lower.mir.instructions.len);
20 defer emit.lower.allocator.free(code_offset_mapping);
21 var relocs: std.ArrayListUnmanaged(Reloc) = .empty;
22 defer relocs.deinit(emit.lower.allocator);
23 var table_relocs: std.ArrayListUnmanaged(TableReloc) = .empty;
24 defer table_relocs.deinit(emit.lower.allocator);
2225 for (0..emit.lower.mir.instructions.len) |mir_i| {
2326 const mir_index: Mir.Inst.Index = @intCast(mir_i);
24 try emit.code_offset_mapping.putNoClobber(
25 emit.lower.allocator,
26 mir_index,
27 @intCast(emit.code.items.len),
28 );
27 code_offset_mapping[mir_index] = @intCast(emit.code.items.len);
2928 const lowered = try emit.lower.lowerMir(mir_index);
3029 var lowered_relocs = lowered.relocs;
3130 for (lowered.insts, 0..) |lowered_inst, lowered_index| {
......@@ -89,13 +88,17 @@ pub fn emitMir(emit: *Emit) Error!void {
8988 lowered_relocs[0].lowered_inst_index == lowered_index) : ({
9089 lowered_relocs = lowered_relocs[1..];
9190 }) switch (lowered_relocs[0].target) {
92 .inst => |target| try emit.relocs.append(emit.lower.allocator, .{
91 .inst => |target| try relocs.append(emit.lower.allocator, .{
9392 .source = start_offset,
9493 .source_offset = end_offset - 4,
9594 .target = target,
9695 .target_offset = lowered_relocs[0].off,
9796 .length = @intCast(end_offset - start_offset),
9897 }),
98 .table => try table_relocs.append(emit.lower.allocator, .{
99 .source_offset = end_offset - 4,
100 .target_offset = lowered_relocs[0].off,
101 }),
99102 .linker_extern_fn => |sym_index| if (emit.lower.bin_file.cast(.elf)) |elf_file| {
100103 // Add relocation to the decl.
101104 const zo = elf_file.zigObjectPtr().?;
......@@ -103,7 +106,7 @@ pub fn emitMir(emit: *Emit) Error!void {
103106 const r_type = @intFromEnum(std.elf.R_X86_64.PLT32);
104107 try atom_ptr.addReloc(gpa, .{
105108 .r_offset = end_offset - 4,
106 .r_info = (@as(u64, @intCast(sym_index)) << 32) | r_type,
109 .r_info = @as(u64, sym_index) << 32 | r_type,
107110 .r_addend = lowered_relocs[0].off - 4,
108111 }, zo);
109112 } else if (emit.lower.bin_file.cast(.macho)) |macho_file| {
......@@ -150,7 +153,7 @@ pub fn emitMir(emit: *Emit) Error!void {
150153 const r_type = @intFromEnum(std.elf.R_X86_64.TLSLD);
151154 try atom.addReloc(gpa, .{
152155 .r_offset = end_offset - 4,
153 .r_info = (@as(u64, @intCast(sym_index)) << 32) | r_type,
156 .r_info = @as(u64, sym_index) << 32 | r_type,
154157 .r_addend = lowered_relocs[0].off - 4,
155158 }, zo);
156159 },
......@@ -161,7 +164,7 @@ pub fn emitMir(emit: *Emit) Error!void {
161164 const r_type = @intFromEnum(std.elf.R_X86_64.DTPOFF32);
162165 try atom.addReloc(gpa, .{
163166 .r_offset = end_offset - 4,
164 .r_info = (@as(u64, @intCast(sym_index)) << 32) | r_type,
167 .r_info = @as(u64, sym_index) << 32 | r_type,
165168 .r_addend = lowered_relocs[0].off,
166169 }, zo);
167170 },
......@@ -176,7 +179,7 @@ pub fn emitMir(emit: *Emit) Error!void {
176179 @intFromEnum(std.elf.R_X86_64.PC32);
177180 try atom.addReloc(gpa, .{
178181 .r_offset = end_offset - 4,
179 .r_info = (@as(u64, @intCast(sym_index)) << 32) | r_type,
182 .r_info = @as(u64, sym_index) << 32 | r_type,
180183 .r_addend = lowered_relocs[0].off - 4,
181184 }, zo);
182185 } else {
......@@ -186,7 +189,7 @@ pub fn emitMir(emit: *Emit) Error!void {
186189 @intFromEnum(std.elf.R_X86_64.@"32");
187190 try atom.addReloc(gpa, .{
188191 .r_offset = end_offset - 4,
189 .r_info = (@as(u64, @intCast(sym_index)) << 32) | r_type,
192 .r_info = @as(u64, sym_index) << 32 | r_type,
190193 .r_addend = lowered_relocs[0].off,
191194 }, zo);
192195 }
......@@ -412,7 +415,7 @@ pub fn emitMir(emit: *Emit) Error!void {
412415 loc_buf[0] = switch (mem.base()) {
413416 .none => .{ .constu = 0 },
414417 .reg => |reg| .{ .breg = reg.dwarfNum() },
415 .frame => unreachable,
418 .frame, .table => unreachable,
416419 .reloc => |sym_index| .{ .addr = .{ .sym = sym_index } },
417420 };
418421 break :base &loc_buf[0];
......@@ -463,13 +466,40 @@ pub fn emitMir(emit: *Emit) Error!void {
463466 }
464467 }
465468 }
466 try emit.fixupRelocs();
467}
469 {
470 // TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
471 // This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
472 // possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
473 // until the entire decl is correctly emitted with all JMP/CALL instructions within range.
474 for (relocs.items) |reloc| {
475 const target = code_offset_mapping[reloc.target];
476 const disp = @as(i64, @intCast(target)) - @as(i64, @intCast(reloc.source + reloc.length)) + reloc.target_offset;
477 std.mem.writeInt(i32, emit.code.items[reloc.source_offset..][0..4], @intCast(disp), .little);
478 }
479 }
480 if (emit.lower.mir.table.len > 0) {
481 if (emit.lower.bin_file.cast(.elf)) |elf_file| {
482 const zo = elf_file.zigObjectPtr().?;
483 const atom = zo.symbol(emit.atom_index).atom(elf_file).?;
468484
469pub fn deinit(emit: *Emit) void {
470 emit.relocs.deinit(emit.lower.allocator);
471 emit.code_offset_mapping.deinit(emit.lower.allocator);
472 emit.* = undefined;
485 const ptr_size = @divExact(emit.lower.target.ptrBitWidth(), 8);
486 var table_offset = std.mem.alignForward(u32, @intCast(emit.code.items.len), ptr_size);
487 for (table_relocs.items) |table_reloc| try atom.addReloc(gpa, .{
488 .r_offset = table_reloc.source_offset,
489 .r_info = @as(u64, emit.atom_index) << 32 | @intFromEnum(std.elf.R_X86_64.@"32"),
490 .r_addend = @as(i64, table_offset) + table_reloc.target_offset,
491 }, zo);
492 for (emit.lower.mir.table) |entry| {
493 try atom.addReloc(gpa, .{
494 .r_offset = table_offset,
495 .r_info = @as(u64, emit.atom_index) << 32 | @intFromEnum(std.elf.R_X86_64.@"64"),
496 .r_addend = code_offset_mapping[entry],
497 }, zo);
498 table_offset += ptr_size;
499 }
500 try emit.code.appendNTimes(gpa, 0, table_offset - emit.code.items.len);
501 } else unreachable;
502 }
473503}
474504
475505fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error {
......@@ -481,7 +511,7 @@ fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error {
481511
482512const Reloc = struct {
483513 /// Offset of the instruction.
484 source: usize,
514 source: u32,
485515 /// Offset of the relocation within the instruction.
486516 source_offset: u32,
487517 /// Target of the relocation.
......@@ -492,18 +522,12 @@ const Reloc = struct {
492522 length: u5,
493523};
494524
495fn fixupRelocs(emit: *Emit) Error!void {
496 // TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
497 // This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
498 // possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
499 // until the entire decl is correctly emitted with all JMP/CALL instructions within range.
500 for (emit.relocs.items) |reloc| {
501 const target = emit.code_offset_mapping.get(reloc.target) orelse
502 return emit.fail("JMP/CALL relocation target not found!", .{});
503 const disp = @as(i64, @intCast(target)) - @as(i64, @intCast(reloc.source + reloc.length)) + reloc.target_offset;
504 std.mem.writeInt(i32, emit.code.items[reloc.source_offset..][0..4], @intCast(disp), .little);
505 }
506}
525const TableReloc = struct {
526 /// Offset of the relocation.
527 source_offset: u32,
528 /// Offset from the start of the table.
529 target_offset: i32,
530};
507531
508532const Loc = struct {
509533 line: u32,
src/arch/x86_64/Encoding.zig+65-23
......@@ -30,9 +30,10 @@ pub fn findByMnemonic(
3030 prefix: Instruction.Prefix,
3131 mnemonic: Mnemonic,
3232 ops: []const Instruction.Operand,
33 target: *const std.Target,
3334) !?Encoding {
34 var input_ops = [1]Op{.none} ** 4;
35 for (input_ops[0..ops.len], ops) |*input_op, op| input_op.* = Op.fromOperand(op);
35 var input_ops: [4]Op = @splat(.none);
36 for (input_ops[0..ops.len], ops) |*input_op, op| input_op.* = Op.fromOperand(op, target);
3637
3738 const rex_required = for (ops) |op| switch (op) {
3839 .reg => |r| switch (r) {
......@@ -57,6 +58,16 @@ pub fn findByMnemonic(
5758 var shortest_enc: ?Encoding = null;
5859 var shortest_len: ?usize = null;
5960 next: for (mnemonic_to_encodings_map[@intFromEnum(mnemonic)]) |data| {
61 if (!switch (data.feature) {
62 .none => true,
63 inline else => |tag| has_features: {
64 comptime var feature_it = std.mem.splitScalar(u8, @tagName(tag), ' ');
65 comptime var features: []const std.Target.x86.Feature = &.{};
66 inline while (comptime feature_it.next()) |feature| features = features ++ .{@field(std.Target.x86.Feature, feature)};
67 break :has_features std.Target.x86.featureSetHasAll(target.cpu.features, features[0..].*);
68 },
69 }) continue;
70
6071 switch (data.mode) {
6172 .none, .short => if (rex_required) continue,
6273 .rex, .rex_short => if (!rex_required) continue,
......@@ -64,7 +75,7 @@ pub fn findByMnemonic(
6475 }
6576 for (input_ops, data.ops) |input_op, data_op| if (!input_op.isSubset(data_op)) continue :next;
6677
67 const enc = Encoding{ .mnemonic = mnemonic, .data = data };
78 const enc: Encoding = .{ .mnemonic = mnemonic, .data = data };
6879 if (shortest_enc) |previous_shortest_enc| {
6980 const len = estimateInstructionLength(prefix, enc, ops);
7081 const previous_shortest_len = shortest_len orelse
......@@ -165,9 +176,14 @@ pub fn format(
165176 for (opc) |byte| try writer.print("{x:0>2} ", .{byte});
166177
167178 switch (encoding.data.op_en) {
168 .zo, .fd, .td, .i, .zi, .d => {},
169 .o, .oi => {
170 const tag = switch (encoding.data.ops[0]) {
179 .z, .fd, .td, .i, .zi, .d => {},
180 .o, .zo, .oz, .oi => {
181 const op = switch (encoding.data.op_en) {
182 .o, .oz, .oi => encoding.data.ops[0],
183 .zo => encoding.data.ops[1],
184 else => unreachable,
185 };
186 const tag = switch (op) {
171187 .r8 => "rb",
172188 .r16 => "rw",
173189 .r32 => "rd",
......@@ -177,7 +193,7 @@ pub fn format(
177193 try writer.print("+{s} ", .{tag});
178194 },
179195 .m, .mi, .m1, .mc, .vmi => try writer.print("/{d} ", .{encoding.modRmExt()}),
180 .mr, .rm, .rmi, .mri, .mrc, .rm0, .rvm, .rvmr, .rvmi, .mvr => try writer.writeAll("/r "),
196 .mr, .rm, .rmi, .mri, .mrc, .rm0, .rvm, .rvmr, .rvmi, .mvr, .rmv => try writer.writeAll("/r "),
181197 }
182198
183199 switch (encoding.data.op_en) {
......@@ -202,7 +218,7 @@ pub fn format(
202218 try writer.print("{s} ", .{tag});
203219 },
204220 .rvmr => try writer.writeAll("/is4 "),
205 .zo, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rm0, .rvm, .mvr => {},
221 .z, .fd, .td, .o, .zo, .oz, .m, .m1, .mc, .mr, .rm, .mrc, .rm0, .rvm, .mvr, .rmv => {},
206222 }
207223
208224 try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
......@@ -239,7 +255,8 @@ pub const Mnemonic = enum {
239255 // General-purpose
240256 adc, add, @"and",
241257 bsf, bsr, bswap, bt, btc, btr, bts,
242 call, cbw, cdq, cdqe, clflush,
258 call, cbw, cdq, cdqe,
259 clac, clc, cld, clflush, cli, clts, clui,
243260 cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
244261 cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
245262 cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz,
......@@ -260,10 +277,12 @@ pub const Mnemonic = enum {
260277 neg, nop, not,
261278 @"or",
262279 pause, pop, popcnt, popfq, push, pushfq,
263 rcl, rcr, ret, rol, ror,
264 sal, sar, sbb,
280 rcl, rcr, ret, rol, ror, rorx,
281 sal, sar, sarx, sbb,
265282 scas, scasb, scasd, scasq, scasw,
266 shl, shld, shr, shrd, sub, syscall,
283 shl, shld, shlx, shr, shrd, shrx,
284 stac, stc, std, sti, stui,
285 sub, syscall,
267286 seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
268287 setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
269288 setnz, seto, setp, setpe, setpo, sets, setz,
......@@ -296,7 +315,7 @@ pub const Mnemonic = enum {
296315 ldmxcsr,
297316 maxps, maxss,
298317 minps, minss,
299 movaps, movhlps, movlhps,
318 movaps, movhlps, movhps, movlhps, movlps,
300319 movmskps,
301320 movss, movups,
302321 mulps, mulss,
......@@ -322,6 +341,7 @@ pub const Mnemonic = enum {
322341 minpd, minsd,
323342 movapd,
324343 movdqa, movdqu,
344 movhpd, movlpd,
325345 movmskpd,
326346 //movsd,
327347 movupd,
......@@ -353,6 +373,7 @@ pub const Mnemonic = enum {
353373 pmovsxbd, pmovsxbq, pmovsxbw, pmovsxdq, pmovsxwd, pmovsxwq,
354374 pmovzxbd, pmovzxbq, pmovzxbw, pmovzxdq, pmovzxwd, pmovzxwq,
355375 pmulld,
376 ptest,
356377 roundpd, roundps, roundsd, roundss,
357378 // SSE4.2
358379 pcmpgtq,
......@@ -383,7 +404,7 @@ pub const Mnemonic = enum {
383404 vmovd,
384405 vmovddup,
385406 vmovdqa, vmovdqu,
386 vmovhlps, vmovlhps,
407 vmovhlps, vmovhpd, vmovhps, vmovlhps, vmovlpd, vmovlps,
387408 vmovmskpd, vmovmskps,
388409 vmovq,
389410 vmovsd,
......@@ -413,6 +434,7 @@ pub const Mnemonic = enum {
413434 vpsrad, vpsraq, vpsraw,
414435 vpsrld, vpsrldq, vpsrlq, vpsrlw,
415436 vpsubb, vpsubd, vpsubq, vpsubsb, vpsubsw, vpsubusb, vpsubusw, vpsubw,
437 vptest,
416438 vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd,
417439 vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd,
418440 vpxor,
......@@ -421,6 +443,7 @@ pub const Mnemonic = enum {
421443 vsqrtpd, vsqrtps, vsqrtsd, vsqrtss,
422444 vstmxcsr,
423445 vsubpd, vsubps, vsubsd, vsubss,
446 vtestpd, vtestps,
424447 vxorpd, vxorps,
425448 // F16C
426449 vcvtph2ps, vcvtps2ph,
......@@ -437,14 +460,14 @@ pub const Mnemonic = enum {
437460
438461pub const OpEn = enum {
439462 // zig fmt: off
440 zo,
441 o, oi,
463 z,
464 o, zo, oz, oi,
442465 i, zi,
443466 d, m,
444467 fd, td,
445468 m1, mc, mi, mr, rm,
446469 rmi, mri, mrc,
447 rm0, vmi, rvm, rvmr, rvmi, mvr,
470 rm0, vmi, rvm, rvmr, rvmi, mvr, rmv,
448471 // zig fmt: on
449472};
450473
......@@ -471,7 +494,7 @@ pub const Op = enum {
471494 ymm, ymm_m256,
472495 // zig fmt: on
473496
474 pub fn fromOperand(operand: Instruction.Operand) Op {
497 pub fn fromOperand(operand: Instruction.Operand, target: *const std.Target) Op {
475498 return switch (operand) {
476499 .none => .none,
477500
......@@ -513,7 +536,7 @@ pub const Op = enum {
513536
514537 .mem => |mem| switch (mem) {
515538 .moffs => .moffs,
516 .sib, .rip => switch (mem.bitSize()) {
539 .sib, .rip => switch (mem.bitSize(target)) {
517540 0 => .m,
518541 8 => .m8,
519542 16 => .m16,
......@@ -557,6 +580,21 @@ pub const Op = enum {
557580 };
558581 }
559582
583 pub fn toReg(op: Op) Register {
584 return switch (op) {
585 else => .none,
586 .al => .al,
587 .ax => .ax,
588 .eax => .eax,
589 .rax => .rax,
590 .cl => .cl,
591 .rip => .rip,
592 .eip => .eip,
593 .ip => .ip,
594 .xmm0 => .xmm0,
595 };
596 }
597
560598 pub fn immBitSize(op: Op) u64 {
561599 return switch (op) {
562600 .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
......@@ -808,6 +846,8 @@ pub const Feature = enum {
808846 avx,
809847 avx2,
810848 bmi,
849 bmi2,
850 cmov,
811851 f16c,
812852 fma,
813853 lzcnt,
......@@ -815,6 +855,7 @@ pub const Feature = enum {
815855 pclmul,
816856 @"pclmul avx",
817857 popcnt,
858 smap,
818859 sse,
819860 sse2,
820861 sse3,
......@@ -822,6 +863,7 @@ pub const Feature = enum {
822863 sse4_2,
823864 ssse3,
824865 sha,
866 uintr,
825867 vaes,
826868 vpclmulqdq,
827869 x87,
......@@ -831,7 +873,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
831873 var inst = Instruction{
832874 .prefix = prefix,
833875 .encoding = encoding,
834 .ops = [1]Operand{.none} ** 4,
876 .ops = @splat(.none),
835877 };
836878 @memcpy(inst.ops[0..ops.len], ops);
837879
......@@ -846,7 +888,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
846888const mnemonic_to_encodings_map = init: {
847889 @setEvalBranchQuota(5_000);
848890 const mnemonic_count = @typeInfo(Mnemonic).@"enum".fields.len;
849 var mnemonic_map: [mnemonic_count][]Data = .{&.{}} ** mnemonic_count;
891 var mnemonic_map: [mnemonic_count][]Data = @splat(&.{});
850892 const encodings = @import("encodings.zig");
851893 for (encodings.table) |entry| mnemonic_map[@intFromEnum(entry[0])].len += 1;
852894 var data_storage: [encodings.table.len]Data = undefined;
......@@ -855,7 +897,7 @@ const mnemonic_to_encodings_map = init: {
855897 value.ptr = data_storage[storage_i..].ptr;
856898 storage_i += value.len;
857899 }
858 var mnemonic_i: [mnemonic_count]usize = .{0} ** mnemonic_count;
900 var mnemonic_i: [mnemonic_count]usize = @splat(0);
859901 const ops_len = @typeInfo(std.meta.FieldType(Data, .ops)).array.len;
860902 const opc_len = @typeInfo(std.meta.FieldType(Data, .opc)).array.len;
861903 for (encodings.table) |entry| {
......@@ -872,7 +914,7 @@ const mnemonic_to_encodings_map = init: {
872914 i.* += 1;
873915 }
874916 const final_storage = data_storage;
875 var final_map: [mnemonic_count][]const Data = .{&.{}} ** mnemonic_count;
917 var final_map: [mnemonic_count][]const Data = @splat(&.{});
876918 storage_i = 0;
877919 for (&final_map, mnemonic_map) |*final_value, value| {
878920 final_value.* = final_storage[storage_i..][0..value.len];
src/arch/x86_64/Lower.zig+39-37
......@@ -1,6 +1,7 @@
11//! This file contains the functionality for lowering x86_64 MIR to Instructions
22
33bin_file: *link.File,
4target: *const std.Target,
45output_mode: std.builtin.OutputMode,
56link_mode: std.builtin.LinkMode,
67pic: bool,
......@@ -56,6 +57,7 @@ pub const Reloc = struct {
5657
5758 const Target = union(enum) {
5859 inst: Mir.Inst.Index,
60 table,
5961 linker_reloc: u32,
6062 linker_tlsld: u32,
6163 linker_dtpoff: u32,
......@@ -193,7 +195,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
193195 .pseudo_probe_align_ri_s => {
194196 try lower.emit(.none, .@"test", &.{
195197 .{ .reg = inst.data.ri.r1 },
196 .{ .imm = Immediate.s(@bitCast(inst.data.ri.i)) },
198 .{ .imm = .s(@bitCast(inst.data.ri.i)) },
197199 });
198200 try lower.emit(.none, .jz, &.{
199201 .{ .imm = lower.reloc(.{ .inst = index + 1 }, 0) },
......@@ -229,14 +231,14 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
229231 }
230232 try lower.emit(.none, .sub, &.{
231233 .{ .reg = inst.data.ri.r1 },
232 .{ .imm = Immediate.s(@bitCast(inst.data.ri.i)) },
234 .{ .imm = .s(@bitCast(inst.data.ri.i)) },
233235 });
234236 assert(lower.result_insts_len <= pseudo_probe_adjust_unrolled_max_insts);
235237 },
236238 .pseudo_probe_adjust_setup_rri_s => {
237239 try lower.emit(.none, .mov, &.{
238240 .{ .reg = inst.data.rri.r2.to32() },
239 .{ .imm = Immediate.s(@bitCast(inst.data.rri.i)) },
241 .{ .imm = .s(@bitCast(inst.data.rri.i)) },
240242 });
241243 try lower.emit(.none, .sub, &.{
242244 .{ .reg = inst.data.rri.r1 },
......@@ -255,7 +257,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
255257 });
256258 try lower.emit(.none, .sub, &.{
257259 .{ .reg = inst.data.rr.r2 },
258 .{ .imm = Immediate.s(page_size) },
260 .{ .imm = .s(page_size) },
259261 });
260262 try lower.emit(.none, .jae, &.{
261263 .{ .imm = lower.reloc(.{ .inst = index }, 0) },
......@@ -347,7 +349,7 @@ pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
347349 return error.LowerFail;
348350}
349351
350pub fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
352pub fn imm(lower: *const Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
351353 return switch (ops) {
352354 .rri_s,
353355 .ri_s,
......@@ -355,7 +357,7 @@ pub fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
355357 .mi_s,
356358 .rmi_s,
357359 .pseudo_dbg_local_ai_s,
358 => Immediate.s(@bitCast(i)),
360 => .s(@bitCast(i)),
359361
360362 .rrri,
361363 .rri_u,
......@@ -368,18 +370,26 @@ pub fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
368370 .rrm,
369371 .rrmi,
370372 .pseudo_dbg_local_ai_u,
371 => Immediate.u(i),
373 => .u(i),
372374
373375 .ri_64,
374376 .pseudo_dbg_local_ai_64,
375 => Immediate.u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
377 => .u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
376378
377379 else => unreachable,
378380 };
379381}
380382
381pub fn mem(lower: Lower, payload: u32) Memory {
382 return lower.mir.resolveFrameLoc(lower.mir.extraData(Mir.Memory, payload).data).decode();
383pub fn mem(lower: *Lower, payload: u32) Memory {
384 var m = lower.mir.resolveFrameLoc(lower.mir.extraData(Mir.Memory, payload).data).decode();
385 switch (m) {
386 .sib => |*sib| switch (sib.base) {
387 else => {},
388 .table => sib.disp = lower.reloc(.table, sib.disp).signed,
389 },
390 else => {},
391 }
392 return m;
383393}
384394
385395fn reloc(lower: *Lower, target: Reloc.Target, off: i32) Immediate {
......@@ -389,7 +399,7 @@ fn reloc(lower: *Lower, target: Reloc.Target, off: i32) Immediate {
389399 .off = off,
390400 };
391401 lower.result_relocs_len += 1;
392 return Immediate.s(0);
402 return .s(0);
393403}
394404
395405fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
......@@ -417,19 +427,17 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
417427 // Here, we currently assume local dynamic TLS vars, and so
418428 // we emit LD model.
419429 _ = lower.reloc(.{ .linker_tlsld = sym_index }, 0);
420 lower.result_insts[lower.result_insts_len] =
421 try Instruction.new(.none, .lea, &[_]Operand{
430 lower.result_insts[lower.result_insts_len] = try .new(.none, .lea, &.{
422431 .{ .reg = .rdi },
423432 .{ .mem = Memory.initRip(mem_op.sib.ptr_size, 0) },
424 });
433 }, lower.target);
425434 lower.result_insts_len += 1;
426435 _ = lower.reloc(.{
427436 .linker_extern_fn = try elf_file.getGlobalSymbol("__tls_get_addr", null),
428437 }, 0);
429 lower.result_insts[lower.result_insts_len] =
430 try Instruction.new(.none, .call, &[_]Operand{
431 .{ .imm = Immediate.s(0) },
432 });
438 lower.result_insts[lower.result_insts_len] = try .new(.none, .call, &.{
439 .{ .imm = .s(0) },
440 }, lower.target);
433441 lower.result_insts_len += 1;
434442 _ = lower.reloc(.{ .linker_dtpoff = sym_index }, 0);
435443 emit_mnemonic = .lea;
......@@ -439,11 +447,10 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
439447 }) };
440448 } else {
441449 // Since we are linking statically, we emit LE model directly.
442 lower.result_insts[lower.result_insts_len] =
443 try Instruction.new(.none, .mov, &[_]Operand{
450 lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
444451 .{ .reg = .rax },
445452 .{ .mem = Memory.initSib(.qword, .{ .base = .{ .reg = .fs } }) },
446 });
453 }, lower.target);
447454 lower.result_insts_len += 1;
448455 _ = lower.reloc(.{ .linker_reloc = sym_index }, 0);
449456 emit_mnemonic = .lea;
......@@ -463,11 +470,10 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
463470 .mov => {
464471 if (elf_sym.flags.is_extern_ptr) {
465472 const reg = ops[0].reg;
466 lower.result_insts[lower.result_insts_len] =
467 try Instruction.new(.none, .mov, &[_]Operand{
473 lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
468474 .{ .reg = reg.to64() },
469475 .{ .mem = Memory.initRip(.qword, 0) },
470 });
476 }, lower.target);
471477 lower.result_insts_len += 1;
472478 break :op .{ .mem = Memory.initSib(mem_op.sib.ptr_size, .{ .base = .{
473479 .reg = reg.to64(),
......@@ -482,7 +488,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
482488 }) },
483489 .lea => {
484490 emit_mnemonic = .mov;
485 break :op .{ .imm = Immediate.s(0) };
491 break :op .{ .imm = .s(0) };
486492 },
487493 .mov => break :op .{ .mem = Memory.initSib(mem_op.sib.ptr_size, .{
488494 .base = .{ .reg = .ds },
......@@ -495,16 +501,14 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
495501
496502 if (macho_sym.flags.tlv) {
497503 _ = lower.reloc(.{ .linker_reloc = sym_index }, 0);
498 lower.result_insts[lower.result_insts_len] =
499 try Instruction.new(.none, .mov, &[_]Operand{
504 lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
500505 .{ .reg = .rdi },
501506 .{ .mem = Memory.initRip(mem_op.sib.ptr_size, 0) },
502 });
507 }, lower.target);
503508 lower.result_insts_len += 1;
504 lower.result_insts[lower.result_insts_len] =
505 try Instruction.new(.none, .call, &[_]Operand{
509 lower.result_insts[lower.result_insts_len] = try .new(.none, .call, &.{
506510 .{ .mem = Memory.initSib(.qword, .{ .base = .{ .reg = .rdi } }) },
507 });
511 }, lower.target);
508512 lower.result_insts_len += 1;
509513 emit_mnemonic = .mov;
510514 break :op .{ .reg = .rax };
......@@ -519,11 +523,10 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
519523 .mov => {
520524 if (macho_sym.flags.is_extern_ptr) {
521525 const reg = ops[0].reg;
522 lower.result_insts[lower.result_insts_len] =
523 try Instruction.new(.none, .mov, &[_]Operand{
526 lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
524527 .{ .reg = reg.to64() },
525528 .{ .mem = Memory.initRip(.qword, 0) },
526 });
529 }, lower.target);
527530 lower.result_insts_len += 1;
528531 break :op .{ .mem = Memory.initSib(mem_op.sib.ptr_size, .{ .base = .{
529532 .reg = reg.to64(),
......@@ -540,8 +543,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
540543 },
541544 };
542545 }
543 lower.result_insts[lower.result_insts_len] =
544 try Instruction.new(emit_prefix, emit_mnemonic, emit_ops);
546 lower.result_insts[lower.result_insts_len] = try .new(emit_prefix, emit_mnemonic, emit_ops, lower.target);
545547 lower.result_insts_len += 1;
546548}
547549
......@@ -743,7 +745,7 @@ fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Er
743745 while (it.next()) |i| {
744746 try lower.emit(.directive, .@".cfi_rel_offset", &.{
745747 .{ .reg = callee_preserved_regs[i] },
746 .{ .imm = Immediate.s(off) },
748 .{ .imm = .s(off) },
747749 });
748750 off += 8;
749751 }
src/arch/x86_64/Mir.zig+64-15
......@@ -9,6 +9,7 @@
99instructions: std.MultiArrayList(Inst).Slice,
1010/// The meaning of this data is determined by `Inst.Tag` value.
1111extra: []const u32,
12table: []const Inst.Index,
1213frame_locs: std.MultiArrayList(FrameLoc).Slice,
1314
1415pub const Inst = struct {
......@@ -22,17 +23,26 @@ pub const Inst = struct {
2223 /// ___
2324 @"_",
2425
25 /// Integer __
26 /// Integer ___
2627 i_,
2728
2829 /// ___ Left
2930 _l,
3031 /// ___ Left Double
3132 _ld,
33 /// ___ Left Without Affecting Flags
34 _lx,
3235 /// ___ Right
3336 _r,
3437 /// ___ Right Double
3538 _rd,
39 /// ___ Right Without Affecting Flags
40 _rx,
41
42 /// ___ Forward
43 _f,
44 /// ___ Reverse
45 //_r,
3646
3747 /// ___ Above
3848 _a,
......@@ -43,6 +53,7 @@ pub const Inst = struct {
4353 /// ___ Below Or Equal
4454 _be,
4555 /// ___ Carry
56 /// ___ Carry Flag
4657 _c,
4758 /// ___ Equal
4859 _e,
......@@ -94,6 +105,14 @@ pub const Inst = struct {
94105 _s,
95106 /// ___ Zero
96107 _z,
108 /// ___ Alignment Check Flag
109 _ac,
110 /// ___ Direction Flag
111 //_d,
112 /// ___ Interrupt Flag
113 _i,
114 /// ___ User Interrupt Flag
115 _ui,
97116
98117 /// ___ Byte
99118 //_b,
......@@ -210,6 +229,10 @@ pub const Inst = struct {
210229 p_q,
211230 /// Packed ___ Double Quadword
212231 p_dq,
232 /// ___ Aligned Packed Integer Values
233 _dqa,
234 /// ___ Unaligned Packed Integer Values
235 _dqu,
213236
214237 /// ___ Scalar Single-Precision Values
215238 _ss,
......@@ -230,6 +253,10 @@ pub const Inst = struct {
230253 v_d,
231254 /// VEX-Encoded ___ QuadWord
232255 v_q,
256 /// VEX-Encoded ___ Aligned Packed Integer Values
257 v_dqa,
258 /// VEX-Encoded ___ Unaligned Packed Integer Values
259 v_dqu,
233260 /// VEX-Encoded ___ Integer Data
234261 v_i128,
235262 /// VEX-Encoded Packed ___
......@@ -287,9 +314,8 @@ pub const Inst = struct {
287314 /// Bitwise logical and of packed double-precision floating-point values
288315 @"and",
289316 /// Bit scan forward
290 bsf,
291317 /// Bit scan reverse
292 bsr,
318 bs,
293319 /// Byte swap
294320 bswap,
295321 /// Bit test
......@@ -305,6 +331,10 @@ pub const Inst = struct {
305331 cdq,
306332 /// Convert doubleword to quadword
307333 cdqe,
334 /// Clear carry flag
335 /// Clear direction flag
336 /// Clear interrupt flag
337 cl,
308338 /// Flush cache line
309339 clflush,
310340 /// Conditional move
......@@ -358,6 +388,8 @@ pub const Inst = struct {
358388 /// Move scalar double-precision floating-point value
359389 /// Move doubleword
360390 /// Move quadword
391 /// Move aligned packed integer values
392 /// Move unaligned packed integer values
361393 mov,
362394 /// Move data after swapping bytes
363395 movbe,
......@@ -401,9 +433,11 @@ pub const Inst = struct {
401433 ret,
402434 /// Rotate left
403435 /// Rotate right
436 /// Rotate right logical without affecting flags
404437 ro,
405438 /// Arithmetic shift left
406439 /// Arithmetic shift right
440 /// Shift left arithmetic without affecting flags
407441 sa,
408442 /// Integer subtraction with borrow
409443 sbb,
......@@ -417,6 +451,8 @@ pub const Inst = struct {
417451 /// Double precision shift left
418452 /// Logical shift right
419453 /// Double precision shift right
454 /// Shift left logical without affecting flags
455 /// Shift right logical without affecting flags
420456 sh,
421457 /// Subtract
422458 /// Subtract packed integers
......@@ -425,6 +461,11 @@ pub const Inst = struct {
425461 /// Subtract packed double-precision floating-point values
426462 /// Subtract scalar double-precision floating-point values
427463 sub,
464 /// Set carry flag
465 /// Set direction flag
466 /// Set interrupt flag
467 /// Store floating-point value
468 st,
428469 /// Store string
429470 sto,
430471 /// Syscall
......@@ -460,8 +501,6 @@ pub const Inst = struct {
460501 ldenv,
461502 /// Store x87 FPU environment
462503 nstenv,
463 /// Store floating-point value
464 st,
465504 /// Store x87 FPU environment
466505 stenv,
467506
......@@ -542,8 +581,14 @@ pub const Inst = struct {
542581 /// Move aligned packed single-precision floating-point values
543582 /// Move aligned packed double-precision floating-point values
544583 mova,
584 /// Move high packed single-precision floating-point values
585 /// Move high packed double-precision floating-point values
586 movh,
545587 /// Move packed single-precision floating-point values high to low
546588 movhl,
589 /// Move low packed single-precision floating-point values
590 /// Move low packed double-precision floating-point values
591 movl,
547592 /// Move packed single-precision floating-point values low to high
548593 movlh,
549594 /// Move unaligned packed single-precision floating-point values
......@@ -601,10 +646,6 @@ pub const Inst = struct {
601646 cvttps2dq,
602647 /// Convert with truncation scalar double-precision floating-point value to doubleword integer
603648 cvttsd2si,
604 /// Move aligned packed integer values
605 movdqa,
606 /// Move unaligned packed integer values
607 movdqu,
608649 /// Packed interleave shuffle of quadruplets of single-precision floating-point values
609650 /// Packed interleave shuffle of pairs of double-precision floating-point values
610651 /// Shuffle packed doublewords
......@@ -1127,11 +1168,13 @@ pub const AirOffset = struct { air_inst: Air.Inst.Index, off: i32 };
11271168
11281169/// Used in conjunction with payload to transfer a list of used registers in a compact manner.
11291170pub const RegisterList = struct {
1130 bitset: BitSet = BitSet.initEmpty(),
1171 bitset: BitSet,
11311172
11321173 const BitSet = IntegerBitSet(32);
11331174 const Self = @This();
11341175
1176 pub const empty: RegisterList = .{ .bitset = .initEmpty() };
1177
11351178 fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt {
11361179 for (registers, 0..) |cpreg, i| {
11371180 if (reg.id() == cpreg.id()) return @intCast(i);
......@@ -1157,8 +1200,12 @@ pub const RegisterList = struct {
11571200 return @intCast(self.bitset.count());
11581201 }
11591202
1160 pub fn size(self: Self) i32 {
1161 return @intCast(self.bitset.count() * 8);
1203 pub fn size(self: Self, target: *const std.Target) i32 {
1204 return @intCast(self.bitset.count() * @as(u4, switch (target.cpu.arch) {
1205 else => unreachable,
1206 .x86 => 4,
1207 .x86_64 => 8,
1208 }));
11621209 }
11631210};
11641211
......@@ -1197,7 +1244,7 @@ pub const Memory = struct {
11971244 size: bits.Memory.Size,
11981245 index: Register,
11991246 scale: bits.Memory.Scale,
1200 _: u16 = undefined,
1247 _: u15 = undefined,
12011248 };
12021249
12031250 pub fn encode(mem: bits.Memory) Memory {
......@@ -1220,7 +1267,7 @@ pub const Memory = struct {
12201267 },
12211268 },
12221269 .base = switch (mem.base) {
1223 .none => undefined,
1270 .none, .table => undefined,
12241271 .reg => |reg| @intFromEnum(reg),
12251272 .frame => |frame_index| @intFromEnum(frame_index),
12261273 .reloc => |sym_index| sym_index,
......@@ -1249,6 +1296,7 @@ pub const Memory = struct {
12491296 .none => .none,
12501297 .reg => .{ .reg = @enumFromInt(mem.base) },
12511298 .frame => .{ .frame = @enumFromInt(mem.base) },
1299 .table => .table,
12521300 .reloc => .{ .reloc = mem.base },
12531301 },
12541302 .scale_index = switch (mem.info.index) {
......@@ -1277,6 +1325,7 @@ pub const Memory = struct {
12771325pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
12781326 mir.instructions.deinit(gpa);
12791327 gpa.free(mir.extra);
1328 gpa.free(mir.table);
12801329 mir.frame_locs.deinit(gpa);
12811330 mir.* = undefined;
12821331}
......@@ -1312,7 +1361,7 @@ pub fn resolveFrameAddr(mir: Mir, frame_addr: bits.FrameAddr) bits.RegisterOffse
13121361
13131362pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory {
13141363 return switch (mem.info.base) {
1315 .none, .reg, .reloc => mem,
1364 .none, .reg, .table, .reloc => mem,
13161365 .frame => if (mir.frame_locs.len > 0) .{
13171366 .info = .{
13181367 .base = .reg,
src/arch/x86_64/abi.zig+78-26
......@@ -242,17 +242,20 @@ pub fn classifySystemV(ty: Type, zcu: *Zcu, target: std.Target, ctx: Context) [8
242242 .sse, .sseup, .sseup, .sseup,
243243 .sseup, .sseup, .sseup, .none,
244244 };
245 // LLVM always returns vectors byval
246 if (bits <= 512 or ctx == .ret) return .{
245 if (bits <= 512 or (ctx == .ret and bits <= @as(u64, if (std.Target.x86.featureSetHas(target.cpu.features, .avx512f))
246 2048
247 else if (std.Target.x86.featureSetHas(target.cpu.features, .avx))
248 1024
249 else
250 512))) return .{
247251 .sse, .sseup, .sseup, .sseup,
248252 .sseup, .sseup, .sseup, .sseup,
249253 };
250254 return memory_class;
251255 },
252256 .optional => {
253 if (ty.isPtrLikeOptional(zcu)) {
254 result[0] = .integer;
255 return result;
257 if (ty.optionalReprIsPayload(zcu)) {
258 return classifySystemV(ty.optionalChild(zcu), zcu, target, ctx);
256259 }
257260 return memory_class;
258261 },
......@@ -405,6 +408,31 @@ fn classifySystemVUnion(
405408 return starting_byte_offset + loaded_union.sizeUnordered(ip);
406409}
407410
411pub const zigcc = struct {
412 pub const stack_align: ?InternPool.Alignment = null;
413 pub const return_in_regs = true;
414 pub const params_in_regs = true;
415
416 const volatile_gpr = gp_regs.len - 5;
417 const volatile_x87 = x87_regs.len - 1;
418 const volatile_sse = sse_avx_regs.len;
419
420 /// Note that .rsp and .rbp also belong to this set, however, we never expect to use them
421 /// for anything else but stack offset tracking therefore we exclude them from this set.
422 pub const callee_preserved_regs = gp_regs[volatile_gpr..] ++ x87_regs[volatile_x87 .. x87_regs.len - 1] ++ sse_avx_regs[volatile_sse..];
423 /// These registers need to be preserved (saved on the stack) and restored by the caller before
424 /// the caller relinquishes control to a subroutine via call instruction (or similar).
425 /// In other words, these registers are free to use by the callee.
426 pub const caller_preserved_regs = gp_regs[0..volatile_gpr] ++ x87_regs[0..volatile_x87] ++ sse_avx_regs[0..volatile_sse];
427
428 const int_param_regs = gp_regs[0 .. volatile_gpr - 1];
429 const x87_param_regs = x87_regs[0..volatile_x87];
430 const sse_param_regs = sse_avx_regs[0..volatile_sse];
431 const int_return_regs = gp_regs[0..volatile_gpr];
432 const x87_return_regs = x87_regs[0..volatile_x87];
433 const sse_return_regs = sse_avx_regs[0..volatile_gpr];
434};
435
408436pub const SysV = struct {
409437 /// Note that .rsp and .rbp also belong to this set, however, we never expect to use them
410438 /// for anything else but stack offset tracking therefore we exclude them from this set.
......@@ -415,9 +443,11 @@ pub const SysV = struct {
415443 pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs;
416444
417445 pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
446 pub const c_abi_x87_param_regs = x87_regs[0..0].*;
418447 pub const c_abi_sse_param_regs = sse_avx_regs[0..8].*;
419448 pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };
420 pub const c_abi_sse_return_regs = sse_avx_regs[0..2].*;
449 pub const c_abi_x87_return_regs = x87_regs[0..2].*;
450 pub const c_abi_sse_return_regs = sse_avx_regs[0..4].*;
421451};
422452
423453pub const Win64 = struct {
......@@ -430,74 +460,96 @@ pub const Win64 = struct {
430460 pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs;
431461
432462 pub const c_abi_int_param_regs = [_]Register{ .rcx, .rdx, .r8, .r9 };
463 pub const c_abi_x87_param_regs = x87_regs[0..0].*;
433464 pub const c_abi_sse_param_regs = sse_avx_regs[0..4].*;
434465 pub const c_abi_int_return_regs = [_]Register{.rax};
466 pub const c_abi_x87_return_regs = x87_regs[0..0].*;
435467 pub const c_abi_sse_return_regs = sse_avx_regs[0..1].*;
436468};
437469
438pub fn resolveCallingConvention(
439 cc: std.builtin.CallingConvention,
440 target: std.Target,
441) std.builtin.CallingConvention {
442 return switch (cc) {
443 .auto => switch (target.os.tag) {
444 else => .{ .x86_64_sysv = .{} },
445 .windows => .{ .x86_64_win = .{} },
446 },
447 else => cc,
448 };
449}
450
451pub fn getCalleePreservedRegs(cc: std.builtin.CallingConvention) []const Register {
470pub fn getCalleePreservedRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
452471 return switch (cc) {
472 .auto => zigcc.callee_preserved_regs,
453473 .x86_64_sysv => &SysV.callee_preserved_regs,
454474 .x86_64_win => &Win64.callee_preserved_regs,
455475 else => unreachable,
456476 };
457477}
458478
459pub fn getCallerPreservedRegs(cc: std.builtin.CallingConvention) []const Register {
479pub fn getCallerPreservedRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
460480 return switch (cc) {
481 .auto => zigcc.caller_preserved_regs,
461482 .x86_64_sysv => &SysV.caller_preserved_regs,
462483 .x86_64_win => &Win64.caller_preserved_regs,
463484 else => unreachable,
464485 };
465486}
466487
467pub fn getCAbiIntParamRegs(cc: std.builtin.CallingConvention) []const Register {
488pub fn getCAbiIntParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
468489 return switch (cc) {
490 .auto => zigcc.int_param_regs,
469491 .x86_64_sysv => &SysV.c_abi_int_param_regs,
470492 .x86_64_win => &Win64.c_abi_int_param_regs,
471493 else => unreachable,
472494 };
473495}
474496
475pub fn getCAbiSseParamRegs(cc: std.builtin.CallingConvention) []const Register {
497pub fn getCAbiX87ParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
498 return switch (cc) {
499 .auto => zigcc.x87_param_regs,
500 .x86_64_sysv => &SysV.c_abi_x87_param_regs,
501 .x86_64_win => &Win64.c_abi_x87_param_regs,
502 else => unreachable,
503 };
504}
505
506pub fn getCAbiSseParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
476507 return switch (cc) {
508 .auto => zigcc.sse_param_regs,
477509 .x86_64_sysv => &SysV.c_abi_sse_param_regs,
478510 .x86_64_win => &Win64.c_abi_sse_param_regs,
479511 else => unreachable,
480512 };
481513}
482514
483pub fn getCAbiIntReturnRegs(cc: std.builtin.CallingConvention) []const Register {
515pub fn getCAbiIntReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
484516 return switch (cc) {
517 .auto => zigcc.int_return_regs,
485518 .x86_64_sysv => &SysV.c_abi_int_return_regs,
486519 .x86_64_win => &Win64.c_abi_int_return_regs,
487520 else => unreachable,
488521 };
489522}
490523
491pub fn getCAbiSseReturnRegs(cc: std.builtin.CallingConvention) []const Register {
524pub fn getCAbiX87ReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
525 return switch (cc) {
526 .auto => zigcc.x87_return_regs,
527 .x86_64_sysv => &SysV.c_abi_x87_return_regs,
528 .x86_64_win => &Win64.c_abi_x87_return_regs,
529 else => unreachable,
530 };
531}
532
533pub fn getCAbiSseReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
492534 return switch (cc) {
535 .auto => zigcc.sse_return_regs,
493536 .x86_64_sysv => &SysV.c_abi_sse_return_regs,
494537 .x86_64_win => &Win64.c_abi_sse_return_regs,
495538 else => unreachable,
496539 };
497540}
498541
542pub fn getCAbiLinkerScratchReg(cc: std.builtin.CallingConvention.Tag) Register {
543 return switch (cc) {
544 .auto => zigcc.int_return_regs[zigcc.int_return_regs.len - 1],
545 .x86_64_sysv => SysV.c_abi_int_return_regs[0],
546 .x86_64_win => Win64.c_abi_int_return_regs[0],
547 else => unreachable,
548 };
549}
550
499551const gp_regs = [_]Register{
500 .rax, .rcx, .rdx, .rbx, .rsi, .rdi, .r8, .r9, .r10, .r11, .r12, .r13, .r14, .r15,
552 .rax, .rdx, .rbx, .rcx, .rsi, .rdi, .r8, .r9, .r10, .r11, .r12, .r13, .r14, .r15,
501553};
502554const x87_regs = [_]Register{
503555 .st0, .st1, .st2, .st3, .st4, .st5, .st6, .st7,
src/arch/x86_64/bits.zig+80-11
......@@ -150,6 +150,31 @@ pub const Condition = enum(u5) {
150150 .nz_or_p => .z_and_np,
151151 };
152152 }
153
154 /// Returns the equivalent condition when the operands are swapped.
155 pub fn commute(cond: Condition) Condition {
156 return switch (cond) {
157 else => cond,
158 .a => .b,
159 .ae => .be,
160 .b => .a,
161 .be => .ae,
162 .c => .a,
163 .g => .l,
164 .ge => .le,
165 .l => .g,
166 .le => .ge,
167 .na => .nb,
168 .nae => .nbe,
169 .nb => .na,
170 .nbe => .nae,
171 .nc => .na,
172 .ng => .nl,
173 .nge => .nle,
174 .nl => .ng,
175 .nle => .nge,
176 };
177 }
153178};
154179
155180pub const Register = enum(u7) {
......@@ -454,37 +479,41 @@ pub const RegisterOffset = struct { reg: Register, off: i32 = 0 };
454479pub const SymbolOffset = struct { sym_index: u32, off: i32 = 0 };
455480
456481pub const Memory = struct {
457 base: Base,
458 mod: Mod,
482 base: Base = .none,
483 mod: Mod = .{ .rm = .{} },
459484
460 pub const Base = union(enum(u2)) {
485 pub const Base = union(enum(u3)) {
461486 none,
462487 reg: Register,
463488 frame: FrameIndex,
489 table,
464490 reloc: u32,
465491
466492 pub const Tag = @typeInfo(Base).@"union".tag_type.?;
467493
468494 pub fn isExtended(self: Base) bool {
469495 return switch (self) {
470 .none, .frame, .reloc => false, // rsp, rbp, and rip are not extended
496 .none, .frame, .table, .reloc => false, // rsp, rbp, and rip are not extended
471497 .reg => |reg| reg.isExtended(),
472498 };
473499 }
474500 };
475501
476502 pub const Mod = union(enum(u1)) {
477 rm: struct {
478 size: Size,
503 rm: Rm,
504 off: u64,
505
506 pub const Rm = struct {
507 size: Size = .none,
479508 index: Register = .none,
480509 scale: Scale = .@"1",
481510 disp: i32 = 0,
482 },
483 off: u64,
511 };
484512 };
485513
486514 pub const Size = enum(u4) {
487515 none,
516 ptr,
488517 byte,
489518 word,
490519 dword,
......@@ -521,9 +550,10 @@ pub const Memory = struct {
521550 };
522551 }
523552
524 pub fn bitSize(s: Size) u64 {
553 pub fn bitSize(s: Size, target: *const std.Target) u64 {
525554 return switch (s) {
526555 .none => 0,
556 .ptr => target.ptrBitWidth(),
527557 .byte => 8,
528558 .word => 16,
529559 .dword => 32,
......@@ -543,11 +573,50 @@ pub const Memory = struct {
543573 ) @TypeOf(writer).Error!void {
544574 if (s == .none) return;
545575 try writer.writeAll(@tagName(s));
546 try writer.writeAll(" ptr");
576 switch (s) {
577 .none => unreachable,
578 .ptr => {},
579 else => {
580 try writer.writeByte(' ');
581 try writer.writeAll("ptr");
582 },
583 }
547584 }
548585 };
549586
550 pub const Scale = enum(u2) { @"1", @"2", @"4", @"8" };
587 pub const Scale = enum(u2) {
588 @"1",
589 @"2",
590 @"4",
591 @"8",
592
593 pub fn fromFactor(factor: u4) Scale {
594 return switch (factor) {
595 else => unreachable,
596 1 => .@"1",
597 2 => .@"2",
598 4 => .@"4",
599 8 => .@"8",
600 };
601 }
602
603 pub fn toFactor(scale: Scale) u4 {
604 return switch (scale) {
605 .@"1" => 1,
606 .@"2" => 2,
607 .@"4" => 4,
608 .@"8" => 8,
609 };
610 }
611
612 pub fn fromLog2(log2: u2) Scale {
613 return @enumFromInt(log2);
614 }
615
616 pub fn toLog2(scale: Scale) u2 {
617 return @intFromEnum(scale);
618 }
619 };
551620};
552621
553622pub const Immediate = union(enum) {
src/arch/x86_64/encoder.zig+94-84
......@@ -138,7 +138,7 @@ pub const Instruction = struct {
138138 .moffs => true,
139139 .rip => false,
140140 .sib => |s| switch (s.base) {
141 .none, .frame, .reloc => false,
141 .none, .frame, .table, .reloc => false,
142142 .reg => |reg| reg.class() == .segment,
143143 },
144144 };
......@@ -161,17 +161,17 @@ pub const Instruction = struct {
161161
162162 pub fn disp(mem: Memory) Immediate {
163163 return switch (mem) {
164 .sib => |s| Immediate.s(s.disp),
165 .rip => |r| Immediate.s(r.disp),
166 .moffs => |m| Immediate.u(m.offset),
164 .sib => |s| .s(s.disp),
165 .rip => |r| .s(r.disp),
166 .moffs => |m| .u(m.offset),
167167 };
168168 }
169169
170 pub fn bitSize(mem: Memory) u64 {
170 pub fn bitSize(mem: Memory, target: *const std.Target) u64 {
171171 return switch (mem) {
172 .rip => |r| r.ptr_size.bitSize(),
173 .sib => |s| s.ptr_size.bitSize(),
174 .moffs => 64,
172 .rip => |r| r.ptr_size.bitSize(target),
173 .sib => |s| s.ptr_size.bitSize(target),
174 .moffs => target.ptrBitWidth(),
175175 };
176176 }
177177 };
......@@ -277,6 +277,7 @@ pub const Instruction = struct {
277277 .none => any = false,
278278 .reg => |reg| try writer.print("{s}", .{@tagName(reg)}),
279279 .frame => |frame_index| try writer.print("{}", .{frame_index}),
280 .table => try writer.print("Table", .{}),
280281 .reloc => |sym_index| try writer.print("Symbol({d})", .{sym_index}),
281282 }
282283 if (mem.scaleIndex()) |si| {
......@@ -314,28 +315,33 @@ pub const Instruction = struct {
314315 }
315316 };
316317
317 pub fn new(prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) !Instruction {
318 pub fn new(
319 prefix: Prefix,
320 mnemonic: Mnemonic,
321 ops: []const Operand,
322 target: *const std.Target,
323 ) !Instruction {
318324 const encoding: Encoding = switch (prefix) {
319 else => (try Encoding.findByMnemonic(prefix, mnemonic, ops)) orelse {
325 else => (try Encoding.findByMnemonic(prefix, mnemonic, ops, target)) orelse {
320326 log.err("no encoding found for: {s} {s} {s} {s} {s} {s}", .{
321327 @tagName(prefix),
322328 @tagName(mnemonic),
323 @tagName(if (ops.len > 0) Encoding.Op.fromOperand(ops[0]) else .none),
324 @tagName(if (ops.len > 1) Encoding.Op.fromOperand(ops[1]) else .none),
325 @tagName(if (ops.len > 2) Encoding.Op.fromOperand(ops[2]) else .none),
326 @tagName(if (ops.len > 3) Encoding.Op.fromOperand(ops[3]) else .none),
329 @tagName(if (ops.len > 0) Encoding.Op.fromOperand(ops[0], target) else .none),
330 @tagName(if (ops.len > 1) Encoding.Op.fromOperand(ops[1], target) else .none),
331 @tagName(if (ops.len > 2) Encoding.Op.fromOperand(ops[2], target) else .none),
332 @tagName(if (ops.len > 3) Encoding.Op.fromOperand(ops[3], target) else .none),
327333 });
328334 return error.InvalidInstruction;
329335 },
330336 .directive => .{
331337 .mnemonic = mnemonic,
332338 .data = .{
333 .op_en = .zo,
339 .op_en = .z,
334340 .ops = .{
335 if (ops.len > 0) Encoding.Op.fromOperand(ops[0]) else .none,
336 if (ops.len > 1) Encoding.Op.fromOperand(ops[1]) else .none,
337 if (ops.len > 2) Encoding.Op.fromOperand(ops[2]) else .none,
338 if (ops.len > 3) Encoding.Op.fromOperand(ops[3]) else .none,
341 if (ops.len > 0) Encoding.Op.fromOperand(ops[0], target) else .none,
342 if (ops.len > 1) Encoding.Op.fromOperand(ops[1], target) else .none,
343 if (ops.len > 2) Encoding.Op.fromOperand(ops[2], target) else .none,
344 if (ops.len > 3) Encoding.Op.fromOperand(ops[3], target) else .none,
339345 },
340346 .opc_len = 0,
341347 .opc = undefined,
......@@ -395,7 +401,7 @@ pub const Instruction = struct {
395401 }
396402
397403 switch (data.op_en) {
398 .zo, .o => {},
404 .z, .o, .zo, .oz => {},
399405 .i, .d => try encodeImm(inst.ops[0].imm, data.ops[0], encoder),
400406 .zi, .oi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder),
401407 .fd => try encoder.imm64(inst.ops[1].mem.moffs.offset),
......@@ -403,7 +409,7 @@ pub const Instruction = struct {
403409 else => {
404410 const mem_op = switch (data.op_en) {
405411 .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
406 .rm, .rmi, .rm0, .vmi => inst.ops[1],
412 .rm, .rmi, .rm0, .vmi, .rmv => inst.ops[1],
407413 .rvm, .rvmr, .rvmi => inst.ops[2],
408414 else => unreachable,
409415 };
......@@ -412,7 +418,7 @@ pub const Instruction = struct {
412418 const rm = switch (data.op_en) {
413419 .m, .mi, .m1, .mc, .vmi => enc.modRmExt(),
414420 .mr, .mri, .mrc => inst.ops[1].reg.lowEnc(),
415 .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi => inst.ops[0].reg.lowEnc(),
421 .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => inst.ops[0].reg.lowEnc(),
416422 .mvr => inst.ops[2].reg.lowEnc(),
417423 else => unreachable,
418424 };
......@@ -422,7 +428,7 @@ pub const Instruction = struct {
422428 const op = switch (data.op_en) {
423429 .m, .mi, .m1, .mc, .vmi => .none,
424430 .mr, .mri, .mrc => inst.ops[1],
425 .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi => inst.ops[0],
431 .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => inst.ops[0],
426432 .mvr => inst.ops[2],
427433 else => unreachable,
428434 };
......@@ -448,7 +454,8 @@ pub const Instruction = struct {
448454 const final = opcode.len - 1;
449455 for (opcode[first..final]) |byte| try encoder.opcode_1byte(byte);
450456 switch (inst.encoding.data.op_en) {
451 .o, .oi => try encoder.opcode_withReg(opcode[final], inst.ops[0].reg.lowEnc()),
457 .o, .oz, .oi => try encoder.opcode_withReg(opcode[final], inst.ops[0].reg.lowEnc()),
458 .zo => try encoder.opcode_withReg(opcode[final], inst.ops[1].reg.lowEnc()),
452459 else => try encoder.opcode_1byte(opcode[final]),
453460 }
454461 }
......@@ -474,7 +481,7 @@ pub const Instruction = struct {
474481 }
475482
476483 const segment_override: ?Register = switch (op_en) {
477 .zo, .i, .zi, .o, .oi, .d => null,
484 .z, .i, .zi, .o, .zo, .oz, .oi, .d => null,
478485 .fd => inst.ops[1].mem.base().reg,
479486 .td => inst.ops[0].mem.base().reg,
480487 .rm, .rmi, .rm0 => if (inst.ops[1].isSegmentRegister())
......@@ -493,7 +500,7 @@ pub const Instruction = struct {
493500 }
494501 else
495502 null,
496 .vmi, .rvm, .rvmr, .rvmi, .mvr => unreachable,
503 .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => unreachable,
497504 };
498505 if (segment_override) |seg| {
499506 legacy.setSegmentOverride(seg);
......@@ -510,11 +517,12 @@ pub const Instruction = struct {
510517 rex.w = inst.encoding.data.mode == .long;
511518
512519 switch (op_en) {
513 .zo, .i, .zi, .fd, .td, .d => {},
514 .o, .oi => rex.b = inst.ops[0].reg.isExtended(),
515 .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0 => {
520 .z, .i, .zi, .fd, .td, .d => {},
521 .o, .oz, .oi => rex.b = inst.ops[0].reg.isExtended(),
522 .zo => rex.b = inst.ops[1].reg.isExtended(),
523 .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .rmv => {
516524 const r_op = switch (op_en) {
517 .rm, .rmi, .rm0 => inst.ops[0],
525 .rm, .rmi, .rm0, .rmv => inst.ops[0],
518526 .mr, .mri, .mrc => inst.ops[1],
519527 else => .none,
520528 };
......@@ -544,11 +552,12 @@ pub const Instruction = struct {
544552 vex.w = inst.encoding.data.mode.isLong();
545553
546554 switch (op_en) {
547 .zo, .i, .zi, .fd, .td, .d => {},
548 .o, .oi => vex.b = inst.ops[0].reg.isExtended(),
549 .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .vmi, .rvm, .rvmr, .rvmi, .mvr => {
555 .z, .i, .zi, .fd, .td, .d => {},
556 .o, .oz, .oi => vex.b = inst.ops[0].reg.isExtended(),
557 .zo => vex.b = inst.ops[1].reg.isExtended(),
558 .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => {
550559 const r_op = switch (op_en) {
551 .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi => inst.ops[0],
560 .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => inst.ops[0],
552561 .mr, .mri, .mrc => inst.ops[1],
553562 .mvr => inst.ops[2],
554563 .m, .mi, .m1, .mc, .vmi => .none,
......@@ -557,7 +566,7 @@ pub const Instruction = struct {
557566 vex.r = r_op.isBaseExtended();
558567
559568 const b_x_op = switch (op_en) {
560 .rm, .rmi, .rm0, .vmi => inst.ops[1],
569 .rm, .rmi, .rm0, .vmi, .rmv => inst.ops[1],
561570 .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
562571 .rvm, .rvmr, .rvmi => inst.ops[2],
563572 else => unreachable,
......@@ -588,6 +597,7 @@ pub const Instruction = struct {
588597 else => {},
589598 .vmi => vex.v = inst.ops[0].reg,
590599 .rvm, .rvmr, .rvmi => vex.v = inst.ops[1].reg,
600 .rmv => vex.v = inst.ops[2].reg,
591601 }
592602
593603 try encoder.vex(vex);
......@@ -608,7 +618,7 @@ pub const Instruction = struct {
608618 switch (mem) {
609619 .moffs => unreachable,
610620 .sib => |sib| switch (sib.base) {
611 .none => {
621 .none, .table => {
612622 try encoder.modRm_SIBDisp0(operand_enc);
613623 if (mem.scaleIndex()) |si| {
614624 const scale = math.log2_int(u4, si.scale);
......@@ -676,11 +686,11 @@ pub const Instruction = struct {
676686 else => unreachable,
677687 },
678688 .frame => if (@TypeOf(encoder).options.allow_frame_locs) {
679 try encoder.modRm_indirectDisp32(operand_enc, undefined);
689 try encoder.modRm_indirectDisp32(operand_enc, 0);
680690 try encoder.disp32(undefined);
681691 } else return error.CannotEncode,
682692 .reloc => if (@TypeOf(encoder).options.allow_symbols) {
683 try encoder.modRm_indirectDisp32(operand_enc, undefined);
693 try encoder.modRm_indirectDisp32(operand_enc, 0);
684694 try encoder.disp32(undefined);
685695 } else return error.CannotEncode,
686696 },
......@@ -1185,7 +1195,7 @@ const TestEncode = struct {
11851195 ) !void {
11861196 var stream = std.io.fixedBufferStream(&enc.buffer);
11871197 var count_writer = std.io.countingWriter(stream.writer());
1188 const inst = try Instruction.new(.none, mnemonic, ops);
1198 const inst: Instruction = try .new(.none, mnemonic, ops);
11891199 try inst.encode(count_writer.writer(), .{});
11901200 enc.index = count_writer.bytes_written;
11911201 }
......@@ -1199,9 +1209,9 @@ test "encode" {
11991209 var buf = std.ArrayList(u8).init(testing.allocator);
12001210 defer buf.deinit();
12011211
1202 const inst = try Instruction.new(.none, .mov, &.{
1212 const inst: Instruction = try .new(.none, .mov, &.{
12031213 .{ .reg = .rbx },
1204 .{ .imm = Instruction.Immediate.u(4) },
1214 .{ .imm = .u(4) },
12051215 });
12061216 try inst.encode(buf.writer(), .{});
12071217 try testing.expectEqualSlices(u8, &.{ 0x48, 0xc7, 0xc3, 0x4, 0x0, 0x0, 0x0 }, buf.items);
......@@ -1211,47 +1221,47 @@ test "lower I encoding" {
12111221 var enc = TestEncode{};
12121222
12131223 try enc.encode(.push, &.{
1214 .{ .imm = Instruction.Immediate.u(0x10) },
1224 .{ .imm = .u(0x10) },
12151225 });
12161226 try expectEqualHexStrings("\x6A\x10", enc.code(), "push 0x10");
12171227
12181228 try enc.encode(.push, &.{
1219 .{ .imm = Instruction.Immediate.u(0x1000) },
1229 .{ .imm = .u(0x1000) },
12201230 });
12211231 try expectEqualHexStrings("\x66\x68\x00\x10", enc.code(), "push 0x1000");
12221232
12231233 try enc.encode(.push, &.{
1224 .{ .imm = Instruction.Immediate.u(0x10000000) },
1234 .{ .imm = .u(0x10000000) },
12251235 });
12261236 try expectEqualHexStrings("\x68\x00\x00\x00\x10", enc.code(), "push 0x10000000");
12271237
12281238 try enc.encode(.adc, &.{
12291239 .{ .reg = .rax },
1230 .{ .imm = Instruction.Immediate.u(0x10000000) },
1240 .{ .imm = .u(0x10000000) },
12311241 });
12321242 try expectEqualHexStrings("\x48\x15\x00\x00\x00\x10", enc.code(), "adc rax, 0x10000000");
12331243
12341244 try enc.encode(.add, &.{
12351245 .{ .reg = .al },
1236 .{ .imm = Instruction.Immediate.u(0x10) },
1246 .{ .imm = .u(0x10) },
12371247 });
12381248 try expectEqualHexStrings("\x04\x10", enc.code(), "add al, 0x10");
12391249
12401250 try enc.encode(.add, &.{
12411251 .{ .reg = .rax },
1242 .{ .imm = Instruction.Immediate.u(0x10) },
1252 .{ .imm = .u(0x10) },
12431253 });
12441254 try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10");
12451255
12461256 try enc.encode(.sbb, &.{
12471257 .{ .reg = .ax },
1248 .{ .imm = Instruction.Immediate.u(0x10) },
1258 .{ .imm = .u(0x10) },
12491259 });
12501260 try expectEqualHexStrings("\x66\x1D\x10\x00", enc.code(), "sbb ax, 0x10");
12511261
12521262 try enc.encode(.xor, &.{
12531263 .{ .reg = .al },
1254 .{ .imm = Instruction.Immediate.u(0x10) },
1264 .{ .imm = .u(0x10) },
12551265 });
12561266 try expectEqualHexStrings("\x34\x10", enc.code(), "xor al, 0x10");
12571267}
......@@ -1261,43 +1271,43 @@ test "lower MI encoding" {
12611271
12621272 try enc.encode(.mov, &.{
12631273 .{ .reg = .r12 },
1264 .{ .imm = Instruction.Immediate.u(0x1000) },
1274 .{ .imm = .u(0x1000) },
12651275 });
12661276 try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
12671277
12681278 try enc.encode(.mov, &.{
12691279 .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .r12 } }) },
1270 .{ .imm = Instruction.Immediate.u(0x10) },
1280 .{ .imm = .u(0x10) },
12711281 });
12721282 try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10");
12731283
12741284 try enc.encode(.mov, &.{
12751285 .{ .reg = .r12 },
1276 .{ .imm = Instruction.Immediate.u(0x1000) },
1286 .{ .imm = .u(0x1000) },
12771287 });
12781288 try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
12791289
12801290 try enc.encode(.mov, &.{
12811291 .{ .reg = .r12 },
1282 .{ .imm = Instruction.Immediate.u(0x1000) },
1292 .{ .imm = .u(0x1000) },
12831293 });
12841294 try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
12851295
12861296 try enc.encode(.mov, &.{
12871297 .{ .reg = .rax },
1288 .{ .imm = Instruction.Immediate.u(0x10) },
1298 .{ .imm = .u(0x10) },
12891299 });
12901300 try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10");
12911301
12921302 try enc.encode(.mov, &.{
12931303 .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r11 } }) },
1294 .{ .imm = Instruction.Immediate.u(0x10) },
1304 .{ .imm = .u(0x10) },
12951305 });
12961306 try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10");
12971307
12981308 try enc.encode(.mov, &.{
12991309 .{ .mem = Instruction.Memory.initRip(.qword, 0x10) },
1300 .{ .imm = Instruction.Immediate.u(0x10) },
1310 .{ .imm = .u(0x10) },
13011311 });
13021312 try expectEqualHexStrings(
13031313 "\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
......@@ -1307,19 +1317,19 @@ test "lower MI encoding" {
13071317
13081318 try enc.encode(.mov, &.{
13091319 .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -8 }) },
1310 .{ .imm = Instruction.Immediate.u(0x10) },
1320 .{ .imm = .u(0x10) },
13111321 });
13121322 try expectEqualHexStrings("\x48\xc7\x45\xf8\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rbp - 8], 0x10");
13131323
13141324 try enc.encode(.mov, &.{
13151325 .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -2 }) },
1316 .{ .imm = Instruction.Immediate.s(-16) },
1326 .{ .imm = .s(-16) },
13171327 });
13181328 try expectEqualHexStrings("\x66\xC7\x45\xFE\xF0\xFF", enc.code(), "mov WORD PTR [rbp - 2], -16");
13191329
13201330 try enc.encode(.mov, &.{
13211331 .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -1 }) },
1322 .{ .imm = Instruction.Immediate.u(0x10) },
1332 .{ .imm = .u(0x10) },
13231333 });
13241334 try expectEqualHexStrings("\xC6\x45\xFF\x10", enc.code(), "mov BYTE PTR [rbp - 1], 0x10");
13251335
......@@ -1329,7 +1339,7 @@ test "lower MI encoding" {
13291339 .disp = 0x10000000,
13301340 .scale_index = .{ .scale = 2, .index = .rcx },
13311341 }) },
1332 .{ .imm = Instruction.Immediate.u(0x10) },
1342 .{ .imm = .u(0x10) },
13331343 });
13341344 try expectEqualHexStrings(
13351345 "\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
......@@ -1339,43 +1349,43 @@ test "lower MI encoding" {
13391349
13401350 try enc.encode(.adc, &.{
13411351 .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) },
1342 .{ .imm = Instruction.Immediate.u(0x10) },
1352 .{ .imm = .u(0x10) },
13431353 });
13441354 try expectEqualHexStrings("\x80\x55\xF0\x10", enc.code(), "adc BYTE PTR [rbp - 0x10], 0x10");
13451355
13461356 try enc.encode(.adc, &.{
13471357 .{ .mem = Instruction.Memory.initRip(.qword, 0) },
1348 .{ .imm = Instruction.Immediate.u(0x10) },
1358 .{ .imm = .u(0x10) },
13491359 });
13501360 try expectEqualHexStrings("\x48\x83\x15\x00\x00\x00\x00\x10", enc.code(), "adc QWORD PTR [rip], 0x10");
13511361
13521362 try enc.encode(.adc, &.{
13531363 .{ .reg = .rax },
1354 .{ .imm = Instruction.Immediate.u(0x10) },
1364 .{ .imm = .u(0x10) },
13551365 });
13561366 try expectEqualHexStrings("\x48\x83\xD0\x10", enc.code(), "adc rax, 0x10");
13571367
13581368 try enc.encode(.add, &.{
13591369 .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .rdx }, .disp = -8 }) },
1360 .{ .imm = Instruction.Immediate.u(0x10) },
1370 .{ .imm = .u(0x10) },
13611371 });
13621372 try expectEqualHexStrings("\x83\x42\xF8\x10", enc.code(), "add DWORD PTR [rdx - 8], 0x10");
13631373
13641374 try enc.encode(.add, &.{
13651375 .{ .reg = .rax },
1366 .{ .imm = Instruction.Immediate.u(0x10) },
1376 .{ .imm = .u(0x10) },
13671377 });
13681378 try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10");
13691379
13701380 try enc.encode(.add, &.{
13711381 .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) },
1372 .{ .imm = Instruction.Immediate.s(-0x10) },
1382 .{ .imm = .s(-0x10) },
13731383 });
13741384 try expectEqualHexStrings("\x48\x83\x45\xF0\xF0", enc.code(), "add QWORD PTR [rbp - 0x10], -0x10");
13751385
13761386 try enc.encode(.@"and", &.{
13771387 .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) },
1378 .{ .imm = Instruction.Immediate.u(0x10) },
1388 .{ .imm = .u(0x10) },
13791389 });
13801390 try expectEqualHexStrings(
13811391 "\x83\x24\x25\x00\x00\x00\x10\x10",
......@@ -1385,7 +1395,7 @@ test "lower MI encoding" {
13851395
13861396 try enc.encode(.@"and", &.{
13871397 .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .es }, .disp = 0x10000000 }) },
1388 .{ .imm = Instruction.Immediate.u(0x10) },
1398 .{ .imm = .u(0x10) },
13891399 });
13901400 try expectEqualHexStrings(
13911401 "\x26\x83\x24\x25\x00\x00\x00\x10\x10",
......@@ -1395,7 +1405,7 @@ test "lower MI encoding" {
13951405
13961406 try enc.encode(.@"and", &.{
13971407 .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r12 }, .disp = 0x10000000 }) },
1398 .{ .imm = Instruction.Immediate.u(0x10) },
1408 .{ .imm = .u(0x10) },
13991409 });
14001410 try expectEqualHexStrings(
14011411 "\x41\x83\xA4\x24\x00\x00\x00\x10\x10",
......@@ -1405,7 +1415,7 @@ test "lower MI encoding" {
14051415
14061416 try enc.encode(.sub, &.{
14071417 .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r11 }, .disp = 0x10000000 }) },
1408 .{ .imm = Instruction.Immediate.u(0x10) },
1418 .{ .imm = .u(0x10) },
14091419 });
14101420 try expectEqualHexStrings(
14111421 "\x41\x83\xAB\x00\x00\x00\x10\x10",
......@@ -1624,14 +1634,14 @@ test "lower RMI encoding" {
16241634 try enc.encode(.imul, &.{
16251635 .{ .reg = .r11 },
16261636 .{ .reg = .r12 },
1627 .{ .imm = Instruction.Immediate.s(-2) },
1637 .{ .imm = .s(-2) },
16281638 });
16291639 try expectEqualHexStrings("\x4D\x6B\xDC\xFE", enc.code(), "imul r11, r12, -2");
16301640
16311641 try enc.encode(.imul, &.{
16321642 .{ .reg = .r11 },
16331643 .{ .mem = Instruction.Memory.initRip(.qword, -16) },
1634 .{ .imm = Instruction.Immediate.s(-1024) },
1644 .{ .imm = .s(-1024) },
16351645 });
16361646 try expectEqualHexStrings(
16371647 "\x4C\x69\x1D\xF0\xFF\xFF\xFF\x00\xFC\xFF\xFF",
......@@ -1642,7 +1652,7 @@ test "lower RMI encoding" {
16421652 try enc.encode(.imul, &.{
16431653 .{ .reg = .bx },
16441654 .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) },
1645 .{ .imm = Instruction.Immediate.s(-1024) },
1655 .{ .imm = .s(-1024) },
16461656 });
16471657 try expectEqualHexStrings(
16481658 "\x66\x69\x5D\xF0\x00\xFC",
......@@ -1653,7 +1663,7 @@ test "lower RMI encoding" {
16531663 try enc.encode(.imul, &.{
16541664 .{ .reg = .bx },
16551665 .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) },
1656 .{ .imm = Instruction.Immediate.u(1024) },
1666 .{ .imm = .u(1024) },
16571667 });
16581668 try expectEqualHexStrings(
16591669 "\x66\x69\x5D\xF0\x00\x04",
......@@ -1769,7 +1779,7 @@ test "lower M encoding" {
17691779 try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0");
17701780
17711781 try enc.encode(.call, &.{
1772 .{ .imm = Instruction.Immediate.s(0) },
1782 .{ .imm = .s(0) },
17731783 });
17741784 try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0");
17751785
......@@ -1828,7 +1838,7 @@ test "lower OI encoding" {
18281838
18291839 try enc.encode(.mov, &.{
18301840 .{ .reg = .rax },
1831 .{ .imm = Instruction.Immediate.u(0x1000000000000000) },
1841 .{ .imm = .u(0x1000000000000000) },
18321842 });
18331843 try expectEqualHexStrings(
18341844 "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
......@@ -1838,7 +1848,7 @@ test "lower OI encoding" {
18381848
18391849 try enc.encode(.mov, &.{
18401850 .{ .reg = .r11 },
1841 .{ .imm = Instruction.Immediate.u(0x1000000000000000) },
1851 .{ .imm = .u(0x1000000000000000) },
18421852 });
18431853 try expectEqualHexStrings(
18441854 "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
......@@ -1848,19 +1858,19 @@ test "lower OI encoding" {
18481858
18491859 try enc.encode(.mov, &.{
18501860 .{ .reg = .r11d },
1851 .{ .imm = Instruction.Immediate.u(0x10000000) },
1861 .{ .imm = .u(0x10000000) },
18521862 });
18531863 try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", enc.code(), "mov r11d, 0x10000000");
18541864
18551865 try enc.encode(.mov, &.{
18561866 .{ .reg = .r11w },
1857 .{ .imm = Instruction.Immediate.u(0x1000) },
1867 .{ .imm = .u(0x1000) },
18581868 });
18591869 try expectEqualHexStrings("\x66\x41\xBB\x00\x10", enc.code(), "mov r11w, 0x1000");
18601870
18611871 try enc.encode(.mov, &.{
18621872 .{ .reg = .r11b },
1863 .{ .imm = Instruction.Immediate.u(0x10) },
1873 .{ .imm = .u(0x10) },
18641874 });
18651875 try expectEqualHexStrings("\x41\xB3\x10", enc.code(), "mov r11b, 0x10");
18661876}
......@@ -1934,7 +1944,7 @@ test "lower NP encoding" {
19341944}
19351945
19361946fn invalidInstruction(mnemonic: Instruction.Mnemonic, ops: []const Instruction.Operand) !void {
1937 const err = Instruction.new(.none, mnemonic, ops);
1947 const err: Instruction = .new(.none, mnemonic, ops);
19381948 try testing.expectError(error.InvalidInstruction, err);
19391949}
19401950
......@@ -1982,12 +1992,12 @@ test "invalid instruction" {
19821992 .{ .reg = .r12d },
19831993 });
19841994 try invalidInstruction(.push, &.{
1985 .{ .imm = Instruction.Immediate.u(0x1000000000000000) },
1995 .{ .imm = .u(0x1000000000000000) },
19861996 });
19871997}
19881998
19891999fn cannotEncode(mnemonic: Instruction.Mnemonic, ops: []const Instruction.Operand) !void {
1990 try testing.expectError(error.CannotEncode, Instruction.new(.none, mnemonic, ops));
2000 try testing.expectError(error.CannotEncode, .new(.none, mnemonic, ops));
19912001}
19922002
19932003test "cannot encode" {
......@@ -2171,7 +2181,7 @@ const Assembler = struct {
21712181
21722182 pub fn assemble(as: *Assembler, writer: anytype) !void {
21732183 while (try as.next()) |parsed_inst| {
2174 const inst = try Instruction.new(.none, parsed_inst.mnemonic, &parsed_inst.ops);
2184 const inst: Instruction = try .new(.none, parsed_inst.mnemonic, &parsed_inst.ops);
21752185 try inst.encode(writer, .{});
21762186 }
21772187 }
src/arch/x86_64/encodings.zig+229-163
......@@ -124,106 +124,118 @@ pub const table = [_]Entry{
124124 .{ .call, .d, &.{ .rel32 }, &.{ 0xe8 }, 0, .none, .none },
125125 .{ .call, .m, &.{ .rm64 }, &.{ 0xff }, 2, .none, .none },
126126
127 .{ .cbw, .zo, &.{ .o16 }, &.{ 0x98 }, 0, .short, .none },
128 .{ .cwde, .zo, &.{ .o32 }, &.{ 0x98 }, 0, .none, .none },
129 .{ .cdqe, .zo, &.{ .o64 }, &.{ 0x98 }, 0, .long, .none },
127 .{ .cbw, .z, &.{ .o16 }, &.{ 0x98 }, 0, .short, .none },
128 .{ .cwde, .z, &.{ .o32 }, &.{ 0x98 }, 0, .none, .none },
129 .{ .cdqe, .z, &.{ .o64 }, &.{ 0x98 }, 0, .long, .none },
130130
131 .{ .cwd, .zo, &.{ .o16 }, &.{ 0x99 }, 0, .short, .none },
132 .{ .cdq, .zo, &.{ .o32 }, &.{ 0x99 }, 0, .none, .none },
133 .{ .cqo, .zo, &.{ .o64 }, &.{ 0x99 }, 0, .long, .none },
131 .{ .cwd, .z, &.{ .o16 }, &.{ 0x99 }, 0, .short, .none },
132 .{ .cdq, .z, &.{ .o32 }, &.{ 0x99 }, 0, .none, .none },
133 .{ .cqo, .z, &.{ .o64 }, &.{ 0x99 }, 0, .long, .none },
134
135 .{ .clac, .z, &.{}, &.{ 0x0f, 0x01, 0xca }, 0, .none, .smap },
136
137 .{ .clc, .z, &.{}, &.{ 0xf8 }, 0, .none, .none },
138
139 .{ .cld, .z, &.{}, &.{ 0xfc }, 0, .none, .none },
134140
135141 .{ .clflush, .m, &.{ .m8 }, &.{ 0x0f, 0xae }, 7, .none, .none },
136142
137 .{ .cmova, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },
138 .{ .cmova, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
139 .{ .cmova, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
140 .{ .cmovae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
141 .{ .cmovae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
142 .{ .cmovae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
143 .{ .cmovb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
144 .{ .cmovb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
145 .{ .cmovb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
146 .{ .cmovbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },
147 .{ .cmovbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
148 .{ .cmovbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
149 .{ .cmovc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
150 .{ .cmovc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
151 .{ .cmovc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
152 .{ .cmove, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },
153 .{ .cmove, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
154 .{ .cmove, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
155 .{ .cmovg, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },
156 .{ .cmovg, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
157 .{ .cmovg, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
158 .{ .cmovge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },
159 .{ .cmovge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
160 .{ .cmovge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
161 .{ .cmovl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },
162 .{ .cmovl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
163 .{ .cmovl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
164 .{ .cmovle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },
165 .{ .cmovle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
166 .{ .cmovle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
167 .{ .cmovna, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },
168 .{ .cmovna, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
169 .{ .cmovna, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
170 .{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
171 .{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
172 .{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
173 .{ .cmovnb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
174 .{ .cmovnb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
175 .{ .cmovnb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
176 .{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },
177 .{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
178 .{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
179 .{ .cmovnc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
180 .{ .cmovnc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
181 .{ .cmovnc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
182 .{ .cmovne, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },
183 .{ .cmovne, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
184 .{ .cmovne, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
185 .{ .cmovng, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },
186 .{ .cmovng, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
187 .{ .cmovng, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
188 .{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },
189 .{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
190 .{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
191 .{ .cmovnl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },
192 .{ .cmovnl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
193 .{ .cmovnl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
194 .{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },
195 .{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
196 .{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
197 .{ .cmovno, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .short, .none },
198 .{ .cmovno, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none, .none },
199 .{ .cmovno, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long, .none },
200 .{ .cmovnp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },
201 .{ .cmovnp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
202 .{ .cmovnp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
203 .{ .cmovns, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .short, .none },
204 .{ .cmovns, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none, .none },
205 .{ .cmovns, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long, .none },
206 .{ .cmovnz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },
207 .{ .cmovnz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
208 .{ .cmovnz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
209 .{ .cmovo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .short, .none },
210 .{ .cmovo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none, .none },
211 .{ .cmovo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long, .none },
212 .{ .cmovp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },
213 .{ .cmovp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
214 .{ .cmovp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
215 .{ .cmovpe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },
216 .{ .cmovpe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
217 .{ .cmovpe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
218 .{ .cmovpo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },
219 .{ .cmovpo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
220 .{ .cmovpo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
221 .{ .cmovs, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .short, .none },
222 .{ .cmovs, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none, .none },
223 .{ .cmovs, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long, .none },
224 .{ .cmovz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },
225 .{ .cmovz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
226 .{ .cmovz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
143 .{ .cli, .z, &.{}, &.{ 0xfa }, 0, .none, .none },
144
145 .{ .clts, .z, &.{}, &.{ 0x0f, 0x06 }, 0, .none, .none },
146
147 .{ .clui, .z, &.{}, &.{ 0xf3, 0x0f, 0x01, 0xee }, 0, .none, .uintr },
148
149 .{ .cmova, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .cmov },
150 .{ .cmova, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .cmov },
151 .{ .cmova, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .cmov },
152 .{ .cmovae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .cmov },
153 .{ .cmovae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .cmov },
154 .{ .cmovae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .cmov },
155 .{ .cmovb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .cmov },
156 .{ .cmovb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .cmov },
157 .{ .cmovb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .cmov },
158 .{ .cmovbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .cmov },
159 .{ .cmovbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .cmov },
160 .{ .cmovbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .cmov },
161 .{ .cmovc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .cmov },
162 .{ .cmovc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .cmov },
163 .{ .cmovc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .cmov },
164 .{ .cmove, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .cmov },
165 .{ .cmove, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .cmov },
166 .{ .cmove, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .cmov },
167 .{ .cmovg, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .cmov },
168 .{ .cmovg, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .cmov },
169 .{ .cmovg, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .cmov },
170 .{ .cmovge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .cmov },
171 .{ .cmovge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .cmov },
172 .{ .cmovge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .cmov },
173 .{ .cmovl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .cmov },
174 .{ .cmovl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .cmov },
175 .{ .cmovl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .cmov },
176 .{ .cmovle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .cmov },
177 .{ .cmovle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .cmov },
178 .{ .cmovle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .cmov },
179 .{ .cmovna, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .cmov },
180 .{ .cmovna, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .cmov },
181 .{ .cmovna, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .cmov },
182 .{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .cmov },
183 .{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .cmov },
184 .{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .cmov },
185 .{ .cmovnb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .cmov },
186 .{ .cmovnb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .cmov },
187 .{ .cmovnb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .cmov },
188 .{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .cmov },
189 .{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .cmov },
190 .{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .cmov },
191 .{ .cmovnc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .cmov },
192 .{ .cmovnc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .cmov },
193 .{ .cmovnc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .cmov },
194 .{ .cmovne, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .cmov },
195 .{ .cmovne, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .cmov },
196 .{ .cmovne, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .cmov },
197 .{ .cmovng, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .cmov },
198 .{ .cmovng, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .cmov },
199 .{ .cmovng, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .cmov },
200 .{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .cmov },
201 .{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .cmov },
202 .{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .cmov },
203 .{ .cmovnl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .cmov },
204 .{ .cmovnl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .cmov },
205 .{ .cmovnl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .cmov },
206 .{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .cmov },
207 .{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .cmov },
208 .{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .cmov },
209 .{ .cmovno, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .short, .cmov },
210 .{ .cmovno, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none, .cmov },
211 .{ .cmovno, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long, .cmov },
212 .{ .cmovnp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .cmov },
213 .{ .cmovnp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .cmov },
214 .{ .cmovnp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .cmov },
215 .{ .cmovns, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .short, .cmov },
216 .{ .cmovns, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none, .cmov },
217 .{ .cmovns, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long, .cmov },
218 .{ .cmovnz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .cmov },
219 .{ .cmovnz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .cmov },
220 .{ .cmovnz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .cmov },
221 .{ .cmovo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .short, .cmov },
222 .{ .cmovo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none, .cmov },
223 .{ .cmovo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long, .cmov },
224 .{ .cmovp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .cmov },
225 .{ .cmovp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .cmov },
226 .{ .cmovp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .cmov },
227 .{ .cmovpe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .cmov },
228 .{ .cmovpe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .cmov },
229 .{ .cmovpe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .cmov },
230 .{ .cmovpo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .cmov },
231 .{ .cmovpo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .cmov },
232 .{ .cmovpo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .cmov },
233 .{ .cmovs, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .short, .cmov },
234 .{ .cmovs, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none, .cmov },
235 .{ .cmovs, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long, .cmov },
236 .{ .cmovz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .cmov },
237 .{ .cmovz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .cmov },
238 .{ .cmovz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .cmov },
227239
228240 .{ .cmp, .zi, &.{ .al, .imm8 }, &.{ 0x3c }, 0, .none, .none },
229241 .{ .cmp, .zi, &.{ .ax, .imm16 }, &.{ 0x3d }, 0, .short, .none },
......@@ -248,15 +260,15 @@ pub const table = [_]Entry{
248260 .{ .cmp, .rm, &.{ .r32, .rm32 }, &.{ 0x3b }, 0, .none, .none },
249261 .{ .cmp, .rm, &.{ .r64, .rm64 }, &.{ 0x3b }, 0, .long, .none },
250262
251 .{ .cmps, .zo, &.{ .m8, .m8 }, &.{ 0xa6 }, 0, .none, .none },
252 .{ .cmps, .zo, &.{ .m16, .m16 }, &.{ 0xa7 }, 0, .short, .none },
253 .{ .cmps, .zo, &.{ .m32, .m32 }, &.{ 0xa7 }, 0, .none, .none },
254 .{ .cmps, .zo, &.{ .m64, .m64 }, &.{ 0xa7 }, 0, .long, .none },
263 .{ .cmps, .z, &.{ .m8, .m8 }, &.{ 0xa6 }, 0, .none, .none },
264 .{ .cmps, .z, &.{ .m16, .m16 }, &.{ 0xa7 }, 0, .short, .none },
265 .{ .cmps, .z, &.{ .m32, .m32 }, &.{ 0xa7 }, 0, .none, .none },
266 .{ .cmps, .z, &.{ .m64, .m64 }, &.{ 0xa7 }, 0, .long, .none },
255267
256 .{ .cmpsb, .zo, &.{}, &.{ 0xa6 }, 0, .none, .none },
257 .{ .cmpsw, .zo, &.{}, &.{ 0xa7 }, 0, .short, .none },
258 .{ .cmpsd, .zo, &.{}, &.{ 0xa7 }, 0, .none, .none },
259 .{ .cmpsq, .zo, &.{}, &.{ 0xa7 }, 0, .long, .none },
268 .{ .cmpsb, .z, &.{}, &.{ 0xa6 }, 0, .none, .none },
269 .{ .cmpsw, .z, &.{}, &.{ 0xa7 }, 0, .short, .none },
270 .{ .cmpsd, .z, &.{}, &.{ 0xa7 }, 0, .none, .none },
271 .{ .cmpsq, .z, &.{}, &.{ 0xa7 }, 0, .long, .none },
260272
261273 .{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .none, .none },
262274 .{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .rex, .none },
......@@ -267,7 +279,7 @@ pub const table = [_]Entry{
267279 .{ .cmpxchg8b, .m, &.{ .m64 }, &.{ 0x0f, 0xc7 }, 1, .none, .none },
268280 .{ .cmpxchg16b, .m, &.{ .m128 }, &.{ 0x0f, 0xc7 }, 1, .long, .none },
269281
270 .{ .cpuid, .zo, &.{}, &.{ 0x0f, 0xa2 }, 0, .none, .none },
282 .{ .cpuid, .z, &.{}, &.{ 0x0f, 0xa2 }, 0, .none, .none },
271283
272284 .{ .dec, .m, &.{ .rm8 }, &.{ 0xfe }, 1, .none, .none },
273285 .{ .dec, .m, &.{ .rm8 }, &.{ 0xfe }, 1, .rex, .none },
......@@ -308,7 +320,7 @@ pub const table = [_]Entry{
308320 .{ .inc, .m, &.{ .rm32 }, &.{ 0xff }, 0, .none, .none },
309321 .{ .inc, .m, &.{ .rm64 }, &.{ 0xff }, 0, .long, .none },
310322
311 .{ .int3, .zo, &.{}, &.{ 0xcc }, 0, .none, .none },
323 .{ .int3, .z, &.{}, &.{ 0xcc }, 0, .none, .none },
312324
313325 .{ .ja, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none, .none },
314326 .{ .jae, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
......@@ -349,23 +361,23 @@ pub const table = [_]Entry{
349361 .{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none, .none },
350362 .{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long, .none },
351363
352 .{ .lfence, .zo, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none, .none },
364 .{ .lfence, .z, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none, .none },
353365
354 .{ .lods, .zo, &.{ .m8 }, &.{ 0xac }, 0, .none, .none },
355 .{ .lods, .zo, &.{ .m16 }, &.{ 0xad }, 0, .short, .none },
356 .{ .lods, .zo, &.{ .m32 }, &.{ 0xad }, 0, .none, .none },
357 .{ .lods, .zo, &.{ .m64 }, &.{ 0xad }, 0, .long, .none },
366 .{ .lods, .z, &.{ .m8 }, &.{ 0xac }, 0, .none, .none },
367 .{ .lods, .z, &.{ .m16 }, &.{ 0xad }, 0, .short, .none },
368 .{ .lods, .z, &.{ .m32 }, &.{ 0xad }, 0, .none, .none },
369 .{ .lods, .z, &.{ .m64 }, &.{ 0xad }, 0, .long, .none },
358370
359 .{ .lodsb, .zo, &.{}, &.{ 0xac }, 0, .none, .none },
360 .{ .lodsw, .zo, &.{}, &.{ 0xad }, 0, .short, .none },
361 .{ .lodsd, .zo, &.{}, &.{ 0xad }, 0, .none, .none },
362 .{ .lodsq, .zo, &.{}, &.{ 0xad }, 0, .long, .none },
371 .{ .lodsb, .z, &.{}, &.{ 0xac }, 0, .none, .none },
372 .{ .lodsw, .z, &.{}, &.{ 0xad }, 0, .short, .none },
373 .{ .lodsd, .z, &.{}, &.{ 0xad }, 0, .none, .none },
374 .{ .lodsq, .z, &.{}, &.{ 0xad }, 0, .long, .none },
363375
364376 .{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .short, .lzcnt },
365377 .{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .lzcnt },
366378 .{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .lzcnt },
367379
368 .{ .mfence, .zo, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none },
380 .{ .mfence, .z, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none },
369381
370382 .{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .none, .none },
371383 .{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .rex, .none },
......@@ -409,15 +421,15 @@ pub const table = [_]Entry{
409421 .{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .movbe },
410422 .{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long, .movbe },
411423
412 .{ .movs, .zo, &.{ .m8, .m8 }, &.{ 0xa4 }, 0, .none, .none },
413 .{ .movs, .zo, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .short, .none },
414 .{ .movs, .zo, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none, .none },
415 .{ .movs, .zo, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long, .none },
424 .{ .movs, .z, &.{ .m8, .m8 }, &.{ 0xa4 }, 0, .none, .none },
425 .{ .movs, .z, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .short, .none },
426 .{ .movs, .z, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none, .none },
427 .{ .movs, .z, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long, .none },
416428
417 .{ .movsb, .zo, &.{}, &.{ 0xa4 }, 0, .none, .none },
418 .{ .movsw, .zo, &.{}, &.{ 0xa5 }, 0, .short, .none },
419 .{ .movsd, .zo, &.{}, &.{ 0xa5 }, 0, .none, .none },
420 .{ .movsq, .zo, &.{}, &.{ 0xa5 }, 0, .long, .none },
429 .{ .movsb, .z, &.{}, &.{ 0xa4 }, 0, .none, .none },
430 .{ .movsw, .z, &.{}, &.{ 0xa5 }, 0, .short, .none },
431 .{ .movsd, .z, &.{}, &.{ 0xa5 }, 0, .none, .none },
432 .{ .movsq, .z, &.{}, &.{ 0xa5 }, 0, .long, .none },
421433
422434 .{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .short, .none },
423435 .{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex_short, .none },
......@@ -453,7 +465,7 @@ pub const table = [_]Entry{
453465 .{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none, .none },
454466 .{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long, .none },
455467
456 .{ .nop, .zo, &.{}, &.{ 0x90 }, 0, .none, .none },
468 .{ .nop, .z, &.{}, &.{ 0x90 }, 0, .none, .none },
457469
458470 .{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .none, .none },
459471 .{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .rex, .none },
......@@ -484,7 +496,7 @@ pub const table = [_]Entry{
484496 .{ .@"or", .rm, &.{ .r32, .rm32 }, &.{ 0x0b }, 0, .none, .none },
485497 .{ .@"or", .rm, &.{ .r64, .rm64 }, &.{ 0x0b }, 0, .long, .none },
486498
487 .{ .pause, .zo, &.{}, &.{ 0xf3, 0x90 }, 0, .none, .none },
499 .{ .pause, .z, &.{}, &.{ 0xf3, 0x90 }, 0, .none, .none },
488500
489501 .{ .pop, .o, &.{ .r16 }, &.{ 0x58 }, 0, .short, .none },
490502 .{ .pop, .o, &.{ .r64 }, &.{ 0x58 }, 0, .none, .none },
......@@ -495,7 +507,7 @@ pub const table = [_]Entry{
495507 .{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .popcnt },
496508 .{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .popcnt },
497509
498 .{ .popfq, .zo, &.{}, &.{ 0x9d }, 0, .none, .none },
510 .{ .popfq, .z, &.{}, &.{ 0x9d }, 0, .none, .none },
499511
500512 .{ .push, .o, &.{ .r16 }, &.{ 0x50 }, 0, .short, .none },
501513 .{ .push, .o, &.{ .r64 }, &.{ 0x50 }, 0, .none, .none },
......@@ -505,9 +517,9 @@ pub const table = [_]Entry{
505517 .{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .short, .none },
506518 .{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none, .none },
507519
508 .{ .pushfq, .zo, &.{}, &.{ 0x9c }, 0, .none, .none },
520 .{ .pushfq, .z, &.{}, &.{ 0x9c }, 0, .none, .none },
509521
510 .{ .ret, .zo, &.{}, &.{ 0xc3 }, 0, .none, .none },
522 .{ .ret, .z, &.{}, &.{ 0xc3 }, 0, .none, .none },
511523
512524 .{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .none, .none },
513525 .{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .rex, .none },
......@@ -628,15 +640,15 @@ pub const table = [_]Entry{
628640 .{ .sbb, .rm, &.{ .r32, .rm32 }, &.{ 0x1b }, 0, .none, .none },
629641 .{ .sbb, .rm, &.{ .r64, .rm64 }, &.{ 0x1b }, 0, .long, .none },
630642
631 .{ .scas, .zo, &.{ .m8 }, &.{ 0xae }, 0, .none, .none },
632 .{ .scas, .zo, &.{ .m16 }, &.{ 0xaf }, 0, .short, .none },
633 .{ .scas, .zo, &.{ .m32 }, &.{ 0xaf }, 0, .none, .none },
634 .{ .scas, .zo, &.{ .m64 }, &.{ 0xaf }, 0, .long, .none },
643 .{ .scas, .z, &.{ .m8 }, &.{ 0xae }, 0, .none, .none },
644 .{ .scas, .z, &.{ .m16 }, &.{ 0xaf }, 0, .short, .none },
645 .{ .scas, .z, &.{ .m32 }, &.{ 0xaf }, 0, .none, .none },
646 .{ .scas, .z, &.{ .m64 }, &.{ 0xaf }, 0, .long, .none },
635647
636 .{ .scasb, .zo, &.{}, &.{ 0xae }, 0, .none, .none },
637 .{ .scasw, .zo, &.{}, &.{ 0xaf }, 0, .short, .none },
638 .{ .scasd, .zo, &.{}, &.{ 0xaf }, 0, .none, .none },
639 .{ .scasq, .zo, &.{}, &.{ 0xaf }, 0, .long, .none },
648 .{ .scasb, .z, &.{}, &.{ 0xae }, 0, .none, .none },
649 .{ .scasw, .z, &.{}, &.{ 0xaf }, 0, .short, .none },
650 .{ .scasd, .z, &.{}, &.{ 0xaf }, 0, .none, .none },
651 .{ .scasq, .z, &.{}, &.{ 0xaf }, 0, .long, .none },
640652
641653 .{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none, .none },
642654 .{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex, .none },
......@@ -699,7 +711,7 @@ pub const table = [_]Entry{
699711 .{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none, .none },
700712 .{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex, .none },
701713
702 .{ .sfence, .zo, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none, .none },
714 .{ .sfence, .z, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none, .none },
703715
704716 .{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none, .none },
705717 .{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex, .none },
......@@ -747,15 +759,25 @@ pub const table = [_]Entry{
747759 .{ .shrd, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xad }, 0, .none, .none },
748760 .{ .shrd, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xad }, 0, .long, .none },
749761
750 .{ .stos, .zo, &.{ .m8 }, &.{ 0xaa }, 0, .none, .none },
751 .{ .stos, .zo, &.{ .m16 }, &.{ 0xab }, 0, .short, .none },
752 .{ .stos, .zo, &.{ .m32 }, &.{ 0xab }, 0, .none, .none },
753 .{ .stos, .zo, &.{ .m64 }, &.{ 0xab }, 0, .long, .none },
762 .{ .stac, .z, &.{}, &.{ 0x0f, 0x01, 0xcb }, 0, .none, .smap },
763
764 .{ .stc, .z, &.{}, &.{ 0xf9 }, 0, .none, .none },
765
766 .{ .std, .z, &.{}, &.{ 0xfd }, 0, .none, .none },
767
768 .{ .sti, .z, &.{}, &.{ 0xfb }, 0, .none, .none },
754769
755 .{ .stosb, .zo, &.{}, &.{ 0xaa }, 0, .none, .none },
756 .{ .stosw, .zo, &.{}, &.{ 0xab }, 0, .short, .none },
757 .{ .stosd, .zo, &.{}, &.{ 0xab }, 0, .none, .none },
758 .{ .stosq, .zo, &.{}, &.{ 0xab }, 0, .long, .none },
770 .{ .stui, .z, &.{}, &.{ 0xf3, 0x0f, 0x01, 0xef }, 0, .none, .uintr },
771
772 .{ .stos, .z, &.{ .m8 }, &.{ 0xaa }, 0, .none, .none },
773 .{ .stos, .z, &.{ .m16 }, &.{ 0xab }, 0, .short, .none },
774 .{ .stos, .z, &.{ .m32 }, &.{ 0xab }, 0, .none, .none },
775 .{ .stos, .z, &.{ .m64 }, &.{ 0xab }, 0, .long, .none },
776
777 .{ .stosb, .z, &.{}, &.{ 0xaa }, 0, .none, .none },
778 .{ .stosw, .z, &.{}, &.{ 0xab }, 0, .short, .none },
779 .{ .stosd, .z, &.{}, &.{ 0xab }, 0, .none, .none },
780 .{ .stosq, .z, &.{}, &.{ 0xab }, 0, .long, .none },
759781
760782 .{ .sub, .zi, &.{ .al, .imm8 }, &.{ 0x2c }, 0, .none, .none },
761783 .{ .sub, .zi, &.{ .ax, .imm16 }, &.{ 0x2d }, 0, .short, .none },
......@@ -780,7 +802,7 @@ pub const table = [_]Entry{
780802 .{ .sub, .rm, &.{ .r32, .rm32 }, &.{ 0x2b }, 0, .none, .none },
781803 .{ .sub, .rm, &.{ .r64, .rm64 }, &.{ 0x2b }, 0, .long, .none },
782804
783 .{ .syscall, .zo, &.{}, &.{ 0x0f, 0x05 }, 0, .none, .none },
805 .{ .syscall, .z, &.{}, &.{ 0x0f, 0x05 }, 0, .none, .none },
784806
785807 .{ .@"test", .zi, &.{ .al, .imm8 }, &.{ 0xa8 }, 0, .none, .none },
786808 .{ .@"test", .zi, &.{ .ax, .imm16 }, &.{ 0xa9 }, 0, .short, .none },
......@@ -801,7 +823,7 @@ pub const table = [_]Entry{
801823 .{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .bmi },
802824 .{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .bmi },
803825
804 .{ .ud2, .zo, &.{}, &.{ 0x0f, 0x0b }, 0, .none, .none },
826 .{ .ud2, .z, &.{}, &.{ 0x0f, 0x0b }, 0, .none, .none },
805827
806828 .{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .none, .none },
807829 .{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .rex, .none },
......@@ -809,12 +831,12 @@ pub const table = [_]Entry{
809831 .{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none, .none },
810832 .{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long, .none },
811833
812 .{ .xchg, .o, &.{ .ax, .r16 }, &.{ 0x90 }, 0, .short, .none },
813 .{ .xchg, .o, &.{ .r16, .ax }, &.{ 0x90 }, 0, .short, .none },
814 .{ .xchg, .o, &.{ .eax, .r32 }, &.{ 0x90 }, 0, .none, .none },
815 .{ .xchg, .o, &.{ .rax, .r64 }, &.{ 0x90 }, 0, .long, .none },
816 .{ .xchg, .o, &.{ .r32, .eax }, &.{ 0x90 }, 0, .none, .none },
817 .{ .xchg, .o, &.{ .r64, .rax }, &.{ 0x90 }, 0, .long, .none },
834 .{ .xchg, .zo, &.{ .ax, .r16 }, &.{ 0x90 }, 0, .short, .none },
835 .{ .xchg, .oz, &.{ .r16, .ax }, &.{ 0x90 }, 0, .short, .none },
836 .{ .xchg, .zo, &.{ .eax, .r32 }, &.{ 0x90 }, 0, .none, .none },
837 .{ .xchg, .zo, &.{ .rax, .r64 }, &.{ 0x90 }, 0, .long, .none },
838 .{ .xchg, .oz, &.{ .r32, .eax }, &.{ 0x90 }, 0, .none, .none },
839 .{ .xchg, .oz, &.{ .r64, .rax }, &.{ 0x90 }, 0, .long, .none },
818840 .{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .none, .none },
819841 .{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .rex, .none },
820842 .{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .none, .none },
......@@ -826,7 +848,7 @@ pub const table = [_]Entry{
826848 .{ .xchg, .rm, &.{ .r32, .rm32 }, &.{ 0x87 }, 0, .none, .none },
827849 .{ .xchg, .rm, &.{ .r64, .rm64 }, &.{ 0x87 }, 0, .long, .none },
828850
829 .{ .xgetbv, .zo, &.{}, &.{ 0x0f, 0x01, 0xd0 }, 0, .none, .none },
851 .{ .xgetbv, .z, &.{}, &.{ 0x0f, 0x01, 0xd0 }, 0, .none, .none },
830852
831853 .{ .xor, .zi, &.{ .al, .imm8 }, &.{ 0x34 }, 0, .none, .none },
832854 .{ .xor, .zi, &.{ .ax, .imm16 }, &.{ 0x35 }, 0, .short, .none },
......@@ -852,9 +874,9 @@ pub const table = [_]Entry{
852874 .{ .xor, .rm, &.{ .r64, .rm64 }, &.{ 0x33 }, 0, .long, .none },
853875
854876 // X87
855 .{ .fabs, .zo, &.{}, &.{ 0xd9, 0xe1 }, 0, .none, .x87 },
877 .{ .fabs, .z, &.{}, &.{ 0xd9, 0xe1 }, 0, .none, .x87 },
856878
857 .{ .fchs, .zo, &.{}, &.{ 0xd9, 0xe0 }, 0, .none, .x87 },
879 .{ .fchs, .z, &.{}, &.{ 0xd9, 0xe0 }, 0, .none, .x87 },
858880
859881 .{ .ffree, .o, &.{ .st }, &.{ 0xdd, 0xc0 }, 0, .none, .x87 },
860882
......@@ -927,8 +949,14 @@ pub const table = [_]Entry{
927949
928950 .{ .movhlps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .none, .sse },
929951
952 .{ .movhps, .rm, &.{ .xmm, .m64 }, &.{ 0x0f, 0x16 }, 0, .none, .sse },
953 .{ .movhps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x17 }, 0, .none, .sse },
954
930955 .{ .movlhps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x16 }, 0, .none, .sse },
931956
957 .{ .movlps, .rm, &.{ .xmm, .m64 }, &.{ 0x0f, 0x12 }, 0, .none, .sse },
958 .{ .movlps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x13 }, 0, .none, .sse },
959
932960 .{ .movmskps, .rm, &.{ .r32, .xmm }, &.{ 0x0f, 0x50 }, 0, .none, .sse },
933961 .{ .movmskps, .rm, &.{ .r64, .xmm }, &.{ 0x0f, 0x50 }, 0, .none, .sse },
934962
......@@ -1037,6 +1065,12 @@ pub const table = [_]Entry{
10371065 .{ .movdqu, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x6f }, 0, .none, .sse2 },
10381066 .{ .movdqu, .mr, &.{ .xmm_m128, .xmm }, &.{ 0xf3, 0x0f, 0x7f }, 0, .none, .sse2 },
10391067
1068 .{ .movhpd, .rm, &.{ .xmm, .m64 }, &.{ 0x66, 0x0f, 0x16 }, 0, .none, .sse2 },
1069 .{ .movhpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x17 }, 0, .none, .sse2 },
1070
1071 .{ .movlpd, .rm, &.{ .xmm, .m64 }, &.{ 0x66, 0x0f, 0x12 }, 0, .none, .sse2 },
1072 .{ .movlpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x13 }, 0, .none, .sse2 },
1073
10401074 .{ .movmskpd, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 },
10411075 .{ .movmskpd, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 },
10421076
......@@ -1251,6 +1285,8 @@ pub const table = [_]Entry{
12511285
12521286 .{ .pmulld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 },
12531287
1288 .{ .ptest, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x17 }, 0, .none, .sse4_1 },
1289
12541290 .{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
12551291
12561292 .{ .roundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .none, .sse4_1 },
......@@ -1287,6 +1323,16 @@ pub const table = [_]Entry{
12871323 .{ .sha256rnds2, .rm0, &.{ .xmm, .xmm_m128, .xmm0 }, &.{ 0x0f, 0x38, 0xcb }, 0, .none, .sha },
12881324
12891325 // AVX
1326 .{ .rorx, .rmi, &.{ .r32, .rm32, .imm8 }, &.{ 0xf2, 0x0f, 0x3a }, 0, .vex_lz_w0, .bmi2 },
1327 .{ .rorx, .rmi, &.{ .r64, .rm64, .imm8 }, &.{ 0xf2, 0x0f, 0x3a }, 0, .vex_lz_w1, .bmi2 },
1328
1329 .{ .sarx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 },
1330 .{ .shlx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 },
1331 .{ .shrx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 },
1332 .{ .sarx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 },
1333 .{ .shlx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 },
1334 .{ .shrx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 },
1335
12901336 .{ .vaddpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_128_wig, .avx },
12911337 .{ .vaddpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_256_wig, .avx },
12921338
......@@ -1474,8 +1520,20 @@ pub const table = [_]Entry{
14741520
14751521 .{ .vmovhlps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
14761522
1523 .{ .vmovhpd, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x66, 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
1524 .{ .vmovhpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x17 }, 0, .vex_128_wig, .avx },
1525
1526 .{ .vmovhps, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
1527 .{ .vmovhps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x17 }, 0, .vex_128_wig, .avx },
1528
14771529 .{ .vmovlhps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
14781530
1531 .{ .vmovlpd, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x66, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
1532 .{ .vmovlpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x13 }, 0, .vex_128_wig, .avx },
1533
1534 .{ .vmovlps, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
1535 .{ .vmovlps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x13 }, 0, .vex_128_wig, .avx },
1536
14791537 .{ .vmovq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .vex_128_wig, .avx },
14801538 .{ .vmovq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .vex_128_wig, .avx },
14811539
......@@ -1571,14 +1629,14 @@ pub const table = [_]Entry{
15711629 .{ .vpextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w0, .avx },
15721630 .{ .vpextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w1, .avx },
15731631
1574 .{ .vpextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x15 }, 0, .vex_128_wig, .avx },
1575 .{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_wig, .avx },
1632 .{ .vpextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .vex_128_w0, .avx },
1633 .{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_w0, .avx },
15761634
1577 .{ .vpinsrb, .rmi, &.{ .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx },
1578 .{ .vpinsrd, .rmi, &.{ .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx },
1579 .{ .vpinsrq, .rmi, &.{ .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx },
1635 .{ .vpinsrb, .rvmi, &.{ .xmm, .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx },
1636 .{ .vpinsrd, .rvmi, &.{ .xmm, .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx },
1637 .{ .vpinsrq, .rvmi, &.{ .xmm, .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx },
15801638
1581 .{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_wig, .avx },
1639 .{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_w0, .avx },
15821640
15831641 .{ .vpmaxsb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3c }, 0, .vex_128_wig, .avx },
15841642 .{ .vpmaxsw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xee }, 0, .vex_128_wig, .avx },
......@@ -1666,6 +1724,9 @@ pub const table = [_]Entry{
16661724 .{ .vpsubusb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd8 }, 0, .vex_128_wig, .avx },
16671725 .{ .vpsubusw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd9 }, 0, .vex_128_wig, .avx },
16681726
1727 .{ .vptest, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x17 }, 0, .vex_128_wig, .avx },
1728 .{ .vptest, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x17 }, 0, .vex_256_wig, .avx },
1729
16691730 .{ .vpunpckhbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_128_wig, .avx },
16701731 .{ .vpunpckhwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_128_wig, .avx },
16711732 .{ .vpunpckhdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_128_wig, .avx },
......@@ -1716,6 +1777,11 @@ pub const table = [_]Entry{
17161777
17171778 .{ .vsubss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx },
17181779
1780 .{ .vtestps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x0e }, 0, .vex_128_w0, .avx },
1781 .{ .vtestps, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x0e }, 0, .vex_256_w0, .avx },
1782 .{ .vtestpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x0f }, 0, .vex_128_w0, .avx },
1783 .{ .vtestpd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x0f }, 0, .vex_256_w0, .avx },
1784
17191785 .{ .vxorpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .vex_128_wig, .avx },
17201786 .{ .vxorpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x57 }, 0, .vex_256_wig, .avx },
17211787
src/codegen/c/Type.zig+5-5
......@@ -1312,10 +1312,10 @@ pub const Pool = struct {
13121312 },
13131313 else => {
13141314 const target = &mod.resolved_target.result;
1315 const abi_align = Type.intAbiAlignment(int_info.bits, target.*, false);
1315 const abi_align = Type.intAbiAlignment(int_info.bits, target.*);
13161316 const abi_align_bytes = abi_align.toByteUnits().?;
13171317 const array_ctype = try pool.getArray(allocator, .{
1318 .len = @divExact(Type.intAbiSize(int_info.bits, target.*, false), abi_align_bytes),
1318 .len = @divExact(Type.intAbiSize(int_info.bits, target.*), abi_align_bytes),
13191319 .elem_ctype = try pool.fromIntInfo(allocator, .{
13201320 .signedness = .unsigned,
13211321 .bits = @intCast(abi_align_bytes * 8),
......@@ -1429,7 +1429,7 @@ pub const Pool = struct {
14291429 .name = .{ .index = .len },
14301430 .ctype = CType.usize,
14311431 .alignas = AlignAs.fromAbiAlignment(
1432 Type.intAbiAlignment(target.ptrBitWidth(), target.*, false),
1432 Type.intAbiAlignment(target.ptrBitWidth(), target.*),
14331433 ),
14341434 },
14351435 };
......@@ -1524,7 +1524,7 @@ pub const Pool = struct {
15241524 .name = .{ .index = .len },
15251525 .ctype = CType.usize,
15261526 .alignas = AlignAs.fromAbiAlignment(
1527 Type.intAbiAlignment(target.ptrBitWidth(), target.*, false),
1527 Type.intAbiAlignment(target.ptrBitWidth(), target.*),
15281528 ),
15291529 },
15301530 };
......@@ -1644,7 +1644,7 @@ pub const Pool = struct {
16441644 .name = .{ .index = .@"error" },
16451645 .ctype = error_set_ctype,
16461646 .alignas = AlignAs.fromAbiAlignment(
1647 Type.intAbiAlignment(error_set_bits, target.*, false),
1647 Type.intAbiAlignment(error_set_bits, target.*),
16481648 ),
16491649 },
16501650 .{
src/codegen/llvm.zig+1-1
......@@ -581,7 +581,7 @@ const DataLayoutBuilder = struct {
581581 switch (kind) {
582582 .integer => {
583583 if (self.target.ptrBitWidth() <= 16 and size >= 128) return;
584 abi = @min(abi, Type.maxIntAlignment(self.target, true) * 8);
584 abi = @min(abi, Type.maxIntAlignment(self.target) * 8);
585585 switch (self.target.cpu.arch) {
586586 .aarch64,
587587 .aarch64_be,
src/dev.zig+1
......@@ -135,6 +135,7 @@ pub const Env = enum {
135135 else => Env.ast_gen.supports(feature),
136136 },
137137 .@"x86_64-linux" => switch (feature) {
138 .build_command,
138139 .stdio_listen,
139140 .incremental,
140141 .x86_64_backend,
src/link/C.zig+1-1
......@@ -396,7 +396,7 @@ fn abiDefines(self: *C, target: std.Target) !std.ArrayList(u8) {
396396 else => {},
397397 }
398398 try writer.print("#define ZIG_TARGET_MAX_INT_ALIGNMENT {d}\n", .{
399 Type.maxIntAlignment(target, false),
399 Type.maxIntAlignment(target),
400400 });
401401 return defines;
402402}
src/link/Elf/Atom.zig+35-32
......@@ -976,6 +976,7 @@ const x86_64 = struct {
976976 it: *RelocsIterator,
977977 ) !void {
978978 dev.check(.x86_64_backend);
979 const t = &elf_file.base.comp.root_mod.resolved_target.result;
979980 const is_static = elf_file.base.isStatic();
980981 const is_dyn_lib = elf_file.isEffectivelyDynLib();
981982
......@@ -1046,7 +1047,7 @@ const x86_64 = struct {
10461047 .GOTTPOFF => {
10471048 const should_relax = blk: {
10481049 if (is_dyn_lib or symbol.flags.import) break :blk false;
1049 if (!x86_64.canRelaxGotTpOff(code.?[r_offset - 3 ..])) break :blk false;
1050 if (!x86_64.canRelaxGotTpOff(code.?[r_offset - 3 ..], t)) break :blk false;
10501051 break :blk true;
10511052 };
10521053 if (!should_relax) {
......@@ -1090,6 +1091,7 @@ const x86_64 = struct {
10901091 stream: anytype,
10911092 ) (error{ InvalidInstruction, CannotEncode } || RelocError)!void {
10921093 dev.check(.x86_64_backend);
1094 const t = &elf_file.base.comp.root_mod.resolved_target.result;
10931095 const diags = &elf_file.base.comp.link_diags;
10941096 const r_type: elf.R_X86_64 = @enumFromInt(rel.r_type());
10951097 const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow;
......@@ -1120,7 +1122,7 @@ const x86_64 = struct {
11201122
11211123 .GOTPCRELX => {
11221124 if (!target.flags.import and !target.isIFunc(elf_file) and !target.isAbs(elf_file)) blk: {
1123 x86_64.relaxGotpcrelx(code[r_offset - 2 ..]) catch break :blk;
1125 x86_64.relaxGotpcrelx(code[r_offset - 2 ..], t) catch break :blk;
11241126 try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little);
11251127 return;
11261128 }
......@@ -1129,7 +1131,7 @@ const x86_64 = struct {
11291131
11301132 .REX_GOTPCRELX => {
11311133 if (!target.flags.import and !target.isIFunc(elf_file) and !target.isAbs(elf_file)) blk: {
1132 x86_64.relaxRexGotpcrelx(code[r_offset - 3 ..]) catch break :blk;
1134 x86_64.relaxRexGotpcrelx(code[r_offset - 3 ..], t) catch break :blk;
11331135 try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little);
11341136 return;
11351137 }
......@@ -1184,7 +1186,7 @@ const x86_64 = struct {
11841186 const S_ = target.tlsDescAddress(elf_file);
11851187 try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little);
11861188 } else {
1187 x86_64.relaxGotPcTlsDesc(code[r_offset - 3 ..]) catch {
1189 x86_64.relaxGotPcTlsDesc(code[r_offset - 3 ..], t) catch {
11881190 var err = try diags.addErrorWithNotes(1);
11891191 try err.addMsg("could not relax {s}", .{@tagName(r_type)});
11901192 err.addNote("in {}:{s} at offset 0x{x}", .{
......@@ -1208,7 +1210,7 @@ const x86_64 = struct {
12081210 const S_ = target.gotTpAddress(elf_file);
12091211 try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little);
12101212 } else {
1211 x86_64.relaxGotTpOff(code[r_offset - 3 ..]);
1213 x86_64.relaxGotTpOff(code[r_offset - 3 ..], t);
12121214 try cwriter.writeInt(i32, @as(i32, @intCast(S - TP)), .little);
12131215 }
12141216 },
......@@ -1269,31 +1271,31 @@ const x86_64 = struct {
12691271 }
12701272 }
12711273
1272 fn relaxGotpcrelx(code: []u8) !void {
1274 fn relaxGotpcrelx(code: []u8, t: *const std.Target) !void {
12731275 dev.check(.x86_64_backend);
12741276 const old_inst = disassemble(code) orelse return error.RelaxFailure;
1275 const inst = switch (old_inst.encoding.mnemonic) {
1276 .call => try Instruction.new(old_inst.prefix, .call, &.{
1277 const inst: Instruction = switch (old_inst.encoding.mnemonic) {
1278 .call => try .new(old_inst.prefix, .call, &.{
12771279 // TODO: hack to force imm32s in the assembler
1278 .{ .imm = Immediate.s(-129) },
1279 }),
1280 .jmp => try Instruction.new(old_inst.prefix, .jmp, &.{
1280 .{ .imm = .s(-129) },
1281 }, t),
1282 .jmp => try .new(old_inst.prefix, .jmp, &.{
12811283 // TODO: hack to force imm32s in the assembler
1282 .{ .imm = Immediate.s(-129) },
1283 }),
1284 .{ .imm = .s(-129) },
1285 }, t),
12841286 else => return error.RelaxFailure,
12851287 };
12861288 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
1287 const nop = try Instruction.new(.none, .nop, &.{});
1289 const nop: Instruction = try .new(.none, .nop, &.{}, t);
12881290 try encode(&.{ nop, inst }, code);
12891291 }
12901292
1291 fn relaxRexGotpcrelx(code: []u8) !void {
1293 fn relaxRexGotpcrelx(code: []u8, t: *const std.Target) !void {
12921294 dev.check(.x86_64_backend);
12931295 const old_inst = disassemble(code) orelse return error.RelaxFailure;
12941296 switch (old_inst.encoding.mnemonic) {
12951297 .mov => {
1296 const inst = try Instruction.new(old_inst.prefix, .lea, &old_inst.ops);
1298 const inst: Instruction = try .new(old_inst.prefix, .lea, &old_inst.ops, t);
12971299 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
12981300 try encode(&.{inst}, code);
12991301 },
......@@ -1398,23 +1400,24 @@ const x86_64 = struct {
13981400 }
13991401 }
14001402
1401 fn canRelaxGotTpOff(code: []const u8) bool {
1403 fn canRelaxGotTpOff(code: []const u8, t: *const std.Target) bool {
14021404 dev.check(.x86_64_backend);
14031405 const old_inst = disassemble(code) orelse return false;
14041406 switch (old_inst.encoding.mnemonic) {
1405 .mov => if (Instruction.new(old_inst.prefix, .mov, &.{
1406 old_inst.ops[0],
1407 // TODO: hack to force imm32s in the assembler
1408 .{ .imm = Immediate.s(-129) },
1409 })) |inst| {
1407 .mov => {
1408 const inst = Instruction.new(old_inst.prefix, .mov, &.{
1409 old_inst.ops[0],
1410 // TODO: hack to force imm32s in the assembler
1411 .{ .imm = .s(-129) },
1412 }, t) catch return false;
14101413 inst.encode(std.io.null_writer, .{}) catch return false;
14111414 return true;
1412 } else |_| return false,
1415 },
14131416 else => return false,
14141417 }
14151418 }
14161419
1417 fn relaxGotTpOff(code: []u8) void {
1420 fn relaxGotTpOff(code: []u8, t: *const std.Target) void {
14181421 dev.check(.x86_64_backend);
14191422 const old_inst = disassemble(code) orelse unreachable;
14201423 switch (old_inst.encoding.mnemonic) {
......@@ -1422,8 +1425,8 @@ const x86_64 = struct {
14221425 const inst = Instruction.new(old_inst.prefix, .mov, &.{
14231426 old_inst.ops[0],
14241427 // TODO: hack to force imm32s in the assembler
1425 .{ .imm = Immediate.s(-129) },
1426 }) catch unreachable;
1428 .{ .imm = .s(-129) },
1429 }, t) catch unreachable;
14271430 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
14281431 encode(&.{inst}, code) catch unreachable;
14291432 },
......@@ -1431,16 +1434,16 @@ const x86_64 = struct {
14311434 }
14321435 }
14331436
1434 fn relaxGotPcTlsDesc(code: []u8) !void {
1437 fn relaxGotPcTlsDesc(code: []u8, target: *const std.Target) !void {
14351438 dev.check(.x86_64_backend);
14361439 const old_inst = disassemble(code) orelse return error.RelaxFailure;
14371440 switch (old_inst.encoding.mnemonic) {
14381441 .lea => {
1439 const inst = try Instruction.new(old_inst.prefix, .mov, &.{
1442 const inst: Instruction = try .new(old_inst.prefix, .mov, &.{
14401443 old_inst.ops[0],
14411444 // TODO: hack to force imm32s in the assembler
1442 .{ .imm = Immediate.s(-129) },
1443 });
1445 .{ .imm = .s(-129) },
1446 }, target);
14441447 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
14451448 try encode(&.{inst}, code);
14461449 },
......@@ -1779,7 +1782,7 @@ const aarch64 = struct {
17791782 const off: u12 = @truncate(@as(u64, @bitCast(S_ + A)));
17801783 aarch64_util.writeAddImmInst(off, code);
17811784 } else {
1782 const old_inst = Instruction{
1785 const old_inst: Instruction = .{
17831786 .add_subtract_immediate = mem.bytesToValue(std.meta.TagPayload(
17841787 Instruction,
17851788 Instruction.add_subtract_immediate,
......@@ -1793,7 +1796,7 @@ const aarch64 = struct {
17931796 },
17941797
17951798 .TLSDESC_CALL => if (!target.flags.has_tlsdesc) {
1796 const old_inst = Instruction{
1799 const old_inst: Instruction = .{
17971800 .unconditional_branch_register = mem.bytesToValue(std.meta.TagPayload(
17981801 Instruction,
17991802 Instruction.unconditional_branch_register,
src/link/MachO.zig+1-1
......@@ -3548,7 +3548,7 @@ pub fn getTarget(self: MachO) std.Target {
35483548pub fn invalidateKernelCache(dir: fs.Dir, sub_path: []const u8) !void {
35493549 const tracy = trace(@src());
35503550 defer tracy.end();
3551 if (comptime builtin.target.isDarwin() and builtin.target.cpu.arch == .aarch64) {
3551 if (builtin.target.isDarwin() and builtin.target.cpu.arch == .aarch64) {
35523552 try dir.copyFile(sub_path, dir, sub_path, .{});
35533553 }
35543554}
src/link/MachO/Atom.zig+7-5
......@@ -640,7 +640,8 @@ fn resolveRelocInner(
640640 macho_file: *MachO,
641641 writer: anytype,
642642) ResolveError!void {
643 const cpu_arch = macho_file.getTarget().cpu.arch;
643 const t = &macho_file.base.comp.root_mod.resolved_target.result;
644 const cpu_arch = t.cpu.arch;
644645 const rel_offset = math.cast(usize, rel.offset - self.off) orelse return error.Overflow;
645646 const P = @as(i64, @intCast(self.getAddress(macho_file))) + @as(i64, @intCast(rel_offset));
646647 const A = rel.addend + rel.getRelocAddend(cpu_arch);
......@@ -747,7 +748,7 @@ fn resolveRelocInner(
747748 const S_: i64 = @intCast(sym.getTlvPtrAddress(macho_file));
748749 try writer.writeInt(i32, @intCast(S_ + A - P), .little);
749750 } else {
750 try x86_64.relaxTlv(code[rel_offset - 3 ..]);
751 try x86_64.relaxTlv(code[rel_offset - 3 ..], t);
751752 try writer.writeInt(i32, @intCast(S + A - P), .little);
752753 }
753754 },
......@@ -893,11 +894,12 @@ fn resolveRelocInner(
893894const x86_64 = struct {
894895 fn relaxGotLoad(self: Atom, code: []u8, rel: Relocation, macho_file: *MachO) ResolveError!void {
895896 dev.check(.x86_64_backend);
897 const t = &macho_file.base.comp.root_mod.resolved_target.result;
896898 const diags = &macho_file.base.comp.link_diags;
897899 const old_inst = disassemble(code) orelse return error.RelaxFail;
898900 switch (old_inst.encoding.mnemonic) {
899901 .mov => {
900 const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops) catch return error.RelaxFail;
902 const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops, t) catch return error.RelaxFail;
901903 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
902904 encode(&.{inst}, code) catch return error.RelaxFail;
903905 },
......@@ -916,12 +918,12 @@ const x86_64 = struct {
916918 }
917919 }
918920
919 fn relaxTlv(code: []u8) error{RelaxFail}!void {
921 fn relaxTlv(code: []u8, t: *const std.Target) error{RelaxFail}!void {
920922 dev.check(.x86_64_backend);
921923 const old_inst = disassemble(code) orelse return error.RelaxFail;
922924 switch (old_inst.encoding.mnemonic) {
923925 .mov => {
924 const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops) catch return error.RelaxFail;
926 const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops, t) catch return error.RelaxFail;
925927 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
926928 encode(&.{inst}, code) catch return error.RelaxFail;
927929 },
src/main.zig+8
......@@ -39,6 +39,11 @@ test {
3939 _ = Package;
4040}
4141
42const thread_stack_size = switch (builtin.zig_backend) {
43 else => std.Thread.SpawnConfig.default_stack_size,
44 .stage2_x86_64 => 32 << 20,
45};
46
4247pub const std_options: std.Options = .{
4348 .wasiCwd = wasi_cwd,
4449 .logFn = log,
......@@ -3320,6 +3325,7 @@ fn buildOutputType(
33203325 .allocator = gpa,
33213326 .n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),
33223327 .track_ids = true,
3328 .stack_size = thread_stack_size,
33233329 });
33243330 defer thread_pool.deinit();
33253331
......@@ -5024,6 +5030,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
50245030 .allocator = gpa,
50255031 .n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),
50265032 .track_ids = true,
5033 .stack_size = thread_stack_size,
50275034 });
50285035 defer thread_pool.deinit();
50295036
......@@ -5460,6 +5467,7 @@ fn jitCmd(
54605467 .allocator = gpa,
54615468 .n_jobs = @min(@max(std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),
54625469 .track_ids = true,
5470 .stack_size = thread_stack_size,
54635471 });
54645472 defer thread_pool.deinit();
54655473
src/print_air.zig+12-12
......@@ -96,8 +96,8 @@ const Writer = struct {
9696 fn writeInst(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
9797 const tag = w.air.instructions.items(.tag)[@intFromEnum(inst)];
9898 try s.writeByteNTimes(' ', w.indent);
99 try s.print("%{d}{c}= {s}(", .{
100 @intFromEnum(inst),
99 try s.print("{}{c}= {s}(", .{
100 inst,
101101 @as(u8, if (if (w.liveness) |liveness| liveness.isUnused(inst) else false) '!' else ' '),
102102 @tagName(tag),
103103 });
......@@ -409,7 +409,7 @@ const Writer = struct {
409409 try s.writeAll("}");
410410
411411 for (liveness_block.deaths) |operand| {
412 try s.print(" %{d}!", .{@intFromEnum(operand)});
412 try s.print(" {}!", .{operand});
413413 }
414414 }
415415
......@@ -728,7 +728,7 @@ const Writer = struct {
728728 try s.writeByteNTimes(' ', w.indent);
729729 for (liveness_condbr.else_deaths, 0..) |operand, i| {
730730 if (i != 0) try s.writeAll(" ");
731 try s.print("%{d}!", .{@intFromEnum(operand)});
731 try s.print("{}!", .{operand});
732732 }
733733 try s.writeAll("\n");
734734 }
......@@ -739,7 +739,7 @@ const Writer = struct {
739739 try s.writeAll("}");
740740
741741 for (liveness_condbr.then_deaths) |operand| {
742 try s.print(" %{d}!", .{@intFromEnum(operand)});
742 try s.print(" {}!", .{operand});
743743 }
744744 }
745745
......@@ -765,7 +765,7 @@ const Writer = struct {
765765 try s.writeByteNTimes(' ', w.indent);
766766 for (liveness_condbr.else_deaths, 0..) |operand, i| {
767767 if (i != 0) try s.writeAll(" ");
768 try s.print("%{d}!", .{@intFromEnum(operand)});
768 try s.print("{}!", .{operand});
769769 }
770770 try s.writeAll("\n");
771771 }
......@@ -776,7 +776,7 @@ const Writer = struct {
776776 try s.writeAll("}");
777777
778778 for (liveness_condbr.then_deaths) |operand| {
779 try s.print(" %{d}!", .{@intFromEnum(operand)});
779 try s.print(" {}!", .{operand});
780780 }
781781 }
782782
......@@ -807,7 +807,7 @@ const Writer = struct {
807807 try s.writeByteNTimes(' ', w.indent);
808808 for (liveness_condbr.then_deaths, 0..) |operand, i| {
809809 if (i != 0) try s.writeAll(" ");
810 try s.print("%{d}!", .{@intFromEnum(operand)});
810 try s.print("{}!", .{operand});
811811 }
812812 try s.writeAll("\n");
813813 }
......@@ -827,7 +827,7 @@ const Writer = struct {
827827 try s.writeByteNTimes(' ', w.indent);
828828 for (liveness_condbr.else_deaths, 0..) |operand, i| {
829829 if (i != 0) try s.writeAll(" ");
830 try s.print("%{d}!", .{@intFromEnum(operand)});
830 try s.print("{}!", .{operand});
831831 }
832832 try s.writeAll("\n");
833833 }
......@@ -884,7 +884,7 @@ const Writer = struct {
884884 try s.writeByteNTimes(' ', w.indent);
885885 for (deaths, 0..) |operand, i| {
886886 if (i != 0) try s.writeAll(" ");
887 try s.print("%{d}!", .{@intFromEnum(operand)});
887 try s.print("{}!", .{operand});
888888 }
889889 try s.writeAll("\n");
890890 }
......@@ -910,7 +910,7 @@ const Writer = struct {
910910 try s.writeByteNTimes(' ', w.indent);
911911 for (deaths, 0..) |operand, i| {
912912 if (i != 0) try s.writeAll(" ");
913 try s.print("%{d}!", .{@intFromEnum(operand)});
913 try s.print("{}!", .{operand});
914914 }
915915 try s.writeAll("\n");
916916 }
......@@ -994,7 +994,7 @@ const Writer = struct {
994994 dies: bool,
995995 ) @TypeOf(s).Error!void {
996996 _ = w;
997 try s.print("%{d}", .{@intFromEnum(inst)});
997 try s.print("{}", .{inst});
998998 if (dies) try s.writeByte('!');
999999 }
10001000
src/register_manager.zig+50-71
......@@ -41,12 +41,12 @@ pub fn RegisterManager(
4141 registers: TrackedRegisters = undefined,
4242 /// Tracks which registers are free (in which case the
4343 /// corresponding bit is set to 1)
44 free_registers: RegisterBitSet = RegisterBitSet.initFull(),
44 free_registers: RegisterBitSet = .initFull(),
4545 /// Tracks all registers allocated in the course of this
4646 /// function
47 allocated_registers: RegisterBitSet = RegisterBitSet.initEmpty(),
47 allocated_registers: RegisterBitSet = .initEmpty(),
4848 /// Tracks registers which are locked from being allocated
49 locked_registers: RegisterBitSet = RegisterBitSet.initEmpty(),
49 locked_registers: RegisterBitSet = .initEmpty(),
5050
5151 const Self = @This();
5252
......@@ -58,11 +58,6 @@ pub fn RegisterManager(
5858 return @alignCast(@fieldParentPtr("register_manager", self));
5959 }
6060
61 fn excludeRegister(reg: Register, register_class: RegisterBitSet) bool {
62 const index = indexOfRegIntoTracked(reg) orelse return true;
63 return !register_class.isSet(index);
64 }
65
6661 fn markRegIndexAllocated(self: *Self, tracked_index: TrackedIndex) void {
6762 self.allocated_registers.set(tracked_index);
6863 }
......@@ -99,8 +94,7 @@ pub fn RegisterManager(
9994 max_id = @max(elem_id, max_id);
10095 }
10196
102 const OptionalIndex = std.math.IntFittingRange(0, set.len);
103 comptime var map = [1]OptionalIndex{set.len} ** (max_id - min_id + 1);
97 comptime var map: [max_id - min_id + 1]std.math.IntFittingRange(0, set.len) = @splat(set.len);
10498 inline for (set, 0..) |elem, elem_index| map[comptime elem.id() - min_id] = elem_index;
10599
106100 const id_index = reg.id() -% min_id;
......@@ -112,6 +106,9 @@ pub fn RegisterManager(
112106 pub fn indexOfRegIntoTracked(reg: Register) ?TrackedIndex {
113107 return indexOfReg(tracked_registers, reg);
114108 }
109 pub inline fn indexOfKnownRegIntoTracked(comptime reg: Register) ?TrackedIndex {
110 return comptime indexOfRegIntoTracked(reg);
111 }
115112
116113 pub fn regAtTrackedIndex(tracked_index: TrackedIndex) Register {
117114 return tracked_registers[tracked_index];
......@@ -124,6 +121,9 @@ pub fn RegisterManager(
124121 pub fn isRegFree(self: Self, reg: Register) bool {
125122 return self.isRegIndexFree(indexOfRegIntoTracked(reg) orelse return true);
126123 }
124 pub fn isKnownRegFree(self: Self, comptime reg: Register) bool {
125 return self.isRegIndexFree(indexOfKnownRegIntoTracked(reg) orelse return true);
126 }
127127
128128 /// Returns whether this register was allocated in the course
129129 /// of this function.
......@@ -143,6 +143,9 @@ pub fn RegisterManager(
143143 pub fn isRegLocked(self: Self, reg: Register) bool {
144144 return self.isRegIndexLocked(indexOfRegIntoTracked(reg) orelse return false);
145145 }
146 pub fn isKnownRegLocked(self: Self, comptime reg: Register) bool {
147 return self.isRegIndexLocked(indexOfKnownRegIntoTracked(reg) orelse return false);
148 }
146149
147150 pub const RegisterLock = struct { tracked_index: TrackedIndex };
148151
......@@ -176,6 +179,9 @@ pub fn RegisterManager(
176179 pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock {
177180 return self.lockRegIndexAssumeUnused(indexOfRegIntoTracked(reg) orelse unreachable);
178181 }
182 pub fn lockKnownRegAssumeUnused(self: *Self, comptime reg: Register) RegisterLock {
183 return self.lockRegIndexAssumeUnused(indexOfKnownRegIntoTracked(reg) orelse unreachable);
184 }
179185
180186 /// Like `lockReg` but locks multiple registers.
181187 pub fn lockRegs(
......@@ -223,28 +229,20 @@ pub fn RegisterManager(
223229 ) ?[count]Register {
224230 comptime assert(count > 0 and count <= tracked_registers.len);
225231
226 var free_and_not_locked_registers = self.free_registers;
227 free_and_not_locked_registers.setIntersection(register_class);
228
229 var unlocked_registers = self.locked_registers;
230 unlocked_registers.toggleAll();
231
232 free_and_not_locked_registers.setIntersection(unlocked_registers);
233
234 if (free_and_not_locked_registers.count() < count) return null;
232 var free_and_unlocked_registers = self.locked_registers;
233 free_and_unlocked_registers.toggleAll();
234 free_and_unlocked_registers.setIntersection(self.free_registers);
235 free_and_unlocked_registers.setIntersection(register_class);
235236
236237 var regs: [count]Register = undefined;
237238 var i: usize = 0;
238 for (tracked_registers) |reg| {
239 if (i >= count) break;
240 if (excludeRegister(reg, register_class)) continue;
241 if (self.isRegLocked(reg)) continue;
242 if (!self.isRegFree(reg)) continue;
243
244 regs[i] = reg;
239 var it = free_and_unlocked_registers.iterator(.{});
240 while (it.next()) |reg_index| {
241 regs[i] = regAtTrackedIndex(@intCast(reg_index));
245242 i += 1;
243 if (i >= count) break;
246244 }
247 assert(i == count);
245 if (i < count) return null;
248246
249247 for (regs, insts) |reg, inst| {
250248 log.debug("tryAllocReg {} for inst {?}", .{ reg, inst });
......@@ -279,46 +277,27 @@ pub fn RegisterManager(
279277 ) AllocationError![count]Register {
280278 comptime assert(count > 0 and count <= tracked_registers.len);
281279
282 var locked_registers = self.locked_registers;
283 locked_registers.setIntersection(register_class);
284
285 if (count > register_class.count() - locked_registers.count()) return error.OutOfRegisters;
286
287280 const result = self.tryAllocRegs(count, insts, register_class) orelse blk: {
281 var unlocked_registers = self.locked_registers;
282 unlocked_registers.toggleAll();
283 unlocked_registers.setIntersection(register_class);
284
288285 // We'll take over the first count registers. Spill
289286 // the instructions that were previously there to a
290287 // stack allocations.
291288 var regs: [count]Register = undefined;
292289 var i: usize = 0;
293 for (tracked_registers) |reg| {
294 if (i >= count) break;
295 if (excludeRegister(reg, register_class)) break;
296 if (self.isRegLocked(reg)) continue;
297
298 log.debug("allocReg {} for inst {?}", .{ reg, insts[i] });
299 regs[i] = reg;
300 self.markRegAllocated(reg);
301 const index = indexOfRegIntoTracked(reg).?; // indexOfReg() on a callee-preserved reg should never return null
302 if (insts[i]) |inst| {
303 // Track the register
304 if (self.isRegFree(reg)) {
305 self.markRegUsed(reg);
306 } else {
307 const spilled_inst = self.registers[index];
308 try self.getFunction().spillInstruction(reg, spilled_inst);
309 }
310 self.registers[index] = inst;
311 } else {
312 // Don't track the register
313 if (!self.isRegFree(reg)) {
314 const spilled_inst = self.registers[index];
315 try self.getFunction().spillInstruction(reg, spilled_inst);
316 self.freeReg(reg);
317 }
318 }
319
290 var it = unlocked_registers.iterator(.{});
291 while (it.next()) |reg_index| {
292 const tracked_index: TrackedIndex = @intCast(reg_index);
293 if (!self.isRegIndexFree(tracked_index) and
294 self.registers[tracked_index].unwrap() == .target) continue;
295 try self.getRegIndex(tracked_index, insts[i]);
296 regs[i] = regAtTrackedIndex(tracked_index);
320297 i += 1;
298 if (i >= count) break;
321299 }
300 if (i < count) return error.OutOfRegisters;
322301
323302 break :blk regs;
324303 };
......@@ -340,7 +319,7 @@ pub fn RegisterManager(
340319 /// Spills the register if it is currently allocated. If a
341320 /// corresponding instruction is passed, will also track this
342321 /// register.
343 fn getRegIndex(
322 pub fn getRegIndex(
344323 self: *Self,
345324 tracked_index: TrackedIndex,
346325 inst: ?Air.Inst.Index,
......@@ -366,13 +345,13 @@ pub fn RegisterManager(
366345 comptime reg: Register,
367346 inst: ?Air.Inst.Index,
368347 ) AllocationError!void {
369 return self.getRegIndex((comptime indexOfRegIntoTracked(reg)) orelse return, inst);
348 return self.getRegIndex(indexOfKnownRegIntoTracked(reg) orelse return, inst);
370349 }
371350
372351 /// Allocates the specified register with the specified
373352 /// instruction. Asserts that the register is free and no
374353 /// spilling is necessary.
375 fn getRegIndexAssumeFree(
354 pub fn getRegIndexAssumeFree(
376355 self: *Self,
377356 tracked_index: TrackedIndex,
378357 inst: ?Air.Inst.Index,
......@@ -391,7 +370,7 @@ pub fn RegisterManager(
391370 }
392371
393372 /// Marks the specified register as free
394 fn freeRegIndex(self: *Self, tracked_index: TrackedIndex) void {
373 pub fn freeRegIndex(self: *Self, tracked_index: TrackedIndex) void {
395374 log.debug("freeing register {}", .{regAtTrackedIndex(tracked_index)});
396375 self.registers[tracked_index] = undefined;
397376 self.markRegIndexFree(tracked_index);
......@@ -420,8 +399,8 @@ const MockRegister1 = enum(u2) {
420399 &MockRegister1.allocatable_registers,
421400 );
422401
423 const gp: RM.RegisterBitSet = blk: {
424 var set = RM.RegisterBitSet.initEmpty();
402 const gp = blk: {
403 var set: RM.RegisterBitSet = .initEmpty();
425404 set.setRangeValue(.{
426405 .start = 0,
427406 .end = allocatable_registers.len,
......@@ -448,8 +427,8 @@ const MockRegister2 = enum(u2) {
448427 &MockRegister2.allocatable_registers,
449428 );
450429
451 const gp: RM.RegisterBitSet = blk: {
452 var set = RM.RegisterBitSet.initEmpty();
430 const gp = blk: {
431 var set: RM.RegisterBitSet = .initEmpty();
453432 set.setRangeValue(.{
454433 .start = 0,
455434 .end = allocatable_registers.len,
......@@ -489,16 +468,16 @@ const MockRegister3 = enum(u3) {
489468 &MockRegister3.allocatable_registers,
490469 );
491470
492 const gp: RM.RegisterBitSet = blk: {
493 var set = RM.RegisterBitSet.initEmpty();
471 const gp = blk: {
472 var set: RM.RegisterBitSet = .initEmpty();
494473 set.setRangeValue(.{
495474 .start = 0,
496475 .end = gp_regs.len,
497476 }, true);
498477 break :blk set;
499478 };
500 const ext: RM.RegisterBitSet = blk: {
501 var set = RM.RegisterBitSet.initEmpty();
479 const ext = blk: {
480 var set: RM.RegisterBitSet = .initEmpty();
502481 set.setRangeValue(.{
503482 .start = gp_regs.len,
504483 .end = allocatable_registers.len,
test/behavior.zig+2
......@@ -110,6 +110,8 @@ test {
110110 _ = @import("behavior/widening.zig");
111111 _ = @import("behavior/abs.zig");
112112
113 _ = @import("behavior/x86_64.zig");
114
113115 if (builtin.cpu.arch == .wasm32) {
114116 _ = @import("behavior/wasm.zig");
115117 }
test/behavior/align.zig+19-33
......@@ -144,31 +144,17 @@ test "alignment and size of structs with 128-bit fields" {
144144 },
145145 },
146146
147 .x86_64 => switch (builtin.zig_backend) {
148 .stage2_x86_64 => .{
149 .a_align = 8,
150 .a_size = 16,
151
152 .b_align = 16,
153 .b_size = 32,
154
155 .u128_align = 8,
156 .u128_size = 16,
157 .u129_align = 8,
158 .u129_size = 24,
159 },
160 else => .{
161 .a_align = 16,
162 .a_size = 16,
147 .x86_64 => .{
148 .a_align = 16,
149 .a_size = 16,
163150
164 .b_align = 16,
165 .b_size = 32,
151 .b_align = 16,
152 .b_size = 32,
166153
167 .u128_align = 16,
168 .u128_size = 16,
169 .u129_align = 16,
170 .u129_size = 32,
171 },
154 .u128_align = 16,
155 .u128_size = 16,
156 .u129_align = 16,
157 .u129_size = 32,
172158 },
173159
174160 .x86,
......@@ -291,8 +277,8 @@ test "function alignment" {
291277 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
292278 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
293279
294 // function alignment is a compile error on wasm32/wasm64
295 if (native_arch == .wasm32 or native_arch == .wasm64) return error.SkipZigTest;
280 // function alignment is a compile error on wasm
281 if (native_arch.isWasm()) return error.SkipZigTest;
296282
297283 const S = struct {
298284 fn alignExpr() align(@sizeOf(usize) * 2) i32 {
......@@ -321,8 +307,8 @@ test "implicitly decreasing fn alignment" {
321307 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
322308 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
323309
324 // function alignment is a compile error on wasm32/wasm64
325 if (native_arch == .wasm32 or native_arch == .wasm64) return error.SkipZigTest;
310 // function alignment is a compile error on wasm
311 if (native_arch.isWasm()) return error.SkipZigTest;
326312
327313 try testImplicitlyDecreaseFnAlign(alignedSmall, 1234);
328314 try testImplicitlyDecreaseFnAlign(alignedBig, 5678);
......@@ -345,9 +331,9 @@ test "@alignCast functions" {
345331 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
346332 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
347333
348 // function alignment is a compile error on wasm32/wasm64
349 if (native_arch == .wasm32 or native_arch == .wasm64) return error.SkipZigTest;
350 if (native_arch == .thumb or native_arch == .thumbeb) return error.SkipZigTest;
334 // function alignment is a compile error on wasm
335 if (native_arch.isWasm()) return error.SkipZigTest;
336 if (native_arch.isThumb()) return error.SkipZigTest;
351337
352338 try expect(fnExpectsOnly1(simple4) == 0x19);
353339}
......@@ -510,9 +496,9 @@ test "align(N) on functions" {
510496 return error.SkipZigTest;
511497 }
512498
513 // function alignment is a compile error on wasm32/wasm64
514 if (native_arch == .wasm32 or native_arch == .wasm64) return error.SkipZigTest;
515 if (native_arch == .thumb or native_arch == .thumbeb) return error.SkipZigTest;
499 // function alignment is a compile error on wasm
500 if (native_arch.isWasm()) return error.SkipZigTest;
501 if (native_arch.isThumb()) return error.SkipZigTest;
516502
517503 try expect((@intFromPtr(&overaligned_fn) & (0x1000 - 1)) == 0);
518504}
test/behavior/asm.zig+1-1
......@@ -178,7 +178,7 @@ test "rw constraint (x86_64)" {
178178}
179179
180180test "asm modifiers (AArch64)" {
181 if (builtin.target.cpu.arch != .aarch64) return error.SkipZigTest;
181 if (!builtin.target.cpu.arch.isAARCH64()) return error.SkipZigTest;
182182 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
183183
184184 if (builtin.zig_backend == .stage2_c and builtin.os.tag == .windows) return error.SkipZigTest; // MSVC doesn't support inline assembly
test/behavior/basic.zig+1-1
......@@ -1169,7 +1169,7 @@ test "arrays and vectors with big integers" {
11691169 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
11701170 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
11711171 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
1172 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1172 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
11731173 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
11741174 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
11751175 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
test/behavior/call.zig+1
......@@ -660,6 +660,7 @@ test "arguments pointed to on stack into tailcall" {
660660
661661 switch (builtin.cpu.arch) {
662662 .wasm32,
663 .wasm64,
663664 .mips,
664665 .mipsel,
665666 .mips64,
test/behavior/cast.zig+2-3
......@@ -124,7 +124,7 @@ test "@floatFromInt(f80)" {
124124 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
125125 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
126126 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
127 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
127 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
128128 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
129129 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
130130
......@@ -1362,7 +1362,7 @@ test "cast f16 to wider types" {
13621362 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
13631363 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
13641364 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1365 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
1365 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
13661366 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
13671367 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
13681368
......@@ -2539,7 +2539,6 @@ test "@intFromBool on vector" {
25392539 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
25402540 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
25412541 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
2542 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
25432542 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
25442543
25452544 const S = struct {
test/behavior/eval.zig+1-1
......@@ -522,7 +522,7 @@ test "runtime 128 bit integer division" {
522522 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
523523 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
524524 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
525 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
525 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
526526 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
527527 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
528528
test/behavior/math.zig+12-8
......@@ -65,6 +65,8 @@ test "@clz" {
6565 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
6666 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
6767 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
68 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
69 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
6870
6971 try testClz();
7072 try comptime testClz();
......@@ -75,6 +77,7 @@ fn testClz() !void {
7577 try expect(testOneClz(u8, 0b00001010) == 4);
7678 try expect(testOneClz(u8, 0b00011010) == 3);
7779 try expect(testOneClz(u8, 0b00000000) == 8);
80 try expect(testOneClz(i8, -1) == 0);
7881}
7982
8083test "@clz big ints" {
......@@ -100,7 +103,7 @@ fn testOneClz(comptime T: type, x: T) u32 {
100103
101104test "@clz vectors" {
102105 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
103 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
106 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
104107 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
105108 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
106109 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -159,6 +162,8 @@ fn testCtz() !void {
159162 try expect(testOneCtz(u8, 0b10100000) == 5);
160163 try expect(testOneCtz(u8, 0b10001010) == 1);
161164 try expect(testOneCtz(u8, 0b00000000) == 8);
165 try expect(testOneCtz(i8, -1) == 0);
166 try expect(testOneCtz(i8, -2) == 1);
162167 try expect(testOneCtz(u16, 0b00000000) == 16);
163168}
164169
......@@ -467,7 +472,6 @@ test "division" {
467472 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
468473 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
469474 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
470 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
471475 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
472476 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
473477
......@@ -583,7 +587,7 @@ fn testFloatDivision() !void {
583587}
584588
585589test "large integer division" {
586 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
590 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
587591 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
588592 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
589593 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
......@@ -780,7 +784,7 @@ test "128-bit multiplication" {
780784 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
781785 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
782786 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
783 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
787 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
784788 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
785789
786790 {
......@@ -1369,7 +1373,7 @@ test "remainder division" {
13691373 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
13701374 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
13711375 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
1372 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
1376 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
13731377 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
13741378
13751379 if (builtin.zig_backend == .stage2_llvm and builtin.os.tag == .windows) {
......@@ -1522,7 +1526,7 @@ test "@round f80" {
15221526 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
15231527 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
15241528 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1525 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
1529 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
15261530 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
15271531 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
15281532
......@@ -1535,7 +1539,7 @@ test "@round f128" {
15351539 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
15361540 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
15371541 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1538 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
1542 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
15391543 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
15401544 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
15411545
......@@ -1712,7 +1716,7 @@ test "mod lazy values" {
17121716
17131717test "@clz works on both vector and scalar inputs" {
17141718 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1715 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1719 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
17161720 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
17171721 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
17181722 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/maximum_minimum.zig+1-1
......@@ -122,7 +122,7 @@ test "@min/max for floats" {
122122 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
123123 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
124124 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
125 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
125 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
126126 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
127127 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
128128
test/behavior/muladd.zig+4-4
......@@ -58,7 +58,7 @@ test "@mulAdd f80" {
5858 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
5959 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
6060 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
61 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
61 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
6262 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
6363 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
6464
......@@ -79,7 +79,7 @@ test "@mulAdd f128" {
7979 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
8080 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
8181 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
82 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
82 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
8383 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
8484 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
8585
......@@ -189,7 +189,7 @@ test "vector f80" {
189189 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
190190 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
191191 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
192 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
192 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
193193 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
194194
195195 try comptime vector80();
......@@ -216,7 +216,7 @@ test "vector f128" {
216216 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
217217 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
218218 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
219 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
219 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
220220 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
221221
222222 try comptime vector128();
test/behavior/optional.zig+1-1
......@@ -57,7 +57,7 @@ fn testNullPtrsEql() !void {
5757
5858test "optional with zero-bit type" {
5959 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
60 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
60 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
6161 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
6262
6363 const S = struct {
test/behavior/packed-union.zig-1
......@@ -138,7 +138,6 @@ test "packed union initialized with a runtime value" {
138138 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
139139 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
140140 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
141 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
142141 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
143142
144143 const Fields = packed struct {
test/behavior/saturating_arithmetic.zig+5-5
......@@ -164,10 +164,10 @@ test "saturating multiplication <= 32 bits" {
164164 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
165165 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
166166 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
167 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
167 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
168168 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
169169
170 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .wasm32) {
170 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isWasm()) {
171171 // https://github.com/ziglang/zig/issues/9660
172172 return error.SkipZigTest;
173173 }
......@@ -264,10 +264,10 @@ test "saturating multiplication" {
264264 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
265265 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
266266 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
267 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
267 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
268268 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
269269
270 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .wasm32) {
270 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isWasm()) {
271271 // https://github.com/ziglang/zig/issues/9660
272272 return error.SkipZigTest;
273273 }
......@@ -311,7 +311,7 @@ test "saturating shift-left" {
311311 try testSatShl(i8, 127, 1, 127);
312312 try testSatShl(i8, -128, 1, -128);
313313 // TODO: remove this check once #9668 is completed
314 if (builtin.cpu.arch != .wasm32) {
314 if (!builtin.cpu.arch.isWasm()) {
315315 // skip testing ints > 64 bits on wasm due to miscompilation / wasmtime ci error
316316 try testSatShl(i128, maxInt(i128), 64, maxInt(i128));
317317 try testSatShl(u128, maxInt(u128), 64, maxInt(u128));
test/behavior/struct.zig+4-4
......@@ -418,8 +418,8 @@ test "packed struct 24bits" {
418418 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
419419 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
420420 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
421 if (builtin.cpu.arch == .wasm32) return error.SkipZigTest; // TODO
422 if (comptime builtin.cpu.arch.isArm()) return error.SkipZigTest; // TODO
421 if (builtin.cpu.arch.isWasm()) return error.SkipZigTest; // TODO
422 if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // TODO
423423 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
424424 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
425425
......@@ -818,7 +818,7 @@ test "non-packed struct with u128 entry in union" {
818818 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
819819 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
820820 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
821 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
821 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
822822 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
823823
824824 const U = union(enum) {
......@@ -941,7 +941,7 @@ test "tuple assigned to variable" {
941941
942942test "comptime struct field" {
943943 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
944 if (comptime builtin.cpu.arch.isArm()) return error.SkipZigTest; // TODO
944 if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // TODO
945945
946946 const T = struct {
947947 a: i32,
test/behavior/union.zig+1-1
......@@ -2246,12 +2246,12 @@ test "matching captures causes union equivalence" {
22462246}
22472247
22482248test "signed enum tag with negative value" {
2249 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
22502249 if (builtin.zig_backend == .stage2_x86) return error.SkipZigTest; // TODO
22512250 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
22522251 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
22532252 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
22542253 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
2254 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
22552255
22562256 const Enum = enum(i8) {
22572257 a = -1,
test/behavior/var_args.zig+5-5
......@@ -100,7 +100,7 @@ test "simple variadic function" {
100100 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
101101 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
102102 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
103 if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {
103 if (builtin.os.tag != .macos and builtin.cpu.arch.isAARCH64()) {
104104 // https://github.com/ziglang/zig/issues/14096
105105 return error.SkipZigTest;
106106 }
......@@ -161,7 +161,7 @@ test "coerce reference to var arg" {
161161 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
162162 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
163163 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
164 if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {
164 if (builtin.os.tag != .macos and builtin.cpu.arch.isAARCH64()) {
165165 // https://github.com/ziglang/zig/issues/14096
166166 return error.SkipZigTest;
167167 }
......@@ -194,7 +194,7 @@ test "variadic functions" {
194194 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
195195 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
196196 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
197 if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {
197 if (builtin.os.tag != .macos and builtin.cpu.arch.isAARCH64()) {
198198 // https://github.com/ziglang/zig/issues/14096
199199 return error.SkipZigTest;
200200 }
......@@ -239,7 +239,7 @@ test "copy VaList" {
239239 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
240240 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
241241 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
242 if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {
242 if (builtin.os.tag != .macos and builtin.cpu.arch.isAARCH64()) {
243243 // https://github.com/ziglang/zig/issues/14096
244244 return error.SkipZigTest;
245245 }
......@@ -273,7 +273,7 @@ test "unused VaList arg" {
273273 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
274274 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
275275 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
276 if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {
276 if (builtin.os.tag != .macos and builtin.cpu.arch.isAARCH64()) {
277277 // https://github.com/ziglang/zig/issues/14096
278278 return error.SkipZigTest;
279279 }
test/behavior/vector.zig+6-9
......@@ -101,7 +101,7 @@ test "vector float operators" {
101101 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
102102 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
103103 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
104 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
104 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
105105 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
106106
107107 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
......@@ -205,7 +205,6 @@ test "array vector coercion - odd sizes" {
205205 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
206206 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
207207 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
208 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
209208 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
210209 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
211210
......@@ -308,7 +307,6 @@ test "tuple to vector" {
308307
309308test "vector casts of sizes not divisible by 8" {
310309 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
311 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
312310 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
313311 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
314312 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -646,7 +644,7 @@ test "vector division operators" {
646644
647645test "vector bitwise not operator" {
648646 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
649 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
647 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
650648 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
651649 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
652650 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -754,7 +752,7 @@ test "vector reduce operation" {
754752 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
755753 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
756754 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
757 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;
755 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
758756 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
759757 if (builtin.cpu.arch.isMIPS64()) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21091
760758
......@@ -989,7 +987,7 @@ test "saturating multiplication" {
989987 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
990988
991989 // TODO: once #9660 has been solved, remove this line
992 if (builtin.target.cpu.arch == .wasm32) return error.SkipZigTest;
990 if (builtin.target.cpu.arch.isWasm()) return error.SkipZigTest;
993991
994992 const S = struct {
995993 fn doTheTest() !void {
......@@ -1256,7 +1254,7 @@ test "byte vector initialized in inline function" {
12561254 if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
12571255
12581256 if (comptime builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .x86_64 and
1259 builtin.cpu.features.isEnabled(@intFromEnum(std.Target.x86.Feature.avx512f)))
1257 std.Target.x86.featureSetHas(builtin.cpu.features, .avx512f))
12601258 {
12611259 // TODO https://github.com/ziglang/zig/issues/13279
12621260 return error.SkipZigTest;
......@@ -1363,7 +1361,7 @@ test "load packed vector element" {
13631361 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
13641362 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
13651363 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1366 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1364 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
13671365 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
13681366
13691367 var x: @Vector(2, u15) = .{ 1, 4 };
......@@ -1411,7 +1409,6 @@ test "store vector with memset" {
14111409 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
14121410 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
14131411 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1414 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
14151412 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
14161413 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; // TODO
14171414 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
test/behavior/wrapping_arithmetic.zig+1-1
......@@ -83,7 +83,7 @@ test "wrapping multiplication" {
8383 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
8484
8585 // TODO: once #9660 has been solved, remove this line
86 if (builtin.cpu.arch == .wasm32) return error.SkipZigTest;
86 if (builtin.cpu.arch.isWasm()) return error.SkipZigTest;
8787
8888 const S = struct {
8989 fn doTheTest() !void {
test/behavior/x86_64.zig created+9
......@@ -0,0 +1,9 @@
1//! CodeGen tests for the x86_64 backend.
2
3test {
4 const builtin = @import("builtin");
5 if (builtin.zig_backend != .stage2_x86_64) return error.SkipZigTest;
6 if (builtin.object_format == .coff) return error.SkipZigTest;
7 _ = @import("x86_64/math.zig");
8 _ = @import("x86_64/mem.zig");
9}
test/behavior/x86_64/build.zig created+115
......@@ -0,0 +1,115 @@
1const std = @import("std");
2pub fn build(b: *std.Build) void {
3 const compiler_rt_lib = b.addStaticLibrary(.{
4 .name = "compiler_rt",
5 .use_llvm = false,
6 .use_lld = false,
7 .root_module = b.createModule(.{
8 .root_source_file = b.addWriteFiles().add("compiler_rt.zig", ""),
9 .target = b.resolveTargetQuery(.{ .cpu_arch = .x86_64 }),
10 }),
11 });
12 compiler_rt_lib.bundle_compiler_rt = true;
13
14 for ([_]std.Target.Query{
15 .{
16 .cpu_arch = .x86_64,
17 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
18 .cpu_features_add = std.Target.x86.featureSet(&.{.bsf_bsr_0_clobbers_result}),
19 //.cpu_features_sub = std.Target.x86.featureSet(&.{.sse}),
20 },
21 .{
22 .cpu_arch = .x86_64,
23 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
24 .cpu_features_add = std.Target.x86.featureSet(&.{.bsf_bsr_0_clobbers_result}),
25 .cpu_features_sub = std.Target.x86.featureSet(&.{
26 .cmov,
27 //.sse,
28 }),
29 },
30 //.{
31 // .cpu_arch = .x86_64,
32 // .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
33 // .cpu_features_sub = std.Target.x86.featureSet(&.{.sse}),
34 //},
35 .{
36 .cpu_arch = .x86_64,
37 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
38 .cpu_features_sub = std.Target.x86.featureSet(&.{.sse2}),
39 },
40 .{
41 .cpu_arch = .x86_64,
42 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
43 },
44 .{
45 .cpu_arch = .x86_64,
46 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
47 .cpu_features_add = std.Target.x86.featureSet(&.{.sse3}),
48 },
49 .{
50 .cpu_arch = .x86_64,
51 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
52 .cpu_features_add = std.Target.x86.featureSet(&.{.ssse3}),
53 },
54 .{
55 .cpu_arch = .x86_64,
56 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
57 .cpu_features_add = std.Target.x86.featureSet(&.{.sse4_1}),
58 },
59 .{
60 .cpu_arch = .x86_64,
61 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
62 .cpu_features_add = std.Target.x86.featureSet(&.{.sse4_2}),
63 },
64 .{
65 .cpu_arch = .x86_64,
66 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v2 },
67 },
68 .{
69 .cpu_arch = .x86_64,
70 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v2 },
71 .cpu_features_add = std.Target.x86.featureSet(&.{.avx}),
72 },
73 .{
74 .cpu_arch = .x86_64,
75 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v3 },
76 .cpu_features_sub = std.Target.x86.featureSet(&.{.avx2}),
77 },
78 .{
79 .cpu_arch = .x86_64,
80 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v3 },
81 },
82 .{
83 .cpu_arch = .x86_64,
84 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v4 },
85 },
86 }) |query| {
87 const target = b.resolveTargetQuery(query);
88 const cpu = query.serializeCpuAlloc(b.allocator) catch @panic("OOM");
89 for ([_][]const u8{
90 "math.zig",
91 "mem.zig",
92 }) |path| {
93 const test_mod = b.createModule(.{
94 .root_source_file = b.path(path),
95 .target = target,
96 });
97 const test_exe = b.addTest(.{
98 .name = std.fs.path.stem(path),
99 .use_llvm = false,
100 .use_lld = false,
101 .root_module = test_mod,
102 });
103 if (!std.Target.x86.featureSetHas(target.result.cpu.features, .sse2)) {
104 test_exe.bundle_compiler_rt = false;
105 test_mod.linkLibrary(compiler_rt_lib);
106 }
107 const test_run = b.addRunArtifact(test_exe);
108 b.default_step.dependOn(&test_run.step);
109 for ([_]*std.Build.Step{
110 &test_exe.step,
111 &test_run.step,
112 }) |step| step.name = b.fmt("{s} {s}", .{ step.name, cpu });
113 }
114 }
115}
test/behavior/x86_64/math.zig created+1103
......@@ -0,0 +1,1103 @@
1fn testUnary(comptime op: anytype) !void {
2 const testType = struct {
3 fn testType(comptime Type: type, comptime imm_arg: Type) !void {
4 const expected = op(Type, imm_arg);
5 try struct {
6 fn testOne(actual: @TypeOf(expected)) !void {
7 if (switch (@typeInfo(@TypeOf(expected))) {
8 else => actual != expected,
9 .vector => @reduce(.Or, actual != expected),
10 }) return error.Unexpected;
11 }
12 noinline fn testOps(mem_arg: Type) !void {
13 var reg_arg = mem_arg;
14 _ = .{&reg_arg};
15 try testOne(op(Type, reg_arg));
16 try testOne(op(Type, mem_arg));
17 try testOne(op(Type, imm_arg));
18 }
19 }.testOps(imm_arg);
20 }
21 }.testType;
22
23 try testType(i0, 0);
24 try testType(u0, 0);
25
26 try testType(i1, -1);
27 try testType(i1, 0);
28 try testType(u1, 0);
29 try testType(u1, 1 << 0);
30
31 try testType(i2, -1 << 1);
32 try testType(i2, -1);
33 try testType(i2, 0);
34 try testType(u2, 0);
35 try testType(u2, 1 << 0);
36 try testType(u2, 1 << 1);
37
38 try testType(i3, -1 << 2);
39 try testType(i3, -1);
40 try testType(i3, 0);
41 try testType(u3, 0);
42 try testType(u3, 1 << 0);
43 try testType(u3, 1 << 1);
44 try testType(u3, 1 << 2);
45
46 try testType(i4, -1 << 3);
47 try testType(i4, -1);
48 try testType(i4, 0);
49 try testType(u4, 0);
50 try testType(u4, 1 << 0);
51 try testType(u4, 1 << 1);
52 try testType(u4, 1 << 2);
53 try testType(u4, 1 << 3);
54
55 try testType(i5, -1 << 4);
56 try testType(i5, -1);
57 try testType(i5, 0);
58 try testType(u5, 0);
59 try testType(u5, 1 << 0);
60 try testType(u5, 1 << 1);
61 try testType(u5, 1 << 3);
62 try testType(u5, 1 << 4);
63
64 try testType(i7, -1 << 6);
65 try testType(i7, -1);
66 try testType(i7, 0);
67 try testType(u7, 0);
68 try testType(u7, 1 << 0);
69 try testType(u7, 1 << 1);
70 try testType(u7, 1 << 5);
71 try testType(u7, 1 << 6);
72
73 try testType(i8, -1 << 7);
74 try testType(i8, -1);
75 try testType(i8, 0);
76 try testType(u8, 0);
77 try testType(u8, 1 << 0);
78 try testType(u8, 1 << 1);
79 try testType(u8, 1 << 6);
80 try testType(u8, 1 << 7);
81
82 try testType(i9, -1 << 8);
83 try testType(i9, -1);
84 try testType(i9, 0);
85 try testType(u9, 0);
86 try testType(u9, 1 << 0);
87 try testType(u9, 1 << 1);
88 try testType(u9, 1 << 7);
89 try testType(u9, 1 << 8);
90
91 try testType(i15, -1 << 14);
92 try testType(i15, -1);
93 try testType(i15, 0);
94 try testType(u15, 0);
95 try testType(u15, 1 << 0);
96 try testType(u15, 1 << 1);
97 try testType(u15, 1 << 13);
98 try testType(u15, 1 << 14);
99
100 try testType(i16, -1 << 15);
101 try testType(i16, -1);
102 try testType(i16, 0);
103 try testType(u16, 0);
104 try testType(u16, 1 << 0);
105 try testType(u16, 1 << 1);
106 try testType(u16, 1 << 14);
107 try testType(u16, 1 << 15);
108
109 try testType(i17, -1 << 16);
110 try testType(i17, -1);
111 try testType(i17, 0);
112 try testType(u17, 0);
113 try testType(u17, 1 << 0);
114 try testType(u17, 1 << 1);
115 try testType(u17, 1 << 15);
116 try testType(u17, 1 << 16);
117
118 try testType(i31, -1 << 30);
119 try testType(i31, -1);
120 try testType(i31, 0);
121 try testType(u31, 0);
122 try testType(u31, 1 << 0);
123 try testType(u31, 1 << 1);
124 try testType(u31, 1 << 29);
125 try testType(u31, 1 << 30);
126
127 try testType(i32, -1 << 31);
128 try testType(i32, -1);
129 try testType(i32, 0);
130 try testType(u32, 0);
131 try testType(u32, 1 << 0);
132 try testType(u32, 1 << 1);
133 try testType(u32, 1 << 30);
134 try testType(u32, 1 << 31);
135
136 try testType(i33, -1 << 32);
137 try testType(i33, -1);
138 try testType(i33, 0);
139 try testType(u33, 0);
140 try testType(u33, 1 << 0);
141 try testType(u33, 1 << 1);
142 try testType(u33, 1 << 31);
143 try testType(u33, 1 << 32);
144
145 try testType(i63, -1 << 62);
146 try testType(i63, -1);
147 try testType(i63, 0);
148 try testType(u63, 0);
149 try testType(u63, 1 << 0);
150 try testType(u63, 1 << 1);
151 try testType(u63, 1 << 61);
152 try testType(u63, 1 << 62);
153
154 try testType(i64, -1 << 63);
155 try testType(i64, -1);
156 try testType(i64, 0);
157 try testType(u64, 0);
158 try testType(u64, 1 << 0);
159 try testType(u64, 1 << 1);
160 try testType(u64, 1 << 62);
161 try testType(u64, 1 << 63);
162
163 try testType(i65, -1 << 64);
164 try testType(i65, -1);
165 try testType(i65, 0);
166 try testType(u65, 0);
167 try testType(u65, 1 << 0);
168 try testType(u65, 1 << 1);
169 try testType(u65, 1 << 63);
170 try testType(u65, 1 << 64);
171
172 try testType(i95, -1 << 94);
173 try testType(i95, -1);
174 try testType(i95, 0);
175 try testType(u95, 0);
176 try testType(u95, 1 << 0);
177 try testType(u95, 1 << 1);
178 try testType(u95, 1 << 93);
179 try testType(u95, 1 << 94);
180
181 try testType(i96, -1 << 95);
182 try testType(i96, -1);
183 try testType(i96, 0);
184 try testType(u96, 0);
185 try testType(u96, 1 << 0);
186 try testType(u96, 1 << 1);
187 try testType(u96, 1 << 94);
188 try testType(u96, 1 << 95);
189
190 try testType(i97, -1 << 96);
191 try testType(i97, -1);
192 try testType(i97, 0);
193 try testType(u97, 0);
194 try testType(u97, 1 << 0);
195 try testType(u97, 1 << 1);
196 try testType(u97, 1 << 95);
197 try testType(u97, 1 << 96);
198
199 try testType(i127, -1 << 126);
200 try testType(i127, -1);
201 try testType(i127, 0);
202 try testType(u127, 0);
203 try testType(u127, 1 << 0);
204 try testType(u127, 1 << 1);
205 try testType(u127, 1 << 125);
206 try testType(u127, 1 << 126);
207
208 try testType(i128, -1 << 127);
209 try testType(i128, -1);
210 try testType(i128, 0);
211 try testType(u128, 0);
212 try testType(u128, 1 << 0);
213 try testType(u128, 1 << 1);
214 try testType(u128, 1 << 126);
215 try testType(u128, 1 << 127);
216
217 try testType(i129, -1 << 128);
218 try testType(i129, -1);
219 try testType(i129, 0);
220 try testType(u129, 0);
221 try testType(u129, 1 << 0);
222 try testType(u129, 1 << 1);
223 try testType(u129, 1 << 127);
224 try testType(u129, 1 << 128);
225
226 try testType(i159, -1 << 158);
227 try testType(i159, -1);
228 try testType(i159, 0);
229 try testType(u159, 0);
230 try testType(u159, 1 << 0);
231 try testType(u159, 1 << 1);
232 try testType(u159, 1 << 157);
233 try testType(u159, 1 << 158);
234
235 try testType(i160, -1 << 159);
236 try testType(i160, -1);
237 try testType(i160, 0);
238 try testType(u160, 0);
239 try testType(u160, 1 << 0);
240 try testType(u160, 1 << 1);
241 try testType(u160, 1 << 158);
242 try testType(u160, 1 << 159);
243
244 try testType(i161, -1 << 160);
245 try testType(i161, -1);
246 try testType(i161, 0);
247 try testType(u161, 0);
248 try testType(u161, 1 << 0);
249 try testType(u161, 1 << 1);
250 try testType(u161, 1 << 159);
251 try testType(u161, 1 << 160);
252
253 try testType(i191, -1 << 190);
254 try testType(i191, -1);
255 try testType(i191, 0);
256 try testType(u191, 0);
257 try testType(u191, 1 << 0);
258 try testType(u191, 1 << 1);
259 try testType(u191, 1 << 189);
260 try testType(u191, 1 << 190);
261
262 try testType(i192, -1 << 191);
263 try testType(i192, -1);
264 try testType(i192, 0);
265 try testType(u192, 0);
266 try testType(u192, 1 << 0);
267 try testType(u192, 1 << 1);
268 try testType(u192, 1 << 190);
269 try testType(u192, 1 << 191);
270
271 try testType(i193, -1 << 192);
272 try testType(i193, -1);
273 try testType(i193, 0);
274 try testType(u193, 0);
275 try testType(u193, 1 << 0);
276 try testType(u193, 1 << 1);
277 try testType(u193, 1 << 191);
278 try testType(u193, 1 << 192);
279
280 try testType(i223, -1 << 222);
281 try testType(i223, -1);
282 try testType(i223, 0);
283 try testType(u223, 0);
284 try testType(u223, 1 << 0);
285 try testType(u223, 1 << 1);
286 try testType(u223, 1 << 221);
287 try testType(u223, 1 << 222);
288
289 try testType(i224, -1 << 223);
290 try testType(i224, -1);
291 try testType(i224, 0);
292 try testType(u224, 0);
293 try testType(u224, 1 << 0);
294 try testType(u224, 1 << 1);
295 try testType(u224, 1 << 222);
296 try testType(u224, 1 << 223);
297
298 try testType(i225, -1 << 224);
299 try testType(i225, -1);
300 try testType(i225, 0);
301 try testType(u225, 0);
302 try testType(u225, 1 << 0);
303 try testType(u225, 1 << 1);
304 try testType(u225, 1 << 223);
305 try testType(u225, 1 << 224);
306
307 try testType(i255, -1 << 254);
308 try testType(i255, -1);
309 try testType(i255, 0);
310 try testType(u255, 0);
311 try testType(u255, 1 << 0);
312 try testType(u255, 1 << 1);
313 try testType(u255, 1 << 253);
314 try testType(u255, 1 << 254);
315
316 try testType(i256, -1 << 255);
317 try testType(i256, -1);
318 try testType(i256, 0);
319 try testType(u256, 0);
320 try testType(u256, 1 << 0);
321 try testType(u256, 1 << 1);
322 try testType(u256, 1 << 254);
323 try testType(u256, 1 << 255);
324
325 try testType(i257, -1 << 256);
326 try testType(i257, -1);
327 try testType(i257, 0);
328 try testType(u257, 0);
329 try testType(u257, 1 << 0);
330 try testType(u257, 1 << 1);
331 try testType(u257, 1 << 255);
332 try testType(u257, 1 << 256);
333
334 try testType(i511, -1 << 510);
335 try testType(i511, -1);
336 try testType(i511, 0);
337 try testType(u511, 0);
338 try testType(u511, 1 << 0);
339 try testType(u511, 1 << 1);
340 try testType(u511, 1 << 509);
341 try testType(u511, 1 << 510);
342
343 try testType(i512, -1 << 511);
344 try testType(i512, -1);
345 try testType(i512, 0);
346 try testType(u512, 0);
347 try testType(u512, 1 << 0);
348 try testType(u512, 1 << 1);
349 try testType(u512, 1 << 510);
350 try testType(u512, 1 << 511);
351
352 try testType(i513, -1 << 512);
353 try testType(i513, -1);
354 try testType(i513, 0);
355 try testType(u513, 0);
356 try testType(u513, 1 << 0);
357 try testType(u513, 1 << 1);
358 try testType(u513, 1 << 511);
359 try testType(u513, 1 << 512);
360
361 try testType(i1023, -1 << 1022);
362 try testType(i1023, -1);
363 try testType(i1023, 0);
364 try testType(u1023, 0);
365 try testType(u1023, 1 << 0);
366 try testType(u1023, 1 << 1);
367 try testType(u1023, 1 << 1021);
368 try testType(u1023, 1 << 1022);
369
370 try testType(i1024, -1 << 1023);
371 try testType(i1024, -1);
372 try testType(i1024, 0);
373 try testType(u1024, 0);
374 try testType(u1024, 1 << 0);
375 try testType(u1024, 1 << 1);
376 try testType(u1024, 1 << 1022);
377 try testType(u1024, 1 << 1023);
378
379 try testType(i1025, -1 << 1024);
380 try testType(i1025, -1);
381 try testType(i1025, 0);
382 try testType(u1025, 0);
383 try testType(u1025, 1 << 0);
384 try testType(u1025, 1 << 1);
385 try testType(u1025, 1 << 1023);
386 try testType(u1025, 1 << 1024);
387
388 try testType(@Vector(3, i0), .{ 0 << 0, 0, 0 });
389 try testType(@Vector(3, u0), .{ 0, 0, 0 << 0 });
390
391 try testType(@Vector(3, i1), .{ -1 << 0, -1, 0 });
392 try testType(@Vector(3, u1), .{ 0, 1, 1 << 0 });
393
394 try testType(@Vector(3, i2), .{ -1 << 1, -1, 0 });
395 try testType(@Vector(3, u2), .{ 0, 1, 1 << 1 });
396
397 try testType(@Vector(3, i3), .{ -1 << 2, -1, 0 });
398 try testType(@Vector(3, u3), .{ 0, 1, 1 << 2 });
399
400 try testType(@Vector(3, i4), .{ -1 << 3, -1, 0 });
401 try testType(@Vector(3, u4), .{ 0, 1, 1 << 3 });
402 try testType(@Vector(1, u4), .{
403 0xb,
404 });
405 try testType(@Vector(2, u4), .{
406 0x3, 0x4,
407 });
408 try testType(@Vector(4, u4), .{
409 0x9, 0x2, 0xf, 0xe,
410 });
411 try testType(@Vector(8, u4), .{
412 0x8, 0x1, 0xb, 0x1, 0xf, 0x5, 0x9, 0x6,
413 });
414 try testType(@Vector(16, u4), .{
415 0xb, 0x6, 0x0, 0x7, 0x8, 0x5, 0x6, 0x9, 0xe, 0xb, 0x3, 0xa, 0xb, 0x5, 0x8, 0xc,
416 });
417 try testType(@Vector(32, u4), .{
418 0xe, 0x6, 0xe, 0xa, 0xb, 0x4, 0xa, 0xb, 0x1, 0x3, 0xb, 0xc, 0x0, 0xb, 0x9, 0x4, 0xd, 0xa, 0xd, 0xd, 0x4, 0x8, 0x8, 0x6, 0xb, 0xe, 0x9, 0x6, 0xc, 0xd, 0x5, 0xd,
419 });
420 try testType(@Vector(64, u4), .{
421 0x1, 0xc, 0xe, 0x9, 0x9, 0xf, 0x3, 0xf, 0x9, 0x9, 0x5, 0x3, 0xb, 0xd, 0xd, 0xf, 0x1, 0x2, 0xf, 0x9, 0x4, 0x4, 0x8, 0x9, 0x2, 0x9, 0x8, 0xe, 0x8, 0xa, 0x4, 0x3,
422 0x4, 0xc, 0xb, 0x6, 0x4, 0x0, 0xa, 0x5, 0x1, 0xa, 0x4, 0xe, 0xa, 0x7, 0xd, 0x0, 0x4, 0xe, 0xe, 0x7, 0x7, 0xa, 0x4, 0x5, 0x6, 0xc, 0x6, 0x2, 0x6, 0xa, 0xe, 0xa,
423 });
424 try testType(@Vector(128, u4), .{
425 0xd, 0x5, 0x6, 0xe, 0x3, 0x3, 0x3, 0xe, 0xd, 0xd, 0x9, 0x0, 0x0, 0xe, 0xa, 0x9, 0x8, 0x7, 0xb, 0x5, 0x7, 0xf, 0xb, 0x8, 0x0, 0xf, 0xb, 0x3, 0xa, 0x2, 0xb, 0xc,
426 0x1, 0x1, 0xc, 0x8, 0x8, 0x6, 0x9, 0x1, 0xb, 0x0, 0x2, 0xb, 0x2, 0x2, 0x7, 0x6, 0x1, 0x1, 0xb, 0x4, 0x6, 0x4, 0x7, 0xc, 0xd, 0xc, 0xa, 0x8, 0x1, 0x7, 0x8, 0xa,
427 0x9, 0xa, 0x1, 0x8, 0x1, 0x7, 0x9, 0x4, 0x5, 0x9, 0xd, 0x0, 0xa, 0xf, 0x3, 0x3, 0x9, 0x2, 0xf, 0x5, 0xb, 0x8, 0x6, 0xb, 0xf, 0x5, 0x8, 0x3, 0x9, 0xf, 0x6, 0x8,
428 0xc, 0x8, 0x3, 0x4, 0xa, 0xe, 0xc, 0x1, 0xe, 0x9, 0x1, 0x8, 0xf, 0x6, 0xc, 0xc, 0x6, 0xf, 0x6, 0xd, 0xb, 0x9, 0xc, 0x3, 0xd, 0xa, 0x6, 0x8, 0x4, 0xa, 0x6, 0x9,
429 });
430 try testType(@Vector(256, u4), .{
431 0x6, 0xc, 0xe, 0x3, 0x8, 0x2, 0xb, 0xd, 0x3, 0xa, 0x3, 0x8, 0xb, 0x8, 0x3, 0x0, 0xb, 0x5, 0x1, 0x3, 0x2, 0x2, 0xf, 0xc, 0x5, 0x1, 0x3, 0xb, 0x1, 0xc, 0x2, 0xd,
432 0xa, 0x8, 0x1, 0xc, 0xb, 0xa, 0x3, 0x1, 0xe, 0x4, 0xf, 0xb, 0xd, 0x8, 0xf, 0xa, 0xc, 0xb, 0xb, 0x0, 0xa, 0xc, 0xf, 0xe, 0x8, 0xd, 0x9, 0x3, 0xa, 0xe, 0x8, 0x7,
433 0x5, 0xa, 0x0, 0xe, 0x0, 0xd, 0x2, 0x2, 0x9, 0x4, 0x8, 0x9, 0x0, 0x4, 0x4, 0x8, 0xe, 0x1, 0xf, 0x1, 0x9, 0x3, 0xf, 0xc, 0xa, 0x0, 0x3, 0x2, 0x4, 0x1, 0x2, 0x3,
434 0xf, 0x2, 0x7, 0xb, 0x5, 0x0, 0xd, 0x3, 0x4, 0xf, 0xa, 0x3, 0xc, 0x2, 0x5, 0xe, 0x7, 0x5, 0xd, 0x7, 0x9, 0x0, 0xd, 0x7, 0x9, 0xd, 0x5, 0x7, 0xf, 0xd, 0xb, 0x4,
435 0x9, 0x6, 0xf, 0xb, 0x1, 0xb, 0x6, 0xb, 0xf, 0x7, 0xf, 0x0, 0x4, 0x7, 0x5, 0xa, 0x8, 0x1, 0xf, 0x9, 0x9, 0x0, 0x6, 0xb, 0x1, 0x2, 0x4, 0x3, 0x2, 0x0, 0x7, 0x0,
436 0x6, 0x7, 0xf, 0x1, 0xe, 0xa, 0x8, 0x2, 0x9, 0xc, 0x1, 0x5, 0x7, 0x1, 0xb, 0x0, 0x1, 0x3, 0xd, 0x3, 0x0, 0x1, 0xa, 0x0, 0x3, 0x7, 0x1, 0x2, 0xb, 0xc, 0x2, 0x9,
437 0x8, 0x8, 0x7, 0x0, 0xd, 0x5, 0x1, 0x5, 0x7, 0x7, 0x2, 0x3, 0x8, 0x7, 0xc, 0x8, 0xf, 0xa, 0xf, 0xf, 0x3, 0x2, 0x0, 0x4, 0x7, 0x5, 0x6, 0xd, 0x6, 0x3, 0xa, 0x4,
438 0x1, 0x1, 0x2, 0xc, 0x3, 0xe, 0x2, 0xc, 0x7, 0x6, 0xe, 0xf, 0xb, 0x8, 0x6, 0x6, 0x9, 0x0, 0x4, 0xb, 0xe, 0x4, 0x2, 0x7, 0xf, 0xc, 0x0, 0x6, 0xd, 0xa, 0xe, 0xc,
439 });
440
441 try testType(@Vector(3, i5), .{ -1 << 4, -1, 0 });
442 try testType(@Vector(3, u5), .{ 0, 1, 1 << 4 });
443
444 try testType(@Vector(3, i7), .{ -1 << 6, -1, 0 });
445 try testType(@Vector(3, u7), .{ 0, 1, 1 << 6 });
446
447 try testType(@Vector(3, i8), .{ -1 << 7, -1, 0 });
448 try testType(@Vector(3, u8), .{ 0, 1, 1 << 7 });
449 try testType(@Vector(1, u8), .{
450 0x33,
451 });
452 try testType(@Vector(2, u8), .{
453 0x66, 0x87,
454 });
455 try testType(@Vector(4, u8), .{
456 0x9d, 0xcb, 0x30, 0x7b,
457 });
458 try testType(@Vector(8, u8), .{
459 0x4b, 0x35, 0x3f, 0x5c, 0xa5, 0x91, 0x23, 0x6d,
460 });
461 try testType(@Vector(16, u8), .{
462 0xb7, 0x57, 0x27, 0x29, 0x58, 0xf8, 0xc9, 0x6c, 0xbe, 0x41, 0xf4, 0xd7, 0x4d, 0x01, 0xf0, 0x37,
463 });
464 try testType(@Vector(32, u8), .{
465 0x5f, 0x61, 0x34, 0xe8, 0x37, 0x12, 0xba, 0x5a, 0x85, 0xf3, 0x3e, 0xa2, 0x0f, 0xd0, 0x65, 0xae,
466 0xed, 0xf5, 0xe8, 0x65, 0x61, 0x28, 0x4a, 0x27, 0x2e, 0x01, 0x40, 0x8c, 0xe3, 0x36, 0x5d, 0xb6,
467 });
468 try testType(@Vector(64, u8), .{
469 0xb0, 0x19, 0x5c, 0xc2, 0x3b, 0x16, 0x70, 0xad, 0x26, 0x45, 0xf2, 0xe1, 0x4f, 0x0f, 0x01, 0x72,
470 0x7f, 0x1f, 0x07, 0x9e, 0xee, 0x9b, 0xb3, 0x38, 0x50, 0xf3, 0x56, 0x73, 0xd0, 0xd1, 0xee, 0xe3,
471 0xeb, 0xf3, 0x1b, 0xe0, 0x77, 0x78, 0x75, 0xc6, 0x19, 0xe4, 0x69, 0xaa, 0x73, 0x08, 0xcd, 0x0c,
472 0xf9, 0xed, 0x94, 0xf8, 0x79, 0x86, 0x63, 0x31, 0xbf, 0xd1, 0xe3, 0x17, 0x2b, 0xb9, 0xa1, 0x72,
473 });
474 try testType(@Vector(128, u8), .{
475 0x2e, 0x93, 0x87, 0x09, 0x4f, 0x68, 0x14, 0xab, 0x3f, 0x04, 0x86, 0xc1, 0x95, 0xe8, 0x74, 0x11,
476 0x57, 0x25, 0xe1, 0x88, 0xc0, 0x96, 0x33, 0x99, 0x15, 0x86, 0x2c, 0x84, 0x2e, 0xd7, 0x57, 0x21,
477 0xd3, 0x18, 0xd5, 0x0e, 0xb4, 0x60, 0xe2, 0x08, 0xce, 0xbc, 0xd5, 0x4d, 0x8f, 0x59, 0x01, 0x67,
478 0x71, 0x0a, 0x74, 0x48, 0xef, 0x39, 0x49, 0x7e, 0xa8, 0x39, 0x34, 0x75, 0x95, 0x3b, 0x38, 0xea,
479 0x60, 0xd7, 0xed, 0x8f, 0xbb, 0xc0, 0x7d, 0xc2, 0x79, 0x2d, 0xbf, 0xa5, 0x64, 0xf4, 0x09, 0x86,
480 0xfb, 0x29, 0xfe, 0xc7, 0xff, 0x62, 0x1a, 0x6f, 0xf8, 0xbd, 0xfe, 0xa4, 0xac, 0x24, 0xcf, 0x56,
481 0x82, 0x69, 0x81, 0x0d, 0xc1, 0x51, 0x8d, 0x85, 0xf4, 0x00, 0xe7, 0x25, 0xab, 0xa5, 0x33, 0x45,
482 0x66, 0x2e, 0x33, 0xc8, 0xf3, 0x35, 0x16, 0x7d, 0x1f, 0xc9, 0xf7, 0x44, 0xab, 0x66, 0x28, 0x0d,
483 });
484
485 try testType(@Vector(3, i9), .{ -1 << 8, -1, 0 });
486 try testType(@Vector(3, u9), .{ 0, 1, 1 << 8 });
487
488 try testType(@Vector(3, i15), .{ -1 << 14, -1, 0 });
489 try testType(@Vector(3, u15), .{ 0, 1, 1 << 14 });
490
491 try testType(@Vector(3, i16), .{ -1 << 15, -1, 0 });
492 try testType(@Vector(3, u16), .{ 0, 1, 1 << 15 });
493 try testType(@Vector(1, u16), .{
494 0x4da6,
495 });
496 try testType(@Vector(2, u16), .{
497 0x04d7, 0x50c6,
498 });
499 try testType(@Vector(4, u16), .{
500 0x4c06, 0xd71f, 0x4d8f, 0xe0a4,
501 });
502 try testType(@Vector(8, u16), .{
503 0xee9a, 0x881d, 0x31fb, 0xd3f7, 0x2c74, 0x6949, 0x4e04, 0x53d7,
504 });
505 try testType(@Vector(16, u16), .{
506 0xeafe, 0x9a7b, 0x0d6f, 0x18cb, 0xaf8f, 0x8ee4, 0xa47e, 0xd39a,
507 0x6572, 0x9c53, 0xf36e, 0x982e, 0x41c1, 0x8682, 0xf5dc, 0x7e01,
508 });
509 try testType(@Vector(32, u16), .{
510 0xdfb3, 0x7de6, 0xd9ed, 0xb42e, 0x95ac, 0x9b5b, 0x0422, 0xdfcd,
511 0x6196, 0x4dbe, 0x1818, 0x8816, 0x75e7, 0xc9b0, 0x92f7, 0x1f71,
512 0xe584, 0x576c, 0x043a, 0x0f31, 0xfc4c, 0x2c87, 0x6b02, 0x0229,
513 0x25b7, 0x53cd, 0x9bab, 0x866b, 0x9008, 0xf0f3, 0xeb21, 0x88e2,
514 });
515 try testType(@Vector(64, u16), .{
516 0x084c, 0x445f, 0xce89, 0xd3ee, 0xb399, 0x315d, 0x8ef8, 0x4f6f,
517 0xf9af, 0xcbc4, 0x0332, 0xcd55, 0xa4dc, 0xbc38, 0x6e33, 0x8ead,
518 0xd15a, 0x5057, 0x58ef, 0x657a, 0xe9f0, 0x1418, 0x2b62, 0x3387,
519 0x1c15, 0x04e1, 0x0276, 0x3783, 0xad9c, 0xea9a, 0x0e5e, 0xe803,
520 0x2ee7, 0x0cf1, 0x30f1, 0xb12a, 0x381b, 0x353d, 0xf637, 0xf853,
521 0x2ac1, 0x7ce8, 0x6a50, 0xcbb8, 0xc9b8, 0x9b25, 0xd1e9, 0xeff0,
522 0xc0a2, 0x8e51, 0xde7a, 0x4e58, 0x5685, 0xeb3f, 0xd29b, 0x66ed,
523 0x3dd5, 0xcb59, 0x6003, 0xf710, 0x943a, 0x7276, 0xe547, 0xe48f,
524 });
525
526 try testType(@Vector(3, i17), .{ -1 << 16, -1, 0 });
527 try testType(@Vector(3, u17), .{ 0, 1, 1 << 16 });
528
529 try testType(@Vector(3, i31), .{ -1 << 30, -1, 0 });
530 try testType(@Vector(3, u31), .{ 0, 1, 1 << 30 });
531
532 try testType(@Vector(3, i32), .{ -1 << 31, -1, 0 });
533 try testType(@Vector(3, u32), .{ 0, 1, 1 << 31 });
534 try testType(@Vector(1, u32), .{
535 0x17e2805c,
536 });
537 try testType(@Vector(2, u32), .{
538 0xdb6aadc5, 0xb1ff3754,
539 });
540 try testType(@Vector(4, u32), .{
541 0xf7897b31, 0x342e1af9, 0x190fd76b, 0x283b5374,
542 });
543 try testType(@Vector(8, u32), .{
544 0x81a0bd16, 0xc55da94e, 0x910f7e7c, 0x078d5ef7,
545 0x0bdb1e4a, 0xf1a96e99, 0xcdd729b5, 0xe6966a1c,
546 });
547 try testType(@Vector(16, u32), .{
548 0xfee812db, 0x29eacbed, 0xaed48136, 0x3053de13,
549 0xbbda20df, 0x6faa274a, 0xe0b5ec3a, 0x1878b0dc,
550 0x98204475, 0x810d8d05, 0x1e6996b6, 0xc543826a,
551 0x53b47d8c, 0xc72c3142, 0x12f7e1f9, 0xf6782e54,
552 });
553 try testType(@Vector(32, u32), .{
554 0xf0cf30d3, 0xe3c587b8, 0xcee44739, 0xe4a0bd72,
555 0x41d44cce, 0x6d7c4259, 0xd85580a5, 0xec4b02d7,
556 0xa366483d, 0x2d7b59d4, 0xe9c0ace4, 0x82cb441c,
557 0xa23958ba, 0x04a70148, 0x3f0d20a3, 0xf9e21e37,
558 0x009fce8b, 0x4a34a229, 0xf09c35cf, 0xc0977d4d,
559 0xcc4d4647, 0xa30f1363, 0x27a65b14, 0xe572c785,
560 0x8f42e320, 0x2b2cdeca, 0x11205bd4, 0x739d26aa,
561 0xcbcc2df0, 0x5f7a3649, 0xbde1b7aa, 0x180a169f,
562 });
563
564 try testType(@Vector(3, i33), .{ -1 << 32, -1, 0 });
565 try testType(@Vector(3, u33), .{ 0, 1, 1 << 32 });
566
567 try testType(@Vector(3, i63), .{ -1 << 62, -1, 0 });
568 try testType(@Vector(3, u63), .{ 0, 1, 1 << 62 });
569
570 try testType(@Vector(3, i64), .{ -1 << 63, -1, 0 });
571 try testType(@Vector(3, u64), .{ 0, 1, 1 << 63 });
572 try testType(@Vector(1, u64), .{
573 0x7d2e439abb0edba7,
574 });
575 try testType(@Vector(2, u64), .{
576 0x3749ee5a2d237b9f, 0x6d8f4c3e1378f389,
577 });
578 try testType(@Vector(4, u64), .{
579 0x03c127040e10d52b, 0xa86fe019072e27eb,
580 0x0a554a47b709cdba, 0xf4342cc597e196c3,
581 });
582 try testType(@Vector(8, u64), .{
583 0xea455c104375a055, 0x5c35d9d945edb2fa,
584 0xc11b73d9d9d546fc, 0x2a9d63aae838dd5b,
585 0xed6603f1f5d574b3, 0x2f37b354c81c1e56,
586 0xbe7f5e2476bc76bd, 0xb0c88eacfffa9a8f,
587 });
588 try testType(@Vector(16, u64), .{
589 0x2258fc04b31f8dbe, 0x3a2e5483003a10d8,
590 0xebf24b31c0460510, 0x15d5b4c09b53ffa5,
591 0x05abf6e744b17cc6, 0x9747b483f2d159fe,
592 0x4616d8b2c8673125, 0x8ae3f91d422447eb,
593 0x18da2f101a9e9776, 0x77a1197fb0441007,
594 0x4ba480c8ec2dd10b, 0xeb99b9c0a1725278,
595 0xd9d0acc5084ecdf0, 0xa0a23317fff4f515,
596 0x0901c59a9a6a408b, 0x7c77ca72e25df033,
597 });
598
599 try testType(@Vector(3, i65), .{ -1 << 64, -1, 0 });
600 try testType(@Vector(3, u65), .{ 0, 1, 1 << 64 });
601
602 try testType(@Vector(3, i127), .{ -1 << 126, -1, 0 });
603 try testType(@Vector(3, u127), .{ 0, 1, 1 << 126 });
604
605 try testType(@Vector(3, i128), .{ -1 << 127, -1, 0 });
606 try testType(@Vector(3, u128), .{ 0, 1, 1 << 127 });
607 try testType(@Vector(1, u128), .{
608 0x809f29e7fbafadc01145e1732590e7d9,
609 });
610 try testType(@Vector(2, u128), .{
611 0x5150ac3438aacd0d51132cc2723b2995,
612 0x151be9c47ad29cf719cf8358dd40165c,
613 });
614 try testType(@Vector(4, u128), .{
615 0x4bae22df929f2f7cb9bd84deaad3e7a8,
616 0x1ed46b2d6e1f3569f56b2ac33d8bc1cb,
617 0xae93ea459d2ccfd5fb794e6d5c31aabb,
618 0xb1177136acf099f550b70949ac202ec4,
619 });
620 try testType(@Vector(8, u128), .{
621 0x7cd78db6baed6bfdf8c5265136c4e0fd,
622 0xa41b8984c6bbde84640068194b7eba98,
623 0xd33102778f2ae1a48d1e9bf8801bbbf0,
624 0x0d59f6de003513a60055c86cbce2c200,
625 0x825579d90012afddfbf04851c0748561,
626 0xc2647c885e9d6f0ee1f5fac5da8ef7f5,
627 0xcb4bbc1f81aa8ee68aa4dc140745687b,
628 0x4ff10f914f74b46c694407f5bf7c7836,
629 });
630
631 try testType(@Vector(3, i129), .{ -1 << 128, -1, 0 });
632 try testType(@Vector(3, u129), .{ 0, 1, 1 << 128 });
633
634 try testType(@Vector(3, i191), .{ -1 << 190, -1, 0 });
635 try testType(@Vector(3, u191), .{ 0, 1, 1 << 190 });
636
637 try testType(@Vector(3, i192), .{ -1 << 191, -1, 0 });
638 try testType(@Vector(3, u192), .{ 0, 1, 1 << 191 });
639 try testType(@Vector(1, u192), .{
640 0xe7baafcb9781626a77571b0539b9471a60c97d6c02106c8b,
641 });
642 try testType(@Vector(2, u192), .{
643 0xbc9510913ed09e2c2aa50ffab9f1bc7b303a87f36e232a83,
644 0x1f37bee446d7712d1ad457c47a66812cb926198d052aee65,
645 });
646 try testType(@Vector(4, u192), .{
647 0xdca6a7cfc19c69efc34022062a8ca36f2569ab3dce001202,
648 0xd25a4529e621c9084181fdb6917c6a32eccc58b63601b35d,
649 0x0a258afd6debbaf8c158f1caa61fed63b31871d13f51b43d,
650 0x6b40a178674fcb82c623ac322f851623d5e993dac97a219a,
651 });
652
653 try testType(@Vector(3, i193), .{ -1 << 192, -1, 0 });
654 try testType(@Vector(3, u193), .{ 0, 1, 1 << 192 });
655
656 try testType(@Vector(3, i255), .{ -1 << 254, -1, 0 });
657 try testType(@Vector(3, u255), .{ 0, 1, 1 << 254 });
658
659 try testType(@Vector(3, i256), .{ -1 << 255, -1, 0 });
660 try testType(@Vector(3, u256), .{ 0, 1, 1 << 255 });
661 try testType(@Vector(1, u256), .{
662 0x230413bb481fa3a997796acf282010c560d1942e7339fd584a0f15a90c83fbda,
663 });
664 try testType(@Vector(2, u256), .{
665 0x3ad569f8d91fdbc9da8ec0e933565919f2feb90b996c90c352b461aa0908e62d,
666 0x0f109696d64647983f1f757042515510729ad1350e862cbf38cb73b5cf99f0f7,
667 });
668 try testType(@Vector(4, u256), .{
669 0x1717c6ded4ac6de282d59f75f068da47d5a47a30f2c5053d2d59e715f9d28b97,
670 0x3087189ce7540e2e0028b80af571ebc6353a00b2917f243a869ed29ecca0adaa,
671 0x1507c6a9d104684bf503cdb08841cf91adab4644306bd67aafff5326604833ce,
672 0x857e134ff9179733c871295b25f824bd3eb562977bad30890964fa0cdc15bb07,
673 });
674
675 try testType(@Vector(3, i257), .{ -1 << 256, -1, 0 });
676 try testType(@Vector(3, u257), .{ 0, 1, 1 << 256 });
677
678 try testType(@Vector(3, i511), .{ -1 << 510, -1, 0 });
679 try testType(@Vector(3, u511), .{ 0, 1, 1 << 510 });
680
681 try testType(@Vector(3, i512), .{ -1 << 511, -1, 0 });
682 try testType(@Vector(3, u512), .{ 0, 1, 1 << 511 });
683 try testType(@Vector(1, u512), .{
684 0xa3ff51a609f1370e5eeb96b05169bf7469e465cf76ac5b4ea8ffd166c1ba3cd94f2dedf0d647a1fe424f3a06e6d7940f03e257f28100970b00bd5528c52b9ae6,
685 });
686 try testType(@Vector(2, u512), .{
687 0xc6d43cd46ae31ab71f9468a895c83bf17516c6b2f1c9b04b9aa113bf7fe1b789eb7d95fcf951f12a9a6f2124589551efdd8c00f528b366a7bfb852faf8f3da53,
688 0xc9099d2bdf8d1a0d30485ec6db4a24cbc0d89a863de30e18313ee1d66f71dd2d26235caaa703286cf4a2b51e1a12ef96d2d944c66c0bd3f0d72dd4cf0fc8100e,
689 });
690
691 try testType(@Vector(3, i513), .{ -1 << 512, -1, 0 });
692 try testType(@Vector(3, u513), .{ 0, 1, 1 << 512 });
693
694 try testType(@Vector(3, i1023), .{ -1 << 1022, -1, 0 });
695 try testType(@Vector(3, u1023), .{ 0, 1, 1 << 1022 });
696
697 try testType(@Vector(3, i1024), .{ -1 << 1023, -1, 0 });
698 try testType(@Vector(3, u1024), .{ 0, 1, 1 << 1023 });
699 try testType(@Vector(1, u1024), .{
700 0xc6cfaa6571139552e1f067402dfc131d9b9a58aafda97198a78764b05138fb68cf26f085b7652f3d5ae0e56aa21732f296a581bb411d4a73795c213de793489fa49b173b9f5c089aa6295ff1fcdc14d491a05035b45d08fc35cd67a83d887a02b8db512f07518132e0ba56533c7d6fbe958255eddf5649bd8aba288c0dd84a25,
701 });
702
703 try testType(@Vector(3, i1025), .{ -1 << 1024, -1, 0 });
704 try testType(@Vector(3, u1025), .{ 0, 1, 1 << 1024 });
705}
706
707fn testBinary(comptime op: anytype) !void {
708 const testType = struct {
709 fn testType(comptime Type: type, comptime imm_lhs: Type, comptime imm_rhs: Type) !void {
710 const expected = op(Type, imm_lhs, imm_rhs);
711 try struct {
712 fn testOne(actual: @TypeOf(expected)) !void {
713 if (switch (@typeInfo(@TypeOf(expected))) {
714 else => actual != expected,
715 .vector => @reduce(.Or, actual != expected),
716 }) return error.Unexpected;
717 }
718 noinline fn testOps(mem_lhs: Type, mem_rhs: Type) !void {
719 var reg_lhs = mem_lhs;
720 var reg_rhs = mem_rhs;
721 _ = .{ &reg_lhs, &reg_rhs };
722 try testOne(op(Type, reg_lhs, reg_rhs));
723 try testOne(op(Type, reg_lhs, mem_rhs));
724 try testOne(op(Type, reg_lhs, imm_rhs));
725 try testOne(op(Type, mem_lhs, reg_rhs));
726 try testOne(op(Type, mem_lhs, mem_rhs));
727 try testOne(op(Type, mem_lhs, imm_rhs));
728 try testOne(op(Type, imm_lhs, reg_rhs));
729 try testOne(op(Type, imm_lhs, mem_rhs));
730 }
731 }.testOps(imm_lhs, imm_rhs);
732 }
733 }.testType;
734
735 try testType(u8, 0xbb, 0x43);
736 try testType(u16, 0xb8bf, 0x626d);
737 try testType(u32, 0x80d7a2c6, 0xbff6a402);
738 try testType(u64, 0x71138bc6b4a38898, 0x1bc4043de9438c7b);
739 try testType(u128, 0xe05fc132ef2cd8affee00a907f0a851f, 0x29f912a72cfc6a7c6973426a9636da9a);
740 try testType(
741 u256,
742 0xb7935f5c2f3b1ae7a422c0a7c446884294b7d5370bada307d2fe5a4c4284a999,
743 0x310e6e196ba4f143b8d285ca6addf7f3bb3344224aff221b27607a31e148be08,
744 );
745 try testType(
746 u258,
747 0x186d5ddaab8cb8cb04e5b41e36f812e039d008baf49f12894c39e29a07796d800,
748 0x2072daba6ffad168826163eb136f6d28ca4360c8e7e5e41e29755e19e4753a4f5,
749 );
750 try testType(
751 u495,
752 0x6eaf4e252b3bf74b75bac59e0b43ca5326bad2a25b3fdb74a67ef132ac5e47d72eebc3316fb2351ee66c50dc5afb92a75cea9b0e35160652c7db39eeb158,
753 0x49fbed744a92b549d8c05bb3512c617d24dd824f3f69bdf3923bc326a75674b85f5b828d2566fab9c86f571d12c2a63c9164feb0d191d27905533d09622a,
754 );
755 try testType(
756 u512,
757 0xe5b1fedca3c77db765e517aabd05ffc524a3a8aff1784bbf67c45b894447ede32b65b9940e78173c591e56e078932d465f235aece7ad47b7f229df7ba8f12295,
758 0x8b4bb7c2969e3b121cc1082c442f8b4330f0a50058438fed56447175bb10178607ecfe425cb54dacc25ef26810f3e04681de1844f1aa8d029aca75d658634806,
759 );
760
761 try testType(@Vector(1, u8), .{
762 0x1f,
763 }, .{
764 0x06,
765 });
766 try testType(@Vector(2, u8), .{
767 0x80, 0x63,
768 }, .{
769 0xe4, 0x28,
770 });
771 try testType(@Vector(4, u8), .{
772 0x83, 0x9e, 0x1e, 0xc1,
773 }, .{
774 0xf0, 0x5c, 0x46, 0x85,
775 });
776 try testType(@Vector(8, u8), .{
777 0x1e, 0x4d, 0x9d, 0x2a, 0x4c, 0x74, 0x0a, 0x83,
778 }, .{
779 0x28, 0x60, 0xa9, 0xb5, 0xd9, 0xa6, 0xf1, 0xb6,
780 });
781 try testType(@Vector(16, u8), .{
782 0xea, 0x80, 0xbb, 0xe8, 0x74, 0x81, 0xc8, 0x66, 0x7b, 0x41, 0x90, 0xcb, 0x30, 0x70, 0x4b, 0x0f,
783 }, .{
784 0x61, 0x26, 0xbe, 0x47, 0x00, 0x9c, 0x55, 0xa5, 0x59, 0xf0, 0xb2, 0x20, 0x30, 0xaf, 0x82, 0x3e,
785 });
786 try testType(@Vector(32, u8), .{
787 0xa1, 0x88, 0xc4, 0xf4, 0x77, 0x0b, 0xf5, 0xbb, 0x09, 0x03, 0xbf, 0xf5, 0xcc, 0x7f, 0x6b, 0x2a,
788 0x4c, 0x05, 0x37, 0xc9, 0x8a, 0xcb, 0x91, 0x23, 0x09, 0x5f, 0xb8, 0x99, 0x4a, 0x75, 0x26, 0xe4,
789 }, .{
790 0xff, 0x0f, 0x99, 0x49, 0xa6, 0x25, 0xa7, 0xd4, 0xc9, 0x2f, 0x97, 0x6a, 0x01, 0xd6, 0x6e, 0x41,
791 0xa4, 0xb5, 0x3c, 0x03, 0xea, 0x82, 0x9c, 0x5f, 0xac, 0x07, 0x16, 0x15, 0x1c, 0x64, 0x25, 0x2f,
792 });
793 try testType(@Vector(64, u8), .{
794 0xaa, 0x08, 0xeb, 0xb2, 0xd7, 0x89, 0x0f, 0x98, 0xda, 0x9f, 0xa6, 0x4e, 0x3c, 0xce, 0x1b, 0x1b,
795 0x9e, 0x5f, 0x2b, 0xd6, 0x59, 0x26, 0x47, 0x05, 0x2a, 0xb7, 0xd1, 0x10, 0xde, 0xd9, 0x84, 0x00,
796 0x07, 0xc0, 0xaa, 0x6e, 0xfa, 0x3b, 0x97, 0x85, 0xa8, 0x42, 0xd7, 0xa5, 0x90, 0xe6, 0x10, 0x1a,
797 0x47, 0x84, 0xe1, 0x3e, 0xb0, 0x70, 0x26, 0x3f, 0xea, 0x24, 0xb8, 0x5f, 0xe3, 0xe3, 0x4c, 0xed,
798 }, .{
799 0x3b, 0xc5, 0xe0, 0x3d, 0x4f, 0x2e, 0x1d, 0xa9, 0xf7, 0x7b, 0xc7, 0xc1, 0x48, 0xc6, 0xe5, 0x9e,
800 0x4d, 0xa8, 0x21, 0x37, 0xa1, 0x1a, 0x95, 0x69, 0x89, 0x2f, 0x15, 0x07, 0x3d, 0x7b, 0x69, 0x89,
801 0xea, 0x87, 0xf0, 0x94, 0x67, 0xf2, 0x3d, 0x04, 0x96, 0x8a, 0xd6, 0x70, 0x7c, 0x16, 0xe7, 0x62,
802 0xf0, 0x8d, 0x96, 0x65, 0xd1, 0x4a, 0x35, 0x3e, 0x7a, 0x67, 0xa6, 0x1f, 0x37, 0x66, 0xe3, 0x45,
803 });
804 try testType(@Vector(128, u8), .{
805 0xa1, 0xd0, 0x7b, 0xf9, 0x7b, 0x77, 0x7b, 0x3d, 0x2d, 0x68, 0xc2, 0x7b, 0xb0, 0xb8, 0xd4, 0x7c,
806 0x1a, 0x1f, 0xd2, 0x92, 0x3e, 0xcb, 0xc1, 0x6b, 0xb9, 0x4d, 0xf1, 0x67, 0x58, 0x8e, 0x77, 0xa6,
807 0xb9, 0xdf, 0x10, 0x6f, 0xbe, 0xe3, 0x33, 0xb6, 0x93, 0x77, 0x80, 0xef, 0x09, 0x9d, 0x61, 0x40,
808 0xa2, 0xf4, 0x52, 0x18, 0x9d, 0xe4, 0xb0, 0xaf, 0x0a, 0xa7, 0x0b, 0x09, 0x67, 0x38, 0x71, 0x04,
809 0x72, 0xa1, 0xd2, 0xfd, 0xf8, 0xf0, 0xa7, 0x23, 0x24, 0x5b, 0x7d, 0xfb, 0x43, 0xba, 0x6c, 0xc4,
810 0x83, 0x46, 0x0e, 0x4d, 0x6c, 0x92, 0xab, 0x4f, 0xd2, 0x70, 0x9d, 0xfe, 0xce, 0xf8, 0x05, 0x9f,
811 0x98, 0x36, 0x9c, 0x90, 0x9a, 0xd0, 0xb5, 0x76, 0x16, 0xe8, 0x25, 0xc2, 0xbd, 0x91, 0xab, 0xf9,
812 0x6f, 0x6c, 0xc5, 0x60, 0xe5, 0x30, 0xf2, 0xb7, 0x59, 0xc4, 0x9c, 0xdd, 0xdf, 0x04, 0x65, 0xd9,
813 }, .{
814 0xed, 0xe1, 0x8a, 0xf6, 0xf3, 0x8b, 0xfd, 0x1d, 0x3c, 0x87, 0xbf, 0xfe, 0x04, 0x52, 0x15, 0x82,
815 0x0b, 0xb0, 0xcf, 0xcf, 0xf8, 0x03, 0x9c, 0xef, 0xc1, 0x76, 0x7e, 0xe3, 0xe9, 0xa8, 0x18, 0x90,
816 0xd4, 0xc4, 0x91, 0x15, 0x68, 0x7f, 0x65, 0xd8, 0xe1, 0xb3, 0x23, 0xc2, 0x7d, 0x84, 0x3b, 0xaf,
817 0x74, 0x69, 0x07, 0x2a, 0x1b, 0x5f, 0x0e, 0x44, 0x0d, 0x2b, 0x9c, 0x82, 0x41, 0xf9, 0x7f, 0xb5,
818 0xc4, 0xd9, 0xcb, 0xd3, 0xc5, 0x31, 0x8b, 0x5f, 0xda, 0x09, 0x9b, 0x29, 0xa3, 0xb7, 0x13, 0x0d,
819 0x55, 0x9b, 0x59, 0x33, 0x2a, 0x59, 0x3a, 0x44, 0x1f, 0xd3, 0x40, 0x4e, 0xde, 0x2c, 0xe4, 0x16,
820 0xfd, 0xc3, 0x02, 0x74, 0xaa, 0x65, 0xfd, 0xc8, 0x2a, 0x8a, 0xdb, 0xae, 0x44, 0x28, 0x62, 0xa4,
821 0x56, 0x4f, 0xf1, 0xaa, 0x0a, 0x0f, 0xdb, 0x1b, 0xc8, 0x45, 0x9b, 0x12, 0xb4, 0x1a, 0xe4, 0xa3,
822 });
823
824 try testType(@Vector(1, u16), .{
825 0x9d6f,
826 }, .{
827 0x44b1,
828 });
829 try testType(@Vector(2, u16), .{
830 0xa0fa, 0xc365,
831 }, .{
832 0xe736, 0xc394,
833 });
834 try testType(@Vector(4, u16), .{
835 0x9608, 0xa558, 0x161b, 0x206f,
836 }, .{
837 0x3088, 0xf25c, 0x7837, 0x9b3f,
838 });
839 try testType(@Vector(8, u16), .{
840 0xcf61, 0xb121, 0x3cf1, 0x3e9f, 0x43a7, 0x8d69, 0x96f5, 0xc11e,
841 }, .{
842 0xee30, 0x82f0, 0x270b, 0x1498, 0x4c60, 0x6e72, 0x0b64, 0x02d4,
843 });
844 try testType(@Vector(16, u16), .{
845 0x9191, 0xd23e, 0xf844, 0xd84a, 0xe907, 0xf1e8, 0x712d, 0x90af,
846 0x6541, 0x3fa6, 0x92eb, 0xe35a, 0xc0c9, 0xcb47, 0xb790, 0x4453,
847 }, .{
848 0x21c3, 0x4039, 0x9b71, 0x60bd, 0xcd7f, 0x2ec8, 0x50ba, 0xe810,
849 0xebd4, 0x06e5, 0xed18, 0x2f66, 0x7e31, 0xe282, 0xad63, 0xb25e,
850 });
851 try testType(@Vector(32, u16), .{
852 0x6b6a, 0x30a9, 0xc267, 0x2231, 0xbf4c, 0x00bc, 0x9c2c, 0x2928,
853 0xecad, 0x82df, 0xcfb0, 0xa4e5, 0x909b, 0x1b05, 0xaf40, 0x1fd9,
854 0xcec6, 0xd8dc, 0xd4b5, 0x6d59, 0x8e3f, 0x4d8a, 0xb83a, 0x808e,
855 0x47e2, 0x5782, 0x59bf, 0xcefc, 0x5179, 0x3f48, 0x93dc, 0x66d2,
856 }, .{
857 0x1be8, 0xe98c, 0xf9b3, 0xb008, 0x2f8d, 0xf087, 0xc9b9, 0x75aa,
858 0xbd16, 0x9540, 0xc5bd, 0x2b2c, 0xd43f, 0x9394, 0x3e1d, 0xf695,
859 0x167d, 0xff7a, 0xf09d, 0xdff8, 0xdfa2, 0xc779, 0x70b7, 0x01bd,
860 0x46b3, 0x995a, 0xb7bc, 0xa79d, 0x5542, 0x961e, 0x37cd, 0x9c2a,
861 });
862 try testType(@Vector(64, u16), .{
863 0x6b87, 0xfd84, 0x436b, 0xe345, 0xfb82, 0x81fc, 0x0992, 0x45f9,
864 0x5527, 0x1f6d, 0xda46, 0x6a16, 0xf6e1, 0x8fb7, 0x3619, 0xdfe3,
865 0x64ce, 0x8ac6, 0x3ae8, 0x30e3, 0xec3b, 0x4ba7, 0x02a4, 0xa694,
866 0x8e68, 0x8f0c, 0x5e30, 0x0e55, 0x6538, 0x9852, 0xea35, 0x7be2,
867 0xdabd, 0x57e6, 0x5b38, 0x0fb2, 0x2604, 0x85e7, 0x6595, 0x8de9,
868 0x49b1, 0xe9a2, 0x3758, 0xa4d9, 0x505b, 0xc9d3, 0xddc5, 0x9a43,
869 0xfd44, 0x50f5, 0x379e, 0x03b6, 0x6375, 0x692f, 0x5586, 0xc717,
870 0x94dd, 0xee06, 0xb32d, 0x0bb9, 0x0e35, 0x5f8f, 0x0ba4, 0x19a8,
871 }, .{
872 0xbeeb, 0x3e54, 0x6486, 0x5167, 0xe432, 0x57cf, 0x9cac, 0x922e,
873 0xd2f8, 0x5614, 0x2e7f, 0x19cf, 0x9a07, 0x0524, 0x168f, 0x4464,
874 0x4def, 0x83ce, 0x97b4, 0xf269, 0xda5f, 0x28c1, 0x9cc3, 0xfa7c,
875 0x25a0, 0x912d, 0x25b2, 0xd60d, 0xcd82, 0x0e03, 0x40cc, 0xc9dc,
876 0x18eb, 0xc609, 0xb06d, 0x29e0, 0xf3c7, 0x997b, 0x8ca2, 0xa750,
877 0xc9bc, 0x8f0e, 0x3916, 0xd905, 0x94f8, 0x397f, 0x98b5, 0xc61d,
878 0x05db, 0x3e7a, 0xf750, 0xe8de, 0x3225, 0x81d9, 0x612e, 0x0a7e,
879 0x2c02, 0xff5b, 0x19ca, 0xbbf5, 0x870e, 0xc9ca, 0x47bb, 0xcfcc,
880 });
881
882 try testType(@Vector(1, u32), .{
883 0x1d0d9cc4,
884 }, .{
885 0xce2d0ab6,
886 });
887 try testType(@Vector(2, u32), .{
888 0x5ab78c03, 0xd21bb513,
889 }, .{
890 0x8a6664eb, 0x79eac37d,
891 });
892 try testType(@Vector(4, u32), .{
893 0x234d576e, 0x4151cc9c, 0x39f558e4, 0xba935a32,
894 }, .{
895 0x398f2a9d, 0x4540f093, 0x9225551c, 0x3bac865b,
896 });
897 try testType(@Vector(8, u32), .{
898 0xb8336635, 0x2fc3182c, 0x27a00123, 0x71587fbe,
899 0x9cbc65d2, 0x6f4bb0e6, 0x362594ce, 0x9971df38,
900 }, .{
901 0x5727e734, 0x972b0313, 0xff25f5dc, 0x924f8e55,
902 0x04920a61, 0xa1c3b334, 0xf52df4b6, 0x5ef72ecc,
903 });
904 try testType(@Vector(16, u32), .{
905 0xfb566f9e, 0x9ad4691a, 0x5b5f9ec0, 0x5a572d2a,
906 0x8f2f226b, 0x2dfc7e33, 0x9fb07e32, 0x9d672a2e,
907 0xbedc3cee, 0x6872428d, 0xbc73a9fd, 0xd4d5f055,
908 0x69c1e9ee, 0x65038deb, 0x1449061a, 0x48412ec2,
909 }, .{
910 0x96cbe946, 0x3f24f60b, 0xaeacdc53, 0x7611a8b4,
911 0x031a67a8, 0x52a26828, 0x75646f4b, 0xb75902c3,
912 0x1f881f08, 0x834e02a4, 0x5e5b40eb, 0xc75c264d,
913 0xa8251e09, 0x28e46bbd, 0x12cb1f31, 0x9a2af615,
914 });
915 try testType(@Vector(32, u32), .{
916 0x131bbb7b, 0xa7311026, 0x9d5e59a0, 0x99b090d6,
917 0xfe969e2e, 0x04547697, 0x357d3250, 0x43be6d7a,
918 0x16ecf5c5, 0xf60febcc, 0x1d1e2602, 0x138a96d2,
919 0x9117ba72, 0x9f185b32, 0xc10e23fd, 0x3e6b7fd8,
920 0x4dc9be70, 0x2ee30047, 0xaffeab60, 0x7172d362,
921 0x6154bfcf, 0x5388dc3e, 0xd6e5a76e, 0x8b782f2d,
922 0xacbef4a2, 0x843aca71, 0x25d8ab5c, 0xe1a63a39,
923 0xc26212e5, 0x0847b84b, 0xb53541e5, 0x0c8e44db,
924 }, .{
925 0x4ad92822, 0x715b623f, 0xa5bed8a7, 0x937447a9,
926 0x7ecb38eb, 0x0a2f3dfc, 0x96f467a2, 0xec882793,
927 0x41a8707f, 0xf7310656, 0x76217b80, 0x2058e5fc,
928 0x26682154, 0x87313e31, 0x4bdc480a, 0x193572ff,
929 0x60b03c75, 0x0fe45908, 0x56c73703, 0xdb86554c,
930 0xdda2dd7d, 0x34371b27, 0xe4e6ad50, 0x422d1828,
931 0x1de3801b, 0xdce268d3, 0x20af9ec8, 0x188a591f,
932 0xf080e943, 0xc8718d14, 0x3f920382, 0x18d101b5,
933 });
934
935 try testType(@Vector(1, u64), .{
936 0x333f593bf9d08546,
937 }, .{
938 0x6918bd767e730778,
939 });
940 try testType(@Vector(2, u64), .{
941 0x4cd89a317b03d430, 0x28998f61842f63a9,
942 }, .{
943 0x6c34db64af0e217e, 0x57aa5d02cd45dceb,
944 });
945 try testType(@Vector(4, u64), .{
946 0x946cf7e7484691c9, 0xf4fc5be2a762fcbf,
947 0x71cc83bc25abaf14, 0xc69cef44c6f833a1,
948 }, .{
949 0x9f90cbd6c3ce1d4e, 0x182f65295dff4e84,
950 0x4dfe62c59fed0040, 0x18402347c1db1999,
951 });
952 try testType(@Vector(8, u64), .{
953 0x92c6281333943e2c, 0xa97750504668efb5,
954 0x234be51057c0181f, 0xefbc1f407f3df4fb,
955 0x8da6cc7c39cebb94, 0xb408f7e56feee497,
956 0x2363f1f8821592ed, 0x01716e800c0619e1,
957 }, .{
958 0xa617426684147e7e, 0x7542da7ebe093a7b,
959 0x3f21d99ac57606b7, 0x65cd36d697d22de4,
960 0xed23d6bdf176c844, 0x2d4573f100ff7b58,
961 0x4968f4d21b49f8ab, 0xf5d9a205d453e933,
962 });
963 try testType(@Vector(16, u64), .{
964 0x2f61a4ee66177b4a, 0xf13b286b279f6a93,
965 0x36b46beb63665318, 0x74294dbde0da98d2,
966 0x3aa872ba60b936eb, 0xe8f698b36e62600b,
967 0x9e8930c21a6a1a76, 0x876998b09b8eb03c,
968 0xa0244771a2ec0adb, 0xb4c72bff3d3ac1a2,
969 0xd70677210830eced, 0x6622abc1734dd72d,
970 0x157e2bb0d57d6596, 0x2aac8192fb7ef973,
971 0xc4a0ca92f34d7b13, 0x04300f8ad1845246,
972 }, .{
973 0xeaf71dcf0eb76f5d, 0x0e84b1b63dc97139,
974 0x0f64cc38d23c94a1, 0x12775cf0816349b7,
975 0xfdcf13387ba48d54, 0xf8d3c672cacd8779,
976 0xe728c1f5eb56ab1e, 0x05931a34877f7a69,
977 0x1861a763c8dafd1f, 0x4ac97573ecd5739f,
978 0x3384414c9bf77b8c, 0x32c15bbd04a5ddc4,
979 0xbfd88aee1d82ed32, 0x20e91c15b701059a,
980 0xed533d18f8657f3f, 0x1ddd7cd7f6bab957,
981 });
982
983 try testType(@Vector(1, u128), .{
984 0x5f11e16b0ca3392f907a857881455d2e,
985 }, .{
986 0xf9142d73b408fd6955922f9fc147f7d7,
987 });
988 try testType(@Vector(2, u128), .{
989 0xee0fb41fabd805923fb21b5c658e3a87,
990 0x2352e74aad6c58b3255ff0bba5aa6552,
991 }, .{
992 0x8d822f9fdd9cb9a5b43513b14419b224,
993 0x1aef2a02704379e38ead4d53d69e4cc4,
994 });
995 try testType(@Vector(4, u128), .{
996 0xc74437a4ea3bbbb193dbf0ea2f0c5281,
997 0x039e4b1640868248780db1834a0027eb,
998 0xb9e8bb34155b2b238da20331d08ff85b,
999 0x863802d34a54c2e6aa71dd0f067c4904,
1000 }, .{
1001 0x7471bae24ff7b84ab107f86ba2b7d1e7,
1002 0x8f34c449d0576e682c20bda74aa6b6c9,
1003 0x1f34c3efa167b61c48c9d5ec01a1a93f,
1004 0x71c8318fcf3ddc7be058c73a52dce9e3,
1005 });
1006 try testType(@Vector(8, u128), .{
1007 0xbf2db71463037f55ee338431f902a906,
1008 0xb7ad317626655f38ab25ae30d8a1aa67,
1009 0x7d3c5a3ffaa607b5560d69ae3fcf7863,
1010 0x009a39a8badf8b628c686dc176aa1273,
1011 0x49dba3744c91304cc7bbbdab61b6c969,
1012 0x6ec664b624f7acf79ce69d80ed7bc85c,
1013 0xe02d7a303c0f00c39010f3b815547f1c,
1014 0xb13e1ee914616f58cffe6acd33d9b5c8,
1015 }, .{
1016 0x2f2d355a071942a7384f82ba72a945b8,
1017 0x61f151b3afec8cb7664f813cecf581d1,
1018 0x5bfbf5484f3a07f0eacc4739ff48af80,
1019 0x59c0abbf8d829cf525a87d5c9c41a38a,
1020 0xdad8b18eb680f0520ca49ebfb5842e22,
1021 0xa05adcaedd9057480b3ba0413d003cec,
1022 0x8b0b4a27fc94a0e90652d19bc755b63d,
1023 0xa858bce5ad0e48c13588a4e170e8667c,
1024 });
1025
1026 try testType(@Vector(1, u256), .{
1027 0x28df37e1f57a56133ba3f5b5b2164ce24eb6c29a8973a597fd91fbee8ab4bafb,
1028 }, .{
1029 0x63f725028cab082b5b1e6cb474428c8c3655cf438f3bb05c7a87f8270198f357,
1030 });
1031 try testType(@Vector(2, u256), .{
1032 0xcc79740b85597ef411e6d7e92049dfaa2328781ea4911540a3dcb512b71c7f3c,
1033 0x51ae46d2f93cbecff1578481f6ddc633dacee94ecaf81597c752c5c5db0ae766,
1034 }, .{
1035 0x257f0107305cb71cef582a9a58612a019f335e390d7998f51f5898f245874a6e,
1036 0x0a95a17323a4d16a715720f122b752785e9877e3dd3d3f9b72cdac3d1139a81f,
1037 });
1038 try testType(@Vector(4, u256), .{
1039 0x19667a6e269342cba437a8904c7ba42a762358d32723723ae2637b01124e63c5,
1040 0x14f7d3599a7edc7bcc46874f68d4291793e6ef72bd1f3763bc5e923f54f2f781,
1041 0x1c939de0ae980b80de773a04088ba45813441336cdfdc281ee356c98d71f653b,
1042 0x39f5d755965382fe13d1b1d6690b8e3827f153f8166768c4ad8a28a963b781f2,
1043 }, .{
1044 0xbe03de37cdcb8126083b4e86cd8a9803121d31b186fd5ce555ad77ce624dd6c7,
1045 0xa0c0730f0d7f141cc959849d09730b049f00693361539f1bc4758270554a60c1,
1046 0x2664bdba8de4eaa36ecee72f6bfec5b4daa6b4e00272d8116f2cc532c29490cc,
1047 0xe47a122bd45d5e7d69722d864a6b795ddee965a0993094f8791dd309d692de8b,
1048 });
1049
1050 try testType(@Vector(1, u512), .{
1051 0x651058c1d89a8f34cfc5e66b6d25294eecfcc4a7e1e4a356eb51ee7d7b2db25378e4afee51b7d18d16e520772a60c50a02d7966f40ced1870b32c658e5821397,
1052 }, .{
1053 0xd726e265ec80cb99510ba4f480ca64e959de5c528a7f54c386ecad22eeeefa845f0fd44b1bd64258a5f868197ee2d8fed59df9c9f0b72e74051a7ff20230880e,
1054 });
1055 try testType(@Vector(2, u512), .{
1056 0x22c8183c95cca8b09fdf541e431b73e9e4a1a5a00dff12381937fab52681d09d38ea25727d7025a2be08942cfa01535759e1644792e347c7901ec94b343c6337,
1057 0x292fdf644e75927e1aea9465ae2f60fb27550cd095f1afdea2cf7855286d26fbeed1c0b9c0474b73cb6b75621f7eadaa2f94ec358179ce2aaa0766df20da1ef3,
1058 }, .{
1059 0xe1cd8c0ca244c6626d4415e10b4ac43fa69e454c529c24fec4b13e6b945684d4ea833709c16c636ca78cffa5c5bf0fe945cd714a9ad695184a6bdad31dec9e31,
1060 0x8fa3d86099e9e2789d72f8e792290356d659ab20ac0414ff94745984c6ae7d986082197bb849889f912e896670aa2c1a11bd7e66e3f650710b0f0a18a1533f90,
1061 });
1062
1063 try testType(@Vector(1, u1024), .{
1064 0x0ca1a0dfaf8bb1da714b457d23c71aef948e66c7cd45c0aa941498a796fb18502ec32f34e885d0a107d44ae81595f8b52c2f0fb38e584b7139903a0e8a823ae20d01ca0662722dd474e7efc40f32d74cc065d97d8a09d0447f1ab6107fa0a57f3f8c866ae872506627ce82f18add79cee8dc69837f4ead3ca770c4d622d7e544,
1065 }, .{
1066 0xf1e3bbe031d59351770a7a501b6e969b2c00d144f17648db3f944b69dfeb7be72e5ff933a061eba4eaa422f8ca09e5a97d0b0dd740fd4076eba8c72d7a278523f399202dc2d043c4e0eb58a2bcd4066e2146e321810b1ee4d3afdddb4f026bcc7905ce17e033a7727b4e08f33b53c63d8c9f763fc6c31d0523eb38c30d5e40bc,
1067 });
1068}
1069
1070inline fn bitNot(comptime Type: type, rhs: Type) @TypeOf(~rhs) {
1071 return ~rhs;
1072}
1073test bitNot {
1074 try testUnary(bitNot);
1075}
1076
1077inline fn clz(comptime Type: type, rhs: Type) @TypeOf(@clz(rhs)) {
1078 return @clz(rhs);
1079}
1080test clz {
1081 try testUnary(clz);
1082}
1083
1084inline fn bitAnd(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs & rhs) {
1085 return lhs & rhs;
1086}
1087test bitAnd {
1088 try testBinary(bitAnd);
1089}
1090
1091inline fn bitOr(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs | rhs) {
1092 return lhs | rhs;
1093}
1094test bitOr {
1095 try testBinary(bitOr);
1096}
1097
1098inline fn bitXor(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs ^ rhs) {
1099 return lhs ^ rhs;
1100}
1101test bitXor {
1102 try testBinary(bitXor);
1103}
test/behavior/x86_64/mem.zig created+37
......@@ -0,0 +1,37 @@
1fn access(comptime array: anytype) !void {
2 var slice: []const @typeInfo(@TypeOf(array)).array.child = undefined;
3 slice = &array;
4 inline for (0.., &array) |ct_index, *elem| {
5 var rt_index: usize = undefined;
6 rt_index = ct_index;
7 if (&(slice.ptr + ct_index)[0] != elem) return error.Unexpected;
8 if (&(slice.ptr + rt_index)[0] != elem) return error.Unexpected;
9 if (&slice.ptr[ct_index..][0] != elem) return error.Unexpected;
10 if (&slice.ptr[rt_index..][0] != elem) return error.Unexpected;
11 if (&slice.ptr[ct_index] != elem) return error.Unexpected;
12 if (&slice.ptr[rt_index] != elem) return error.Unexpected;
13 if (&slice[ct_index..].ptr[0] != elem) return error.Unexpected;
14 if (&slice[rt_index..].ptr[0] != elem) return error.Unexpected;
15 if (&slice[ct_index] != elem) return error.Unexpected;
16 if (&slice[rt_index] != elem) return error.Unexpected;
17 if (slice.ptr[ct_index] != elem.*) return error.Unexpected;
18 if (slice.ptr[rt_index] != elem.*) return error.Unexpected;
19 if (slice[ct_index] != elem.*) return error.Unexpected;
20 if (slice[rt_index] != elem.*) return error.Unexpected;
21 }
22}
23test access {
24 try access([3]u8{ 0xdb, 0xef, 0xbd });
25 try access([3]u16{ 0x340e, 0x3654, 0x88d7 });
26 try access([3]u32{ 0xd424c2c0, 0x2d6ac466, 0x5a0cfaba });
27 try access([3]u64{
28 0x9327a4f5221666a6,
29 0x5c34d3ddd84a8b12,
30 0xbae087f39f649260,
31 });
32 try access([3]u128{
33 0x601cf010065444d4d42d5536dd9b95db,
34 0xa03f592fcaa22d40af23a0c735531e3c,
35 0x5da44907b31602b95c2d93f0b582ceab,
36 });
37}
tools/lldb_pretty_printers.py+1-1
......@@ -383,7 +383,7 @@ def InstRef_SummaryProvider(value, _=None):
383383 'InternPool.Index(%d)' % value.unsigned if value.unsigned < 0x80000000 else 'instructions[%d]' % (value.unsigned - 0x80000000))
384384
385385def InstIndex_SummaryProvider(value, _=None):
386 return 'instructions[%d]' % value.unsigned
386 return 'instructions[%d]' % value.unsigned if value.unsigned < 0x80000000 else 'temps[%d]' % (value.unsigned - 0x80000000)
387387
388388class zig_DeclIndex_SynthProvider:
389389 def __init__(self, value, _=None): self.value = value
tools/update_cpu_features.zig+135-3
......@@ -902,8 +902,8 @@ const llvm_targets = [_]LlvmTarget{
902902 .features = &.{ "v8a", "exynos" },
903903 },
904904 },
905 // LLVM removed support for v2 and v3 but zig wants to support targeting old hardware
906905 .extra_features = &.{
906 // LLVM removed support for v2 and v3 but zig wants to support targeting old hardware
907907 .{
908908 .zig_name = "v2",
909909 .desc = "ARMv2 architecture",
......@@ -1043,10 +1043,22 @@ const llvm_targets = [_]LlvmTarget{
10431043 .llvm_name = "64bit-mode",
10441044 .omit = true,
10451045 },
1046 .{
1047 .llvm_name = "alderlake",
1048 .extra_deps = &.{ "smap", "smep" },
1049 },
10461050 .{
10471051 .llvm_name = "amdfam10",
10481052 .extra_deps = &.{"3dnowa"},
10491053 },
1054 .{
1055 .llvm_name = "arrowlake",
1056 .extra_deps = &.{ "smap", "smep" },
1057 },
1058 .{
1059 .llvm_name = "arrowlake-s",
1060 .extra_deps = &.{ "smap", "smep" },
1061 },
10501062 .{
10511063 .llvm_name = "athlon",
10521064 .extra_deps = &.{"3dnowa"},
......@@ -1081,16 +1093,64 @@ const llvm_targets = [_]LlvmTarget{
10811093 },
10821094 .{
10831095 .llvm_name = "barcelona",
1084 .extra_deps = &.{"3dnowa"},
1096 .extra_deps = &.{ "3dnowa", "smap", "smep" },
1097 },
1098 .{
1099 .llvm_name = "broadwell",
1100 .extra_deps = &.{ "smap", "smep" },
10851101 },
10861102 .{
10871103 .llvm_name = "c3",
10881104 .extra_deps = &.{"3dnow"},
10891105 },
1106 .{
1107 .llvm_name = "cannonlake",
1108 .extra_deps = &.{ "smap", "smep" },
1109 },
1110 .{
1111 .llvm_name = "cascadelake",
1112 .extra_deps = &.{ "smap", "smep" },
1113 },
1114 .{
1115 .llvm_name = "emeraldrapids",
1116 .extra_deps = &.{ "smap", "smep" },
1117 },
10901118 .{
10911119 .llvm_name = "geode",
10921120 .extra_deps = &.{"3dnowa"},
10931121 },
1122 .{
1123 .llvm_name = "goldmont",
1124 .extra_deps = &.{ "smap", "smep" },
1125 },
1126 .{
1127 .llvm_name = "goldmont_plus",
1128 .extra_deps = &.{ "smap", "smep" },
1129 },
1130 .{
1131 .llvm_name = "haswell",
1132 .extra_deps = &.{"smep"},
1133 },
1134 .{
1135 .llvm_name = "i386",
1136 .extra_deps = &.{"bsf_bsr_0_clobbers_result"},
1137 },
1138 .{
1139 .llvm_name = "i486",
1140 .extra_deps = &.{"bsf_bsr_0_clobbers_result"},
1141 },
1142 .{
1143 .llvm_name = "icelake_client",
1144 .extra_deps = &.{ "smap", "smep" },
1145 },
1146 .{
1147 .llvm_name = "icelake_server",
1148 .extra_deps = &.{ "smap", "smep" },
1149 },
1150 .{
1151 .llvm_name = "ivybridge",
1152 .extra_deps = &.{"smep"},
1153 },
10941154 .{
10951155 .llvm_name = "k6-2",
10961156 .extra_deps = &.{"3dnow"},
......@@ -1127,6 +1187,10 @@ const llvm_targets = [_]LlvmTarget{
11271187 .llvm_name = "lakemont",
11281188 .extra_deps = &.{"soft_float"},
11291189 },
1190 .{
1191 .llvm_name = "meteorlake",
1192 .extra_deps = &.{ "smap", "smep" },
1193 },
11301194 .{
11311195 .llvm_name = "opteron",
11321196 .extra_deps = &.{"3dnowa"},
......@@ -1135,6 +1199,38 @@ const llvm_targets = [_]LlvmTarget{
11351199 .llvm_name = "opteron-sse3",
11361200 .extra_deps = &.{"3dnowa"},
11371201 },
1202 .{
1203 .llvm_name = "raptorlake",
1204 .extra_deps = &.{ "smap", "smep" },
1205 },
1206 .{
1207 .llvm_name = "rocketlake",
1208 .extra_deps = &.{ "smap", "smep" },
1209 },
1210 .{
1211 .llvm_name = "sapphirerapids",
1212 .extra_deps = &.{ "smap", "smep" },
1213 },
1214 .{
1215 .llvm_name = "silvermont",
1216 .extra_deps = &.{"smep"},
1217 },
1218 .{
1219 .llvm_name = "skx",
1220 .extra_deps = &.{ "smap", "smep" },
1221 },
1222 .{
1223 .llvm_name = "skylake",
1224 .extra_deps = &.{ "smap", "smep" },
1225 },
1226 .{
1227 .llvm_name = "skylake_avx512",
1228 .extra_deps = &.{ "smap", "smep" },
1229 },
1230 .{
1231 .llvm_name = "tigerlake",
1232 .extra_deps = &.{ "smap", "smep" },
1233 },
11381234 .{
11391235 .llvm_name = "winchip2",
11401236 .extra_deps = &.{"3dnow"},
......@@ -1143,9 +1239,29 @@ const llvm_targets = [_]LlvmTarget{
11431239 .llvm_name = "sse4.2",
11441240 .extra_deps = &.{"crc32"},
11451241 },
1242 .{
1243 .llvm_name = "znver1",
1244 .extra_deps = &.{ "smap", "smep" },
1245 },
1246 .{
1247 .llvm_name = "znver2",
1248 .extra_deps = &.{ "smap", "smep" },
1249 },
1250 .{
1251 .llvm_name = "znver3",
1252 .extra_deps = &.{ "smap", "smep" },
1253 },
1254 .{
1255 .llvm_name = "znver4",
1256 .extra_deps = &.{ "smap", "smep" },
1257 },
1258 .{
1259 .llvm_name = "znver5",
1260 .extra_deps = &.{ "smap", "smep" },
1261 },
11461262 },
1147 // Features removed from LLVM
11481263 .extra_features = &.{
1264 // Features removed from LLVM
11491265 .{
11501266 .zig_name = "3dnow",
11511267 .desc = "Enable 3DNow! instructions",
......@@ -1171,6 +1287,22 @@ const llvm_targets = [_]LlvmTarget{
11711287 .desc = "Prefetch with Intent to Write and T1 Hint",
11721288 .deps = &.{},
11731289 },
1290 // Custom Zig features
1291 .{
1292 .zig_name = "bsf_bsr_0_clobbers_result",
1293 .desc = "BSF/BSR may clobber the lower 32-bits of the result register when the source is zero",
1294 .deps = &.{},
1295 },
1296 .{
1297 .zig_name = "smap",
1298 .desc = "Enable Supervisor Mode Access Prevention",
1299 .deps = &.{},
1300 },
1301 .{
1302 .zig_name = "smep",
1303 .desc = "Enable Supervisor Mode Execution Prevention",
1304 .deps = &.{},
1305 },
11741306 },
11751307 .omit_cpus = &.{
11761308 // LLVM defines a bunch of dumb aliases with foreach loops in X86.td.