authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-11-09 06:09:42-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-16 20:42:08-05:00
log7c713251cadf831fb2009ae8908b099f218cf42c
tree3dd3dc0c5a7990218dc382c17db0dcd870563498
parentbeadf702b8d0421cc84c47ebc9644ce07e22c306

x86_64: looped instructions

5 files changed, 1274 insertions(+), 866 deletions(-)

lib/std/crypto/chacha20.zig+1-3
......@@ -499,11 +499,9 @@ fn ChaChaNonVecImpl(comptime rounds_nb: usize) type {
499499fn ChaChaImpl(comptime rounds_nb: usize) type {
500500 switch (builtin.cpu.arch) {
501501 .x86_64 => {
502 if (builtin.zig_backend == .stage2_x86_64) return ChaChaNonVecImpl(rounds_nb);
503
504502 const has_avx2 = std.Target.x86.featureSetHas(builtin.cpu.features, .avx2);
505503 const has_avx512f = std.Target.x86.featureSetHas(builtin.cpu.features, .avx512f);
506 if (has_avx512f) return ChaChaVecImpl(rounds_nb, 4);
504 if (builtin.zig_backend != .stage2_x86_64 and has_avx512f) return ChaChaVecImpl(rounds_nb, 4);
507505 if (has_avx2) return ChaChaVecImpl(rounds_nb, 2);
508506 return ChaChaVecImpl(rounds_nb, 1);
509507 },
lib/std/crypto/tls/Client.zig+7-56
......@@ -356,14 +356,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
356356 if (ciphertext.len > cleartext_fragment_buf.len) return error.TlsRecordOverflow;
357357 const cleartext = cleartext_fragment_buf[0..ciphertext.len];
358358 const auth_tag = record_decoder.array(P.AEAD.tag_length).*;
359 const nonce = if (builtin.zig_backend == .stage2_x86_64 and
360 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
361 nonce: {
362 var nonce = pv.server_handshake_iv;
363 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
364 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ read_seq, .big);
365 break :nonce nonce;
366 } else nonce: {
359 const nonce = nonce: {
367360 const V = @Vector(P.AEAD.nonce_length, u8);
368361 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
369362 const operand: V = pad ++ @as([8]u8, @bitCast(big(read_seq)));
......@@ -400,14 +393,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
400393 const record_iv = record_decoder.array(P.record_iv_length).*;
401394 const masked_read_seq = read_seq &
402395 comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);
403 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and
404 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
405 nonce: {
406 var nonce = pv.app_cipher.server_write_IV ++ record_iv;
407 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
408 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ masked_read_seq, .big);
409 break :nonce nonce;
410 } else nonce: {
396 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
411397 const V = @Vector(P.AEAD.nonce_length, u8);
412398 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
413399 const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));
......@@ -750,14 +736,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
750736 .app_cipher = std.mem.bytesToValue(P.Tls_1_2, &key_block),
751737 } };
752738 const pv = &p.version.tls_1_2;
753 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and
754 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
755 nonce: {
756 var nonce = pv.app_cipher.client_write_IV ++ pv.app_cipher.client_salt;
757 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
758 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ write_seq, .big);
759 break :nonce nonce;
760 } else nonce: {
739 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
761740 const V = @Vector(P.AEAD.nonce_length, u8);
762741 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
763742 const operand: V = pad ++ @as([8]u8, @bitCast(big(write_seq)));
......@@ -1043,14 +1022,7 @@ fn prepareCiphertextRecord(
10431022 ciphertext_end += ciphertext_len;
10441023 const auth_tag = ciphertext_buf[ciphertext_end..][0..P.AEAD.tag_length];
10451024 ciphertext_end += auth_tag.len;
1046 const nonce = if (builtin.zig_backend == .stage2_x86_64 and
1047 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1048 nonce: {
1049 var nonce = pv.client_iv;
1050 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1051 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.write_seq, .big);
1052 break :nonce nonce;
1053 } else nonce: {
1025 const nonce = nonce: {
10541026 const V = @Vector(P.AEAD.nonce_length, u8);
10551027 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
10561028 const operand: V = pad ++ std.mem.toBytes(big(c.write_seq));
......@@ -1098,14 +1070,7 @@ fn prepareCiphertextRecord(
10981070 const ad = std.mem.toBytes(big(c.write_seq)) ++ record_header[0 .. 1 + 2] ++ int(u16, message_len);
10991071 const record_iv = ciphertext_buf[ciphertext_end..][0..P.record_iv_length];
11001072 ciphertext_end += P.record_iv_length;
1101 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and
1102 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1103 nonce: {
1104 var nonce = pv.client_write_IV ++ pv.client_salt;
1105 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1106 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.write_seq, .big);
1107 break :nonce nonce;
1108 } else nonce: {
1073 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
11091074 const V = @Vector(P.AEAD.nonce_length, u8);
11101075 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
11111076 const operand: V = pad ++ @as([8]u8, @bitCast(big(c.write_seq)));
......@@ -1374,14 +1339,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove
13741339 const ciphertext = frag[in..][0..ciphertext_len];
13751340 in += ciphertext_len;
13761341 const auth_tag = frag[in..][0..P.AEAD.tag_length].*;
1377 const nonce = if (builtin.zig_backend == .stage2_x86_64 and
1378 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1379 nonce: {
1380 var nonce = pv.server_iv;
1381 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1382 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.read_seq, .big);
1383 break :nonce nonce;
1384 } else nonce: {
1342 const nonce = nonce: {
13851343 const V = @Vector(P.AEAD.nonce_length, u8);
13861344 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
13871345 const operand: V = pad ++ std.mem.toBytes(big(c.read_seq));
......@@ -1409,14 +1367,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove
14091367 in += P.record_iv_length;
14101368 const masked_read_seq = c.read_seq &
14111369 comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);
1412 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and
1413 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1414 nonce: {
1415 var nonce = pv.server_write_IV ++ record_iv;
1416 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1417 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ masked_read_seq, .big);
1418 break :nonce nonce;
1419 } else nonce: {
1370 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
14201371 const V = @Vector(P.AEAD.nonce_length, u8);
14211372 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
14221373 const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));
lib/std/http/protocol.zig-1
......@@ -4,7 +4,6 @@ const testing = std.testing;
44const mem = std.mem;
55
66const assert = std.debug.assert;
7const use_vectors = builtin.zig_backend != .stage2_x86_64;
87
98pub const State = enum {
109 invalid,
src/arch/x86_64/CodeGen.zig+1261-803
......@@ -104,7 +104,7 @@ const Owner = union(enum) {
104104 nav_index: InternPool.Nav.Index,
105105 lazy_sym: link.File.LazySymbol,
106106
107 fn getSymbolIndex(owner: Owner, ctx: *Self) !u32 {
107 fn getSymbolIndex(owner: Owner, ctx: *CodeGen) !u32 {
108108 const pt = ctx.pt;
109109 switch (owner) {
110110 .nav_index => |nav_index| if (ctx.bin_file.cast(.elf)) |elf_file| {
......@@ -394,7 +394,7 @@ pub const MCValue = union(enum) {
394394 };
395395 }
396396
397 fn mem(mcv: MCValue, function: *Self, size: Memory.Size) !Memory {
397 fn mem(mcv: MCValue, function: *CodeGen, mod_rm: Memory.Mod.Rm) !Memory {
398398 return switch (mcv) {
399399 .none,
400400 .unreach,
......@@ -420,22 +420,28 @@ pub const MCValue = union(enum) {
420420 .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{
421421 .base = .{ .reg = .ds },
422422 .mod = .{ .rm = .{
423 .size = size,
424 .disp = small_addr,
423 .size = mod_rm.size,
424 .index = mod_rm.index,
425 .scale = mod_rm.scale,
426 .disp = small_addr + mod_rm.disp,
425427 } },
426428 } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } },
427429 .indirect => |reg_off| .{
428430 .base = .{ .reg = reg_off.reg },
429431 .mod = .{ .rm = .{
430 .size = size,
431 .disp = reg_off.off,
432 .size = mod_rm.size,
433 .index = mod_rm.index,
434 .scale = mod_rm.scale,
435 .disp = reg_off.off + mod_rm.disp,
432436 } },
433437 },
434438 .load_frame => |frame_addr| .{
435439 .base = .{ .frame = frame_addr.index },
436440 .mod = .{ .rm = .{
437 .size = size,
438 .disp = frame_addr.off,
441 .size = mod_rm.size,
442 .index = mod_rm.index,
443 .scale = mod_rm.scale,
444 .disp = frame_addr.off + mod_rm.disp,
439445 } },
440446 },
441447 .load_symbol => |sym_off| {
......@@ -443,12 +449,14 @@ pub const MCValue = union(enum) {
443449 return .{
444450 .base = .{ .reloc = sym_off.sym_index },
445451 .mod = .{ .rm = .{
446 .size = size,
447 .disp = sym_off.off,
452 .size = mod_rm.size,
453 .index = mod_rm.index,
454 .scale = mod_rm.scale,
455 .disp = sym_off.off + mod_rm.disp,
448456 } },
449457 };
450458 },
451 .air_ref => |ref| (try function.resolveInst(ref)).mem(function, size),
459 .air_ref => |ref| (try function.resolveInst(ref)).mem(function, mod_rm),
452460 };
453461 }
454462
......@@ -537,7 +545,7 @@ const InstTracking = struct {
537545 return self.short.getCondition();
538546 }
539547
540 fn spill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {
548 fn spill(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void {
541549 if (std.meta.eql(self.long, self.short)) return; // Already spilled
542550 // Allocate or reuse frame index
543551 switch (self.long) {
......@@ -586,7 +594,7 @@ const InstTracking = struct {
586594 };
587595 }
588596
589 fn trackSpill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {
597 fn trackSpill(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void {
590598 try function.freeValue(self.short);
591599 self.reuseFrame();
592600 tracking_log.debug("{} => {} (spilled)", .{ inst, self.* });
......@@ -633,7 +641,7 @@ const InstTracking = struct {
633641
634642 fn materialize(
635643 self: *InstTracking,
636 function: *Self,
644 function: *CodeGen,
637645 inst: Air.Inst.Index,
638646 target: InstTracking,
639647 ) !void {
......@@ -643,7 +651,7 @@ const InstTracking = struct {
643651
644652 fn materializeUnsafe(
645653 self: InstTracking,
646 function: *Self,
654 function: *CodeGen,
647655 inst: Air.Inst.Index,
648656 target: InstTracking,
649657 ) !void {
......@@ -675,7 +683,7 @@ const InstTracking = struct {
675683 }
676684 }
677685
678 fn die(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {
686 fn die(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void {
679687 if (self.short == .dead) return;
680688 try function.freeValue(self.short);
681689 self.short = .{ .dead = function.scope_generation };
......@@ -684,7 +692,7 @@ const InstTracking = struct {
684692
685693 fn reuse(
686694 self: *InstTracking,
687 function: *Self,
695 function: *CodeGen,
688696 new_inst: ?Air.Inst.Index,
689697 old_inst: Air.Inst.Index,
690698 ) void {
......@@ -692,7 +700,7 @@ const InstTracking = struct {
692700 tracking_log.debug("{?} => {} (reuse {})", .{ new_inst, self.*, old_inst });
693701 }
694702
695 fn liveOut(self: *InstTracking, function: *Self, inst: Air.Inst.Index) void {
703 fn liveOut(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) void {
696704 for (self.getRegs()) |reg| {
697705 if (function.register_manager.isRegFree(reg)) {
698706 tracking_log.debug("{} => {} (live-out)", .{ inst, self.* });
......@@ -789,7 +797,7 @@ const BlockData = struct {
789797 }
790798};
791799
792const Self = @This();
800const CodeGen = @This();
793801
794802pub fn generate(
795803 bin_file: *link.File,
......@@ -809,7 +817,7 @@ pub fn generate(
809817 const fn_type: Type = .fromInterned(func.ty);
810818 const mod = zcu.navFileScope(func.owner_nav).mod;
811819
812 var function: Self = .{
820 var function: CodeGen = .{
813821 .gpa = gpa,
814822 .pt = pt,
815823 .air = air,
......@@ -962,7 +970,7 @@ pub fn generateLazy(
962970 const gpa = comp.gpa;
963971 // This function is for generating global code, so we use the root module.
964972 const mod = comp.root_mod;
965 var function: Self = .{
973 var function: CodeGen = .{
966974 .gpa = gpa,
967975 .pt = pt,
968976 .air = undefined,
......@@ -1050,7 +1058,7 @@ fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Format
10501058}
10511059
10521060const FormatAirData = struct {
1053 self: *Self,
1061 self: *CodeGen,
10541062 inst: Air.Inst.Index,
10551063};
10561064fn formatAir(
......@@ -1066,12 +1074,12 @@ fn formatAir(
10661074 data.self.liveness,
10671075 );
10681076}
1069fn fmtAir(self: *Self, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) {
1077fn fmtAir(self: *CodeGen, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) {
10701078 return .{ .data = .{ .self = self, .inst = inst } };
10711079}
10721080
10731081const FormatWipMirData = struct {
1074 self: *Self,
1082 self: *CodeGen,
10751083 inst: Mir.Inst.Index,
10761084};
10771085fn formatWipMir(
......@@ -1192,12 +1200,12 @@ fn formatWipMir(
11921200 }
11931201 }
11941202}
1195fn fmtWipMir(self: *Self, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {
1203fn fmtWipMir(self: *CodeGen, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {
11961204 return .{ .data = .{ .self = self, .inst = inst } };
11971205}
11981206
11991207const FormatTrackingData = struct {
1200 self: *Self,
1208 self: *CodeGen,
12011209};
12021210fn formatTracking(
12031211 data: FormatTrackingData,
......@@ -1208,26 +1216,26 @@ fn formatTracking(
12081216 var it = data.self.inst_tracking.iterator();
12091217 while (it.next()) |entry| try writer.print("\n{} = {}", .{ entry.key_ptr.*, entry.value_ptr.* });
12101218}
1211fn fmtTracking(self: *Self) std.fmt.Formatter(formatTracking) {
1219fn fmtTracking(self: *CodeGen) std.fmt.Formatter(formatTracking) {
12121220 return .{ .data = .{ .self = self } };
12131221}
12141222
1215fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
1223fn addInst(self: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
12161224 const gpa = self.gpa;
12171225 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
12181226 const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);
12191227 self.mir_instructions.appendAssumeCapacity(inst);
1220 wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});
1228 if (inst.ops != .pseudo_dead_none) wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});
12211229 return result_index;
12221230}
12231231
1224fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {
1232fn addExtra(self: *CodeGen, extra: anytype) Allocator.Error!u32 {
12251233 const fields = std.meta.fields(@TypeOf(extra));
12261234 try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);
12271235 return self.addExtraAssumeCapacity(extra);
12281236}
12291237
1230fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1238fn addExtraAssumeCapacity(self: *CodeGen, extra: anytype) u32 {
12311239 const fields = std.meta.fields(@TypeOf(extra));
12321240 const result: u32 = @intCast(self.mir_extra.items.len);
12331241 inline for (fields) |field| {
......@@ -1241,7 +1249,7 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
12411249 return result;
12421250}
12431251
1244fn asmOps(self: *Self, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {
1252fn asmOps(self: *CodeGen, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {
12451253 return switch (ops[0]) {
12461254 .none => self.asmOpOnly(tag),
12471255 .reg => |reg0| switch (ops[1]) {
......@@ -1316,7 +1324,7 @@ fn asmOps(self: *Self, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {
13161324}
13171325
13181326/// A `cc` of `.z_and_np` clobbers `reg2`!
1319fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: Register) !void {
1327fn asmCmovccRegisterRegister(self: *CodeGen, cc: Condition, reg1: Register, reg2: Register) !void {
13201328 _ = try self.addInst(.{
13211329 .tag = switch (cc) {
13221330 else => .cmov,
......@@ -1339,7 +1347,7 @@ fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: R
13391347}
13401348
13411349/// A `cc` of `.z_and_np` is not supported by this encoding!
1342fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory) !void {
1350fn asmCmovccRegisterMemory(self: *CodeGen, cc: Condition, reg: Register, m: Memory) !void {
13431351 _ = try self.addInst(.{
13441352 .tag = switch (cc) {
13451353 else => .cmov,
......@@ -1363,7 +1371,7 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)
13631371 });
13641372}
13651373
1366fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
1374fn asmSetccRegister(self: *CodeGen, cc: Condition, reg: Register) !void {
13671375 _ = try self.addInst(.{
13681376 .tag = switch (cc) {
13691377 else => .set,
......@@ -1387,7 +1395,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
13871395 });
13881396}
13891397
1390fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
1398fn asmSetccMemory(self: *CodeGen, cc: Condition, m: Memory) !void {
13911399 const payload = try self.addExtra(Mir.Memory.encode(m));
13921400 _ = try self.addInst(.{
13931401 .tag = switch (cc) {
......@@ -1412,7 +1420,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
14121420 });
14131421}
14141422
1415fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
1423fn asmJmpReloc(self: *CodeGen, target: Mir.Inst.Index) !Mir.Inst.Index {
14161424 return self.addInst(.{
14171425 .tag = .jmp,
14181426 .ops = .inst,
......@@ -1422,7 +1430,7 @@ fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
14221430 });
14231431}
14241432
1425fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Index {
1433fn asmJccReloc(self: *CodeGen, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Index {
14261434 return self.addInst(.{
14271435 .tag = switch (cc) {
14281436 else => .j,
......@@ -1443,7 +1451,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind
14431451 });
14441452}
14451453
1446fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
1454fn asmReloc(self: *CodeGen, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
14471455 _ = try self.addInst(.{
14481456 .tag = tag[1],
14491457 .ops = .inst,
......@@ -1454,7 +1462,7 @@ fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
14541462 });
14551463}
14561464
1457fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
1465fn asmPlaceholder(self: *CodeGen) !Mir.Inst.Index {
14581466 return self.addInst(.{
14591467 .tag = .pseudo,
14601468 .ops = .pseudo_dead_none,
......@@ -1464,7 +1472,7 @@ fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
14641472
14651473const MirTagAir = enum { dbg_local };
14661474
1467fn asmAir(self: *Self, tag: MirTagAir, inst: Air.Inst.Index) !void {
1475fn asmAir(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index) !void {
14681476 _ = try self.addInst(.{
14691477 .tag = .pseudo,
14701478 .ops = switch (tag) {
......@@ -1474,7 +1482,7 @@ fn asmAir(self: *Self, tag: MirTagAir, inst: Air.Inst.Index) !void {
14741482 });
14751483}
14761484
1477fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immediate) !void {
1485fn asmAirImmediate(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index, imm: Immediate) !void {
14781486 switch (imm) {
14791487 .signed => |s| _ = try self.addInst(.{
14801488 .tag = .pseudo,
......@@ -1528,7 +1536,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed
15281536}
15291537
15301538fn asmAirRegisterImmediate(
1531 self: *Self,
1539 self: *CodeGen,
15321540 tag: MirTagAir,
15331541 inst: Air.Inst.Index,
15341542 reg: Register,
......@@ -1550,7 +1558,7 @@ fn asmAirRegisterImmediate(
15501558}
15511559
15521560fn asmAirFrameAddress(
1553 self: *Self,
1561 self: *CodeGen,
15541562 tag: MirTagAir,
15551563 inst: Air.Inst.Index,
15561564 frame_addr: bits.FrameAddr,
......@@ -1567,7 +1575,7 @@ fn asmAirFrameAddress(
15671575 });
15681576}
15691577
1570fn asmAirMemory(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !void {
1578fn asmAirMemory(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !void {
15711579 _ = try self.addInst(.{
15721580 .tag = .pseudo,
15731581 .ops = switch (tag) {
......@@ -1580,7 +1588,7 @@ fn asmAirMemory(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !v
15801588 });
15811589}
15821590
1583fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {
1591fn asmOpOnly(self: *CodeGen, tag: Mir.Inst.FixedTag) !void {
15841592 _ = try self.addInst(.{
15851593 .tag = tag[1],
15861594 .ops = .none,
......@@ -1590,7 +1598,7 @@ fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {
15901598 });
15911599}
15921600
1593fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
1601fn asmPseudo(self: *CodeGen, ops: Mir.Inst.Ops) !void {
15941602 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
15951603 std.mem.endsWith(u8, @tagName(ops), "_none"));
15961604 _ = try self.addInst(.{
......@@ -1600,7 +1608,7 @@ fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
16001608 });
16011609}
16021610
1603fn asmPseudoRegister(self: *Self, ops: Mir.Inst.Ops, reg: Register) !void {
1611fn asmPseudoRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register) !void {
16041612 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
16051613 std.mem.endsWith(u8, @tagName(ops), "_r"));
16061614 _ = try self.addInst(.{
......@@ -1610,7 +1618,7 @@ fn asmPseudoRegister(self: *Self, ops: Mir.Inst.Ops, reg: Register) !void {
16101618 });
16111619}
16121620
1613fn asmPseudoImmediate(self: *Self, ops: Mir.Inst.Ops, imm: Immediate) !void {
1621fn asmPseudoImmediate(self: *CodeGen, ops: Mir.Inst.Ops, imm: Immediate) !void {
16141622 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
16151623 std.mem.endsWith(u8, @tagName(ops), "_i_s"));
16161624 _ = try self.addInst(.{
......@@ -1620,7 +1628,7 @@ fn asmPseudoImmediate(self: *Self, ops: Mir.Inst.Ops, imm: Immediate) !void {
16201628 });
16211629}
16221630
1623fn asmPseudoRegisterRegister(self: *Self, ops: Mir.Inst.Ops, reg1: Register, reg2: Register) !void {
1631fn asmPseudoRegisterRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg1: Register, reg2: Register) !void {
16241632 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
16251633 std.mem.endsWith(u8, @tagName(ops), "_rr"));
16261634 _ = try self.addInst(.{
......@@ -1630,7 +1638,7 @@ fn asmPseudoRegisterRegister(self: *Self, ops: Mir.Inst.Ops, reg1: Register, reg
16301638 });
16311639}
16321640
1633fn asmPseudoRegisterImmediate(self: *Self, ops: Mir.Inst.Ops, reg: Register, imm: Immediate) !void {
1641fn asmPseudoRegisterImmediate(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register, imm: Immediate) !void {
16341642 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
16351643 std.mem.endsWith(u8, @tagName(ops), "_ri_s"));
16361644 _ = try self.addInst(.{
......@@ -1640,7 +1648,7 @@ fn asmPseudoRegisterImmediate(self: *Self, ops: Mir.Inst.Ops, reg: Register, imm
16401648 });
16411649}
16421650
1643fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {
1651fn asmRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register) !void {
16441652 _ = try self.addInst(.{
16451653 .tag = tag[1],
16461654 .ops = .r,
......@@ -1651,7 +1659,7 @@ fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {
16511659 });
16521660}
16531661
1654fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
1662fn asmImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
16551663 _ = try self.addInst(.{
16561664 .tag = tag[1],
16571665 .ops = switch (imm) {
......@@ -1676,7 +1684,7 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
16761684 });
16771685}
16781686
1679fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {
1687fn asmRegisterRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {
16801688 _ = try self.addInst(.{
16811689 .tag = tag[1],
16821690 .ops = .rr,
......@@ -1688,7 +1696,7 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2
16881696 });
16891697}
16901698
1691fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
1699fn asmRegisterImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
16921700 const ops: Mir.Inst.Ops, const i: u32 = switch (imm) {
16931701 .signed => |s| .{ .ri_s, @bitCast(s) },
16941702 .unsigned => |u| if (std.math.cast(u32, u)) |small|
......@@ -1709,7 +1717,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm:
17091717}
17101718
17111719fn asmRegisterRegisterRegister(
1712 self: *Self,
1720 self: *CodeGen,
17131721 tag: Mir.Inst.FixedTag,
17141722 reg1: Register,
17151723 reg2: Register,
......@@ -1728,7 +1736,7 @@ fn asmRegisterRegisterRegister(
17281736}
17291737
17301738fn asmRegisterRegisterRegisterRegister(
1731 self: *Self,
1739 self: *CodeGen,
17321740 tag: Mir.Inst.FixedTag,
17331741 reg1: Register,
17341742 reg2: Register,
......@@ -1749,7 +1757,7 @@ fn asmRegisterRegisterRegisterRegister(
17491757}
17501758
17511759fn asmRegisterRegisterRegisterImmediate(
1752 self: *Self,
1760 self: *CodeGen,
17531761 tag: Mir.Inst.FixedTag,
17541762 reg1: Register,
17551763 reg2: Register,
......@@ -1774,7 +1782,7 @@ fn asmRegisterRegisterRegisterImmediate(
17741782}
17751783
17761784fn asmRegisterRegisterImmediate(
1777 self: *Self,
1785 self: *CodeGen,
17781786 tag: Mir.Inst.FixedTag,
17791787 reg1: Register,
17801788 reg2: Register,
......@@ -1801,7 +1809,7 @@ fn asmRegisterRegisterImmediate(
18011809}
18021810
18031811fn asmRegisterRegisterMemory(
1804 self: *Self,
1812 self: *CodeGen,
18051813 tag: Mir.Inst.FixedTag,
18061814 reg1: Register,
18071815 reg2: Register,
......@@ -1820,7 +1828,7 @@ fn asmRegisterRegisterMemory(
18201828}
18211829
18221830fn asmRegisterRegisterMemoryRegister(
1823 self: *Self,
1831 self: *CodeGen,
18241832 tag: Mir.Inst.FixedTag,
18251833 reg1: Register,
18261834 reg2: Register,
......@@ -1840,7 +1848,7 @@ fn asmRegisterRegisterMemoryRegister(
18401848 });
18411849}
18421850
1843fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1851fn asmMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory) !void {
18441852 _ = try self.addInst(.{
18451853 .tag = tag[1],
18461854 .ops = .m,
......@@ -1851,7 +1859,7 @@ fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
18511859 });
18521860}
18531861
1854fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
1862fn asmRegisterMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
18551863 _ = try self.addInst(.{
18561864 .tag = tag[1],
18571865 .ops = .rm,
......@@ -1864,7 +1872,7 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo
18641872}
18651873
18661874fn asmRegisterMemoryRegister(
1867 self: *Self,
1875 self: *CodeGen,
18681876 tag: Mir.Inst.FixedTag,
18691877 reg1: Register,
18701878 m: Memory,
......@@ -1883,7 +1891,7 @@ fn asmRegisterMemoryRegister(
18831891}
18841892
18851893fn asmRegisterMemoryImmediate(
1886 self: *Self,
1894 self: *CodeGen,
18871895 tag: Mir.Inst.FixedTag,
18881896 reg: Register,
18891897 m: Memory,
......@@ -1928,7 +1936,7 @@ fn asmRegisterMemoryImmediate(
19281936}
19291937
19301938fn asmRegisterRegisterMemoryImmediate(
1931 self: *Self,
1939 self: *CodeGen,
19321940 tag: Mir.Inst.FixedTag,
19331941 reg1: Register,
19341942 reg2: Register,
......@@ -1948,7 +1956,7 @@ fn asmRegisterRegisterMemoryImmediate(
19481956 });
19491957}
19501958
1951fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
1959fn asmMemoryRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
19521960 _ = try self.addInst(.{
19531961 .tag = tag[1],
19541962 .ops = .mr,
......@@ -1960,7 +1968,7 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Regist
19601968 });
19611969}
19621970
1963fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
1971fn asmMemoryImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
19641972 const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
19651973 .signed => |s| @bitCast(s),
19661974 .unsigned => |u| @intCast(u),
......@@ -1982,7 +1990,7 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed
19821990}
19831991
19841992fn asmMemoryRegisterRegister(
1985 self: *Self,
1993 self: *CodeGen,
19861994 tag: Mir.Inst.FixedTag,
19871995 m: Memory,
19881996 reg1: Register,
......@@ -2001,7 +2009,7 @@ fn asmMemoryRegisterRegister(
20012009}
20022010
20032011fn asmMemoryRegisterImmediate(
2004 self: *Self,
2012 self: *CodeGen,
20052013 tag: Mir.Inst.FixedTag,
20062014 m: Memory,
20072015 reg: Register,
......@@ -2019,7 +2027,7 @@ fn asmMemoryRegisterImmediate(
20192027 });
20202028}
20212029
2022fn gen(self: *Self) InnerError!void {
2030fn gen(self: *CodeGen) InnerError!void {
20232031 const pt = self.pt;
20242032 const zcu = pt.zcu;
20252033 const fn_info = zcu.typeToFunc(self.fn_type).?;
......@@ -2229,7 +2237,7 @@ fn gen(self: *Self) InnerError!void {
22292237 });
22302238}
22312239
2232fn checkInvariantsAfterAirInst(self: *Self) void {
2240fn checkInvariantsAfterAirInst(self: *CodeGen) void {
22332241 assert(!self.register_manager.lockedRegsExist());
22342242
22352243 if (std.debug.runtime_safety) {
......@@ -2245,13 +2253,13 @@ fn checkInvariantsAfterAirInst(self: *Self) void {
22452253 }
22462254}
22472255
2248fn genBodyBlock(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2256fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
22492257 try self.asmPseudo(.pseudo_dbg_enter_block_none);
22502258 try self.genBody(body);
22512259 try self.asmPseudo(.pseudo_dbg_leave_block_none);
22522260}
22532261
2254fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2262fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
22552263 const pt = self.pt;
22562264 const zcu = pt.zcu;
22572265 const ip = &zcu.intern_pool;
......@@ -2461,15 +2469,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
24612469 try slot.moveTo(inst, self);
24622470 },
24632471 .assembly => try self.airAsm(inst),
2464 inline .bit_and, .bit_or, .xor => |tag| if (use_old or self.typeOfIndex(inst).abiSize(zcu) > @as(
2465 u64,
2466 if (!self.typeOfIndex(inst).isVector(zcu))
2467 8
2468 else if (!self.hasFeature(.avx2))
2469 16
2470 else
2471 32,
2472 )) try self.airBinOp(inst, tag) else {
2472 inline .bit_and, .bit_or, .xor => |tag| if (use_old) try self.airBinOp(inst, tag) else {
24732473 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
24742474 var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
24752475 try self.spillEflagsIfOccupied();
......@@ -2481,30 +2481,109 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
24812481 };
24822482 var res: [1]Temp = undefined;
24832483 try self.select(&res, &.{ &ops[0], &ops[1] }, &.{
2484 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .ymm, .mem }, .features = &.{.avx2} },
2485 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .mem, .ymm }, .commute = .{ 1, 2 }, .features = &.{.avx2} },
2486 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .ymm, .ymm }, .features = &.{.avx2} },
2487 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .xmm, .mem }, .features = &.{.avx} },
2488 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .mem, .xmm }, .commute = .{ 1, 2 }, .features = &.{.avx} },
2489 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .xmm, .xmm }, .features = &.{.avx} },
2490 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .{ .match = 0 }, .mem }, .features = &.{.sse2} },
2491 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .mem, .{ .match = 0 } }, .features = &.{.sse2} },
2492 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .{ .match = 0 }, .xmm }, .features = &.{.sse2} },
2493 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .xmm, .{ .match = 0 } }, .features = &.{.sse2} },
2494 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .{ .match = 0 }, .mem }, .features = &.{.mmx} },
2495 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .mem, .{ .match = 0 } }, .features = &.{.mmx} },
2496 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .{ .match = 0 }, .mm }, .features = &.{.mmx} },
2497 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .mm, .{ .match = 0 } }, .features = &.{.mmx} },
2498 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .{ .match = 0 }, .simm32 } },
2499 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .simm32, .{ .match = 0 } } },
2500 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .{ .match = 0 }, .gpr } },
2501 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .gpr, .{ .match = 0 } } },
2502 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .simm32 } },
2503 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .simm32, .{ .match = 0 } } },
2504 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .mem } },
2505 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .mem, .{ .match = 0 } } },
2506 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .gpr } },
2507 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .gpr, .{ .match = 0 } } },
2484 .{
2485 .required_features = &.{.avx2},
2486 .mir_tag = .{ .vp_, mir_tag },
2487 .patterns = &.{
2488 .{ .ops = &.{ .ymm, .ymm, .mem } },
2489 .{ .ops = &.{ .ymm, .mem, .ymm }, .commute = .{ 1, 2 } },
2490 .{ .ops = &.{ .ymm, .ymm, .ymm } },
2491 },
2492 },
2493 .{
2494 .required_features = &.{.avx},
2495 .mir_tag = .{ .vp_, mir_tag },
2496 .patterns = &.{
2497 .{ .ops = &.{ .xmm, .xmm, .mem } },
2498 .{ .ops = &.{ .xmm, .mem, .xmm }, .commute = .{ 1, 2 } },
2499 .{ .ops = &.{ .xmm, .xmm, .xmm } },
2500 },
2501 },
2502 .{
2503 .required_features = &.{.sse2},
2504 .mir_tag = .{ .p_, mir_tag },
2505 .patterns = &.{
2506 .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .mem } },
2507 .{ .ops = &.{ .xmm, .mem, .{ .implicit = 0 } } },
2508 .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .xmm } },
2509 .{ .ops = &.{ .xmm, .xmm, .{ .implicit = 0 } } },
2510 },
2511 },
2512 .{
2513 .required_features = &.{.mmx},
2514 .mir_tag = .{ .p_, mir_tag },
2515 .patterns = &.{
2516 .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mem } },
2517 .{ .ops = &.{ .mm, .mem, .{ .implicit = 0 } } },
2518 .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mm } },
2519 .{ .ops = &.{ .mm, .mm, .{ .implicit = 0 } } },
2520 },
2521 },
2522 .{
2523 .mir_tag = .{ ._, mir_tag },
2524 .patterns = &.{
2525 .{ .ops = &.{ .mem, .{ .implicit = 0 }, .simm32 } },
2526 .{ .ops = &.{ .mem, .simm32, .{ .implicit = 0 } } },
2527 .{ .ops = &.{ .mem, .{ .implicit = 0 }, .gpr } },
2528 .{ .ops = &.{ .mem, .gpr, .{ .implicit = 0 } } },
2529 .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .simm32 } },
2530 .{ .ops = &.{ .gpr, .simm32, .{ .implicit = 0 } } },
2531 .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .mem } },
2532 .{ .ops = &.{ .gpr, .mem, .{ .implicit = 0 } } },
2533 .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .gpr } },
2534 .{ .ops = &.{ .gpr, .gpr, .{ .implicit = 0 } } },
2535 },
2536 },
2537
2538 .{
2539 .required_features = &.{.avx2},
2540 .loop = .bitwise,
2541 .mir_tag = .{ .vp_, mir_tag },
2542 .patterns = &.{
2543 .{ .ops = &.{ .ymm_limb, .{ .explicit = 0 }, .mem_limb } },
2544 .{ .ops = &.{ .ymm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
2545 .{ .ops = &.{ .ymm_limb, .ymm_limb, .mem_limb } },
2546 },
2547 },
2548 .{
2549 .required_features = &.{.avx},
2550 .loop = .bitwise,
2551 .mir_tag = .{ .vp_, mir_tag },
2552 .patterns = &.{
2553 .{ .ops = &.{ .xmm_limb, .{ .explicit = 0 }, .mem_limb } },
2554 .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
2555 .{ .ops = &.{ .xmm_limb, .xmm_limb, .mem_limb } },
2556 },
2557 },
2558 .{
2559 .required_features = &.{.sse2},
2560 .loop = .bitwise,
2561 .mir_tag = .{ .p_, mir_tag },
2562 .patterns = &.{
2563 .{ .ops = &.{ .xmm_limb, .{ .implicit = 0 }, .mem_limb } },
2564 .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .implicit = 0 } } },
2565 },
2566 },
2567 .{
2568 .required_features = &.{.mmx},
2569 .loop = .bitwise,
2570 .mir_tag = .{ .p_, mir_tag },
2571 .patterns = &.{
2572 .{ .ops = &.{ .mm_limb, .{ .implicit = 0 }, .mem_limb } },
2573 .{ .ops = &.{ .mm_limb, .mem_limb, .{ .implicit = 0 } } },
2574 },
2575 },
2576 .{
2577 .loop = .bitwise,
2578 .mir_tag = .{ ._, mir_tag },
2579 .patterns = &.{
2580 .{ .ops = &.{ .mem_limb, .{ .implicit = 0 }, .gpr_limb } },
2581 .{ .ops = &.{ .mem_limb, .gpr_limb, .{ .implicit = 0 } } },
2582 .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .mem_limb } },
2583 .{ .ops = &.{ .gpr_limb, .mem_limb, .{ .implicit = 0 } } },
2584 .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .gpr_limb } },
2585 },
2586 },
25082587 });
25092588 if (ops[0].index != res[0].index) try ops[0].die(self);
25102589 if (ops[1].index != res[0].index) try ops[1].die(self);
......@@ -2620,12 +2699,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
26202699 while (try ops[0].toLea(self)) {}
26212700 try self.asmMemoryImmediate(
26222701 .{ ._, .cmp },
2623 try ops[0].tracking(self).short.deref().mem(self, if (!opt_repr_is_pl)
2702 try ops[0].tracking(self).short.deref().mem(self, .{ .size = if (!opt_repr_is_pl)
26242703 .byte
26252704 else if (opt_child_ty.isSlice(zcu))
26262705 .qword
26272706 else
2628 .fromSize(opt_child_abi_size)),
2707 .fromSize(opt_child_abi_size) }),
26292708 .u(0),
26302709 );
26312710 var is_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });
......@@ -2643,12 +2722,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
26432722 while (try ops[0].toLea(self)) {}
26442723 try self.asmMemoryImmediate(
26452724 .{ ._, .cmp },
2646 try ops[0].tracking(self).short.deref().mem(self, if (!opt_repr_is_pl)
2725 try ops[0].tracking(self).short.deref().mem(self, .{ .size = if (!opt_repr_is_pl)
26472726 .byte
26482727 else if (opt_child_ty.isSlice(zcu))
26492728 .qword
26502729 else
2651 .fromSize(opt_child_abi_size)),
2730 .fromSize(opt_child_abi_size) }),
26522731 .u(0),
26532732 );
26542733 var is_non_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });
......@@ -2666,7 +2745,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
26662745 while (try ops[0].toLea(self)) {}
26672746 try self.asmMemoryImmediate(
26682747 .{ ._, .cmp },
2669 try ops[0].tracking(self).short.deref().mem(self, self.memSize(eu_err_ty)),
2748 try ops[0].tracking(self).short.deref().mem(self, .{ .size = self.memSize(eu_err_ty) }),
26702749 .u(0),
26712750 );
26722751 var is_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });
......@@ -2684,7 +2763,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
26842763 while (try ops[0].toLea(self)) {}
26852764 try self.asmMemoryImmediate(
26862765 .{ ._, .cmp },
2687 try ops[0].tracking(self).short.deref().mem(self, self.memSize(eu_err_ty)),
2766 try ops[0].tracking(self).short.deref().mem(self, .{ .size = self.memSize(eu_err_ty) }),
26882767 .u(0),
26892768 );
26902769 var is_non_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });
......@@ -2950,7 +3029,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
29503029 verbose_tracking_log.debug("{}", .{self.fmtTracking()});
29513030}
29523031
2953fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
3032fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
29543033 const pt = self.pt;
29553034 const zcu = pt.zcu;
29563035 const ip = &zcu.intern_pool;
......@@ -3017,7 +3096,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
30173096 }
30183097}
30193098
3020fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void {
3099fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void {
30213100 for (value.getRegs()) |reg| try self.register_manager.getReg(reg, inst);
30223101 switch (value) {
30233102 else => {},
......@@ -3025,21 +3104,23 @@ fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void {
30253104 }
30263105}
30273106
3028fn getValueIfFree(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
3107fn getValueIfFree(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) void {
30293108 for (value.getRegs()) |reg| if (self.register_manager.isRegFree(reg))
30303109 self.register_manager.getRegAssumeFree(reg, inst);
30313110}
30323111
3033fn freeValue(self: *Self, value: MCValue) !void {
3112fn freeReg(self: *CodeGen, reg: Register) !void {
3113 self.register_manager.freeReg(reg);
3114 if (reg.class() == .x87) try self.asmRegister(.{ .f_, .free }, reg);
3115}
3116
3117fn freeValue(self: *CodeGen, value: MCValue) !void {
30343118 switch (value) {
3035 .register => |reg| {
3036 self.register_manager.freeReg(reg);
3037 if (reg.class() == .x87) try self.asmRegister(.{ .f_, .free }, reg);
3038 },
3039 .register_pair => |regs| for (regs) |reg| self.register_manager.freeReg(reg),
3040 .register_offset => |reg_off| self.register_manager.freeReg(reg_off.reg),
3119 .register => |reg| try self.freeReg(reg),
3120 .register_pair => |regs| for (regs) |reg| try self.freeReg(reg),
3121 .register_offset, .indirect => |reg_off| try self.freeReg(reg_off.reg),
30413122 .register_overflow => |reg_ov| {
3042 self.register_manager.freeReg(reg_ov.reg);
3123 try self.freeReg(reg_ov.reg);
30433124 self.eflags_inst = null;
30443125 },
30453126 .eflags => self.eflags_inst = null,
......@@ -3047,16 +3128,16 @@ fn freeValue(self: *Self, value: MCValue) !void {
30473128 }
30483129}
30493130
3050fn feed(self: *Self, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) !void {
3131fn feed(self: *CodeGen, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) !void {
30513132 if (bt.feed()) if (operand.toIndex()) |inst| try self.processDeath(inst);
30523133}
30533134
30543135/// Asserts there is already capacity to insert into top branch inst_table.
3055fn processDeath(self: *Self, inst: Air.Inst.Index) !void {
3136fn processDeath(self: *CodeGen, inst: Air.Inst.Index) !void {
30563137 try self.inst_tracking.getPtr(inst).?.die(self, inst);
30573138}
30583139
3059fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
3140fn finishAirResult(self: *CodeGen, inst: Air.Inst.Index, result: MCValue) void {
30603141 if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) {
30613142 .none, .dead, .unreach => {},
30623143 else => unreachable, // Why didn't the result die?
......@@ -3071,7 +3152,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
30713152}
30723153
30733154fn finishAir(
3074 self: *Self,
3155 self: *CodeGen,
30753156 inst: Air.Inst.Index,
30763157 result: MCValue,
30773158 operands: [Liveness.bpi - 1]Air.Inst.Ref,
......@@ -3092,7 +3173,7 @@ const FrameLayout = struct {
30923173};
30933174
30943175fn setFrameLoc(
3095 self: *Self,
3176 self: *CodeGen,
30963177 frame_index: FrameIndex,
30973178 base: Register,
30983179 offset: *i32,
......@@ -3107,7 +3188,7 @@ fn setFrameLoc(
31073188 offset.* += self.frame_allocs.items(.abi_size)[frame_i];
31083189}
31093190
3110fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayout {
3191fn computeFrameLayout(self: *CodeGen, cc: std.builtin.CallingConvention) !FrameLayout {
31113192 const frame_allocs_len = self.frame_allocs.len;
31123193 try self.frame_locs.resize(self.gpa, frame_allocs_len);
31133194 const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);
......@@ -3173,16 +3254,16 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
31733254 };
31743255}
31753256
3176fn getFrameAddrAlignment(self: *Self, frame_addr: bits.FrameAddr) InternPool.Alignment {
3257fn getFrameAddrAlignment(self: *CodeGen, frame_addr: bits.FrameAddr) InternPool.Alignment {
31773258 const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align;
31783259 return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off)));
31793260}
31803261
3181fn getFrameAddrSize(self: *Self, frame_addr: bits.FrameAddr) u32 {
3262fn getFrameAddrSize(self: *CodeGen, frame_addr: bits.FrameAddr) u32 {
31823263 return self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_size - @as(u31, @intCast(frame_addr.off));
31833264}
31843265
3185fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
3266fn allocFrameIndex(self: *CodeGen, alloc: FrameAlloc) !FrameIndex {
31863267 const frame_allocs_slice = self.frame_allocs.slice();
31873268 const frame_size = frame_allocs_slice.items(.abi_size);
31883269 const frame_align = frame_allocs_slice.items(.abi_align);
......@@ -3205,7 +3286,7 @@ fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
32053286}
32063287
32073288/// Use a pointer instruction as the basis for allocating stack memory.
3208fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
3289fn allocMemPtr(self: *CodeGen, inst: Air.Inst.Index) !FrameIndex {
32093290 const pt = self.pt;
32103291 const zcu = pt.zcu;
32113292 const ptr_ty = self.typeOfIndex(inst);
......@@ -3218,15 +3299,15 @@ fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
32183299 }));
32193300}
32203301
3221fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
3302fn allocRegOrMem(self: *CodeGen, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
32223303 return self.allocRegOrMemAdvanced(self.typeOfIndex(inst), inst, reg_ok);
32233304}
32243305
3225fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue {
3306fn allocTempRegOrMem(self: *CodeGen, elem_ty: Type, reg_ok: bool) !MCValue {
32263307 return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok);
32273308}
32283309
3229fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
3310fn allocRegOrMemAdvanced(self: *CodeGen, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
32303311 const pt = self.pt;
32313312 const zcu = pt.zcu;
32323313 const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse {
......@@ -3260,7 +3341,7 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
32603341 return .{ .load_frame = .{ .index = frame_index } };
32613342}
32623343
3263fn regClassForType(self: *Self, ty: Type) Register.Class {
3344fn regClassForType(self: *CodeGen, ty: Type) Register.Class {
32643345 const pt = self.pt;
32653346 const zcu = pt.zcu;
32663347 return switch (ty.zigTypeTag(zcu)) {
......@@ -3289,7 +3370,7 @@ fn regSetForRegClass(rc: Register.Class) RegisterManager.RegisterBitSet {
32893370 };
32903371}
32913372
3292fn regSetForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {
3373fn regSetForType(self: *CodeGen, ty: Type) RegisterManager.RegisterBitSet {
32933374 return regSetForRegClass(self.regClassForType(ty));
32943375}
32953376
......@@ -3301,14 +3382,14 @@ const State = struct {
33013382 scope_generation: u32,
33023383};
33033384
3304fn initRetroactiveState(self: *Self) State {
3385fn initRetroactiveState(self: *CodeGen) State {
33053386 var state: State = undefined;
33063387 state.inst_tracking_len = @intCast(self.inst_tracking.count());
33073388 state.scope_generation = self.scope_generation;
33083389 return state;
33093390}
33103391
3311fn saveRetroactiveState(self: *Self, state: *State) !void {
3392fn saveRetroactiveState(self: *CodeGen, state: *State) !void {
33123393 try self.spillEflagsIfOccupied();
33133394 const free_registers = self.register_manager.free_registers;
33143395 var it = free_registers.iterator(.{ .kind = .unset });
......@@ -3320,13 +3401,13 @@ fn saveRetroactiveState(self: *Self, state: *State) !void {
33203401 state.free_registers = free_registers;
33213402}
33223403
3323fn saveState(self: *Self) !State {
3404fn saveState(self: *CodeGen) !State {
33243405 var state = self.initRetroactiveState();
33253406 try self.saveRetroactiveState(&state);
33263407 return state;
33273408}
33283409
3329fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, comptime opts: struct {
3410fn restoreState(self: *CodeGen, state: State, deaths: []const Air.Inst.Index, comptime opts: struct {
33303411 emit_instructions: bool,
33313412 update_tracking: bool,
33323413 resurrect: bool,
......@@ -3414,7 +3495,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
34143495 }
34153496}
34163497
3417pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
3498pub fn spillInstruction(self: *CodeGen, reg: Register, inst: Air.Inst.Index) !void {
34183499 const tracking = self.inst_tracking.getPtr(inst) orelse return;
34193500 for (tracking.getRegs()) |tracked_reg| {
34203501 if (tracked_reg.id() == reg.id()) break;
......@@ -3423,7 +3504,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
34233504 try tracking.trackSpill(self, inst);
34243505}
34253506
3426pub fn spillEflagsIfOccupied(self: *Self) !void {
3507pub fn spillEflagsIfOccupied(self: *CodeGen) !void {
34273508 if (self.eflags_inst) |inst| {
34283509 self.eflags_inst = null;
34293510 const tracking = self.inst_tracking.getPtr(inst).?;
......@@ -3433,7 +3514,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
34333514 }
34343515}
34353516
3436pub fn spillCallerPreservedRegs(self: *Self, cc: std.builtin.CallingConvention) !void {
3517pub fn spillCallerPreservedRegs(self: *CodeGen, cc: std.builtin.CallingConvention) !void {
34373518 switch (cc) {
34383519 .x86_64_sysv => try self.spillRegisters(abi.getCallerPreservedRegs(.{ .x86_64_sysv = .{} })),
34393520 .x86_64_win => try self.spillRegisters(abi.getCallerPreservedRegs(.{ .x86_64_win = .{} })),
......@@ -3441,14 +3522,14 @@ pub fn spillCallerPreservedRegs(self: *Self, cc: std.builtin.CallingConvention)
34413522 }
34423523}
34433524
3444pub fn spillRegisters(self: *Self, comptime registers: []const Register) !void {
3525pub fn spillRegisters(self: *CodeGen, comptime registers: []const Register) !void {
34453526 inline for (registers) |reg| try self.register_manager.getKnownReg(reg, null);
34463527}
34473528
34483529/// Copies a value to a register without tracking the register. The register is not considered
34493530/// allocated. A second call to `copyToTmpRegister` may return the same register.
34503531/// This can have a side effect of spilling instructions to the stack to free up a register.
3451fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
3532fn copyToTmpRegister(self: *CodeGen, ty: Type, mcv: MCValue) !Register {
34523533 const reg = try self.register_manager.allocReg(null, self.regSetForType(ty));
34533534 try self.genSetReg(reg, ty, mcv, .{});
34543535 return reg;
......@@ -3459,7 +3540,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
34593540/// This can have a side effect of spilling instructions to the stack to free up a register.
34603541/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
34613542fn copyToRegisterWithInstTracking(
3462 self: *Self,
3543 self: *CodeGen,
34633544 reg_owner: Air.Inst.Index,
34643545 ty: Type,
34653546 mcv: MCValue,
......@@ -3469,12 +3550,12 @@ fn copyToRegisterWithInstTracking(
34693550 return MCValue{ .register = reg };
34703551}
34713552
3472fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
3553fn airAlloc(self: *CodeGen, inst: Air.Inst.Index) !void {
34733554 const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };
34743555 return self.finishAir(inst, result, .{ .none, .none, .none });
34753556}
34763557
3477fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
3558fn airRetPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
34783559 const result: MCValue = switch (self.ret_mcv.long) {
34793560 else => unreachable,
34803561 .none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },
......@@ -3490,7 +3571,7 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
34903571 return self.finishAir(inst, result, .{ .none, .none, .none });
34913572}
34923573
3493fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
3574fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void {
34943575 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
34953576 const dst_ty = self.typeOfIndex(inst);
34963577 const dst_bits = dst_ty.floatBits(self.target.*);
......@@ -3562,7 +3643,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
35623643 .{ .v_ss, .cvtsd2 },
35633644 dst_reg,
35643645 dst_reg,
3565 try src_mcv.mem(self, .qword),
3646 try src_mcv.mem(self, .{ .size = .qword }),
35663647 ) else try self.asmRegisterRegisterRegister(
35673648 .{ .v_ss, .cvtsd2 },
35683649 dst_reg,
......@@ -3574,7 +3655,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
35743655 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
35753656 .{ ._ss, .cvtsd2 },
35763657 dst_reg,
3577 try src_mcv.mem(self, .qword),
3658 try src_mcv.mem(self, .{ .size = .qword }),
35783659 ) else try self.asmRegisterRegister(
35793660 .{ ._ss, .cvtsd2 },
35803661 dst_reg,
......@@ -3589,7 +3670,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
35893670 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
35903671}
35913672
3592fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
3673fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void {
35933674 const pt = self.pt;
35943675 const zcu = pt.zcu;
35953676 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -3674,7 +3755,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
36743755 .{ .v_sd, .cvtss2 },
36753756 dst_alias,
36763757 dst_alias,
3677 try src_mcv.mem(self, self.memSize(src_ty)),
3758 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
36783759 ) else try self.asmRegisterRegisterRegister(
36793760 .{ .v_sd, .cvtss2 },
36803761 dst_alias,
......@@ -3687,7 +3768,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
36873768 2...4 => if (src_mcv.isMemory()) try self.asmRegisterMemory(
36883769 .{ .v_pd, .cvtps2 },
36893770 dst_alias,
3690 try src_mcv.mem(self, self.memSize(src_ty)),
3771 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
36913772 ) else try self.asmRegisterRegister(
36923773 .{ .v_pd, .cvtps2 },
36933774 dst_alias,
......@@ -3704,7 +3785,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
37043785 else => break :result null,
37053786 },
37063787 dst_alias,
3707 try src_mcv.mem(self, self.memSize(src_ty)),
3788 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
37083789 ) else try self.asmRegisterRegister(
37093790 switch (vec_len) {
37103791 1 => .{ ._sd, .cvtss2 },
......@@ -3725,7 +3806,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
37253806 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
37263807}
37273808
3728fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3809fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void {
37293810 const pt = self.pt;
37303811 const zcu = pt.zcu;
37313812 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -3841,7 +3922,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
38413922 if (src_mcv.isMemory()) try self.asmRegisterMemory(
38423923 mir_tag,
38433924 dst_alias,
3844 try src_mcv.mem(self, self.memSize(src_ty)),
3925 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
38453926 ) else try self.asmRegisterRegister(
38463927 mir_tag,
38473928 dst_alias,
......@@ -3984,7 +4065,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
39844065 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
39854066}
39864067
3987fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
4068fn airTrunc(self: *CodeGen, inst: Air.Inst.Index) !void {
39884069 const pt = self.pt;
39894070 const zcu = pt.zcu;
39904071 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -4080,7 +4161,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
40804161 .{ .vp_, .@"and" },
40814162 dst_alias,
40824163 dst_alias,
4083 try splat_addr_mcv.deref().mem(self, .fromSize(splat_abi_size)),
4164 try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }),
40844165 );
40854166 if (src_abi_size > 16) {
40864167 const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
......@@ -4104,7 +4185,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
41044185 try self.asmRegisterMemory(
41054186 .{ .p_, .@"and" },
41064187 dst_alias,
4107 try splat_addr_mcv.deref().mem(self, .fromSize(splat_abi_size)),
4188 try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }),
41084189 );
41094190 try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
41104191 }
......@@ -4130,7 +4211,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
41304211 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
41314212}
41324213
4133fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {
4214fn airIntFromBool(self: *CodeGen, inst: Air.Inst.Index) !void {
41344215 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
41354216 const ty = self.typeOfIndex(inst);
41364217
......@@ -4143,7 +4224,7 @@ fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {
41434224 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
41444225}
41454226
4146fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
4227fn airSlice(self: *CodeGen, inst: Air.Inst.Index) !void {
41474228 const zcu = self.pt.zcu;
41484229 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
41494230 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
......@@ -4167,13 +4248,13 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
41674248 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
41684249}
41694250
4170fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
4251fn airUnOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
41714252 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
41724253 const dst_mcv = try self.genUnOp(inst, tag, ty_op.operand);
41734254 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
41744255}
41754256
4176fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
4257fn airBinOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
41774258 const pt = self.pt;
41784259 const zcu = pt.zcu;
41794260 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -4208,14 +4289,14 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
42084289 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
42094290}
42104291
4211fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
4292fn airPtrArithmetic(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
42124293 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
42134294 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
42144295 const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
42154296 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
42164297}
42174298
4218fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 {
4299fn activeIntBits(self: *CodeGen, dst_air: Air.Inst.Ref) u16 {
42194300 const pt = self.pt;
42204301 const zcu = pt.zcu;
42214302 const air_tag = self.air.instructions.items(.tag);
......@@ -4250,7 +4331,7 @@ fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 {
42504331 return dst_info.bits;
42514332}
42524333
4253fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
4334fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
42544335 const pt = self.pt;
42554336 const zcu = pt.zcu;
42564337 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -4314,7 +4395,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
43144395 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
43154396 .{ ._, .mov },
43164397 tmp_reg,
4317 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
4398 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
43184399 ) else try self.asmRegisterRegister(
43194400 .{ ._, .mov },
43204401 tmp_reg,
......@@ -4338,7 +4419,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
43384419 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
43394420 .{ ._, .xor },
43404421 tmp_reg,
4341 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
4422 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
43424423 ) else try self.asmRegisterRegister(
43434424 .{ ._, .xor },
43444425 tmp_reg,
......@@ -4441,12 +4522,12 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
44414522 try self.asmRegisterMemory(
44424523 .{ ._, .add },
44434524 tmp_regs[0],
4444 try mat_rhs_mcv.mem(self, .qword),
4525 try mat_rhs_mcv.mem(self, .{ .size = .qword }),
44454526 );
44464527 try self.asmRegisterMemory(
44474528 .{ ._, .adc },
44484529 tmp_regs[1],
4449 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
4530 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
44504531 );
44514532 } else for (
44524533 [_]Mir.Inst.Tag{ .add, .adc },
......@@ -4476,7 +4557,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
44764557 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
44774558}
44784559
4479fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
4560fn airAddSat(self: *CodeGen, inst: Air.Inst.Index) !void {
44804561 const pt = self.pt;
44814562 const zcu = pt.zcu;
44824563 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -4577,7 +4658,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
45774658 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
45784659}
45794660
4580fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
4661fn airSubSat(self: *CodeGen, inst: Air.Inst.Index) !void {
45814662 const pt = self.pt;
45824663 const zcu = pt.zcu;
45834664 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -4671,7 +4752,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
46714752 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
46724753}
46734754
4674fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4755fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void {
46754756 const pt = self.pt;
46764757 const zcu = pt.zcu;
46774758 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -4715,7 +4796,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
47154796 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
47164797 .{ ._, .mov },
47174798 tmp_reg,
4718 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
4799 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
47194800 ) else try self.asmRegisterRegister(
47204801 .{ ._, .mov },
47214802 tmp_reg,
......@@ -4739,7 +4820,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
47394820 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
47404821 .{ ._, .xor },
47414822 tmp_reg,
4742 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
4823 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
47434824 ) else try self.asmRegisterRegister(
47444825 .{ ._, .xor },
47454826 tmp_reg,
......@@ -4748,7 +4829,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
47484829
47494830 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
47504831 try self.asmRegister(.{ ._, .not }, tmp_reg);
4751 try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), .s(0));
4832 try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .{ .size = .dword }), .s(0));
47524833 try self.freeValue(overflow);
47534834 try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
47544835 try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, .u(63));
......@@ -4818,7 +4899,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
48184899 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
48194900}
48204901
4821fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4902fn airAddSubWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
48224903 const pt = self.pt;
48234904 const zcu = pt.zcu;
48244905 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -4883,7 +4964,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
48834964 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
48844965}
48854966
4886fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4967fn airShlWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
48874968 const pt = self.pt;
48884969 const zcu = pt.zcu;
48894970 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -4961,7 +5042,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
49615042}
49625043
49635044fn genSetFrameTruncatedOverflowCompare(
4964 self: *Self,
5045 self: *CodeGen,
49655046 tuple_ty: Type,
49665047 frame_index: FrameIndex,
49675048 src_mcv: MCValue,
......@@ -5039,7 +5120,7 @@ fn genSetFrameTruncatedOverflowCompare(
50395120 );
50405121}
50415122
5042fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
5123fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
50435124 const pt = self.pt;
50445125 const zcu = pt.zcu;
50455126 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -5214,7 +5295,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
52145295 );
52155296 try self.asmMemoryImmediate(
52165297 .{ ._, .cmp },
5217 try overflow.mem(self, self.memSize(Type.c_int)),
5298 try overflow.mem(self, .{ .size = self.memSize(Type.c_int) }),
52185299 .s(0),
52195300 );
52205301 try self.genSetMem(
......@@ -5270,7 +5351,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
52705351 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
52715352 .{ ._, .mov },
52725353 .rax,
5273 try mat_lhs_mcv.mem(self, .qword),
5354 try mat_lhs_mcv.mem(self, .{ .size = .qword }),
52745355 ) else try self.asmRegisterRegister(
52755356 .{ ._, .mov },
52765357 .rax,
......@@ -5279,7 +5360,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
52795360 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
52805361 .{ ._, .mov },
52815362 tmp_regs[0],
5282 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
5363 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
52835364 ) else try self.asmRegisterRegister(
52845365 .{ ._, .mov },
52855366 tmp_regs[0],
......@@ -5290,7 +5371,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
52905371 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);
52915372 try self.asmSetccRegister(.o, tmp_regs[2].to8());
52925373 if (mat_rhs_mcv.isMemory())
5293 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
5374 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
52945375 else
52955376 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
52965377 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
......@@ -5299,7 +5380,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
52995380 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
53005381 .{ ._, .mov },
53015382 tmp_regs[0],
5302 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
5383 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
53035384 ) else try self.asmRegisterRegister(
53045385 .{ ._, .mov },
53055386 tmp_regs[0],
......@@ -5316,7 +5397,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
53165397 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
53175398 .{ .i_, .mul },
53185399 tmp_regs[0],
5319 try mat_rhs_mcv.mem(self, .qword),
5400 try mat_rhs_mcv.mem(self, .{ .size = .qword }),
53205401 ) else try self.asmRegisterRegister(
53215402 .{ .i_, .mul },
53225403 tmp_regs[0],
......@@ -5416,7 +5497,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
54165497/// Generates signed or unsigned integer multiplication/division.
54175498/// Clobbers .rax and .rdx registers.
54185499/// Quotient is saved in .rax and remainder in .rdx.
5419fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
5500fn genIntMulDivOpMir(self: *CodeGen, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
54205501 const pt = self.pt;
54215502 const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));
54225503 const bit_size: u32 = @intCast(self.regBitSize(ty));
......@@ -5455,7 +5536,10 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
54555536 };
54565537 switch (mat_rhs) {
54575538 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),
5458 .memory, .indirect, .load_frame => try self.asmMemory(tag, try mat_rhs.mem(self, .fromSize(abi_size))),
5539 .memory, .indirect, .load_frame => try self.asmMemory(
5540 tag,
5541 try mat_rhs.mem(self, .{ .size = .fromSize(abi_size) }),
5542 ),
54595543 else => unreachable,
54605544 }
54615545 if (tag[1] == .div and bit_size == 8) try self.asmRegisterRegister(.{ ._, .mov }, .dl, .ah);
......@@ -5463,7 +5547,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
54635547
54645548/// Always returns a register.
54655549/// Clobbers .rax and .rdx registers.
5466fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
5550fn genInlineIntDivFloor(self: *CodeGen, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
54675551 const pt = self.pt;
54685552 const zcu = pt.zcu;
54695553 const abi_size: u32 = @intCast(ty.abiSize(zcu));
......@@ -5516,7 +5600,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
55165600 return MCValue{ .register = divisor };
55175601}
55185602
5519fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
5603fn airShlShrBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
55205604 const pt = self.pt;
55215605 const zcu = pt.zcu;
55225606 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -5784,14 +5868,14 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
57845868 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
57855869}
57865870
5787fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
5871fn airShlSat(self: *CodeGen, inst: Air.Inst.Index) !void {
57885872 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
57895873 _ = bin_op;
57905874 return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
57915875 //return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
57925876}
57935877
5794fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
5878fn airOptionalPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
57955879 const zcu = self.pt.zcu;
57965880 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
57975881 const result: MCValue = result: {
......@@ -5824,7 +5908,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
58245908 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
58255909}
58265910
5827fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
5911fn airOptionalPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
58285912 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
58295913
58305914 const dst_ty = self.typeOfIndex(inst);
......@@ -5837,7 +5921,7 @@ fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
58375921 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
58385922}
58395923
5840fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5924fn airOptionalPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void {
58415925 const pt = self.pt;
58425926 const zcu = pt.zcu;
58435927 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -5878,7 +5962,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
58785962 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
58795963}
58805964
5881fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
5965fn airUnwrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void {
58825966 const pt = self.pt;
58835967 const zcu = pt.zcu;
58845968 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -5923,7 +6007,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
59236007 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
59246008}
59256009
5926fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
6010fn airUnwrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
59276011 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
59286012 const operand_ty = self.typeOf(ty_op.operand);
59296013 const operand = try self.resolveInst(ty_op.operand);
......@@ -5932,7 +6016,7 @@ fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
59326016}
59336017
59346018// *(E!T) -> E
5935fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6019fn airUnwrapErrUnionErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
59366020 const pt = self.pt;
59376021 const zcu = pt.zcu;
59386022 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -5972,7 +6056,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
59726056}
59736057
59746058// *(E!T) -> *T
5975fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
6059fn airUnwrapErrUnionPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
59766060 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
59776061 const operand_ty = self.typeOf(ty_op.operand);
59786062 const operand = try self.resolveInst(ty_op.operand);
......@@ -5980,7 +6064,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
59806064 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
59816065}
59826066
5983fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
6067fn airErrUnionPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void {
59846068 const pt = self.pt;
59856069 const zcu = pt.zcu;
59866070 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -6037,7 +6121,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
60376121}
60386122
60396123fn genUnwrapErrUnionPayloadMir(
6040 self: *Self,
6124 self: *CodeGen,
60416125 maybe_inst: ?Air.Inst.Index,
60426126 err_union_ty: Type,
60436127 err_union: MCValue,
......@@ -6087,7 +6171,7 @@ fn genUnwrapErrUnionPayloadMir(
60876171}
60886172
60896173fn genUnwrapErrUnionPayloadPtrMir(
6090 self: *Self,
6174 self: *CodeGen,
60916175 maybe_inst: ?Air.Inst.Index,
60926176 ptr_ty: Type,
60936177 ptr_mcv: MCValue,
......@@ -6110,23 +6194,23 @@ fn genUnwrapErrUnionPayloadPtrMir(
61106194 return result;
61116195}
61126196
6113fn airErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {
6197fn airErrReturnTrace(self: *CodeGen, inst: Air.Inst.Index) !void {
61146198 _ = inst;
61156199 return self.fail("TODO implement airErrReturnTrace for {}", .{self.target.cpu.arch});
61166200 //return self.finishAir(inst, result, .{ .none, .none, .none });
61176201}
61186202
6119fn airSetErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {
6203fn airSetErrReturnTrace(self: *CodeGen, inst: Air.Inst.Index) !void {
61206204 _ = inst;
61216205 return self.fail("TODO implement airSetErrReturnTrace for {}", .{self.target.cpu.arch});
61226206}
61236207
6124fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) !void {
6208fn airSaveErrReturnTraceIndex(self: *CodeGen, inst: Air.Inst.Index) !void {
61256209 _ = inst;
61266210 return self.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{self.target.cpu.arch});
61276211}
61286212
6129fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
6213fn airWrapOptional(self: *CodeGen, inst: Air.Inst.Index) !void {
61306214 const pt = self.pt;
61316215 const zcu = pt.zcu;
61326216 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -6181,7 +6265,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
61816265}
61826266
61836267/// T to E!T
6184fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
6268fn airWrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
61856269 const pt = self.pt;
61866270 const zcu = pt.zcu;
61876271 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -6205,7 +6289,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
62056289}
62066290
62076291/// E to E!T
6208fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
6292fn airWrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void {
62096293 const pt = self.pt;
62106294 const zcu = pt.zcu;
62116295 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -6228,7 +6312,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
62286312 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
62296313}
62306314
6231fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
6315fn airSlicePtr(self: *CodeGen, inst: Air.Inst.Index) !void {
62326316 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
62336317 const result = result: {
62346318 const src_mcv = try self.resolveInst(ty_op.operand);
......@@ -6251,7 +6335,7 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
62516335 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
62526336}
62536337
6254fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
6338fn airSliceLen(self: *CodeGen, inst: Air.Inst.Index) !void {
62556339 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
62566340 const result = result: {
62576341 const src_mcv = try self.resolveInst(ty_op.operand);
......@@ -6279,7 +6363,7 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
62796363 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
62806364}
62816365
6282fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
6366fn airPtrSliceLenPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
62836367 const pt = self.pt;
62846368 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
62856369
......@@ -6314,7 +6398,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
63146398 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
63156399}
63166400
6317fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
6401fn airPtrSlicePtrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
63186402 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
63196403
63206404 const dst_ty = self.typeOfIndex(inst);
......@@ -6327,7 +6411,7 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
63276411 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
63286412}
63296413
6330fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Register {
6414fn elemOffset(self: *CodeGen, index_ty: Type, index: MCValue, elem_size: u64) !Register {
63316415 const reg: Register = blk: {
63326416 switch (index) {
63336417 .immediate => |imm| {
......@@ -6347,7 +6431,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
63476431 return reg;
63486432}
63496433
6350fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
6434fn genSliceElemPtr(self: *CodeGen, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
63516435 const pt = self.pt;
63526436 const zcu = pt.zcu;
63536437 const slice_ty = self.typeOf(lhs);
......@@ -6384,7 +6468,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
63846468 return MCValue{ .register = addr_reg.to64() };
63856469}
63866470
6387fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
6471fn airSliceElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
63886472 const pt = self.pt;
63896473 const zcu = pt.zcu;
63906474 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -6403,14 +6487,14 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
64036487 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
64046488}
64056489
6406fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
6490fn airSliceElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
64076491 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
64086492 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
64096493 const dst_mcv = try self.genSliceElemPtr(extra.lhs, extra.rhs);
64106494 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
64116495}
64126496
6413fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
6497fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
64146498 const pt = self.pt;
64156499 const zcu = pt.zcu;
64166500 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -6463,7 +6547,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
64636547 },
64646548 .load_frame => try self.asmMemoryRegister(
64656549 .{ ._, .bt },
6466 try array_mcv.mem(self, .qword),
6550 try array_mcv.mem(self, .{ .size = .qword }),
64676551 index_reg.to64(),
64686552 ),
64696553 .memory, .load_symbol, .load_direct, .load_got, .load_tlv => try self.asmMemoryRegister(
......@@ -6540,7 +6624,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
65406624 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
65416625}
65426626
6543fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
6627fn airPtrElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
65446628 const pt = self.pt;
65456629 const zcu = pt.zcu;
65466630 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -6591,7 +6675,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
65916675 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
65926676}
65936677
6594fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
6678fn airPtrElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
65956679 const pt = self.pt;
65966680 const zcu = pt.zcu;
65976681 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -6637,7 +6721,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
66376721 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
66386722}
66396723
6640fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
6724fn airSetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void {
66416725 const pt = self.pt;
66426726 const zcu = pt.zcu;
66436727 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -6682,7 +6766,7 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
66826766 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
66836767}
66846768
6685fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
6769fn airGetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void {
66866770 const zcu = self.pt.zcu;
66876771 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
66886772
......@@ -6739,7 +6823,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
67396823 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
67406824}
67416825
6742fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6826fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
67436827 const pt = self.pt;
67446828 const zcu = pt.zcu;
67456829 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -6936,7 +7020,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
69367020 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
69377021}
69387022
6939fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
7023fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void {
69407024 const pt = self.pt;
69417025 const zcu = pt.zcu;
69427026 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -7098,7 +7182,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
70987182 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
70997183}
71007184
7101fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
7185fn airPopCount(self: *CodeGen, inst: Air.Inst.Index) !void {
71027186 const pt = self.pt;
71037187 const zcu = pt.zcu;
71047188 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -7157,7 +7241,7 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
71577241}
71587242
71597243fn genPopCount(
7160 self: *Self,
7244 self: *CodeGen,
71617245 dst_reg: Register,
71627246 src_ty: Type,
71637247 src_mcv: MCValue,
......@@ -7249,7 +7333,7 @@ fn genPopCount(
72497333}
72507334
72517335fn genByteSwap(
7252 self: *Self,
7336 self: *CodeGen,
72537337 inst: Air.Inst.Index,
72547338 src_ty: Type,
72557339 src_mcv: MCValue,
......@@ -7308,7 +7392,7 @@ fn genByteSwap(
73087392 try self.asmRegisterMemory(
73097393 .{ ._, if (has_movbe) .movbe else .mov },
73107394 dst_reg.to64(),
7311 try src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .qword),
7395 try src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .{ .size = .qword }),
73127396 );
73137397 if (!has_movbe) try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
73147398 } else {
......@@ -7414,7 +7498,7 @@ fn genByteSwap(
74147498 return dst_mcv;
74157499}
74167500
7417fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
7501fn airByteSwap(self: *CodeGen, inst: Air.Inst.Index) !void {
74187502 const pt = self.pt;
74197503 const zcu = pt.zcu;
74207504 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -7437,7 +7521,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
74377521 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
74387522}
74397523
7440fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7524fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void {
74417525 const pt = self.pt;
74427526 const zcu = pt.zcu;
74437527 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -7560,7 +7644,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
75607644 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
75617645}
75627646
7563fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) !void {
7647fn floatSign(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) !void {
75647648 const pt = self.pt;
75657649 const zcu = pt.zcu;
75667650 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
......@@ -7623,7 +7707,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
76237707 else => unreachable,
76247708 });
76257709 const sign_mem: Memory = if (sign_mcv.isMemory())
7626 try sign_mcv.mem(self, .fromSize(abi_size))
7710 try sign_mcv.mem(self, .{ .size = .fromSize(abi_size) })
76277711 else
76287712 .{
76297713 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
......@@ -7688,7 +7772,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
76887772 return self.finishAir(inst, result, .{ operand, .none, .none });
76897773}
76907774
7691fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
7775fn airFloatSign(self: *CodeGen, inst: Air.Inst.Index) !void {
76927776 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
76937777 const ty = self.typeOf(un_op);
76947778 return self.floatSign(inst, un_op, ty);
......@@ -7713,7 +7797,7 @@ const RoundMode = packed struct(u5) {
77137797 } = .normal,
77147798};
77157799
7716fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void {
7800fn airRound(self: *CodeGen, inst: Air.Inst.Index, mode: RoundMode) !void {
77177801 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
77187802 const ty = self.typeOf(un_op);
77197803
......@@ -7737,7 +7821,7 @@ fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void {
77377821 return self.finishAir(inst, result, .{ un_op, .none, .none });
77387822}
77397823
7740fn getRoundTag(self: *Self, ty: Type) ?Mir.Inst.FixedTag {
7824fn getRoundTag(self: *CodeGen, ty: Type) ?Mir.Inst.FixedTag {
77417825 const pt = self.pt;
77427826 const zcu = pt.zcu;
77437827 return if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(zcu)) {
......@@ -7770,7 +7854,7 @@ fn getRoundTag(self: *Self, ty: Type) ?Mir.Inst.FixedTag {
77707854 } else null;
77717855}
77727856
7773fn genRoundLibcall(self: *Self, ty: Type, src_mcv: MCValue, mode: RoundMode) !MCValue {
7857fn genRoundLibcall(self: *CodeGen, ty: Type, src_mcv: MCValue, mode: RoundMode) !MCValue {
77747858 const pt = self.pt;
77757859 const zcu = pt.zcu;
77767860 if (self.getRoundTag(ty)) |_| return .none;
......@@ -7795,7 +7879,7 @@ fn genRoundLibcall(self: *Self, ty: Type, src_mcv: MCValue, mode: RoundMode) !MC
77957879 } }, &.{ty}, &.{src_mcv});
77967880}
77977881
7798fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: RoundMode) !void {
7882fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: RoundMode) !void {
77997883 const pt = self.pt;
78007884 const mir_tag = self.getRoundTag(ty) orelse {
78017885 const result = try self.genRoundLibcall(ty, src_mcv, mode);
......@@ -7808,7 +7892,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
78087892 mir_tag,
78097893 dst_alias,
78107894 dst_alias,
7811 try src_mcv.mem(self, .fromSize(abi_size)),
7895 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
78127896 .u(@as(u5, @bitCast(mode))),
78137897 ) else try self.asmRegisterRegisterRegisterImmediate(
78147898 mir_tag,
......@@ -7823,7 +7907,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
78237907 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
78247908 mir_tag,
78257909 dst_alias,
7826 try src_mcv.mem(self, .fromSize(abi_size)),
7910 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
78277911 .u(@as(u5, @bitCast(mode))),
78287912 ) else try self.asmRegisterRegisterImmediate(
78297913 mir_tag,
......@@ -7837,7 +7921,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
78377921 }
78387922}
78397923
7840fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7924fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void {
78417925 const pt = self.pt;
78427926 const zcu = pt.zcu;
78437927 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -7865,7 +7949,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
78657949 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
78667950 .l,
78677951 registerAlias(dst_mcv.register, cmov_abi_size),
7868 try src_mcv.mem(self, .fromSize(cmov_abi_size)),
7952 try src_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }),
78697953 ),
78707954 else => {
78717955 const val_reg = try self.copyToTmpRegister(ty, src_mcv);
......@@ -7931,7 +8015,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
79318015
79328016 try self.asmMemoryImmediate(
79338017 .{ ._, .cmp },
7934 try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .qword),
8018 try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .{ .size = .qword }),
79358019 .u(0),
79368020 );
79378021 const positive = try self.asmJccReloc(.ns, undefined);
......@@ -8024,7 +8108,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
80248108 if (src_mcv.isMemory()) try self.asmRegisterMemory(
80258109 mir_tag,
80268110 dst_alias,
8027 try src_mcv.mem(self, self.memSize(ty)),
8111 try src_mcv.mem(self, .{ .size = self.memSize(ty) }),
80288112 ) else try self.asmRegisterRegister(
80298113 mir_tag,
80308114 dst_alias,
......@@ -8038,7 +8122,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
80388122 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
80398123}
80408124
8041fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
8125fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void {
80428126 const pt = self.pt;
80438127 const zcu = pt.zcu;
80448128 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
......@@ -8131,9 +8215,9 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
81318215 if (src_mcv.isMemory()) try self.asmRegisterMemory(
81328216 .{ .v_ps, .cvtph2 },
81338217 wide_reg,
8134 try src_mcv.mem(self, .fromSize(
8218 try src_mcv.mem(self, .{ .size = .fromSize(
81358219 @intCast(@divExact(wide_reg.bitSize(), 16)),
8136 )),
8220 ) }),
81378221 ) else try self.asmRegisterRegister(
81388222 .{ .v_ps, .cvtph2 },
81398223 wide_reg,
......@@ -8177,7 +8261,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
81778261 mir_tag,
81788262 dst_reg,
81798263 dst_reg,
8180 try src_mcv.mem(self, .fromSize(abi_size)),
8264 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
81818265 ) else try self.asmRegisterRegisterRegister(
81828266 mir_tag,
81838267 dst_reg,
......@@ -8190,7 +8274,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
81908274 else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
81918275 mir_tag,
81928276 dst_reg,
8193 try src_mcv.mem(self, .fromSize(abi_size)),
8277 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
81948278 ) else try self.asmRegisterRegister(
81958279 mir_tag,
81968280 dst_reg,
......@@ -8205,7 +8289,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
82058289 return self.finishAir(inst, result, .{ un_op, .none, .none });
82068290}
82078291
8208fn airUnaryMath(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
8292fn airUnaryMath(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
82098293 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
82108294 const ty = self.typeOf(un_op);
82118295 var callee_buf: ["__round?".len]u8 = undefined;
......@@ -8234,7 +8318,7 @@ fn airUnaryMath(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
82348318}
82358319
82368320fn reuseOperand(
8237 self: *Self,
8321 self: *CodeGen,
82388322 inst: Air.Inst.Index,
82398323 operand: Air.Inst.Ref,
82408324 op_index: Liveness.OperandInt,
......@@ -8244,7 +8328,7 @@ fn reuseOperand(
82448328}
82458329
82468330fn reuseOperandAdvanced(
8247 self: *Self,
8331 self: *CodeGen,
82488332 inst: Air.Inst.Index,
82498333 operand: Air.Inst.Ref,
82508334 op_index: Liveness.OperandInt,
......@@ -8282,7 +8366,7 @@ fn reuseOperandAdvanced(
82828366 return true;
82838367}
82848368
8285fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
8369fn packedLoad(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
82868370 const pt = self.pt;
82878371 const zcu = pt.zcu;
82888372
......@@ -8390,7 +8474,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
83908474 try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg }, .{});
83918475}
83928476
8393fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
8477fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
83948478 const pt = self.pt;
83958479 const zcu = pt.zcu;
83968480 const dst_ty = ptr_ty.childType(zcu);
......@@ -8433,7 +8517,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro
84338517 }
84348518}
84358519
8436fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
8520fn airLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
84378521 const pt = self.pt;
84388522 const zcu = pt.zcu;
84398523 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -8492,7 +8576,7 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
84928576 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
84938577}
84948578
8495fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
8579fn packedStore(self: *CodeGen, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
84968580 const pt = self.pt;
84978581 const zcu = pt.zcu;
84988582 const ptr_info = ptr_ty.ptrInfo(zcu);
......@@ -8593,7 +8677,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
85938677}
85948678
85958679fn store(
8596 self: *Self,
8680 self: *CodeGen,
85978681 ptr_ty: Type,
85988682 ptr_mcv: MCValue,
85998683 src_mcv: MCValue,
......@@ -8641,7 +8725,7 @@ fn store(
86418725 }
86428726}
86438727
8644fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8728fn airStore(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
86458729 const pt = self.pt;
86468730 const zcu = pt.zcu;
86478731 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -8667,20 +8751,20 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
86678751 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
86688752}
86698753
8670fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
8754fn airStructFieldPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
86718755 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
86728756 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
86738757 const result = try self.fieldPtr(inst, extra.struct_operand, extra.field_index);
86748758 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
86758759}
86768760
8677fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, field_index: u8) !void {
8761fn airStructFieldPtrIndex(self: *CodeGen, inst: Air.Inst.Index, field_index: u8) !void {
86788762 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
86798763 const result = try self.fieldPtr(inst, ty_op.operand, field_index);
86808764 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
86818765}
86828766
8683fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_index: u32) !MCValue {
8767fn fieldPtr(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_index: u32) !MCValue {
86848768 const ptr_field_ty = self.typeOfIndex(inst);
86858769
86868770 const src_mcv = try self.resolveInst(operand);
......@@ -8692,7 +8776,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_inde
86928776 return dst_mcv.offset(self.fieldOffset(self.typeOf(operand), ptr_field_ty, field_index));
86938777}
86948778
8695fn fieldOffset(self: *Self, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u32) i32 {
8779fn fieldOffset(self: *CodeGen, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u32) i32 {
86968780 const pt = self.pt;
86978781 const zcu = pt.zcu;
86988782 const agg_ty = ptr_agg_ty.childType(zcu);
......@@ -8704,7 +8788,7 @@ fn fieldOffset(self: *Self, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u
87048788 };
87058789}
87068790
8707fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8791fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void {
87088792 const pt = self.pt;
87098793 const zcu = pt.zcu;
87108794 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -8993,7 +9077,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
89939077 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
89949078}
89959079
8996fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
9080fn airFieldParentPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
89979081 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
89989082 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
89999083
......@@ -9008,7 +9092,7 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
90089092 return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });
90099093}
90109094
9011fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
9095fn genUnOp(self: *CodeGen, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
90129096 const pt = self.pt;
90139097 const zcu = pt.zcu;
90149098 const src_ty = self.typeOf(src_air);
......@@ -9097,7 +9181,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
90979181 return dst_mcv;
90989182}
90999183
9100fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
9184fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
91019185 const pt = self.pt;
91029186 const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));
91039187 if (abi_size > 8) return self.fail("TODO implement {} for {}", .{ mir_tag, dst_ty.fmt(pt) });
......@@ -9133,14 +9217,14 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
91339217 },
91349218 .indirect, .load_frame => try self.asmMemory(
91359219 mir_tag,
9136 try dst_mcv.mem(self, .fromSize(abi_size)),
9220 try dst_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
91379221 ),
91389222 }
91399223}
91409224
91419225/// Clobbers .rcx for non-immediate shift value.
91429226fn genShiftBinOpMir(
9143 self: *Self,
9227 self: *CodeGen,
91449228 tag: Mir.Inst.FixedTag,
91459229 lhs_ty: Type,
91469230 lhs_mcv: MCValue,
......@@ -9773,7 +9857,7 @@ fn genShiftBinOpMir(
97739857/// Clobbers .rcx for non-immediate rhs, therefore care is needed to spill .rcx upfront.
97749858/// Asserts .rcx is free.
97759859fn genShiftBinOp(
9776 self: *Self,
9860 self: *CodeGen,
97779861 air_tag: Air.Inst.Tag,
97789862 maybe_inst: ?Air.Inst.Index,
97799863 lhs_mcv: MCValue,
......@@ -9832,7 +9916,7 @@ fn genShiftBinOp(
98329916/// Clobbers .rax and .rdx therefore care is needed to spill .rax and .rdx upfront.
98339917/// Asserts .rax and .rdx are free.
98349918fn genMulDivBinOp(
9835 self: *Self,
9919 self: *CodeGen,
98369920 tag: Air.Inst.Tag,
98379921 maybe_inst: ?Air.Inst.Index,
98389922 dst_ty: Type,
......@@ -9891,27 +9975,27 @@ fn genMulDivBinOp(
98919975 defer self.register_manager.unlockReg(tmp_lock);
98929976
98939977 if (mat_lhs_mcv.isMemory())
9894 try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .qword))
9978 try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .{ .size = .qword }))
98959979 else
98969980 try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);
98979981 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
98989982 .{ ._, .mov },
98999983 tmp_reg,
9900 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
9984 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
99019985 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);
99029986 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);
99039987 if (mat_rhs_mcv.isMemory())
9904 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
9988 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
99059989 else
99069990 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
99079991 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
99089992 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
99099993 .{ ._, .mov },
99109994 tmp_reg,
9911 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
9995 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
99129996 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);
99139997 if (mat_rhs_mcv.isMemory())
9914 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .qword))
9998 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
99159999 else
991610000 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);
991710001 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
......@@ -10238,7 +10322,7 @@ fn genMulDivBinOp(
1023810322}
1023910323
1024010324fn genBinOp(
10241 self: *Self,
10325 self: *CodeGen,
1024210326 maybe_inst: ?Air.Inst.Index,
1024310327 air_tag: Air.Inst.Tag,
1024410328 lhs_air: Air.Inst.Ref,
......@@ -10334,7 +10418,7 @@ fn genBinOp(
1033410418 .{ .vp_w, .insr },
1033510419 dst_reg,
1033610420 dst_reg,
10337 try rhs_mcv.mem(self, .word),
10421 try rhs_mcv.mem(self, .{ .size = .word }),
1033810422 .u(1),
1033910423 ) else try self.asmRegisterRegisterRegister(
1034010424 .{ .vp_, .unpcklwd },
......@@ -10359,7 +10443,7 @@ fn genBinOp(
1035910443 mir_tag,
1036010444 dst_reg,
1036110445 dst_reg,
10362 try src_mcv.mem(self, .fromBitSize(float_bits)),
10446 try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }),
1036310447 ) else try self.asmRegisterRegisterRegister(
1036410448 mir_tag,
1036510449 dst_reg,
......@@ -10378,7 +10462,7 @@ fn genBinOp(
1037810462 if (src_mcv.isMemory()) try self.asmRegisterMemory(
1037910463 mir_tag,
1038010464 dst_reg,
10381 try src_mcv.mem(self, .fromBitSize(float_bits)),
10465 try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }),
1038210466 ) else try self.asmRegisterRegister(
1038310467 mir_tag,
1038410468 dst_reg,
......@@ -10651,22 +10735,22 @@ fn genBinOp(
1065110735 try self.asmRegisterMemory(
1065210736 .{ ._, .cmp },
1065310737 dst_regs[0],
10654 try src_mcv.mem(self, .qword),
10738 try src_mcv.mem(self, .{ .size = .qword }),
1065510739 );
1065610740 try self.asmRegisterMemory(
1065710741 .{ ._, .sbb },
1065810742 tmp_reg,
10659 try src_mcv.address().offset(8).deref().mem(self, .qword),
10743 try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
1066010744 );
1066110745 try self.asmCmovccRegisterMemory(
1066210746 cc,
1066310747 dst_regs[0],
10664 try src_mcv.mem(self, .qword),
10748 try src_mcv.mem(self, .{ .size = .qword }),
1066510749 );
1066610750 try self.asmCmovccRegisterMemory(
1066710751 cc,
1066810752 dst_regs[1],
10669 try src_mcv.address().offset(8).deref().mem(self, .qword),
10753 try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
1067010754 );
1067110755 } else {
1067210756 try self.asmRegisterRegister(
......@@ -10829,7 +10913,7 @@ fn genBinOp(
1082910913 .{ .vp_w, .insr },
1083010914 dst_reg,
1083110915 dst_reg,
10832 try src_mcv.mem(self, .word),
10916 try src_mcv.mem(self, .{ .size = .word }),
1083310917 .u(1),
1083410918 ) else try self.asmRegisterRegisterRegister(
1083510919 .{ .vp_, .unpcklwd },
......@@ -11275,7 +11359,7 @@ fn genBinOp(
1127511359 .{ .vp_w, .insr },
1127611360 dst_reg,
1127711361 dst_reg,
11278 try src_mcv.mem(self, .word),
11362 try src_mcv.mem(self, .{ .size = .word }),
1127911363 .u(1),
1128011364 ) else try self.asmRegisterRegisterRegister(
1128111365 .{ .vp_, .unpcklwd },
......@@ -11321,7 +11405,7 @@ fn genBinOp(
1132111405 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
1132211406 .{ .vp_d, .insr },
1132311407 dst_reg,
11324 try src_mcv.mem(self, .dword),
11408 try src_mcv.mem(self, .{ .size = .dword }),
1132511409 .u(1),
1132611410 ) else try self.asmRegisterRegisterRegister(
1132711411 .{ .v_ps, .unpckl },
......@@ -11373,7 +11457,7 @@ fn genBinOp(
1137311457 if (src_mcv.isMemory()) try self.asmRegisterMemory(
1137411458 .{ .v_ps, .cvtph2 },
1137511459 tmp_reg,
11376 try src_mcv.mem(self, .qword),
11460 try src_mcv.mem(self, .{ .size = .qword }),
1137711461 ) else try self.asmRegisterRegister(
1137811462 .{ .v_ps, .cvtph2 },
1137911463 tmp_reg,
......@@ -11416,7 +11500,7 @@ fn genBinOp(
1141611500 if (src_mcv.isMemory()) try self.asmRegisterMemory(
1141711501 .{ .v_ps, .cvtph2 },
1141811502 tmp_reg,
11419 try src_mcv.mem(self, .xword),
11503 try src_mcv.mem(self, .{ .size = .xword }),
1142011504 ) else try self.asmRegisterRegister(
1142111505 .{ .v_ps, .cvtph2 },
1142211506 tmp_reg,
......@@ -11579,10 +11663,10 @@ fn genBinOp(
1157911663 mir_tag,
1158011664 dst_reg,
1158111665 lhs_reg,
11582 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11666 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
1158311667 else => .fromSize(abi_size),
1158411668 .vector => .fromBitSize(dst_reg.bitSize()),
11585 }),
11669 } }),
1158611670 ) else try self.asmRegisterRegisterRegister(
1158711671 mir_tag,
1158811672 dst_reg,
......@@ -11597,10 +11681,10 @@ fn genBinOp(
1159711681 if (src_mcv.isMemory()) try self.asmRegisterMemory(
1159811682 mir_tag,
1159911683 dst_reg,
11600 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11684 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
1160111685 else => .fromSize(abi_size),
1160211686 .vector => .fromBitSize(dst_reg.bitSize()),
11603 }),
11687 } }),
1160411688 ) else try self.asmRegisterRegister(
1160511689 mir_tag,
1160611690 dst_reg,
......@@ -11625,10 +11709,10 @@ fn genBinOp(
1162511709 mir_tag,
1162611710 dst_reg,
1162711711 lhs_reg,
11628 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11712 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
1162911713 else => .fromSize(abi_size),
1163011714 .vector => .fromBitSize(dst_reg.bitSize()),
11631 }),
11715 } }),
1163211716 imm,
1163311717 ) else try self.asmRegisterRegisterRegisterImmediate(
1163411718 mir_tag,
......@@ -11645,10 +11729,10 @@ fn genBinOp(
1164511729 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
1164611730 mir_tag,
1164711731 dst_reg,
11648 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11732 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
1164911733 else => .fromSize(abi_size),
1165011734 .vector => .fromBitSize(dst_reg.bitSize()),
11651 }),
11735 } }),
1165211736 imm,
1165311737 ) else try self.asmRegisterRegisterImmediate(
1165411738 mir_tag,
......@@ -11854,7 +11938,7 @@ fn genBinOp(
1185411938 const unsigned_ty = try lhs_ty.toUnsigned(pt);
1185511939 const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty));
1185611940 const not_mem: Memory = if (not_mcv.isMemory())
11857 try not_mcv.mem(self, .fromSize(abi_size))
11941 try not_mcv.mem(self, .{ .size = .fromSize(abi_size) })
1185811942 else
1185911943 .{ .base = .{
1186011944 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
......@@ -11915,7 +11999,7 @@ fn genBinOp(
1191511999}
1191612000
1191712001fn genBinOpMir(
11918 self: *Self,
12002 self: *CodeGen,
1191912003 mir_tag: Mir.Inst.FixedTag,
1192012004 ty: Type,
1192112005 dst_mcv: MCValue,
......@@ -12341,7 +12425,7 @@ fn genBinOpMir(
1234112425
1234212426/// Performs multi-operand integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.
1234312427/// Does not support byte-size operands.
12344fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
12428fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
1234512429 const pt = self.pt;
1234612430 const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));
1234712431 try self.spillEflagsIfOccupied();
......@@ -12468,7 +12552,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
1246812552 }
1246912553}
1247012554
12471fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12555fn airArg(self: *CodeGen, inst: Air.Inst.Index) !void {
1247212556 const pt = self.pt;
1247312557 const zcu = pt.zcu;
1247412558 // skip zero-bit arguments as they don't have a corresponding arg instruction
......@@ -12522,7 +12606,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1252212606 const dst_mcv = try self.allocRegOrMem(inst, false);
1252312607 if (regs_frame_addr.regs > 0) try self.asmMemoryRegister(
1252412608 .{ ._, .mov },
12525 try dst_mcv.mem(self, .byte),
12609 try dst_mcv.mem(self, .{ .size = .byte }),
1252612610 prev_reg.to8(),
1252712611 );
1252812612 try self.genInlineMemset(
......@@ -12554,7 +12638,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1255412638 const unset = try self.asmJccReloc(.e, undefined);
1255512639 try self.asmMemoryRegister(
1255612640 .{ ._s, .bt },
12557 try dst_mcv.mem(self, .dword),
12641 try dst_mcv.mem(self, .{ .size = .dword }),
1255812642 index_reg.to32(),
1255912643 );
1256012644 self.performReloc(unset);
......@@ -12578,7 +12662,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1257812662 return self.finishAir(inst, result, .{ .none, .none, .none });
1257912663}
1258012664
12581fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {
12665fn airDbgArg(self: *CodeGen, inst: Air.Inst.Index) !void {
1258212666 // skip zero-bit arguments as they don't have a corresponding arg instruction
1258312667 var arg_index = self.arg_index;
1258412668 while (self.args[arg_index] == .none) arg_index += 1;
......@@ -12594,13 +12678,13 @@ fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {
1259412678 } else try self.airDbgVarArgs();
1259512679}
1259612680
12597fn airDbgVarArgs(self: *Self) !void {
12681fn airDbgVarArgs(self: *CodeGen) !void {
1259812682 if (self.pt.zcu.typeToFunc(self.fn_type).?.is_var_args)
1259912683 try self.asmPseudo(.pseudo_dbg_var_args_none);
1260012684}
1260112685
1260212686fn genLocalDebugInfo(
12603 self: *Self,
12687 self: *CodeGen,
1260412688 inst: Air.Inst.Index,
1260512689 mcv: MCValue,
1260612690) !void {
......@@ -12656,19 +12740,19 @@ fn genLocalDebugInfo(
1265612740 }
1265712741}
1265812742
12659fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
12743fn airRetAddr(self: *CodeGen, inst: Air.Inst.Index) !void {
1266012744 const dst_mcv = try self.allocRegOrMem(inst, true);
1266112745 try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});
1266212746 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
1266312747}
1266412748
12665fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
12749fn airFrameAddress(self: *CodeGen, inst: Air.Inst.Index) !void {
1266612750 const dst_mcv = try self.allocRegOrMem(inst, true);
1266712751 try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }, .{});
1266812752 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
1266912753}
1267012754
12671fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
12755fn airCall(self: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
1267212756 if (modifier == .always_tail) return self.fail("TODO implement tail calls for x86_64", .{});
1267312757
1267412758 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
......@@ -12702,7 +12786,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
1270212786 return self.finishAirResult(inst, result);
1270312787}
1270412788
12705fn genCall(self: *Self, info: union(enum) {
12789fn genCall(self: *CodeGen, info: union(enum) {
1270612790 air: Air.Inst.Ref,
1270712791 lib: struct {
1270812792 return_type: InternPool.Index,
......@@ -12808,7 +12892,7 @@ fn genCall(self: *Self, info: union(enum) {
1280812892 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
1280912893 defer self.register_manager.unlockReg(index_lock);
1281012894
12811 const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .dword) else .{
12895 const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .{ .size = .dword }) else .{
1281212896 .base = .{ .reg = try self.copyToTmpRegister(
1281312897 Type.usize,
1281412898 switch (src_arg) {
......@@ -12900,7 +12984,7 @@ fn genCall(self: *Self, info: union(enum) {
1290012984 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
1290112985 }, .{}),
1290212986 .elementwise_regs_then_frame => |regs_frame_addr| {
12903 const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .dword) else .{
12987 const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .{ .size = .dword }) else .{
1290412988 .base = .{ .reg = try self.copyToTmpRegister(
1290512989 Type.usize,
1290612990 switch (src_arg) {
......@@ -13006,7 +13090,7 @@ fn genCall(self: *Self, info: union(enum) {
1300613090 return call_info.return_value.short;
1300713091}
1300813092
13009fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
13093fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
1301013094 const pt = self.pt;
1301113095 const zcu = pt.zcu;
1301213096 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
......@@ -13042,7 +13126,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1304213126 try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
1304313127}
1304413128
13045fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
13129fn airRetLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
1304613130 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1304713131 const ptr = try self.resolveInst(un_op);
1304813132
......@@ -13062,7 +13146,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
1306213146 try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
1306313147}
1306413148
13065fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void {
13149fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !void {
1306613150 const pt = self.pt;
1306713151 const zcu = pt.zcu;
1306813152 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -13145,7 +13229,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
1314513229 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
1314613230 .{ ._, .mov },
1314713231 temp_lhs_reg.to8(),
13148 try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte),
13232 try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }),
1314913233 ) else {
1315013234 try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{});
1315113235 try self.asmRegisterImmediate(
......@@ -13158,7 +13242,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
1315813242 const payload_compare = payload_compare: {
1315913243 if (rhs_mcv.isMemory()) {
1316013244 const rhs_mem =
13161 try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte);
13245 try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte });
1316213246 try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8());
1316313247 const payload_compare = try self.asmJccReloc(.nz, undefined);
1316413248 try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem);
......@@ -13459,7 +13543,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
1345913543 .{ .vp_w, .insr },
1346013544 tmp1_reg,
1346113545 dst_reg.to128(),
13462 try src_mcv.mem(self, .word),
13546 try src_mcv.mem(self, .{ .size = .word }),
1346313547 .u(1),
1346413548 ) else try self.asmRegisterRegisterRegister(
1346513549 .{ .vp_, .unpcklwd },
......@@ -13505,7 +13589,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
1350513589 return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });
1350613590}
1350713591
13508fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
13592fn airCmpVector(self: *CodeGen, inst: Air.Inst.Index) !void {
1350913593 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1351013594 const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
1351113595 const dst_mcv = try self.genBinOp(
......@@ -13517,7 +13601,7 @@ fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
1351713601 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
1351813602}
1351913603
13520fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
13604fn airCmpLtErrorsLen(self: *CodeGen, inst: Air.Inst.Index) !void {
1352113605 const pt = self.pt;
1352213606 const zcu = pt.zcu;
1352313607 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
......@@ -13550,7 +13634,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
1355013634 return self.finishAir(inst, .{ .eflags = .b }, .{ un_op, .none, .none });
1355113635}
1355213636
13553fn airTry(self: *Self, inst: Air.Inst.Index) !void {
13637fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void {
1355413638 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
1355513639 const extra = self.air.extraData(Air.Try, pl_op.payload);
1355613640 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
......@@ -13559,7 +13643,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void {
1355913643 return self.finishAir(inst, result, .{ .none, .none, .none });
1356013644}
1356113645
13562fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {
13646fn airTryPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1356313647 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1356413648 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
1356513649 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
......@@ -13569,7 +13653,7 @@ fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {
1356913653}
1357013654
1357113655fn genTry(
13572 self: *Self,
13656 self: *CodeGen,
1357313657 inst: Air.Inst.Index,
1357413658 operand: Air.Inst.Ref,
1357513659 body: []const Air.Inst.Index,
......@@ -13615,7 +13699,7 @@ fn genTry(
1361513699 return result;
1361613700}
1361713701
13618fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
13702fn airDbgStmt(self: *CodeGen, inst: Air.Inst.Index) !void {
1361913703 const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
1362013704 _ = try self.addInst(.{
1362113705 .tag = .pseudo,
......@@ -13627,14 +13711,14 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
1362713711 });
1362813712}
1362913713
13630fn airDbgEmptyStmt(self: *Self) !void {
13714fn airDbgEmptyStmt(self: *CodeGen) !void {
1363113715 if (self.mir_instructions.len > 0 and
1363213716 self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] == .pseudo_dbg_line_stmt_line_column)
1363313717 self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] = .pseudo_dbg_line_line_column;
1363413718 try self.asmOpOnly(.{ ._, .nop });
1363513719}
1363613720
13637fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
13721fn airDbgInlineBlock(self: *CodeGen, inst: Air.Inst.Index) !void {
1363813722 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1363913723 const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
1364013724 const old_inline_func = self.inline_func;
......@@ -13653,13 +13737,13 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
1365313737 });
1365413738}
1365513739
13656fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
13740fn airDbgVar(self: *CodeGen, inst: Air.Inst.Index) !void {
1365713741 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
1365813742 try self.genLocalDebugInfo(inst, try self.resolveInst(pl_op.operand));
1365913743 return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
1366013744}
1366113745
13662fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
13746fn genCondBrMir(self: *CodeGen, ty: Type, mcv: MCValue) !Mir.Inst.Index {
1366313747 const pt = self.pt;
1366413748 const abi_size = ty.abiSize(pt.zcu);
1366513749 switch (mcv) {
......@@ -13686,7 +13770,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
1368613770 }
1368713771}
1368813772
13689fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
13773fn airCondBr(self: *CodeGen, inst: Air.Inst.Index) !void {
1369013774 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
1369113775 const cond = try self.resolveInst(pl_op.operand);
1369213776 const cond_ty = self.typeOf(pl_op.operand);
......@@ -13731,7 +13815,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
1373113815 // We already took care of pl_op.operand earlier, so there's nothing left to do.
1373213816}
1373313817
13734fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
13818fn isNull(self: *CodeGen, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
1373513819 const pt = self.pt;
1373613820 const zcu = pt.zcu;
1373713821 switch (opt_mcv) {
......@@ -13862,7 +13946,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
1386213946 }
1386313947}
1386413948
13865fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
13949fn isNullPtr(self: *CodeGen, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
1386613950 const pt = self.pt;
1386713951 const zcu = pt.zcu;
1386813952 const opt_ty = ptr_ty.childType(zcu);
......@@ -13899,7 +13983,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
1389913983 return .{ .eflags = .e };
1390013984}
1390113985
13902fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
13986fn isErr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
1390313987 const pt = self.pt;
1390413988 const zcu = pt.zcu;
1390513989 const err_ty = eu_ty.errorUnionSet(zcu);
......@@ -13948,7 +14032,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)
1394814032 return MCValue{ .eflags = .a };
1394914033}
1395014034
13951fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
14035fn isErrPtr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
1395214036 const pt = self.pt;
1395314037 const zcu = pt.zcu;
1395414038 const eu_ty = ptr_ty.childType(zcu);
......@@ -13981,7 +14065,7 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV
1398114065 return MCValue{ .eflags = .a };
1398214066}
1398314067
13984fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
14068fn isNonErr(self: *CodeGen, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
1398514069 const is_err_res = try self.isErr(inst, eu_ty, eu_mcv);
1398614070 switch (is_err_res) {
1398714071 .eflags => |cc| {
......@@ -13996,7 +14080,7 @@ fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MC
1399614080 }
1399714081}
1399814082
13999fn isNonErrPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
14083fn isNonErrPtr(self: *CodeGen, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
1400014084 const is_err_res = try self.isErrPtr(inst, ptr_ty, ptr_mcv);
1400114085 switch (is_err_res) {
1400214086 .eflags => |cc| {
......@@ -14011,7 +14095,7 @@ fn isNonErrPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue
1401114095 }
1401214096}
1401314097
14014fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
14098fn airIsNull(self: *CodeGen, inst: Air.Inst.Index) !void {
1401514099 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1401614100 const operand = try self.resolveInst(un_op);
1401714101 const ty = self.typeOf(un_op);
......@@ -14019,7 +14103,7 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
1401914103 return self.finishAir(inst, result, .{ un_op, .none, .none });
1402014104}
1402114105
14022fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
14106fn airIsNullPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1402314107 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1402414108 const operand = try self.resolveInst(un_op);
1402514109 const ty = self.typeOf(un_op);
......@@ -14027,7 +14111,7 @@ fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
1402714111 return self.finishAir(inst, result, .{ un_op, .none, .none });
1402814112}
1402914113
14030fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
14114fn airIsNonNull(self: *CodeGen, inst: Air.Inst.Index) !void {
1403114115 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1403214116 const operand = try self.resolveInst(un_op);
1403314117 const ty = self.typeOf(un_op);
......@@ -14039,7 +14123,7 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
1403914123 return self.finishAir(inst, result, .{ un_op, .none, .none });
1404014124}
1404114125
14042fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
14126fn airIsNonNullPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1404314127 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1404414128 const operand = try self.resolveInst(un_op);
1404514129 const ty = self.typeOf(un_op);
......@@ -14050,7 +14134,7 @@ fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
1405014134 return self.finishAir(inst, result, .{ un_op, .none, .none });
1405114135}
1405214136
14053fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
14137fn airIsErr(self: *CodeGen, inst: Air.Inst.Index) !void {
1405414138 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1405514139 const operand = try self.resolveInst(un_op);
1405614140 const ty = self.typeOf(un_op);
......@@ -14058,7 +14142,7 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
1405814142 return self.finishAir(inst, result, .{ un_op, .none, .none });
1405914143}
1406014144
14061fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
14145fn airIsErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1406214146 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1406314147 const operand = try self.resolveInst(un_op);
1406414148 const ty = self.typeOf(un_op);
......@@ -14066,7 +14150,7 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
1406614150 return self.finishAir(inst, result, .{ un_op, .none, .none });
1406714151}
1406814152
14069fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
14153fn airIsNonErr(self: *CodeGen, inst: Air.Inst.Index) !void {
1407014154 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1407114155 const operand = try self.resolveInst(un_op);
1407214156 const ty = self.typeOf(un_op);
......@@ -14074,7 +14158,7 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
1407414158 return self.finishAir(inst, result, .{ un_op, .none, .none });
1407514159}
1407614160
14077fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
14161fn airIsNonErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1407814162 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1407914163 const operand = try self.resolveInst(un_op);
1408014164 const ty = self.typeOf(un_op);
......@@ -14082,7 +14166,7 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
1408214166 return self.finishAir(inst, result, .{ un_op, .none, .none });
1408314167}
1408414168
14085fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
14169fn airLoop(self: *CodeGen, inst: Air.Inst.Index) !void {
1408614170 // A loop is a setup to be able to jump back to the beginning.
1408714171 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1408814172 const loop = self.air.extraData(Air.Block, ty_pl.payload);
......@@ -14100,7 +14184,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
1410014184 try self.genBodyBlock(body);
1410114185}
1410214186
14103fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
14187fn airBlock(self: *CodeGen, inst: Air.Inst.Index) !void {
1410414188 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1410514189 const extra = self.air.extraData(Air.Block, ty_pl.payload);
1410614190 try self.asmPseudo(.pseudo_dbg_enter_block_none);
......@@ -14108,7 +14192,7 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
1410814192 try self.asmPseudo(.pseudo_dbg_leave_block_none);
1410914193}
1411014194
14111fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
14195fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
1411214196 // A block is a setup to be able to jump to the end.
1411314197 const inst_tracking_i = self.inst_tracking.count();
1411414198 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(.unreach));
......@@ -14137,7 +14221,7 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
1413714221 self.getValueIfFree(tracking.short, inst);
1413814222}
1413914223
14140fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwitch, condition: MCValue) !void {
14224fn lowerSwitchBr(self: *CodeGen, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwitch, condition: MCValue) !void {
1414114225 const zcu = self.pt.zcu;
1414214226 const condition_ty = self.typeOf(switch_br.operand);
1414314227 const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.cases_len + 1);
......@@ -14255,7 +14339,7 @@ fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwit
1425514339 }
1425614340}
1425714341
14258fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
14342fn airSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void {
1425914343 const switch_br = self.air.unwrapSwitch(inst);
1426014344 const condition = try self.resolveInst(switch_br.operand);
1426114345
......@@ -14271,7 +14355,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
1427114355 // We already took care of pl_op.operand earlier, so there's nothing left to do
1427214356}
1427314357
14274fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
14358fn airLoopSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void {
1427514359 const switch_br = self.air.unwrapSwitch(inst);
1427614360 const condition = try self.resolveInst(switch_br.operand);
1427714361
......@@ -14309,7 +14393,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
1430914393 try self.processDeath(inst);
1431014394}
1431114395
14312fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {
14396fn airSwitchDispatch(self: *CodeGen, inst: Air.Inst.Index) !void {
1431314397 const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
1431414398
1431514399 const block_ty = self.typeOfIndex(br.block_inst);
......@@ -14354,7 +14438,7 @@ fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {
1435414438 try self.freeValue(block_tracking.short);
1435514439}
1435614440
14357fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {
14441fn performReloc(self: *CodeGen, reloc: Mir.Inst.Index) void {
1435814442 const next_inst: u32 = @intCast(self.mir_instructions.len);
1435914443 switch (self.mir_instructions.items(.tag)[reloc]) {
1436014444 .j, .jmp => {},
......@@ -14367,7 +14451,7 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {
1436714451 self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;
1436814452}
1436914453
14370fn airBr(self: *Self, inst: Air.Inst.Index) !void {
14454fn airBr(self: *CodeGen, inst: Air.Inst.Index) !void {
1437114455 const zcu = self.pt.zcu;
1437214456 const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
1437314457
......@@ -14426,7 +14510,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
1442614510 try self.freeValue(block_tracking.short);
1442714511}
1442814512
14429fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {
14513fn airRepeat(self: *CodeGen, inst: Air.Inst.Index) !void {
1443014514 const loop_inst = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat.loop_inst;
1443114515 const repeat_info = self.loops.get(loop_inst).?;
1443214516 try self.restoreState(repeat_info.state, &.{}, .{
......@@ -14438,7 +14522,7 @@ fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {
1443814522 _ = try self.asmJmpReloc(repeat_info.target);
1443914523}
1444014524
14441fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14525fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
1444214526 const pt = self.pt;
1444314527 const zcu = pt.zcu;
1444414528 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -15110,7 +15194,7 @@ const MoveStrategy = union(enum) {
1511015194 extract: Mir.Inst.FixedTag,
1511115195 };
1511215196
15113 pub fn read(strat: MoveStrategy, self: *Self, dst_reg: Register, src_mem: Memory) !void {
15197 pub fn read(strat: MoveStrategy, self: *CodeGen, dst_reg: Register, src_mem: Memory) !void {
1511415198 switch (strat) {
1511515199 .move => |tag| try self.asmRegisterMemory(tag, switch (tag[1]) {
1511615200 else => dst_reg,
......@@ -15136,7 +15220,7 @@ const MoveStrategy = union(enum) {
1513615220 ),
1513715221 }
1513815222 }
15139 pub fn write(strat: MoveStrategy, self: *Self, dst_mem: Memory, src_reg: Register) !void {
15223 pub fn write(strat: MoveStrategy, self: *CodeGen, dst_mem: Memory, src_reg: Register) !void {
1514015224 switch (strat) {
1514115225 .move => |tag| try self.asmMemoryRegister(tag, dst_mem, src_reg),
1514215226 .x87_load_store => {
......@@ -15152,7 +15236,7 @@ const MoveStrategy = union(enum) {
1515215236 }
1515315237 }
1515415238};
15155fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !MoveStrategy {
15239fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) !MoveStrategy {
1515615240 const pt = self.pt;
1515715241 const zcu = pt.zcu;
1515815242 switch (class) {
......@@ -15383,7 +15467,7 @@ const CopyOptions = struct {
1538315467 safety: bool = false,
1538415468};
1538515469
15386fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) InnerError!void {
15470fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) InnerError!void {
1538715471 const pt = self.pt;
1538815472
1538915473 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
......@@ -15496,7 +15580,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
1549615580}
1549715581
1549815582fn genSetReg(
15499 self: *Self,
15583 self: *CodeGen,
1550015584 dst_reg: Register,
1550115585 ty: Type,
1550215586 src_mcv: MCValue,
......@@ -15781,7 +15865,7 @@ fn genSetReg(
1578115865}
1578215866
1578315867fn genSetMem(
15784 self: *Self,
15868 self: *CodeGen,
1578515869 base: Memory.Base,
1578615870 disp: i32,
1578715871 ty: Type,
......@@ -16012,7 +16096,7 @@ fn genSetMem(
1601216096 }
1601316097}
1601416098
16015fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {
16099fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {
1601616100 try self.spillRegisters(&.{ .rsi, .rdi, .rcx });
1601716101 try self.genSetReg(.rsi, Type.usize, src_ptr, .{});
1601816102 try self.genSetReg(.rdi, Type.usize, dst_ptr, .{});
......@@ -16021,7 +16105,7 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue
1602116105}
1602216106
1602316107fn genInlineMemset(
16024 self: *Self,
16108 self: *CodeGen,
1602516109 dst_ptr: MCValue,
1602616110 value: MCValue,
1602716111 len: MCValue,
......@@ -16035,7 +16119,7 @@ fn genInlineMemset(
1603516119}
1603616120
1603716121fn genExternSymbolRef(
16038 self: *Self,
16122 self: *CodeGen,
1603916123 comptime tag: Mir.Inst.Tag,
1604016124 lib: ?[]const u8,
1604116125 callee: []const u8,
......@@ -16061,7 +16145,7 @@ fn genExternSymbolRef(
1606116145}
1606216146
1606316147fn genLazySymbolRef(
16064 self: *Self,
16148 self: *CodeGen,
1606516149 comptime tag: Mir.Inst.Tag,
1606616150 reg: Register,
1606716151 lazy_sym: link.File.LazySymbol,
......@@ -16159,7 +16243,7 @@ fn genLazySymbolRef(
1615916243 }
1616016244}
1616116245
16162fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {
16246fn airIntFromPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
1616316247 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1616416248 const result = result: {
1616516249 // TODO: handle case where the operand is a slice not a raw pointer
......@@ -16174,7 +16258,7 @@ fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {
1617416258 return self.finishAir(inst, result, .{ un_op, .none, .none });
1617516259}
1617616260
16177fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
16261fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
1617816262 const pt = self.pt;
1617916263 const zcu = pt.zcu;
1618016264 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -16239,7 +16323,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
1623916323 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1624016324}
1624116325
16242fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
16326fn airArrayToSlice(self: *CodeGen, inst: Air.Inst.Index) !void {
1624316327 const pt = self.pt;
1624416328 const zcu = pt.zcu;
1624516329 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -16264,7 +16348,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
1626416348 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1626516349}
1626616350
16267fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
16351fn airFloatFromInt(self: *CodeGen, inst: Air.Inst.Index) !void {
1626816352 const pt = self.pt;
1626916353 const zcu = pt.zcu;
1627016354 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -16344,7 +16428,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
1634416428 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1634516429}
1634616430
16347fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
16431fn airIntFromFloat(self: *CodeGen, inst: Air.Inst.Index) !void {
1634816432 const pt = self.pt;
1634916433 const zcu = pt.zcu;
1635016434 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -16416,7 +16500,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
1641616500 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1641716501}
1641816502
16419fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
16503fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void {
1642016504 const pt = self.pt;
1642116505 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1642216506 const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
......@@ -16463,7 +16547,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1646316547 const ptr_mcv = try self.resolveInst(extra.ptr);
1646416548 const mem_size: Memory.Size = .fromSize(val_abi_size);
1646516549 const ptr_mem: Memory = switch (ptr_mcv) {
16466 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
16550 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }),
1646716551 else => .{
1646816552 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
1646916553 .mod = .{ .rm = .{ .size = mem_size } },
......@@ -16504,7 +16588,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1650416588}
1650516589
1650616590fn atomicOp(
16507 self: *Self,
16591 self: *CodeGen,
1650816592 ptr_mcv: MCValue,
1650916593 val_mcv: MCValue,
1651016594 ptr_ty: Type,
......@@ -16530,7 +16614,7 @@ fn atomicOp(
1653016614 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));
1653116615 const mem_size: Memory.Size = .fromSize(val_abi_size);
1653216616 const ptr_mem: Memory = switch (ptr_mcv) {
16533 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
16617 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }),
1653416618 else => .{
1653516619 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
1653616620 .mod = .{ .rm = .{ .size = mem_size } },
......@@ -16649,7 +16733,7 @@ fn atomicOp(
1664916733 mir_tag,
1665016734 sse_reg.to128(),
1665116735 sse_reg.to128(),
16652 try val_mcv.mem(self, self.memSize(val_ty)),
16736 try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }),
1665316737 ) else try self.asmRegisterRegisterRegister(
1665416738 mir_tag,
1665516739 sse_reg.to128(),
......@@ -16662,7 +16746,7 @@ fn atomicOp(
1666216746 ._ss, ._sd => if (val_mcv.isMemory()) try self.asmRegisterMemory(
1666316747 mir_tag,
1666416748 sse_reg.to128(),
16665 try val_mcv.mem(self, self.memSize(val_ty)),
16749 try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }),
1666616750 ) else try self.asmRegisterRegister(
1666716751 mir_tag,
1666816752 sse_reg.to128(),
......@@ -16717,7 +16801,7 @@ fn atomicOp(
1671716801 try self.asmCmovccRegisterMemory(
1671816802 cc,
1671916803 registerAlias(tmp_reg, cmov_abi_size),
16720 try val_mcv.mem(self, .fromSize(cmov_abi_size)),
16804 try val_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }),
1672116805 );
1672216806 },
1672316807 else => {
......@@ -16773,8 +16857,8 @@ fn atomicOp(
1677316857 .reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),
1677416858 } },
1677516859 };
16776 const val_lo_mem = try val_mem_mcv.mem(self, .qword);
16777 const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .qword);
16860 const val_lo_mem = try val_mem_mcv.mem(self, .{ .size = .qword });
16861 const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .{ .size = .qword });
1677816862 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
1677916863 try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
1678016864 try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
......@@ -16862,7 +16946,7 @@ fn atomicOp(
1686216946 }
1686316947}
1686416948
16865fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
16949fn airAtomicRmw(self: *CodeGen, inst: Air.Inst.Index) !void {
1686616950 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
1686716951 const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
1686816952
......@@ -16883,7 +16967,7 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
1688316967 return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
1688416968}
1688516969
16886fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
16970fn airAtomicLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
1688716971 const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
1688816972
1688916973 const ptr_ty = self.typeOf(atomic_load.ptr);
......@@ -16904,7 +16988,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
1690416988 return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });
1690516989}
1690616990
16907fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
16991fn airAtomicStore(self: *CodeGen, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
1690816992 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1690916993
1691016994 const ptr_ty = self.typeOf(bin_op.lhs);
......@@ -16917,7 +17001,7 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr
1691717001 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1691817002}
1691917003
16920fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
17004fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
1692117005 const pt = self.pt;
1692217006 const zcu = pt.zcu;
1692317007 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -17059,7 +17143,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1705917143 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
1706017144}
1706117145
17062fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
17146fn airMemcpy(self: *CodeGen, inst: Air.Inst.Index) !void {
1706317147 const pt = self.pt;
1706417148 const zcu = pt.zcu;
1706517149 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -17107,7 +17191,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
1710717191 else => try self.asmRegisterMemoryImmediate(
1710817192 .{ .i_, .mul },
1710917193 len_reg,
17110 try dst.address().offset(8).deref().mem(self, .qword),
17194 try dst.address().offset(8).deref().mem(self, .{ .size = .qword }),
1711117195 .s(@intCast(dst_ty.childType(zcu).abiSize(zcu))),
1711217196 ),
1711317197 }
......@@ -17139,7 +17223,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
1713917223 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
1714017224}
1714117225
17142fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
17226fn airTagName(self: *CodeGen, inst: Air.Inst.Index) !void {
1714317227 const pt = self.pt;
1714417228 const zcu = pt.zcu;
1714517229 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
......@@ -17179,7 +17263,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
1717917263 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
1718017264}
1718117265
17182fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
17266fn airErrorName(self: *CodeGen, inst: Air.Inst.Index) !void {
1718317267 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1718417268
1718517269 const err_ty = self.typeOf(un_op);
......@@ -17281,7 +17365,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
1728117365 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
1728217366}
1728317367
17284fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17368fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void {
1728517369 const pt = self.pt;
1728617370 const zcu = pt.zcu;
1728717371 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -17318,7 +17402,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1731817402 else => cc: {
1731917403 try self.asmMemoryImmediate(
1732017404 .{ ._, .@"test" },
17321 try src_mcv.mem(self, .byte),
17405 try src_mcv.mem(self, .{ .size = .byte }),
1732217406 .u(1),
1732317407 );
1732417408 break :cc .nz;
......@@ -17362,7 +17446,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1736217446 if (src_mcv.isMemory()) try self.asmRegisterMemory(
1736317447 mir_tag,
1736417448 registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))),
17365 try src_mcv.mem(self, self.memSize(scalar_ty)),
17449 try src_mcv.mem(self, .{ .size = self.memSize(scalar_ty) }),
1736617450 ) else {
1736717451 if (mir_tag[0] == .v_i128) break :avx2;
1736817452 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
......@@ -17438,7 +17522,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1743817522 if (src_mcv.isMemory()) try self.asmRegisterMemory(
1743917523 .{ .v_ss, .broadcast },
1744017524 dst_reg.to128(),
17441 try src_mcv.mem(self, .dword),
17525 try src_mcv.mem(self, .{ .size = .dword }),
1744217526 ) else {
1744317527 const src_reg = if (src_mcv.isRegister())
1744417528 src_mcv.getReg().?
......@@ -17475,7 +17559,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1747517559 if (src_mcv.isMemory()) try self.asmRegisterMemory(
1747617560 .{ .v_ss, .broadcast },
1747717561 dst_reg.to256(),
17478 try src_mcv.mem(self, .dword),
17562 try src_mcv.mem(self, .{ .size = .dword }),
1747917563 ) else {
1748017564 const src_reg = if (src_mcv.isRegister())
1748117565 src_mcv.getReg().?
......@@ -17521,7 +17605,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1752117605 if (src_mcv.isMemory()) try self.asmRegisterMemory(
1752217606 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
1752317607 dst_reg.to128(),
17524 try src_mcv.mem(self, .qword),
17608 try src_mcv.mem(self, .{ .size = .qword }),
1752517609 ) else try self.asmRegisterRegister(
1752617610 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
1752717611 dst_reg.to128(),
......@@ -17546,7 +17630,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1754617630 if (src_mcv.isMemory()) try self.asmRegisterMemory(
1754717631 .{ .v_sd, .broadcast },
1754817632 dst_reg.to256(),
17549 try src_mcv.mem(self, .qword),
17633 try src_mcv.mem(self, .{ .size = .qword }),
1755017634 ) else {
1755117635 const src_reg = if (src_mcv.isRegister())
1755217636 src_mcv.getReg().?
......@@ -17589,7 +17673,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1758917673 if (src_mcv.isMemory()) try self.asmRegisterMemory(
1759017674 .{ .v_f128, .broadcast },
1759117675 dst_reg.to256(),
17592 try src_mcv.mem(self, .xword),
17676 try src_mcv.mem(self, .{ .size = .xword }),
1759317677 ) else {
1759417678 const src_reg = if (src_mcv.isRegister())
1759517679 src_mcv.getReg().?
......@@ -17616,7 +17700,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1761617700 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1761717701}
1761817702
17619fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17703fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
1762017704 const pt = self.pt;
1762117705 const zcu = pt.zcu;
1762217706 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
......@@ -17695,7 +17779,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1769517779 else => unreachable,
1769617780 }, .broadcast },
1769717781 mask_alias,
17698 if (pred_mcv.isMemory()) try pred_mcv.mem(self, .byte) else .{
17782 if (pred_mcv.isMemory()) try pred_mcv.mem(self, .{ .size = .byte }) else .{
1769917783 .base = .{ .reg = (try self.copyToTmpRegister(
1770017784 Type.usize,
1770117785 pred_mcv.address(),
......@@ -17893,7 +17977,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1789317977 mir_tag,
1789417978 dst_alias,
1789517979 rhs_alias,
17896 try lhs_mcv.mem(self, self.memSize(ty)),
17980 try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }),
1789717981 mask_alias,
1789817982 ) else try self.asmRegisterRegisterRegisterRegister(
1789917983 mir_tag,
......@@ -17908,7 +17992,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1790817992 } else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
1790917993 mir_tag,
1791017994 dst_alias,
17911 try lhs_mcv.mem(self, self.memSize(ty)),
17995 try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }),
1791217996 mask_alias,
1791317997 ) else try self.asmRegisterRegisterRegister(
1791417998 mir_tag,
......@@ -17933,7 +18017,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1793318017 if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
1793418018 .{ mir_fixes, .andn },
1793518019 mask_alias,
17936 try rhs_mcv.mem(self, .fromSize(abi_size)),
18020 try rhs_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
1793718021 ) else try self.asmRegisterRegister(
1793818022 .{ mir_fixes, .andn },
1793918023 mask_alias,
......@@ -17949,7 +18033,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1794918033 return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
1795018034}
1795118035
17952fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18036fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void {
1795318037 const pt = self.pt;
1795418038 const zcu = pt.zcu;
1795518039 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -18074,7 +18158,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1807418158 mir_tag,
1807518159 dst_alias,
1807618160 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
18077 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
18161 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
1807818162 ) else try self.asmRegisterRegisterRegister(
1807918163 mir_tag,
1808018164 dst_alias,
......@@ -18086,7 +18170,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1808618170 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
1808718171 mir_tag,
1808818172 dst_alias,
18089 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
18173 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
1809018174 ) else try self.asmRegisterRegister(
1809118175 mir_tag,
1809218176 dst_alias,
......@@ -18135,7 +18219,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1813518219 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
1813618220 .{ if (has_avx) .vp_d else .p_d, .shuf },
1813718221 dst_alias,
18138 try src_mcv.mem(self, .fromSize(max_abi_size)),
18222 try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
1813918223 .u(control),
1814018224 ) else try self.asmRegisterRegisterImmediate(
1814118225 .{ if (has_avx) .vp_d else .p_d, .shuf },
......@@ -18192,7 +18276,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1819218276 .{ .v_ps, .shuf },
1819318277 dst_alias,
1819418278 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
18195 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
18279 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
1819618280 .u(control),
1819718281 ) else try self.asmRegisterRegisterRegisterImmediate(
1819818282 .{ .v_ps, .shuf },
......@@ -18206,7 +18290,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1820618290 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
1820718291 .{ ._ps, .shuf },
1820818292 dst_alias,
18209 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
18293 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
1821018294 .u(control),
1821118295 ) else try self.asmRegisterRegisterImmediate(
1821218296 .{ ._ps, .shuf },
......@@ -18259,7 +18343,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1825918343 .{ .v_pd, .shuf },
1826018344 dst_alias,
1826118345 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
18262 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
18346 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
1826318347 .u(control),
1826418348 ) else try self.asmRegisterRegisterRegisterImmediate(
1826518349 .{ .v_pd, .shuf },
......@@ -18273,7 +18357,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1827318357 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
1827418358 .{ ._pd, .shuf },
1827518359 dst_alias,
18276 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
18360 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
1827718361 .u(control),
1827818362 ) else try self.asmRegisterRegisterImmediate(
1827918363 .{ ._pd, .shuf },
......@@ -18329,7 +18413,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1832918413 .{ .vp_d, .blend },
1833018414 registerAlias(dst_reg, dst_abi_size),
1833118415 registerAlias(lhs_reg, dst_abi_size),
18332 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18416 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
1833318417 .u(expanded_control),
1833418418 ) else try self.asmRegisterRegisterRegisterImmediate(
1833518419 .{ .vp_d, .blend },
......@@ -18384,7 +18468,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1838418468 lhs_mcv.getReg().?
1838518469 else
1838618470 dst_reg, dst_abi_size),
18387 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18471 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
1838818472 .u(expanded_control),
1838918473 ) else try self.asmRegisterRegisterRegisterImmediate(
1839018474 .{ .vp_w, .blend },
......@@ -18401,7 +18485,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1840118485 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
1840218486 .{ .p_w, .blend },
1840318487 registerAlias(dst_reg, dst_abi_size),
18404 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18488 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
1840518489 .u(expanded_control),
1840618490 ) else try self.asmRegisterRegisterImmediate(
1840718491 .{ .p_w, .blend },
......@@ -18445,7 +18529,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1844518529 lhs_mcv.getReg().?
1844618530 else
1844718531 dst_reg, dst_abi_size),
18448 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18532 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
1844918533 .u(expanded_control),
1845018534 ) else try self.asmRegisterRegisterRegisterImmediate(
1845118535 switch (elem_abi_size) {
......@@ -18470,7 +18554,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1847018554 else => unreachable,
1847118555 },
1847218556 registerAlias(dst_reg, dst_abi_size),
18473 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18557 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
1847418558 .u(expanded_control),
1847518559 ) else try self.asmRegisterRegisterImmediate(
1847618560 switch (elem_abi_size) {
......@@ -18560,7 +18644,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1856018644 registerAlias(lhs_mcv.getReg().?, dst_abi_size)
1856118645 else
1856218646 dst_alias,
18563 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18647 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
1856418648 select_mask_alias,
1856518649 ) else try self.asmRegisterRegisterRegisterRegister(
1856618650 mir_tag,
......@@ -18577,7 +18661,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1857718661 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
1857818662 mir_tag,
1857918663 dst_alias,
18580 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18664 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
1858118665 select_mask_alias,
1858218666 ) else try self.asmRegisterRegisterRegister(
1858318667 mir_tag,
......@@ -18620,7 +18704,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1862018704 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
1862118705 .{ mir_fixes, .andn },
1862218706 mask_alias,
18623 try lhs_mcv.mem(self, .fromSize(dst_abi_size)),
18707 try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
1862418708 ) else try self.asmRegisterRegister(
1862518709 .{ mir_fixes, .andn },
1862618710 mask_alias,
......@@ -18757,7 +18841,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1875718841 return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
1875818842}
1875918843
18760fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18844fn airReduce(self: *CodeGen, inst: Air.Inst.Index) !void {
1876118845 const pt = self.pt;
1876218846 const zcu = pt.zcu;
1876318847 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
......@@ -18776,7 +18860,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
1877618860 .Or => {
1877718861 if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
1877818862 .{ ._, .@"test" },
18779 try operand_mcv.mem(self, .fromSize(abi_size)),
18863 try operand_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
1878018864 .u(mask),
1878118865 ) else {
1878218866 const operand_reg = registerAlias(if (operand_mcv.isRegister())
......@@ -18815,7 +18899,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
1881518899 return self.finishAir(inst, result, .{ reduce.operand, .none, .none });
1881618900}
1881718901
18818fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
18902fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
1881918903 const pt = self.pt;
1882018904 const zcu = pt.zcu;
1882118905 const result_ty = self.typeOfIndex(inst);
......@@ -19007,7 +19091,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
1900719091 return self.finishAirResult(inst, result);
1900819092}
1900919093
19010fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
19094fn airUnionInit(self: *CodeGen, inst: Air.Inst.Index) !void {
1901119095 const pt = self.pt;
1901219096 const zcu = pt.zcu;
1901319097 const ip = &zcu.intern_pool;
......@@ -19053,12 +19137,12 @@ fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
1905319137 return self.finishAir(inst, result, .{ extra.init, .none, .none });
1905419138}
1905519139
19056fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
19140fn airPrefetch(self: *CodeGen, inst: Air.Inst.Index) !void {
1905719141 const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
1905819142 return self.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });
1905919143}
1906019144
19061fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
19145fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void {
1906219146 const pt = self.pt;
1906319147 const zcu = pt.zcu;
1906419148 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
......@@ -19219,14 +19303,14 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
1921919303 mir_tag,
1922019304 mop1_reg,
1922119305 mop2_reg,
19222 try mops[2].mem(self, .fromSize(abi_size)),
19306 try mops[2].mem(self, .{ .size = .fromSize(abi_size) }),
1922319307 );
1922419308 break :result mops[0];
1922519309 };
1922619310 return self.finishAir(inst, result, ops);
1922719311}
1922819312
19229fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
19313fn airVaStart(self: *CodeGen, inst: Air.Inst.Index) !void {
1923019314 const pt = self.pt;
1923119315 const zcu = pt.zcu;
1923219316 const va_list_ty = self.air.instructions.items(.data)[@intFromEnum(inst)].ty;
......@@ -19284,7 +19368,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
1928419368 return self.finishAir(inst, result, .{ .none, .none, .none });
1928519369}
1928619370
19287fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
19371fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void {
1928819372 const pt = self.pt;
1928919373 const zcu = pt.zcu;
1929019374 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -19451,7 +19535,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
1945119535 .{ .v_ss, .cvtsd2 },
1945219536 dst_reg,
1945319537 dst_reg,
19454 try promote_mcv.mem(self, .qword),
19538 try promote_mcv.mem(self, .{ .size = .qword }),
1945519539 ) else try self.asmRegisterRegisterRegister(
1945619540 .{ .v_ss, .cvtsd2 },
1945719541 dst_reg,
......@@ -19463,7 +19547,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
1946319547 ) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(
1946419548 .{ ._ss, .cvtsd2 },
1946519549 dst_reg,
19466 try promote_mcv.mem(self, .qword),
19550 try promote_mcv.mem(self, .{ .size = .qword }),
1946719551 ) else try self.asmRegisterRegister(
1946819552 .{ ._ss, .cvtsd2 },
1946919553 dst_reg,
......@@ -19480,7 +19564,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
1948019564 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1948119565}
1948219566
19483fn airVaCopy(self: *Self, inst: Air.Inst.Index) !void {
19567fn airVaCopy(self: *CodeGen, inst: Air.Inst.Index) !void {
1948419568 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1948519569 const ptr_va_list_ty = self.typeOf(ty_op.operand);
1948619570
......@@ -19489,12 +19573,12 @@ fn airVaCopy(self: *Self, inst: Air.Inst.Index) !void {
1948919573 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
1949019574}
1949119575
19492fn airVaEnd(self: *Self, inst: Air.Inst.Index) !void {
19576fn airVaEnd(self: *CodeGen, inst: Air.Inst.Index) !void {
1949319577 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1949419578 return self.finishAir(inst, .unreach, .{ un_op, .none, .none });
1949519579}
1949619580
19497fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
19581fn resolveInst(self: *CodeGen, ref: Air.Inst.Ref) InnerError!MCValue {
1949819582 const zcu = self.pt.zcu;
1949919583 const ty = self.typeOf(ref);
1950019584
......@@ -19543,7 +19627,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
1954319627 }
1954419628}
1954519629
19546fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) *InstTracking {
19630fn getResolvedInstValue(self: *CodeGen, inst: Air.Inst.Index) *InstTracking {
1954719631 const tracking = self.inst_tracking.getPtr(inst).?;
1954819632 return switch (tracking.short) {
1954919633 .none, .unreach, .dead => unreachable,
......@@ -19556,7 +19640,7 @@ fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) *InstTracking {
1955619640/// A potential opportunity for future optimization here would be keeping track
1955719641/// of the fact that the instruction is available both as an immediate
1955819642/// and as a register.
19559fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCValue {
19643fn limitImmediateType(self: *CodeGen, operand: Air.Inst.Ref, comptime T: type) !MCValue {
1956019644 const mcv = try self.resolveInst(operand);
1956119645 const ti = @typeInfo(T).int;
1956219646 switch (mcv) {
......@@ -19572,7 +19656,7 @@ fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCV
1957219656 return mcv;
1957319657}
1957419658
19575fn genTypedValue(self: *Self, val: Value) InnerError!MCValue {
19659fn genTypedValue(self: *CodeGen, val: Value) InnerError!MCValue {
1957619660 const pt = self.pt;
1957719661 return switch (try codegen.genTypedValue(self.bin_file, pt, self.src_loc, val, self.target.*)) {
1957819662 .mcv => |mcv| switch (mcv) {
......@@ -19599,7 +19683,7 @@ const CallMCValues = struct {
1959919683 gp_count: u32,
1960019684 fp_count: u32,
1960119685
19602 fn deinit(self: *CallMCValues, func: *Self) void {
19686 fn deinit(self: *CallMCValues, func: *CodeGen) void {
1960319687 func.gpa.free(self.args);
1960419688 self.* = undefined;
1960519689 }
......@@ -19607,7 +19691,7 @@ const CallMCValues = struct {
1960719691
1960819692/// Caller must call `CallMCValues.deinit`.
1960919693fn resolveCallingConventionValues(
19610 self: *Self,
19694 self: *CodeGen,
1961119695 fn_info: InternPool.Key.FuncType,
1961219696 var_args: []const Type,
1961319697 stack_frame_base: FrameIndex,
......@@ -19889,7 +19973,7 @@ fn resolveCallingConventionValues(
1988919973 return result;
1989019974}
1989119975
19892fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
19976fn fail(self: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
1989319977 @branchHint(.cold);
1989419978 const zcu = self.pt.zcu;
1989519979 switch (self.owner) {
......@@ -19899,7 +19983,7 @@ fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMem
1989919983 return error.CodegenFail;
1990019984}
1990119985
19902fn failMsg(self: *Self, msg: *Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } {
19986fn failMsg(self: *CodeGen, msg: *Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } {
1990319987 @branchHint(.cold);
1990419988 const zcu = self.pt.zcu;
1990519989 switch (self.owner) {
......@@ -19960,7 +20044,7 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
1996020044 };
1996120045}
1996220046
19963fn memSize(self: *Self, ty: Type) Memory.Size {
20047fn memSize(self: *CodeGen, ty: Type) Memory.Size {
1996420048 const zcu = self.pt.zcu;
1996520049 return switch (ty.zigTypeTag(zcu)) {
1996620050 .float => .fromBitSize(ty.floatBits(self.target.*)),
......@@ -19968,7 +20052,7 @@ fn memSize(self: *Self, ty: Type) Memory.Size {
1996820052 };
1996920053}
1997020054
19971fn splitType(self: *Self, ty: Type) ![2]Type {
20055fn splitType(self: *CodeGen, ty: Type) ![2]Type {
1997220056 const pt = self.pt;
1997320057 const zcu = pt.zcu;
1997420058 const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
......@@ -19998,7 +20082,7 @@ fn splitType(self: *Self, ty: Type) ![2]Type {
1999820082
1999920083/// Truncates the value in the register in place.
2000020084/// Clobbers any remaining bits.
20001fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
20085fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {
2000220086 const pt = self.pt;
2000320087 const zcu = pt.zcu;
2000420088 const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.builtin.Type.Int{
......@@ -20046,7 +20130,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
2004620130 }
2004720131}
2004820132
20049fn regBitSize(self: *Self, ty: Type) u64 {
20133fn regBitSize(self: *CodeGen, ty: Type) u64 {
2005020134 const zcu = self.pt.zcu;
2005120135 const abi_size = ty.abiSize(zcu);
2005220136 return switch (ty.zigTypeTag(zcu)) {
......@@ -20065,27 +20149,27 @@ fn regBitSize(self: *Self, ty: Type) u64 {
2006520149 };
2006620150}
2006720151
20068fn regExtraBits(self: *Self, ty: Type) u64 {
20152fn regExtraBits(self: *CodeGen, ty: Type) u64 {
2006920153 return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
2007020154}
2007120155
20072fn hasFeature(self: *Self, feature: std.Target.x86.Feature) bool {
20156fn hasFeature(self: *CodeGen, feature: std.Target.x86.Feature) bool {
2007320157 return std.Target.x86.featureSetHas(self.target.cpu.features, feature);
2007420158}
20075fn hasAnyFeatures(self: *Self, features: anytype) bool {
20159fn hasAnyFeatures(self: *CodeGen, features: anytype) bool {
2007620160 return std.Target.x86.featureSetHasAny(self.target.cpu.features, features);
2007720161}
20078fn hasAllFeatures(self: *Self, features: anytype) bool {
20162fn hasAllFeatures(self: *CodeGen, features: anytype) bool {
2007920163 return std.Target.x86.featureSetHasAll(self.target.cpu.features, features);
2008020164}
2008120165
20082fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {
20166fn typeOf(self: *CodeGen, inst: Air.Inst.Ref) Type {
2008320167 const pt = self.pt;
2008420168 const zcu = pt.zcu;
2008520169 return self.air.typeOf(inst, &zcu.intern_pool);
2008620170}
2008720171
20088fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {
20172fn typeOfIndex(self: *CodeGen, inst: Air.Inst.Index) Type {
2008920173 const pt = self.pt;
2009020174 const zcu = pt.zcu;
2009120175 const temp: Temp = .{ .index = inst };
......@@ -20118,7 +20202,7 @@ fn floatCompilerRtAbiName(float_bits: u32) u8 {
2011820202 };
2011920203}
2012020204
20121fn floatCompilerRtAbiType(self: *Self, ty: Type, other_ty: Type) Type {
20205fn floatCompilerRtAbiType(self: *CodeGen, ty: Type, other_ty: Type) Type {
2012220206 if (ty.toIntern() == .f16_type and
2012320207 (other_ty.toIntern() == .f32_type or other_ty.toIntern() == .f64_type) and
2012420208 self.target.isDarwin()) return Type.u16;
......@@ -20145,7 +20229,7 @@ fn floatLibcAbiSuffix(ty: Type) []const u8 {
2014520229 };
2014620230}
2014720231
20148fn promoteInt(self: *Self, ty: Type) Type {
20232fn promoteInt(self: *CodeGen, ty: Type) Type {
2014920233 const pt = self.pt;
2015020234 const zcu = pt.zcu;
2015120235 const int_info: InternPool.Key.IntType = switch (ty.toIntern()) {
......@@ -20165,7 +20249,7 @@ fn promoteInt(self: *Self, ty: Type) Type {
2016520249 return ty;
2016620250}
2016720251
20168fn promoteVarArg(self: *Self, ty: Type) Type {
20252fn promoteVarArg(self: *CodeGen, ty: Type) Type {
2016920253 if (!ty.isRuntimeFloat()) return self.promoteInt(ty);
2017020254 switch (ty.floatBits(self.target.*)) {
2017120255 32, 64 => return Type.f64,
......@@ -20181,7 +20265,7 @@ fn promoteVarArg(self: *Self, ty: Type) Type {
2018120265const Temp = struct {
2018220266 index: Air.Inst.Index,
2018320267
20184 fn unwrap(temp: Temp, self: *Self) union(enum) {
20268 fn unwrap(temp: Temp, cg: *CodeGen) union(enum) {
2018520269 ref: Air.Inst.Ref,
2018620270 temp: Index,
2018720271 } {
......@@ -20189,38 +20273,69 @@ const Temp = struct {
2018920273 .ref => |ref| return .{ .ref = ref },
2019020274 .target => |target_index| {
2019120275 const temp_index: Index = @enumFromInt(target_index);
20192 assert(temp_index.isValid(self));
20276 assert(temp_index.isValid(cg));
2019320277 return .{ .temp = temp_index };
2019420278 },
2019520279 }
2019620280 }
2019720281
20198 fn typeOf(temp: Temp, self: *Self) Type {
20199 return switch (temp.unwrap(self)) {
20200 .ref => |ref| self.typeOf(ref),
20201 .temp => |temp_index| temp_index.typeOf(self),
20282 fn typeOf(temp: Temp, cg: *CodeGen) Type {
20283 return switch (temp.unwrap(cg)) {
20284 .ref => |ref| cg.typeOf(ref),
20285 .temp => |temp_index| temp_index.typeOf(cg),
2020220286 };
2020320287 }
2020420288
20205 fn isMut(temp: Temp, self: *Self) bool {
20206 return temp.unwrap(self) == .temp;
20289 fn isMut(temp: Temp, cg: *CodeGen) bool {
20290 return switch (temp.unwrap(cg)) {
20291 .ref => false,
20292 .temp => |temp_index| switch (temp_index.tracking(cg).short) {
20293 .none,
20294 .unreach,
20295 .dead,
20296 .undef,
20297 .immediate,
20298 .eflags,
20299 .register_offset,
20300 .memory,
20301 .load_symbol,
20302 .lea_symbol,
20303 .indirect,
20304 .load_direct,
20305 .lea_direct,
20306 .load_got,
20307 .lea_got,
20308 .load_tlv,
20309 .lea_tlv,
20310 .lea_frame,
20311 .elementwise_regs_then_frame,
20312 .reserved_frame,
20313 .air_ref,
20314 => false,
20315 .register,
20316 .register_pair,
20317 .register_overflow,
20318 => true,
20319 .load_frame => |frame_addr| !frame_addr.index.isNamed(),
20320 },
20321 };
2020720322 }
2020820323
20209 fn tracking(temp: Temp, self: *Self) InstTracking {
20210 return self.inst_tracking.get(temp.index).?;
20324 fn tracking(temp: Temp, cg: *CodeGen) InstTracking {
20325 return cg.inst_tracking.get(temp.index).?;
2021120326 }
2021220327
20213 fn getOffset(temp: Temp, off: i32, self: *Self) !Temp {
20214 const new_temp_index = self.next_temp_index;
20215 self.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
20216 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20217 switch (temp.tracking(self).short) {
20328 fn getOffset(temp: Temp, off: i32, cg: *CodeGen) !Temp {
20329 const new_temp_index = cg.next_temp_index;
20330 cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
20331 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20332 switch (temp.tracking(cg).short) {
2021820333 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
2021920334 .register => |reg| {
2022020335 const new_reg =
20221 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20222 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20223 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
20336 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20337 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20338 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
2022420339 .base = .{ .reg = reg.to64() },
2022520340 .mod = .{ .rm = .{
2022620341 .size = .qword,
......@@ -20230,9 +20345,9 @@ const Temp = struct {
2023020345 },
2023120346 .register_offset => |reg_off| {
2023220347 const new_reg =
20233 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20234 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20235 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
20348 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20349 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20350 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
2023620351 .base = .{ .reg = reg_off.reg.to64() },
2023720352 .mod = .{ .rm = .{
2023820353 .size = .qword,
......@@ -20240,18 +20355,18 @@ const Temp = struct {
2024020355 } },
2024120356 });
2024220357 },
20243 .lea_symbol => |sym_off| new_temp_index.tracking(self).* = .init(.{ .lea_symbol = .{
20358 .lea_symbol => |sym_off| new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = .{
2024420359 .sym_index = sym_off.sym_index,
2024520360 .off = sym_off.off + off,
2024620361 } }),
2024720362 .load_frame => |frame_addr| {
2024820363 const new_reg =
20249 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20250 new_temp_index.tracking(self).* = .init(.{ .register_offset = .{
20364 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20365 new_temp_index.tracking(cg).* = .init(.{ .register_offset = .{
2025120366 .reg = new_reg,
2025220367 .off = off,
2025320368 } });
20254 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
20369 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
2025520370 .base = .{ .frame = frame_addr.index },
2025620371 .mod = .{ .rm = .{
2025720372 .size = .qword,
......@@ -20259,7 +20374,7 @@ const Temp = struct {
2025920374 } },
2026020375 });
2026120376 },
20262 .lea_frame => |frame_addr| new_temp_index.tracking(self).* = .init(.{ .lea_frame = .{
20377 .lea_frame => |frame_addr| new_temp_index.tracking(cg).* = .init(.{ .lea_frame = .{
2026320378 .index = frame_addr.index,
2026420379 .off = frame_addr.off + off,
2026520380 } }),
......@@ -20267,16 +20382,16 @@ const Temp = struct {
2026720382 return .{ .index = new_temp_index.toIndex() };
2026820383 }
2026920384
20270 fn toOffset(temp: *Temp, off: i32, self: *Self) !void {
20385 fn toOffset(temp: *Temp, off: i32, cg: *CodeGen) !void {
2027120386 if (off == 0) return;
20272 switch (temp.unwrap(self)) {
20387 switch (temp.unwrap(cg)) {
2027320388 .ref => {},
2027420389 .temp => |temp_index| {
20275 const temp_tracking = temp_index.tracking(self);
20390 const temp_tracking = temp_index.tracking(cg);
2027620391 switch (temp_tracking.short) {
2027720392 else => {},
2027820393 .register => |reg| {
20279 try self.freeValue(temp_tracking.long);
20394 try cg.freeValue(temp_tracking.long);
2028020395 temp_tracking.* = .init(.{ .register_offset = .{
2028120396 .reg = reg,
2028220397 .off = off,
......@@ -20284,7 +20399,7 @@ const Temp = struct {
2028420399 return;
2028520400 },
2028620401 .register_offset => |reg_off| {
20287 try self.freeValue(temp_tracking.long);
20402 try cg.freeValue(temp_tracking.long);
2028820403 temp_tracking.* = .init(.{ .register_offset = .{
2028920404 .reg = reg_off.reg,
2029020405 .off = reg_off.off + off,
......@@ -20310,39 +20425,39 @@ const Temp = struct {
2031020425 }
2031120426 },
2031220427 }
20313 const new_temp = try temp.getOffset(off, self);
20314 try temp.die(self);
20428 const new_temp = try temp.getOffset(off, cg);
20429 try temp.die(cg);
2031520430 temp.* = new_temp;
2031620431 }
2031720432
20318 fn getLimb(temp: Temp, limb_index: u28, self: *Self) !Temp {
20319 const new_temp_index = self.next_temp_index;
20320 self.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
20321 switch (temp.tracking(self).short) {
20433 fn getLimb(temp: Temp, limb_index: u28, cg: *CodeGen) !Temp {
20434 const new_temp_index = cg.next_temp_index;
20435 cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
20436 switch (temp.tracking(cg).short) {
2032220437 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
2032320438 .immediate => |imm| {
2032420439 assert(limb_index == 0);
20325 new_temp_index.tracking(self).* = .init(.{ .immediate = imm });
20440 new_temp_index.tracking(cg).* = .init(.{ .immediate = imm });
2032620441 },
2032720442 .register => |reg| {
2032820443 assert(limb_index == 0);
2032920444 const new_reg =
20330 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20331 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20332 try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());
20445 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20446 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20447 try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());
2033320448 },
2033420449 .register_pair => |regs| {
2033520450 const new_reg =
20336 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20337 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20338 try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());
20451 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20452 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20453 try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());
2033920454 },
2034020455 .register_offset => |reg_off| {
2034120456 assert(limb_index == 0);
2034220457 const new_reg =
20343 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20344 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20345 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
20458 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20459 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20460 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
2034620461 .base = .{ .reg = reg_off.reg.to64() },
2034720462 .mod = .{ .rm = .{
2034820463 .size = .qword,
......@@ -20352,9 +20467,9 @@ const Temp = struct {
2035220467 },
2035320468 .load_symbol => |sym_off| {
2035420469 const new_reg =
20355 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20356 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20357 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
20470 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20471 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20472 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
2035820473 .base = .{ .reloc = sym_off.sym_index },
2035920474 .mod = .{ .rm = .{
2036020475 .size = .qword,
......@@ -20364,13 +20479,13 @@ const Temp = struct {
2036420479 },
2036520480 .lea_symbol => |sym_off| {
2036620481 assert(limb_index == 0);
20367 new_temp_index.tracking(self).* = .init(.{ .lea_symbol = sym_off });
20482 new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = sym_off });
2036820483 },
2036920484 .load_frame => |frame_addr| {
2037020485 const new_reg =
20371 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20372 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20373 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
20486 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20487 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20488 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
2037420489 .base = .{ .frame = frame_addr.index },
2037520490 .mod = .{ .rm = .{
2037620491 .size = .qword,
......@@ -20380,23 +20495,23 @@ const Temp = struct {
2038020495 },
2038120496 .lea_frame => |frame_addr| {
2038220497 assert(limb_index == 0);
20383 new_temp_index.tracking(self).* = .init(.{ .lea_frame = frame_addr });
20498 new_temp_index.tracking(cg).* = .init(.{ .lea_frame = frame_addr });
2038420499 },
2038520500 }
20386 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20501 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
2038720502 return .{ .index = new_temp_index.toIndex() };
2038820503 }
2038920504
20390 fn toLimb(temp: *Temp, limb_index: u28, self: *Self) !void {
20391 switch (temp.unwrap(self)) {
20505 fn toLimb(temp: *Temp, limb_index: u28, cg: *CodeGen) !void {
20506 switch (temp.unwrap(cg)) {
2039220507 .ref => {},
2039320508 .temp => |temp_index| {
20394 const temp_tracking = temp_index.tracking(self);
20509 const temp_tracking = temp_index.tracking(cg);
2039520510 switch (temp_tracking.short) {
2039620511 else => {},
2039720512 .register, .lea_symbol, .lea_frame => {
2039820513 assert(limb_index == 0);
20399 self.temp_type[@intFromEnum(temp_index)] = Type.usize;
20514 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
2040020515 return;
2040120516 },
2040220517 .register_pair => |regs| {
......@@ -20406,9 +20521,9 @@ const Temp = struct {
2040620521 temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(),
2040720522 }
2040820523 for (regs, 0..) |reg, reg_index| if (reg_index != limb_index)
20409 self.register_manager.freeReg(reg);
20524 cg.register_manager.freeReg(reg);
2041020525 temp_tracking.* = .init(.{ .register = regs[limb_index] });
20411 self.temp_type[@intFromEnum(temp_index)] = Type.usize;
20526 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
2041220527 return;
2041320528 },
2041420529 .load_symbol => |sym_off| {
......@@ -20417,7 +20532,7 @@ const Temp = struct {
2041720532 .sym_index = sym_off.sym_index,
2041820533 .off = sym_off.off + @as(u31, limb_index) * 8,
2041920534 } });
20420 self.temp_type[@intFromEnum(temp_index)] = Type.usize;
20535 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
2042120536 return;
2042220537 },
2042320538 .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) {
......@@ -20426,102 +20541,102 @@ const Temp = struct {
2042620541 .index = frame_addr.index,
2042720542 .off = frame_addr.off + @as(u31, limb_index) * 8,
2042820543 } });
20429 self.temp_type[@intFromEnum(temp_index)] = Type.usize;
20544 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
2043020545 return;
2043120546 },
2043220547 }
2043320548 },
2043420549 }
20435 const new_temp = try temp.getLimb(limb_index, self);
20436 try temp.die(self);
20550 const new_temp = try temp.getLimb(limb_index, cg);
20551 try temp.die(cg);
2043720552 temp.* = new_temp;
2043820553 }
2043920554
20440 fn toReg(temp: *Temp, new_reg: Register, self: *Self) !bool {
20441 const val, const ty = switch (temp.unwrap(self)) {
20442 .ref => |ref| .{ temp.tracking(self).short, self.typeOf(ref) },
20443 .temp => |temp_index| val: {
20444 const temp_tracking = temp_index.tracking(self);
20555 fn toReg(temp: *Temp, new_reg: Register, cg: *CodeGen) !bool {
20556 const val, const ty = val_ty: switch (temp.unwrap(cg)) {
20557 .ref => |ref| .{ temp.tracking(cg).short, cg.typeOf(ref) },
20558 .temp => |temp_index| {
20559 const temp_tracking = temp_index.tracking(cg);
2044520560 if (temp_tracking.short == .register and
2044620561 temp_tracking.short.register == new_reg) return false;
20447 break :val .{ temp_tracking.short, temp_index.typeOf(self) };
20562 break :val_ty .{ temp_tracking.short, temp_index.typeOf(cg) };
2044820563 },
2044920564 };
20450 const new_temp_index = self.next_temp_index;
20451 self.temp_type[@intFromEnum(new_temp_index)] = ty;
20452 try self.genSetReg(new_reg, ty, val, .{});
20453 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20454 try temp.die(self);
20455 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20565 const new_temp_index = cg.next_temp_index;
20566 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
20567 try cg.genSetReg(new_reg, ty, val, .{});
20568 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20569 try temp.die(cg);
20570 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
2045620571 temp.* = .{ .index = new_temp_index.toIndex() };
2045720572 return true;
2045820573 }
2045920574
20460 fn toAnyReg(temp: *Temp, self: *Self) !bool {
20461 const val, const ty = switch (temp.unwrap(self)) {
20462 .ref => |ref| .{ temp.tracking(self).short, self.typeOf(ref) },
20575 fn toAnyReg(temp: *Temp, cg: *CodeGen) !bool {
20576 const val, const ty = switch (temp.unwrap(cg)) {
20577 .ref => |ref| .{ temp.tracking(cg).short, cg.typeOf(ref) },
2046320578 .temp => |temp_index| val: {
20464 const temp_tracking = temp_index.tracking(self);
20579 const temp_tracking = temp_index.tracking(cg);
2046520580 if (temp_tracking.short == .register) return false;
20466 break :val .{ temp_tracking.short, temp_index.typeOf(self) };
20581 break :val .{ temp_tracking.short, temp_index.typeOf(cg) };
2046720582 },
2046820583 };
20469 const new_temp_index = self.next_temp_index;
20470 self.temp_type[@intFromEnum(new_temp_index)] = ty;
20584 const new_temp_index = cg.next_temp_index;
20585 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
2047120586 const new_reg =
20472 try self.register_manager.allocReg(new_temp_index.toIndex(), self.regSetForType(ty));
20473 try self.genSetReg(new_reg, ty, val, .{});
20474 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20475 try temp.die(self);
20476 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20587 try cg.register_manager.allocReg(new_temp_index.toIndex(), cg.regSetForType(ty));
20588 try cg.genSetReg(new_reg, ty, val, .{});
20589 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20590 try temp.die(cg);
20591 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
2047720592 temp.* = .{ .index = new_temp_index.toIndex() };
2047820593 return true;
2047920594 }
2048020595
20481 fn toRegClass(temp: *Temp, rc: Register.Class, self: *Self) !bool {
20482 const val, const ty = switch (temp.unwrap(self)) {
20483 .ref => |ref| .{ temp.tracking(self).short, self.typeOf(ref) },
20596 fn toRegClass(temp: *Temp, rc: Register.Class, cg: *CodeGen) !bool {
20597 const val, const ty = switch (temp.unwrap(cg)) {
20598 .ref => |ref| .{ temp.tracking(cg).short, cg.typeOf(ref) },
2048420599 .temp => |temp_index| val: {
20485 const temp_tracking = temp_index.tracking(self);
20600 const temp_tracking = temp_index.tracking(cg);
2048620601 switch (temp_tracking.short) {
2048720602 else => {},
2048820603 .register => |reg| if (reg.class() == rc) return false,
2048920604 }
20490 break :val .{ temp_tracking.short, temp_index.typeOf(self) };
20605 break :val .{ temp_tracking.short, temp_index.typeOf(cg) };
2049120606 },
2049220607 };
20493 const new_temp_index = self.next_temp_index;
20494 self.temp_type[@intFromEnum(new_temp_index)] = ty;
20495 const new_reg = try self.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc));
20496 try self.genSetReg(new_reg, ty, val, .{});
20497 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20498 try temp.die(self);
20499 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20608 const new_temp_index = cg.next_temp_index;
20609 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
20610 const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc));
20611 try cg.genSetReg(new_reg, ty, val, .{});
20612 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20613 try temp.die(cg);
20614 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
2050020615 temp.* = .{ .index = new_temp_index.toIndex() };
2050120616 return true;
2050220617 }
2050320618
20504 fn toPair(first_temp: *Temp, second_temp: *Temp, self: *Self) !void {
20619 fn toPair(first_temp: *Temp, second_temp: *Temp, cg: *CodeGen) !void {
2050520620 while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| {
20506 if (try part_temp.toAnyReg(self)) break;
20621 if (try part_temp.toAnyReg(cg)) break;
2050720622 } else break;
20508 const first_temp_tracking = first_temp.unwrap(self).temp.tracking(self);
20509 const second_temp_tracking = second_temp.unwrap(self).temp.tracking(self);
20623 const first_temp_tracking = first_temp.unwrap(cg).temp.tracking(cg);
20624 const second_temp_tracking = second_temp.unwrap(cg).temp.tracking(cg);
2051020625 const result: MCValue = .{ .register_pair = .{
2051120626 first_temp_tracking.short.register,
2051220627 second_temp_tracking.short.register,
2051320628 } };
20514 const result_temp_index = self.next_temp_index;
20629 const result_temp_index = cg.next_temp_index;
2051520630 const result_temp: Temp = .{ .index = result_temp_index.toIndex() };
20516 assert(self.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
20517 assert(self.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
20518 self.temp_type[@intFromEnum(result_temp_index)] = Type.slice_const_u8;
20519 result_temp_index.tracking(self).* = .init(result);
20631 assert(cg.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
20632 assert(cg.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
20633 cg.temp_type[@intFromEnum(result_temp_index)] = Type.slice_const_u8;
20634 result_temp_index.tracking(cg).* = .init(result);
2052020635 first_temp.* = result_temp;
2052120636 }
2052220637
20523 fn toLea(temp: *Temp, self: *Self) !bool {
20524 switch (temp.tracking(self).short) {
20638 fn toLea(temp: *Temp, cg: *CodeGen) !bool {
20639 switch (temp.tracking(cg).short) {
2052520640 .none,
2052620641 .unreach,
2052720642 .dead,
......@@ -20548,65 +20663,83 @@ const Temp = struct {
2054820663 .load_got,
2054920664 .load_tlv,
2055020665 .load_frame,
20551 => return temp.toAnyReg(self),
20666 => return temp.toAnyReg(cg),
2055220667 .lea_symbol => |sym_off| {
2055320668 const off = sym_off.off;
2055420669 if (off == 0) return false;
20555 try temp.toOffset(-off, self);
20556 while (try temp.toAnyReg(self)) {}
20557 try temp.toOffset(off, self);
20670 try temp.toOffset(-off, cg);
20671 while (try temp.toAnyReg(cg)) {}
20672 try temp.toOffset(off, cg);
2055820673 return true;
2055920674 },
2056020675 }
2056120676 }
2056220677
20563 fn load(ptr: *Temp, val_ty: Type, self: *Self) !Temp {
20564 const val_abi_size: u32 = @intCast(val_ty.abiSize(self.pt.zcu));
20565 const val = try self.tempAlloc(val_ty);
20566 switch (val.tracking(self).short) {
20678 fn toBase(temp: *Temp, cg: *CodeGen) !bool {
20679 const temp_tracking = temp.tracking(cg);
20680 switch (temp_tracking.short) {
20681 else => {},
20682 .indirect, .load_frame => return false,
20683 }
20684 const new_temp_index = cg.next_temp_index;
20685 cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg);
20686 const new_reg =
20687 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20688 try cg.genSetReg(new_reg, Type.usize, temp_tracking.short.address(), .{});
20689 new_temp_index.tracking(cg).* = .init(.{ .indirect = .{ .reg = new_reg } });
20690 try temp.die(cg);
20691 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20692 temp.* = .{ .index = new_temp_index.toIndex() };
20693 return true;
20694 }
20695
20696 fn load(ptr: *Temp, val_ty: Type, cg: *CodeGen) !Temp {
20697 const val_abi_size: u32 = @intCast(val_ty.abiSize(cg.pt.zcu));
20698 const val = try cg.tempAlloc(val_ty);
20699 switch (val.tracking(cg).short) {
2056720700 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
2056820701 .register => |val_reg| {
20569 while (try ptr.toLea(self)) {}
20702 while (try ptr.toLea(cg)) {}
2057020703 switch (val_reg.class()) {
20571 .general_purpose => try self.asmRegisterMemory(
20704 .general_purpose => try cg.asmRegisterMemory(
2057220705 .{ ._, .mov },
2057320706 registerAlias(val_reg, val_abi_size),
20574 try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),
20707 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }),
2057520708 ),
2057620709 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
2057720710 }
2057820711 },
2057920712 .load_frame => |val_frame_addr| {
20580 var val_ptr = try self.tempFromValue(Type.usize, .{ .lea_frame = val_frame_addr });
20581 var len = try self.tempFromValue(Type.usize, .{ .immediate = val_abi_size });
20582 try val_ptr.memcpy(ptr, &len, self);
20583 try val_ptr.die(self);
20584 try len.die(self);
20713 var val_ptr = try cg.tempFromValue(Type.usize, .{ .lea_frame = val_frame_addr });
20714 var len = try cg.tempFromValue(Type.usize, .{ .immediate = val_abi_size });
20715 try val_ptr.memcpy(ptr, &len, cg);
20716 try val_ptr.die(cg);
20717 try len.die(cg);
2058520718 },
2058620719 }
2058720720 return val;
2058820721 }
2058920722
20590 fn store(ptr: *Temp, val: *Temp, self: *Self) !void {
20591 const val_ty = val.typeOf(self);
20592 const val_abi_size: u32 = @intCast(val_ty.abiSize(self.pt.zcu));
20593 val: switch (val.tracking(self).short) {
20723 fn store(ptr: *Temp, val: *Temp, cg: *CodeGen) !void {
20724 const val_ty = val.typeOf(cg);
20725 const val_abi_size: u32 = @intCast(val_ty.abiSize(cg.pt.zcu));
20726 val: switch (val.tracking(cg).short) {
2059420727 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
2059520728 .immediate => |imm| if (std.math.cast(i32, imm)) |s| {
20596 while (try ptr.toLea(self)) {}
20597 try self.asmMemoryImmediate(
20729 while (try ptr.toLea(cg)) {}
20730 try cg.asmMemoryImmediate(
2059820731 .{ ._, .mov },
20599 try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),
20732 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }),
2060020733 .s(s),
2060120734 );
2060220735 } else continue :val .{ .register = undefined },
2060320736 .register => {
20604 while (try ptr.toLea(self) or try val.toAnyReg(self)) {}
20605 const val_reg = val.tracking(self).short.register;
20737 while (try ptr.toLea(cg) or try val.toAnyReg(cg)) {}
20738 const val_reg = val.tracking(cg).short.register;
2060620739 switch (val_reg.class()) {
20607 .general_purpose => try self.asmMemoryRegister(
20740 .general_purpose => try cg.asmMemoryRegister(
2060820741 .{ ._, .mov },
20609 try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),
20742 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }),
2061020743 registerAlias(val_reg, val_abi_size),
2061120744 ),
2061220745 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
......@@ -20615,64 +20748,64 @@ const Temp = struct {
2061520748 }
2061620749 }
2061720750
20618 fn memcpy(dst: *Temp, src: *Temp, len: *Temp, self: *Self) !void {
20751 fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) !void {
2061920752 while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| {
20620 if (try temp.toReg(reg, self)) break;
20753 if (try temp.toReg(reg, cg)) break;
2062120754 } else break;
20622 try self.asmOpOnly(.{ .@"rep _sb", .mov });
20755 try cg.asmOpOnly(.{ .@"rep _sb", .mov });
2062320756 }
2062420757
2062520758 // i, m, r
20626 fn add(lhs: *Temp, rhs: *Temp, self: *Self) !Temp {
20627 const res_index = self.next_temp_index;
20759 fn add(lhs: *Temp, rhs: *Temp, cg: *CodeGen) !Temp {
20760 const res_index = cg.next_temp_index;
2062820761 var res: Temp = .{ .index = res_index.toIndex() };
20629 try self.select(&.{ &res, lhs, rhs }, .{ ._, .add }, &.{
20762 try cg.select(&.{ &res, lhs, rhs }, .{ ._, .add }, &.{
2063020763 .{ .ops = &.{ .{ .match = 1 }, .r, .i } },
2063120764 .{ .ops = &.{ .{ .match = 1 }, .m, .i } },
2063220765 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },
2063320766 .{ .ops = &.{ .{ .match = 1 }, .m, .r } },
2063420767 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },
2063520768 });
20636 self.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
20637 self.temp_type[@intFromEnum(res_index)] = lhs.typeOf(self);
20769 cg.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
20770 cg.temp_type[@intFromEnum(res_index)] = lhs.typeOf(cg);
2063820771 return res;
2063920772 }
2064020773
20641 fn mul(lhs: *Temp, rhs: *Temp, self: *Self) !Temp {
20642 const res_index = self.next_temp_index;
20643 var res: Temp = .{ .index = self.next_temp_index.toIndex() };
20644 try self.select(&.{ &res, lhs, rhs }, .{ .i_, .mul }, &.{
20774 fn mul(lhs: *Temp, rhs: *Temp, cg: *CodeGen) !Temp {
20775 const res_index = cg.next_temp_index;
20776 var res: Temp = .{ .index = cg.next_temp_index.toIndex() };
20777 try cg.select(&.{ &res, lhs, rhs }, .{ .i_, .mul }, &.{
2064520778 .{ .ops = &.{ .r, .m, .i } },
2064620779 .{ .ops = &.{ .r, .r, .i } },
2064720780 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },
2064820781 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },
2064920782 });
20650 self.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
20651 self.temp_type[@intFromEnum(res_index)] = lhs.typeOf(self);
20783 cg.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
20784 cg.temp_type[@intFromEnum(res_index)] = lhs.typeOf(cg);
2065220785 return res;
2065320786 }
2065420787
20655 fn moveTo(temp: Temp, inst: Air.Inst.Index, self: *Self) !void {
20656 if (self.liveness.isUnused(inst)) try temp.die(self) else switch (temp.unwrap(self)) {
20788 fn moveTo(temp: Temp, inst: Air.Inst.Index, cg: *CodeGen) !void {
20789 if (cg.liveness.isUnused(inst)) try temp.die(cg) else switch (temp.unwrap(cg)) {
2065720790 .ref => {
20658 const result = try self.allocRegOrMem(inst, true);
20659 try self.genCopy(self.typeOfIndex(inst), result, temp.tracking(self).short, .{});
20791 const result = try cg.allocRegOrMem(inst, true);
20792 try cg.genCopy(cg.typeOfIndex(inst), result, temp.tracking(cg).short, .{});
2066020793 tracking_log.debug("{} => {} (birth)", .{ inst, result });
20661 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));
20794 cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));
2066220795 },
2066320796 .temp => |temp_index| {
20664 const temp_tracking = temp_index.tracking(self);
20797 const temp_tracking = temp_index.tracking(cg);
2066520798 tracking_log.debug("{} => {} (birth)", .{ inst, temp_tracking.short });
20666 self.inst_tracking.putAssumeCapacityNoClobber(inst, temp_tracking.*);
20667 assert(self.reuseTemp(inst, temp_index.toIndex(), temp_tracking));
20799 cg.inst_tracking.putAssumeCapacityNoClobber(inst, temp_tracking.*);
20800 assert(cg.reuseTemp(inst, temp_index.toIndex(), temp_tracking));
2066820801 },
2066920802 }
2067020803 }
2067120804
20672 fn die(temp: Temp, self: *Self) !void {
20673 switch (temp.unwrap(self)) {
20805 fn die(temp: Temp, cg: *CodeGen) !void {
20806 switch (temp.unwrap(cg)) {
2067420807 .ref => {},
20675 .temp => |temp_index| try temp_index.tracking(self).die(self, temp_index.toIndex()),
20808 .temp => |temp_index| try temp_index.tracking(cg).die(cg, temp_index.toIndex()),
2067620809 }
2067720810 }
2067820811
......@@ -20687,17 +20820,17 @@ const Temp = struct {
2068720820 return @enumFromInt(index.toTargetIndex());
2068820821 }
2068920822
20690 fn tracking(index: Index, self: *Self) *InstTracking {
20691 return &self.inst_tracking.values()[@intFromEnum(index)];
20823 fn tracking(index: Index, cg: *CodeGen) *InstTracking {
20824 return &cg.inst_tracking.values()[@intFromEnum(index)];
2069220825 }
2069320826
20694 fn isValid(index: Index, self: *Self) bool {
20695 return index.tracking(self).short != .dead;
20827 fn isValid(index: Index, cg: *CodeGen) bool {
20828 return index.tracking(cg).short != .dead;
2069620829 }
2069720830
20698 fn typeOf(index: Index, self: *Self) Type {
20699 assert(index.isValid(self));
20700 return self.temp_type[@intFromEnum(index)];
20831 fn typeOf(index: Index, cg: *CodeGen) Type {
20832 assert(index.isValid(cg));
20833 return cg.temp_type[@intFromEnum(index)];
2070120834 }
2070220835
2070320836 const max = std.math.maxInt(@typeInfo(Index).@"enum".tag_type);
......@@ -20723,17 +20856,17 @@ const Temp = struct {
2072320856 };
2072420857};
2072520858
20726fn resetTemps(self: *Self) void {
20727 for (0..@intFromEnum(self.next_temp_index)) |temp_index| {
20859fn resetTemps(cg: *CodeGen) void {
20860 for (0..@intFromEnum(cg.next_temp_index)) |temp_index| {
2072820861 const temp: Temp.Index = @enumFromInt(temp_index);
20729 assert(!temp.isValid(self));
20730 self.temp_type[temp_index] = undefined;
20862 assert(!temp.isValid(cg));
20863 cg.temp_type[temp_index] = undefined;
2073120864 }
20732 self.next_temp_index = @enumFromInt(0);
20865 cg.next_temp_index = @enumFromInt(0);
2073320866}
2073420867
2073520868fn reuseTemp(
20736 self: *Self,
20869 cg: *CodeGen,
2073720870 new_inst: Air.Inst.Index,
2073820871 old_inst: Air.Inst.Index,
2073920872 tracking: *InstTracking,
......@@ -20743,79 +20876,80 @@ fn reuseTemp(
2074320876 .register_pair,
2074420877 .register_offset,
2074520878 .register_overflow,
20879 .indirect,
2074620880 => for (tracking.short.getRegs()) |tracked_reg| {
2074720881 if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) |tracked_index| {
20748 self.register_manager.registers[tracked_index] = new_inst;
20882 cg.register_manager.registers[tracked_index] = new_inst;
2074920883 }
2075020884 },
2075120885 .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,
2075220886 else => {},
2075320887 }
2075420888 switch (tracking.short) {
20755 .eflags, .register_overflow => self.eflags_inst = new_inst,
20889 .eflags, .register_overflow => cg.eflags_inst = new_inst,
2075620890 else => {},
2075720891 }
20758 tracking.reuse(self, new_inst, old_inst);
20892 tracking.reuse(cg, new_inst, old_inst);
2075920893 return true;
2076020894}
2076120895
20762fn tempAlloc(self: *Self, ty: Type) !Temp {
20763 const temp_index = self.next_temp_index;
20764 temp_index.tracking(self).* = .init(
20765 try self.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true),
20896fn tempAlloc(cg: *CodeGen, ty: Type) !Temp {
20897 const temp_index = cg.next_temp_index;
20898 temp_index.tracking(cg).* = .init(
20899 try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true),
2076620900 );
20767 self.temp_type[@intFromEnum(temp_index)] = ty;
20768 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
20901 cg.temp_type[@intFromEnum(temp_index)] = ty;
20902 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
2076920903 return .{ .index = temp_index.toIndex() };
2077020904}
2077120905
20772fn tempAllocReg(self: *Self, ty: Type, rc: RegisterManager.RegisterBitSet) !Temp {
20773 const temp_index = self.next_temp_index;
20774 temp_index.tracking(self).* = .init(
20775 .{ .register = try self.register_manager.allocReg(temp_index.toIndex(), rc) },
20906fn tempAllocReg(cg: *CodeGen, ty: Type, rc: RegisterManager.RegisterBitSet) !Temp {
20907 const temp_index = cg.next_temp_index;
20908 temp_index.tracking(cg).* = .init(
20909 .{ .register = try cg.register_manager.allocReg(temp_index.toIndex(), rc) },
2077620910 );
20777 self.temp_type[@intFromEnum(temp_index)] = ty;
20778 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
20911 cg.temp_type[@intFromEnum(temp_index)] = ty;
20912 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
2077920913 return .{ .index = temp_index.toIndex() };
2078020914}
2078120915
20782fn tempFromValue(self: *Self, ty: Type, value: MCValue) !Temp {
20783 const temp_index = self.next_temp_index;
20784 temp_index.tracking(self).* = .init(value);
20785 self.temp_type[@intFromEnum(temp_index)] = ty;
20786 try self.getValue(value, temp_index.toIndex());
20787 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
20916fn tempFromValue(cg: *CodeGen, ty: Type, value: MCValue) !Temp {
20917 const temp_index = cg.next_temp_index;
20918 temp_index.tracking(cg).* = .init(value);
20919 cg.temp_type[@intFromEnum(temp_index)] = ty;
20920 try cg.getValue(value, temp_index.toIndex());
20921 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
2078820922 return .{ .index = temp_index.toIndex() };
2078920923}
2079020924
2079120925fn tempFromOperand(
20792 self: *Self,
20926 cg: *CodeGen,
2079320927 inst: Air.Inst.Index,
2079420928 op_index: Liveness.OperandInt,
2079520929 op_ref: Air.Inst.Ref,
2079620930) !Temp {
20797 const zcu = self.pt.zcu;
20931 const zcu = cg.pt.zcu;
2079820932 const ip = &zcu.intern_pool;
2079920933
20800 if (!self.liveness.operandDies(inst, op_index)) {
20934 if (!cg.liveness.operandDies(inst, op_index)) {
2080120935 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };
2080220936 const val = op_ref.toInterned().?;
20803 const gop = try self.const_tracking.getOrPut(self.gpa, val);
20937 const gop = try cg.const_tracking.getOrPut(cg.gpa, val);
2080420938 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
20805 const const_mcv = try self.genTypedValue(.fromInterned(val));
20939 const const_mcv = try cg.genTypedValue(.fromInterned(val));
2080620940 switch (const_mcv) {
20807 .lea_tlv => |tlv_sym| switch (self.bin_file.tag) {
20941 .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) {
2080820942 .elf, .macho => {
20809 if (self.mod.pic) {
20810 try self.spillRegisters(&.{ .rdi, .rax });
20943 if (cg.mod.pic) {
20944 try cg.spillRegisters(&.{ .rdi, .rax });
2081120945 } else {
20812 try self.spillRegisters(&.{.rax});
20946 try cg.spillRegisters(&.{.rax});
2081320947 }
20814 const frame_index = try self.allocFrameIndex(.init(.{
20948 const frame_index = try cg.allocFrameIndex(.init(.{
2081520949 .size = 8,
2081620950 .alignment = .@"8",
2081720951 }));
20818 try self.genSetMem(
20952 try cg.genSetMem(
2081920953 .{ .frame = frame_index },
2082020954 0,
2082120955 Type.usize,
......@@ -20829,24 +20963,24 @@ fn tempFromOperand(
2082920963 else => break :init const_mcv,
2083020964 }
2083120965 });
20832 return self.tempFromValue(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
20966 return cg.tempFromValue(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
2083320967 }
2083420968
20835 const temp_index = self.next_temp_index;
20969 const temp_index = cg.next_temp_index;
2083620970 const temp: Temp = .{ .index = temp_index.toIndex() };
2083720971 const op_inst = op_ref.toIndex().?;
20838 const tracking = self.getResolvedInstValue(op_inst);
20839 temp_index.tracking(self).* = tracking.*;
20840 if (!self.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
20841 self.temp_type[@intFromEnum(temp_index)] = self.typeOf(op_ref);
20842 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
20972 const tracking = cg.getResolvedInstValue(op_inst);
20973 temp_index.tracking(cg).* = tracking.*;
20974 if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
20975 cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref);
20976 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
2084320977 return temp;
2084420978}
2084520979
20846inline fn tempsFromOperands(self: *Self, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp {
20980inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp {
2084720981 var temps: [op_refs.len]Temp = undefined;
2084820982 inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| {
20849 temp.* = try self.tempFromOperand(inst, op_index, op_ref);
20983 temp.* = try cg.tempFromOperand(inst, op_index, op_ref);
2085020984 }
2085120985 return temps;
2085220986}
......@@ -20859,15 +20993,55 @@ const Operand = union(enum) {
2085920993 inst: Mir.Inst.Index,
2086020994};
2086120995
20996const SelectLoop = struct {
20997 element_reloc: Mir.Inst.Index,
20998 element_offset: union(enum) {
20999 unused,
21000 known: u31,
21001 temp: Temp,
21002 },
21003 element_size: ?u13,
21004 limb_reloc: Mir.Inst.Index,
21005 limb_offset: union(enum) {
21006 unused,
21007 known: u31,
21008 temp: Temp,
21009 },
21010 limb_size: ?u8,
21011 remaining_size: ?u64,
21012};
21013
2086221014const Pattern = struct {
20863 tag: Mir.Inst.FixedTag,
2086421015 ops: []const Op,
2086521016 commute: struct { u8, u8 } = .{ 0, 0 },
20866 features: []const std.Target.x86.Feature = &.{},
21017
21018 const Set = struct {
21019 required_features: []const std.Target.x86.Feature = &.{},
21020 loop: enum {
21021 /// only execute the instruction once
21022 once,
21023 /// execute the instruction on all groups of non-overlapping bits in the entire value
21024 bitwise,
21025 /// for each element, execute the instruction on each limb, propogating the carry flag
21026 limbwise_carry,
21027 /// for each element, execute the instruction on pairs of limbs, starting from the
21028 /// least significant, propogating a limb
21029 limbwise_pairs_forward,
21030 /// for each element, execute the instruction on pairs of limbs, starting from the
21031 /// most significant, propogating a limb
21032 limbwise_pairs_reverse,
21033 /// for each element, execute the instruction
21034 elementwise,
21035 } = .once,
21036 mir_tag: Mir.Inst.FixedTag,
21037 patterns: []const Pattern,
21038 };
2086721039
2086821040 const Op = union(enum) {
20869 /// match another operand
20870 match: u8,
21041 /// reuse another operand
21042 implicit: u8,
21043 /// repeat another operand
21044 explicit: u8,
2087121045 /// any general purpose register
2087221046 gpr,
2087321047 /// any 64-bit mmx register
......@@ -20878,165 +21052,449 @@ const Pattern = struct {
2087821052 ymm,
2087921053 /// any memory
2088021054 mem,
21055 /// a limb stored in a gpr
21056 gpr_limb,
21057 /// a limb stored in a 64-bit mmx register
21058 mm_limb,
21059 /// a limb stored in a 128-bit sse register
21060 xmm_limb,
21061 /// a limb stored in a 256-bit sse register
21062 ymm_limb,
21063 /// a limb stored in memory
21064 mem_limb,
2088121065 /// specific immediate
2088221066 imm: i8,
2088321067 /// any immediate signed extended from 32 bits
2088421068 simm32,
2088521069
20886 fn matches(op: Op, is_mut: bool, temp: Temp, self: *Self) bool {
21070 fn matches(op: Op, is_mut: bool, temp: Temp, cg: *CodeGen) bool {
21071 const abi_size = temp.typeOf(cg).abiSize(cg.pt.zcu);
2088721072 return switch (op) {
20888 .match => unreachable,
20889 .gpr => switch (temp.tracking(self).short) {
21073 .implicit, .explicit => unreachable,
21074 .gpr => abi_size <= 8 and switch (temp.tracking(cg).short) {
2089021075 .register => |reg| reg.class() == .general_purpose,
2089121076 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and
2089221077 reg_off.off == 0,
20893 else => self.regClassForType(temp.typeOf(self)) == .general_purpose,
21078 else => cg.regClassForType(temp.typeOf(cg)) == .general_purpose,
2089421079 },
20895 .mm => switch (temp.tracking(self).short) {
21080 .mm => abi_size <= 8 and switch (temp.tracking(cg).short) {
2089621081 .register => |reg| reg.class() == .mmx,
2089721082 .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0,
20898 else => self.regClassForType(temp.typeOf(self)) == .mmx,
21083 else => cg.regClassForType(temp.typeOf(cg)) == .mmx,
2089921084 },
20900 .xmm => switch (temp.tracking(self).short) {
21085 .xmm => abi_size > 8 and abi_size <= 16 and switch (temp.tracking(cg).short) {
2090121086 .register => |reg| reg.class() == .sse,
2090221087 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
20903 else => self.regClassForType(temp.typeOf(self)) == .sse,
21088 else => cg.regClassForType(temp.typeOf(cg)) == .sse,
2090421089 },
20905 .ymm => switch (temp.tracking(self).short) {
21090 .ymm => abi_size > 16 and abi_size <= 32 and switch (temp.tracking(cg).short) {
2090621091 .register => |reg| reg.class() == .sse,
2090721092 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
20908 else => self.regClassForType(temp.typeOf(self)) == .sse,
20909 } and temp.typeOf(self).abiSize(self.pt.zcu) > 16,
20910 .mem => (!is_mut or temp.isMut(self)) and temp.tracking(self).short.isMemory(),
20911 .imm => |specific_imm| if (is_mut) unreachable else switch (temp.tracking(self).short) {
21093 else => cg.regClassForType(temp.typeOf(cg)) == .sse,
21094 },
21095 .mem, .mem_limb => (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(),
21096 .gpr_limb => abi_size > 8,
21097 .mm_limb => abi_size > 8 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory() and cg.regClassForType(temp.typeOf(cg)) == .mmx,
21098 .xmm_limb => abi_size > 16 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(),
21099 .ymm_limb => abi_size > 32 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(),
21100 .imm => |specific_imm| if (is_mut) unreachable else switch (temp.tracking(cg).short) {
2091221101 .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm,
2091321102 else => false,
2091421103 },
20915 .simm32 => if (is_mut) unreachable else switch (temp.tracking(self).short) {
20916 .immediate => |imm| temp.typeOf(self).abiSize(self.pt.zcu) <= 4 or
20917 std.math.cast(i32, @as(i64, @bitCast(imm))) != null,
21104 .simm32 => if (is_mut) unreachable else switch (temp.tracking(cg).short) {
21105 .immediate => |imm| abi_size <= 4 or std.math.cast(i32, @as(i64, @bitCast(imm))) != null,
2091821106 else => false,
2091921107 },
2092021108 };
2092121109 }
2092221110 };
2092321111};
20924fn select(self: *Self, dst_temps: []Temp, src_temps: []const *Temp, patterns: []const Pattern) !void {
20925 patterns: for (patterns) |pattern| {
20926 for (pattern.features) |feature| if (!self.hasFeature(feature)) continue :patterns;
20927 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| if (!switch (src_op) {
20928 .match => |match_index| pattern.ops[match_index],
20929 else => src_op,
20930 }.matches(src_op == .match, src_temp.*, self)) continue :patterns;
20931 while (true) for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
20932 if (switch (switch (src_op) {
20933 .match => |match_index| pattern.ops[match_index],
20934 else => src_op,
20935 }) {
20936 .match => unreachable,
20937 .gpr => try src_temp.toRegClass(.general_purpose, self),
20938 .mm => try src_temp.toRegClass(.mmx, self),
20939 .xmm, .ymm => try src_temp.toRegClass(.sse, self),
20940 .mem, .imm, .simm32 => false,
20941 }) break;
20942 } else break;
20943 var mir_ops: [4]Operand = @splat(.none);
20944 var mir_ops_len = dst_temps.len;
20945 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
20946 const mir_op, const matched_src_op = op: switch (src_op) {
20947 .match => |match_index| {
20948 dst_temps[match_index] = src_temp.*;
20949 break :op .{ &mir_ops[match_index], pattern.ops[match_index] };
20950 },
20951 else => {
20952 defer mir_ops_len += 1;
20953 break :op .{ &mir_ops[mir_ops_len], src_op };
20954 },
20955 };
20956 const src_mcv = src_temp.tracking(self).short;
20957 mir_op.* = switch (matched_src_op) {
20958 .match => unreachable,
20959 .gpr => .{ .reg = registerAlias(
20960 src_mcv.register,
20961 @intCast(src_temp.typeOf(self).abiSize(self.pt.zcu)),
20962 ) },
20963 .mm => .{ .reg = src_mcv.register },
20964 .xmm => .{ .reg = src_mcv.register.to128() },
20965 .ymm => .{ .reg = src_mcv.register.to256() },
20966 .mem => .{ .mem = try src_mcv.mem(self, self.memSize(src_temp.typeOf(self))) },
20967 .imm => |imm| .{ .imm = .s(imm) },
20968 .simm32 => switch (src_temp.typeOf(self).abiSize(self.pt.zcu)) {
20969 else => unreachable,
20970 1 => .{ .imm = if (std.math.cast(i8, @as(i64, @bitCast(src_mcv.immediate)))) |small|
20971 .s(small)
20972 else
20973 .u(@as(u8, @intCast(src_mcv.immediate))) },
20974 2 => .{ .imm = if (std.math.cast(i16, @as(i64, @bitCast(src_mcv.immediate)))) |small|
20975 .s(small)
20976 else
20977 .u(@as(u16, @intCast(src_mcv.immediate))) },
20978 3...8 => .{ .imm = if (std.math.cast(i32, @as(i64, @bitCast(src_mcv.immediate)))) |small|
20979 .s(small)
20980 else
20981 .u(@as(u32, @intCast(src_mcv.immediate))) },
20982 },
20983 };
20984 }
20985 for (
20986 pattern.ops[0..dst_temps.len],
20987 dst_temps,
20988 mir_ops[0..dst_temps.len],
20989 ) |dst_op, *dst_temp, *mir_op| {
20990 if (mir_op.* != .none) continue;
20991 const ty = src_temps[0].typeOf(self);
20992 switch (dst_op) {
20993 .match => |match_index| {
20994 dst_temp.* = dst_temps[match_index];
20995 mir_op.* = mir_ops[match_index];
20996 },
20997 .gpr => {
20998 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.gp);
20999 mir_op.* = .{ .reg = registerAlias(
21000 dst_temp.tracking(self).short.register,
21001 @intCast(ty.abiSize(self.pt.zcu)),
21002 ) };
21003 },
21004 .mm => @panic("TODO"),
21005 .xmm => {
21006 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.sse);
21007 mir_op.* = .{ .reg = dst_temp.tracking(self).short.register.to128() };
21008 },
21009 .ymm => {
21010 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.sse);
21011 mir_op.* = .{ .reg = dst_temp.tracking(self).short.register.to256() };
21112fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_sets: []const Pattern.Set) !void {
21113 var loop: SelectLoop = .{
21114 .element_reloc = undefined,
21115 .element_offset = .unused,
21116 .element_size = null,
21117 .limb_reloc = undefined,
21118 .limb_offset = .unused,
21119 .limb_size = null,
21120 .remaining_size = null,
21121 };
21122 var extra_temps: [4]?Temp = @splat(null);
21123 pattern_sets: for (pattern_sets) |pattern_set| {
21124 for (pattern_set.required_features) |required_feature| if (!cg.hasFeature(required_feature)) continue :pattern_sets;
21125 patterns: for (pattern_set.patterns) |pattern| {
21126 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
21127 const ref_src_op, const is_mut = switch (src_op) {
21128 .implicit, .explicit => |op_index| .{ pattern.ops[op_index], true },
21129 else => .{ src_op, false },
21130 };
21131 if (!ref_src_op.matches(is_mut, src_temp.*, cg)) continue :patterns;
21132 }
21133 while (true) for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
21134 if (switch (switch (src_op) {
21135 .implicit, .explicit => |op_index| pattern.ops[op_index],
21136 else => src_op,
21137 }) {
21138 .implicit, .explicit => unreachable,
21139 .gpr => try src_temp.toRegClass(.general_purpose, cg),
21140 .mm => try src_temp.toRegClass(.mmx, cg),
21141 .xmm, .ymm => try src_temp.toRegClass(.sse, cg),
21142 .mem, .imm, .simm32 => false,
21143 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb, .mem_limb => switch (src_temp.tracking(cg).short) {
21144 .register_pair => false,
21145 else => try src_temp.toBase(cg),
21146 },
21147 }) break;
21148 } else break;
21149 var mir_ops_len = dst_temps.len;
21150 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
21151 const ref_src_op, const extra_temp = op: switch (src_op) {
21152 .implicit => |op_index| {
21153 dst_temps[op_index] = if (src_temp.isMut(cg))
21154 src_temp.*
21155 else
21156 try cg.tempAlloc(src_temp.typeOf(cg));
21157 break :op .{ pattern.ops[op_index], &extra_temps[op_index] };
21158 },
21159 .explicit => |op_index| {
21160 dst_temps[op_index] = if (src_temp.isMut(cg))
21161 src_temp.*
21162 else
21163 try cg.tempAlloc(src_temp.typeOf(cg));
21164 defer mir_ops_len += 1;
21165 break :op .{ pattern.ops[op_index], &extra_temps[mir_ops_len] };
21166 },
21167 else => {
21168 defer mir_ops_len += 1;
21169 break :op .{ src_op, &extra_temps[mir_ops_len] };
21170 },
21171 };
21172 const limb_size: u8, const rc = switch (ref_src_op) {
21173 else => continue,
21174 .gpr_limb => .{ 8, abi.RegisterClass.gp },
21175 .mm_limb => .{ 8, @panic("TODO") },
21176 .xmm_limb => .{ 16, abi.RegisterClass.sse },
21177 .ymm_limb => .{ 32, abi.RegisterClass.sse },
21178 };
21179 assert(loop.limb_size == null or loop.limb_size == limb_size);
21180 loop.limb_size = limb_size;
21181 loop.remaining_size = loop.remaining_size orelse src_temp.typeOf(cg).abiSize(cg.pt.zcu);
21182 switch (src_temp.tracking(cg).short) {
21183 .register_pair => switch (loop.limb_offset) {
21184 .unused, .temp => loop.limb_offset = .{ .known = 0 },
21185 .known => {},
21186 },
21187 else => {
21188 switch (loop.limb_offset) {
21189 .unused => loop.limb_offset = .{ .temp = undefined },
21190 .known, .temp => {},
21191 }
21192 extra_temp.* = try cg.tempAllocReg(Type.usize, rc);
21193 },
21194 }
21195 }
21196 switch (loop.element_offset) {
21197 .unused, .known => {},
21198 .temp => |*element_offset| {
21199 element_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp);
21200 const element_offset_reg = element_offset.tracking(cg).short.register;
21201 try cg.asmRegisterRegister(.{ ._, .xor }, element_offset_reg.to32(), element_offset_reg.to32());
21202 loop.element_reloc = @intCast(cg.mir_instructions.len);
2101221203 },
21013 .mem => @panic("TODO"),
21014 .imm, .simm32 => unreachable, // unmodifiable destination
2101521204 }
21016 }
21017 std.mem.swap(Operand, &mir_ops[pattern.commute[0]], &mir_ops[pattern.commute[1]]);
21018 self.asmOps(pattern.tag, mir_ops) catch |err| switch (err) {
21019 error.InvalidInstruction => {
21020 const fixes = @tagName(pattern.tag[0]);
21021 const fixes_replace = std.mem.indexOfScalar(u8, fixes, '_').?;
21022 return self.fail(
21023 "invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'",
21024 .{
21025 fixes[0..fixes_replace],
21026 @tagName(pattern.tag[1]),
21027 fixes[fixes_replace + 1 ..],
21028 @tagName(mir_ops[0]),
21029 @tagName(mir_ops[1]),
21030 @tagName(mir_ops[2]),
21031 @tagName(mir_ops[3]),
21205 while (true) {
21206 switch (loop.limb_offset) {
21207 .unused, .known => {},
21208 .temp => |*limb_offset| {
21209 limb_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp);
21210 const limb_offset_reg = limb_offset.tracking(cg).short.register;
21211 try cg.asmRegisterRegister(.{ ._, .xor }, limb_offset_reg.to32(), limb_offset_reg.to32());
21212 loop.limb_reloc = @intCast(cg.mir_instructions.len);
2103221213 },
21033 );
21034 },
21035 else => |e| return e,
21036 };
21037 return;
21214 }
21215 while (true) {
21216 var mir_ops: [4]Operand = @splat(.none);
21217 mir_ops_len = dst_temps.len;
21218 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
21219 const mir_op, const ref_src_op, const extra_temp = op: switch (src_op) {
21220 .implicit => |op_index| .{ &mir_ops[op_index], pattern.ops[op_index], &extra_temps[op_index] },
21221 .explicit => |op_index| {
21222 defer mir_ops_len += 1;
21223 break :op .{ &mir_ops[mir_ops_len], pattern.ops[op_index], &extra_temps[mir_ops_len] };
21224 },
21225 else => {
21226 defer mir_ops_len += 1;
21227 break :op .{ &mir_ops[mir_ops_len], src_op, &extra_temps[mir_ops_len] };
21228 },
21229 };
21230 const src_mcv = src_temp.tracking(cg).short;
21231 switch (ref_src_op) {
21232 else => {},
21233 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (src_mcv) {
21234 .register_pair => {},
21235 else => try cg.asmRegisterMemory(
21236 switch (ref_src_op) {
21237 else => unreachable,
21238 .gpr_limb => .{ ._, .mov },
21239 .mm_limb => .{ ._q, .mov },
21240 .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu },
21241 },
21242 registerAlias(extra_temp.*.?.tracking(cg).short.register, loop.limb_size.?),
21243 try src_mcv.mem(cg, switch (loop.limb_offset) {
21244 .unused => unreachable,
21245 .known => |limb_offset| .{
21246 .size = .fromSize(loop.limb_size.?),
21247 .disp = limb_offset,
21248 },
21249 .temp => |limb_offset| .{
21250 .size = .fromSize(loop.limb_size.?),
21251 .index = limb_offset.tracking(cg).short.register.to64(),
21252 },
21253 }),
21254 ),
21255 },
21256 }
21257 mir_op.* = switch (ref_src_op) {
21258 .implicit, .explicit => unreachable,
21259 .gpr => .{ .reg = registerAlias(
21260 src_mcv.register,
21261 @intCast(src_temp.typeOf(cg).abiSize(cg.pt.zcu)),
21262 ) },
21263 .mm => .{ .reg = src_mcv.register },
21264 .xmm => .{ .reg = src_mcv.register.to128() },
21265 .ymm => .{ .reg = src_mcv.register.to256() },
21266 .mem => .{ .mem = try src_mcv.mem(cg, .{ .size = cg.memSize(src_temp.typeOf(cg)) }) },
21267 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (src_mcv) {
21268 .register_pair => |src_regs| switch (loop.limb_offset) {
21269 .unused => unreachable,
21270 .known => |limb_offset| .{ .reg = registerAlias(
21271 src_regs[@divExact(limb_offset, loop.limb_size.?)],
21272 loop.limb_size.?,
21273 ) },
21274 .temp => unreachable,
21275 },
21276 else => .{ .reg = registerAlias(
21277 extra_temp.*.?.tracking(cg).short.register,
21278 loop.limb_size.?,
21279 ) },
21280 },
21281 .mem_limb => .{ .mem = switch (src_mcv) {
21282 .register_pair => unreachable,
21283 else => switch (loop.limb_offset) {
21284 .unused => unreachable,
21285 .known => |limb_offset| try src_mcv.mem(cg, .{
21286 .size = .fromSize(loop.limb_size.?),
21287 .disp = limb_offset,
21288 }),
21289 .temp => |limb_offset| try src_mcv.mem(cg, .{
21290 .size = .fromSize(loop.limb_size.?),
21291 .index = limb_offset.tracking(cg).short.register.to64(),
21292 }),
21293 },
21294 } },
21295 .imm => |imm| .{ .imm = .s(imm) },
21296 .simm32 => switch (src_temp.typeOf(cg).abiSize(cg.pt.zcu)) {
21297 else => unreachable,
21298 1 => .{ .imm = if (std.math.cast(i8, @as(i64, @bitCast(src_mcv.immediate)))) |small|
21299 .s(small)
21300 else
21301 .u(@as(u8, @intCast(src_mcv.immediate))) },
21302 2 => .{ .imm = if (std.math.cast(i16, @as(i64, @bitCast(src_mcv.immediate)))) |small|
21303 .s(small)
21304 else
21305 .u(@as(u16, @intCast(src_mcv.immediate))) },
21306 3...8 => .{ .imm = if (std.math.cast(i32, @as(i64, @bitCast(src_mcv.immediate)))) |small|
21307 .s(small)
21308 else
21309 .u(@as(u32, @intCast(src_mcv.immediate))) },
21310 },
21311 };
21312 }
21313 for (
21314 pattern.ops[0..dst_temps.len],
21315 dst_temps,
21316 mir_ops[0..dst_temps.len],
21317 extra_temps[0..dst_temps.len],
21318 ) |dst_op, *dst_temp, *mir_op, *extra_temp| {
21319 if (mir_op.* != .none) continue;
21320 const ty = src_temps[0].typeOf(cg);
21321 switch (dst_op) {
21322 .implicit => unreachable,
21323 .explicit => |op_index| {
21324 dst_temp.* = dst_temps[op_index];
21325 mir_op.* = mir_ops[op_index];
21326 },
21327 .gpr => {
21328 dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.gp);
21329 mir_op.* = .{ .reg = registerAlias(
21330 dst_temp.tracking(cg).short.register,
21331 @intCast(ty.abiSize(cg.pt.zcu)),
21332 ) };
21333 },
21334 .mm => @panic("TODO"),
21335 .xmm => {
21336 dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.sse);
21337 mir_op.* = .{ .reg = dst_temp.tracking(cg).short.register.to128() };
21338 },
21339 .ymm => {
21340 dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.sse);
21341 mir_op.* = .{ .reg = dst_temp.tracking(cg).short.register.to256() };
21342 },
21343 .mem => @panic("TODO"),
21344 .gpr_limb => {
21345 dst_temp.* = try cg.tempAlloc(ty);
21346 extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp);
21347 mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to64() };
21348 },
21349 .mm_limb => {
21350 dst_temp.* = try cg.tempAlloc(ty);
21351 extra_temp.* = try cg.tempAllocReg(Type.usize, @panic("TODO"));
21352 mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register };
21353 },
21354 .xmm_limb => {
21355 dst_temp.* = try cg.tempAlloc(ty);
21356 extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.sse);
21357 mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to128() };
21358 },
21359 .ymm_limb => {
21360 dst_temp.* = try cg.tempAlloc(ty);
21361 extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.sse);
21362 mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to256() };
21363 },
21364 .mem_limb => {
21365 dst_temp.* = try cg.tempAlloc(ty);
21366 mir_op.* = .{ .mem = try dst_temp.tracking(cg).short.mem(cg, switch (loop.limb_offset) {
21367 .unused => unreachable,
21368 .known => |limb_offset| .{
21369 .size = .fromSize(loop.limb_size.?),
21370 .disp = limb_offset,
21371 },
21372 .temp => |limb_offset| .{
21373 .size = .fromSize(loop.limb_size.?),
21374 .index = limb_offset.tracking(cg).short.register.to64(),
21375 },
21376 }) };
21377 },
21378 .imm, .simm32 => unreachable, // unmodifiable destination
21379 }
21380 }
21381 std.mem.swap(Operand, &mir_ops[pattern.commute[0]], &mir_ops[pattern.commute[1]]);
21382 cg.asmOps(pattern_set.mir_tag, mir_ops) catch |err| switch (err) {
21383 error.InvalidInstruction => {
21384 const fixes = @tagName(pattern_set.mir_tag[0]);
21385 const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?;
21386 return cg.fail(
21387 "invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'",
21388 .{
21389 fixes[0..fixes_blank],
21390 @tagName(pattern_set.mir_tag[1]),
21391 fixes[fixes_blank + 1 ..],
21392 @tagName(mir_ops[0]),
21393 @tagName(mir_ops[1]),
21394 @tagName(mir_ops[2]),
21395 @tagName(mir_ops[3]),
21396 },
21397 );
21398 },
21399 else => |e| return e,
21400 };
21401 for (
21402 extra_temps[0..dst_temps.len],
21403 pattern.ops[0..dst_temps.len],
21404 dst_temps,
21405 ) |maybe_extra_temp, dst_op, dst_temp| if (maybe_extra_temp) |extra_temp| switch (dst_op) {
21406 else => {},
21407 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (dst_temp.tracking(cg).short) {
21408 .register_pair => |dst_regs| switch (loop.limb_offset) {
21409 .unused => unreachable,
21410 .known => |limb_offset| try cg.asmRegisterRegister(
21411 .{ ._, .mov },
21412 dst_regs[@divExact(limb_offset, loop.limb_size.?)].to64(),
21413 extra_temp.tracking(cg).short.register.to64(),
21414 ),
21415 .temp => unreachable,
21416 },
21417 else => |dst_mcv| try cg.asmMemoryRegister(
21418 switch (dst_op) {
21419 else => unreachable,
21420 .gpr_limb => .{ ._, .mov },
21421 .mm_limb => .{ ._q, .mov },
21422 .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu },
21423 },
21424 try dst_mcv.mem(cg, switch (loop.limb_offset) {
21425 .unused => unreachable,
21426 .known => |limb_offset| .{
21427 .size = .fromSize(loop.limb_size.?),
21428 .disp = limb_offset,
21429 },
21430 .temp => |limb_offset| .{
21431 .size = .fromSize(loop.limb_size.?),
21432 .index = limb_offset.tracking(cg).short.register.to64(),
21433 },
21434 }),
21435 registerAlias(extra_temp.tracking(cg).short.register, loop.limb_size.?),
21436 ),
21437 },
21438 };
21439 switch (pattern_set.loop) {
21440 .once => break :pattern_sets,
21441 .bitwise => {},
21442 .limbwise_carry => @panic("TODO"),
21443 .limbwise_pairs_forward => @panic("TODO"),
21444 .limbwise_pairs_reverse => @panic("TODO"),
21445 .elementwise => @panic("TODO"),
21446 }
21447 switch (loop.limb_offset) {
21448 .unused => break,
21449 .known => |*limb_offset| {
21450 limb_offset.* += loop.limb_size.?;
21451 loop.remaining_size.? -= loop.limb_size.?;
21452 if (loop.remaining_size.? < loop.limb_size.? or
21453 (loop.element_size != null and limb_offset.* >= loop.element_size.?))
21454 {
21455 limb_offset.* = 0;
21456 break;
21457 }
21458 },
21459 .temp => |limb_offset| {
21460 const limb_offset_reg = limb_offset.tracking(cg).short.register;
21461 try cg.asmRegisterMemory(.{ ._, .lea }, limb_offset_reg.to32(), .{
21462 .base = .{ .reg = limb_offset_reg.to64() },
21463 .mod = .{ .rm = .{
21464 .size = .qword,
21465 .disp = loop.limb_size.?,
21466 } },
21467 });
21468 try cg.asmRegisterImmediate(
21469 .{ ._, .cmp },
21470 limb_offset_reg.to32(),
21471 .u(loop.element_size orelse loop.remaining_size.?),
21472 );
21473 _ = try cg.asmJccReloc(.b, loop.limb_reloc);
21474 try limb_offset.die(cg);
21475 break;
21476 },
21477 }
21478 }
21479 switch (loop.element_offset) {
21480 .unused => break :pattern_sets,
21481 .known => |*element_offset| {
21482 if (loop.remaining_size.? == 0) break :pattern_sets;
21483 element_offset.* += loop.element_size.?;
21484 },
21485 .temp => |element_offset| {
21486 if (true) @panic("TODO");
21487 try element_offset.die(cg);
21488 if (loop.remaining_size.? == 0) break :pattern_sets;
21489 break;
21490 },
21491 }
21492 }
21493 }
21494 } else {
21495 log.err("failed to select:", .{});
21496 for (src_temps) |src_temp| log.err("{}", .{src_temp.tracking(cg)});
21497 return cg.fail("failed to select", .{});
2103821498 }
21039 log.err("failed to select:", .{});
21040 for (src_temps) |src_temp| log.err("{}", .{src_temp.tracking(self)});
21041 return self.fail("failed to select", .{});
21499 for (extra_temps) |extra_temp| if (extra_temp) |temp| try temp.die(cg);
2104221500}
src/arch/x86_64/bits.zig+5-3
......@@ -474,13 +474,15 @@ pub const Memory = struct {
474474 };
475475
476476 pub const Mod = union(enum(u1)) {
477 rm: struct {
477 rm: Rm,
478 off: u64,
479
480 pub const Rm = struct {
478481 size: Size,
479482 index: Register = .none,
480483 scale: Scale = .@"1",
481484 disp: i32 = 0,
482 },
483 off: u64,
485 };
484486 };
485487
486488 pub const Size = enum(u4) {