authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-11-08 08:32:00-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-16 20:42:08-05:00
logbeadf702b8d0421cc84c47ebc9644ce07e22c306
tree2a9db54d61ab71db479248d06350dfa1f3f99d2a
parentaf1191ea8ba55a6eab53e0b561355bb116bdbf2d

x86_64: rewrite arithmetic

4 files changed, 693 insertions(+), 359 deletions(-)

lib/std/mem.zig+2-2
......@@ -883,8 +883,8 @@ fn SliceTo(comptime T: type, comptime end: std.meta.Elem(T)) type {
883883 @compileError("invalid type given to std.mem.sliceTo: " ++ @typeName(T));
884884}
885885
886/// Takes an array, a pointer to an array, a sentinel-terminated pointer, or a slice and
887/// iterates searching for the first occurrence of `end`, returning the scanned slice.
886/// Takes a pointer to an array, a sentinel-terminated pointer, or a slice and iterates searching for
887/// the first occurrence of `end`, returning the scanned slice.
888888/// If `end` is not found, the full length of the array/slice/sentinel terminated pointer is returned.
889889/// If the pointer type is sentinel terminated and `end` matches that terminator, the
890890/// resulting slice is also sentinel terminated.
src/arch/x86_64/CodeGen.zig+679-345
......@@ -767,7 +767,7 @@ const FrameAlloc = struct {
767767 .size = spill_size,
768768 .pad = @intCast(spill_size - abi_size),
769769 .alignment = ty.abiAlignment(zcu).maxStrict(
770 InternPool.Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
770 .fromNonzeroByteUnits(@min(spill_size, 8)),
771771 ),
772772 });
773773 }
......@@ -806,7 +806,7 @@ pub fn generate(
806806 const gpa = zcu.gpa;
807807 const ip = &zcu.intern_pool;
808808 const func = zcu.funcInfo(func_index);
809 const fn_type = Type.fromInterned(func.ty);
809 const fn_type: Type = .fromInterned(func.ty);
810810 const mod = zcu.navFileScope(func.owner_nav).mod;
811811
812812 var function: Self = .{
......@@ -846,7 +846,7 @@ pub fn generate(
846846 try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max);
847847 for (0..Temp.Index.max) |temp_index| {
848848 const temp: Temp.Index = @enumFromInt(temp_index);
849 function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), InstTracking.init(.none));
849 function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), .init(.none));
850850 }
851851
852852 wip_mir_log.debug("{}:", .{fmtNav(func.owner_nav, ip)});
......@@ -854,11 +854,11 @@ pub fn generate(
854854 try function.frame_allocs.resize(gpa, FrameIndex.named_count);
855855 function.frame_allocs.set(
856856 @intFromEnum(FrameIndex.stack_frame),
857 FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),
857 .init(.{ .size = 0, .alignment = .@"1" }),
858858 );
859859 function.frame_allocs.set(
860860 @intFromEnum(FrameIndex.call_frame),
861 FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),
861 .init(.{ .size = 0, .alignment = .@"1" }),
862862 );
863863
864864 const fn_info = zcu.typeToFunc(fn_type).?;
......@@ -871,20 +871,19 @@ pub fn generate(
871871
872872 function.args = call_info.args;
873873 function.ret_mcv = call_info.return_value;
874 function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), FrameAlloc.init(.{
874 function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), .init(.{
875875 .size = Type.usize.abiSize(zcu),
876876 .alignment = Type.usize.abiAlignment(zcu).min(call_info.stack_align),
877877 }));
878 function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{
878 function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), .init(.{
879879 .size = Type.usize.abiSize(zcu),
880 .alignment = InternPool.Alignment.min(
881 call_info.stack_align,
882 InternPool.Alignment.fromNonzeroByteUnits(function.target.stackAlignment()),
880 .alignment = call_info.stack_align.min(
881 .fromNonzeroByteUnits(function.target.stackAlignment()),
883882 ),
884883 }));
885884 function.frame_allocs.set(
886885 @intFromEnum(FrameIndex.args_frame),
887 FrameAlloc.init(.{
886 .init(.{
888887 .size = call_info.stack_byte_count,
889888 .alignment = call_info.stack_align,
890889 }),
......@@ -1242,6 +1241,80 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
12421241 return result;
12431242}
12441243
1244fn asmOps(self: *Self, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {
1245 return switch (ops[0]) {
1246 .none => self.asmOpOnly(tag),
1247 .reg => |reg0| switch (ops[1]) {
1248 .none => self.asmRegister(tag, reg0),
1249 .reg => |reg1| switch (ops[2]) {
1250 .none => self.asmRegisterRegister(tag, reg0, reg1),
1251 .reg => |reg2| switch (ops[3]) {
1252 .none => self.asmRegisterRegisterRegister(tag, reg0, reg1, reg2),
1253 .reg => |reg3| self.asmRegisterRegisterRegisterRegister(tag, reg0, reg1, reg2, reg3),
1254 .imm => |imm3| self.asmRegisterRegisterRegisterImmediate(tag, reg0, reg1, reg2, imm3),
1255 else => error.InvalidInstruction,
1256 },
1257 .mem => |mem2| switch (ops[3]) {
1258 .none => self.asmRegisterRegisterMemory(tag, reg0, reg1, mem2),
1259 .reg => |reg3| self.asmRegisterRegisterMemoryRegister(tag, reg0, reg1, mem2, reg3),
1260 .imm => |imm3| self.asmRegisterRegisterMemoryImmediate(tag, reg0, reg1, mem2, imm3),
1261 else => error.InvalidInstruction,
1262 },
1263 .imm => |imm2| switch (ops[3]) {
1264 .none => self.asmRegisterRegisterImmediate(tag, reg0, reg1, imm2),
1265 else => error.InvalidInstruction,
1266 },
1267 else => error.InvalidInstruction,
1268 },
1269 .mem => |mem1| switch (ops[2]) {
1270 .none => self.asmRegisterMemory(tag, reg0, mem1),
1271 .reg => |reg2| switch (ops[3]) {
1272 .none => self.asmRegisterMemoryRegister(tag, reg0, mem1, reg2),
1273 else => error.InvalidInstruction,
1274 },
1275 .imm => |imm2| switch (ops[3]) {
1276 .none => self.asmRegisterMemoryImmediate(tag, reg0, mem1, imm2),
1277 else => error.InvalidInstruction,
1278 },
1279 else => error.InvalidInstruction,
1280 },
1281 .imm => |imm1| switch (ops[2]) {
1282 .none => self.asmRegisterImmediate(tag, reg0, imm1),
1283 else => error.InvalidInstruction,
1284 },
1285 else => error.InvalidInstruction,
1286 },
1287 .mem => |mem0| switch (ops[1]) {
1288 .none => self.asmMemory(tag, mem0),
1289 .reg => |reg1| switch (ops[2]) {
1290 .none => self.asmMemoryRegister(tag, mem0, reg1),
1291 .reg => |reg2| switch (ops[3]) {
1292 .none => self.asmMemoryRegisterRegister(tag, mem0, reg1, reg2),
1293 else => error.InvalidInstruction,
1294 },
1295 .imm => |imm2| switch (ops[3]) {
1296 .none => self.asmMemoryRegisterImmediate(tag, mem0, reg1, imm2),
1297 else => error.InvalidInstruction,
1298 },
1299 else => error.InvalidInstruction,
1300 },
1301 .imm => |imm1| switch (ops[2]) {
1302 .none => self.asmMemoryImmediate(tag, mem0, imm1),
1303 else => error.InvalidInstruction,
1304 },
1305 else => error.InvalidInstruction,
1306 },
1307 .imm => |imm0| switch (ops[1]) {
1308 .none => self.asmImmediate(tag, imm0),
1309 else => error.InvalidInstruction,
1310 },
1311 .inst => |inst0| switch (ops[1]) {
1312 .none => self.asmReloc(tag, inst0),
1313 else => error.InvalidInstruction,
1314 },
1315 };
1316}
1317
12451318/// A `cc` of `.z_and_np` clobbers `reg2`!
12461319fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: Register) !void {
12471320 _ = try self.addInst(.{
......@@ -1256,7 +1329,7 @@ fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: R
12561329 },
12571330 .data = .{ .rr = .{
12581331 .fixes = switch (cc) {
1259 else => Mir.Inst.Fixes.fromCondition(cc),
1332 else => .fromCondition(cc),
12601333 .z_and_np, .nz_or_p => ._,
12611334 },
12621335 .r1 = reg1,
......@@ -1280,7 +1353,7 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)
12801353 },
12811354 .data = .{ .rx = .{
12821355 .fixes = switch (cc) {
1283 else => Mir.Inst.Fixes.fromCondition(cc),
1356 else => .fromCondition(cc),
12841357 .z_and_np => unreachable,
12851358 .nz_or_p => ._,
12861359 },
......@@ -1303,7 +1376,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
13031376 },
13041377 .data = switch (cc) {
13051378 else => .{ .r = .{
1306 .fixes = Mir.Inst.Fixes.fromCondition(cc),
1379 .fixes = .fromCondition(cc),
13071380 .r1 = reg,
13081381 } },
13091382 .z_and_np, .nz_or_p => .{ .rr = .{
......@@ -1328,7 +1401,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
13281401 },
13291402 .data = switch (cc) {
13301403 else => .{ .x = .{
1331 .fixes = Mir.Inst.Fixes.fromCondition(cc),
1404 .fixes = .fromCondition(cc),
13321405 .payload = payload,
13331406 } },
13341407 .z_and_np, .nz_or_p => .{ .rx = .{
......@@ -1362,7 +1435,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind
13621435 },
13631436 .data = .{ .inst = .{
13641437 .fixes = switch (cc) {
1365 else => Mir.Inst.Fixes.fromCondition(cc),
1438 else => .fromCondition(cc),
13661439 .z_and_np, .nz_or_p => ._,
13671440 },
13681441 .inst = target,
......@@ -1969,7 +2042,7 @@ fn gen(self: *Self) InnerError!void {
19692042 // The address where to store the return value for the caller is in a
19702043 // register which the callee is free to clobber. Therefore, we purposely
19712044 // spill it to stack immediately.
1972 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(Type.usize, zcu));
2045 const frame_index = try self.allocFrameIndex(.initSpill(Type.usize, zcu));
19732046 try self.genSetMem(
19742047 .{ .frame = frame_index },
19752048 0,
......@@ -1986,7 +2059,7 @@ fn gen(self: *Self) InnerError!void {
19862059 if (fn_info.is_var_args) switch (cc) {
19872060 .x86_64_sysv => {
19882061 const info = &self.va_info.sysv;
1989 const reg_save_area_fi = try self.allocFrameIndex(FrameAlloc.init(.{
2062 const reg_save_area_fi = try self.allocFrameIndex(.init(.{
19902063 .size = abi.SysV.c_abi_int_param_regs.len * 8 +
19912064 abi.SysV.c_abi_sse_param_regs.len * 16,
19922065 .alignment = .@"16",
......@@ -2192,7 +2265,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
21922265 wip_mir_log.debug("{}", .{self.fmtAir(inst)});
21932266 verbose_tracking_log.debug("{}", .{self.fmtTracking()});
21942267
2195 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
2268 self.reused_operands = .initEmpty();
21962269 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
21972270
21982271 try self.airArg(inst);
......@@ -2210,7 +2283,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
22102283 wip_mir_log.debug("{}", .{self.fmtAir(inst)});
22112284 verbose_tracking_log.debug("{}", .{self.fmtTracking()});
22122285
2213 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
2286 self.reused_operands = .initEmpty();
22142287 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
22152288 switch (air_tags[@intFromEnum(inst)]) {
22162289 // zig fmt: off
......@@ -2223,9 +2296,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
22232296 .sub_wrap,
22242297 .bool_and,
22252298 .bool_or,
2226 .bit_and,
2227 .bit_or,
2228 .xor,
22292299 .min,
22302300 .max,
22312301 => |tag| try self.airBinOp(inst, tag),
......@@ -2391,6 +2461,55 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
23912461 try slot.moveTo(inst, self);
23922462 },
23932463 .assembly => try self.airAsm(inst),
2464 inline .bit_and, .bit_or, .xor => |tag| if (use_old or self.typeOfIndex(inst).abiSize(zcu) > @as(
2465 u64,
2466 if (!self.typeOfIndex(inst).isVector(zcu))
2467 8
2468 else if (!self.hasFeature(.avx2))
2469 16
2470 else
2471 32,
2472 )) try self.airBinOp(inst, tag) else {
2473 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
2474 var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
2475 try self.spillEflagsIfOccupied();
2476 const mir_tag: Mir.Inst.Tag = switch (tag) {
2477 else => unreachable,
2478 .bit_and => .@"and",
2479 .bit_or => .@"or",
2480 .xor => .xor,
2481 };
2482 var res: [1]Temp = undefined;
2483 try self.select(&res, &.{ &ops[0], &ops[1] }, &.{
2484 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .ymm, .mem }, .features = &.{.avx2} },
2485 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .mem, .ymm }, .commute = .{ 1, 2 }, .features = &.{.avx2} },
2486 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .ymm, .ymm }, .features = &.{.avx2} },
2487 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .xmm, .mem }, .features = &.{.avx} },
2488 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .mem, .xmm }, .commute = .{ 1, 2 }, .features = &.{.avx} },
2489 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .xmm, .xmm }, .features = &.{.avx} },
2490 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .{ .match = 0 }, .mem }, .features = &.{.sse2} },
2491 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .mem, .{ .match = 0 } }, .features = &.{.sse2} },
2492 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .{ .match = 0 }, .xmm }, .features = &.{.sse2} },
2493 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .xmm, .{ .match = 0 } }, .features = &.{.sse2} },
2494 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .{ .match = 0 }, .mem }, .features = &.{.mmx} },
2495 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .mem, .{ .match = 0 } }, .features = &.{.mmx} },
2496 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .{ .match = 0 }, .mm }, .features = &.{.mmx} },
2497 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .mm, .{ .match = 0 } }, .features = &.{.mmx} },
2498 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .{ .match = 0 }, .simm32 } },
2499 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .simm32, .{ .match = 0 } } },
2500 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .{ .match = 0 }, .gpr } },
2501 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .gpr, .{ .match = 0 } } },
2502 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .simm32 } },
2503 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .simm32, .{ .match = 0 } } },
2504 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .mem } },
2505 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .mem, .{ .match = 0 } } },
2506 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .gpr } },
2507 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .gpr, .{ .match = 0 } } },
2508 });
2509 if (ops[0].index != res[0].index) try ops[0].die(self);
2510 if (ops[1].index != res[0].index) try ops[1].die(self);
2511 try res[0].moveTo(inst, self);
2512 },
23942513 .block => if (use_old) try self.airBlock(inst) else {
23952514 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
23962515 const extra = self.air.extraData(Air.Block, ty_pl.payload);
......@@ -2506,7 +2625,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
25062625 else if (opt_child_ty.isSlice(zcu))
25072626 .qword
25082627 else
2509 Memory.Size.fromSize(opt_child_abi_size)),
2628 .fromSize(opt_child_abi_size)),
25102629 .u(0),
25112630 );
25122631 var is_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });
......@@ -2529,7 +2648,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
25292648 else if (opt_child_ty.isSlice(zcu))
25302649 .qword
25312650 else
2532 Memory.Size.fromSize(opt_child_abi_size)),
2651 .fromSize(opt_child_abi_size)),
25332652 .u(0),
25342653 );
25352654 var is_non_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });
......@@ -2646,31 +2765,51 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
26462765 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
26472766 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
26482767 var ops = try self.tempsFromOperands(inst, .{extra.struct_operand});
2649 try ops[0].toOffset(self.fieldOffset(self.typeOf(extra.struct_operand), self.typeOfIndex(inst), extra.field_index), self);
2768 try ops[0].toOffset(self.fieldOffset(
2769 self.typeOf(extra.struct_operand),
2770 self.typeOfIndex(inst),
2771 extra.field_index,
2772 ), self);
26502773 try ops[0].moveTo(inst, self);
26512774 },
26522775 .struct_field_ptr_index_0 => if (use_old) try self.airStructFieldPtrIndex(inst, 0) else {
26532776 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
26542777 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
2655 try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 0), self);
2778 try ops[0].toOffset(self.fieldOffset(
2779 self.typeOf(ty_op.operand),
2780 self.typeOfIndex(inst),
2781 0,
2782 ), self);
26562783 try ops[0].moveTo(inst, self);
26572784 },
26582785 .struct_field_ptr_index_1 => if (use_old) try self.airStructFieldPtrIndex(inst, 1) else {
26592786 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
26602787 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
2661 try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 1), self);
2788 try ops[0].toOffset(self.fieldOffset(
2789 self.typeOf(ty_op.operand),
2790 self.typeOfIndex(inst),
2791 1,
2792 ), self);
26622793 try ops[0].moveTo(inst, self);
26632794 },
26642795 .struct_field_ptr_index_2 => if (use_old) try self.airStructFieldPtrIndex(inst, 2) else {
26652796 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
26662797 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
2667 try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 2), self);
2798 try ops[0].toOffset(self.fieldOffset(
2799 self.typeOf(ty_op.operand),
2800 self.typeOfIndex(inst),
2801 2,
2802 ), self);
26682803 try ops[0].moveTo(inst, self);
26692804 },
26702805 .struct_field_ptr_index_3 => if (use_old) try self.airStructFieldPtrIndex(inst, 3) else {
26712806 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
26722807 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
2673 try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 3), self);
2808 try ops[0].toOffset(self.fieldOffset(
2809 self.typeOf(ty_op.operand),
2810 self.typeOfIndex(inst),
2811 3,
2812 ), self);
26742813 try ops[0].moveTo(inst, self);
26752814 },
26762815 .slice => if (use_old) try self.airSlice(inst) else {
......@@ -2775,7 +2914,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
27752914 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
27762915 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
27772916 var ops = try self.tempsFromOperands(inst, .{extra.field_ptr});
2778 try ops[0].toOffset(-self.fieldOffset(self.typeOfIndex(inst), self.typeOf(extra.field_ptr), extra.field_index), self);
2917 try ops[0].toOffset(-self.fieldOffset(
2918 self.typeOfIndex(inst),
2919 self.typeOf(extra.field_ptr),
2920 extra.field_index,
2921 ), self);
27792922 try ops[0].moveTo(inst, self);
27802923 },
27812924
......@@ -2813,7 +2956,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
28132956 const ip = &zcu.intern_pool;
28142957 switch (Type.fromInterned(lazy_sym.ty).zigTypeTag(zcu)) {
28152958 .@"enum" => {
2816 const enum_ty = Type.fromInterned(lazy_sym.ty);
2959 const enum_ty: Type = .fromInterned(lazy_sym.ty);
28172960 wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)});
28182961
28192962 const resolved_cc = abi.resolveCallingConvention(.auto, self.target.*);
......@@ -2919,7 +3062,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
29193062 else => unreachable, // Why didn't the result die?
29203063 } else {
29213064 tracking_log.debug("{} => {} (birth)", .{ inst, result });
2922 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
3065 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));
29233066 // In some cases, an operand may be reused as the result.
29243067 // If that operand died and was a register, it was freed by
29253068 // processDeath, so we have to "re-allocate" the register.
......@@ -3067,7 +3210,7 @@ fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
30673210 const zcu = pt.zcu;
30683211 const ptr_ty = self.typeOfIndex(inst);
30693212 const val_ty = ptr_ty.childType(zcu);
3070 return self.allocFrameIndex(FrameAlloc.init(.{
3213 return self.allocFrameIndex(.init(.{
30713214 .size = std.math.cast(u32, val_ty.abiSize(zcu)) orelse {
30723215 return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(pt)});
30733216 },
......@@ -3107,35 +3250,49 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
31073250 },
31083251 else => 8,
31093252 })) {
3110 if (self.register_manager.tryAllocReg(inst, self.regClassForType(ty))) |reg| {
3253 if (self.register_manager.tryAllocReg(inst, self.regSetForType(ty))) |reg| {
31113254 return MCValue{ .register = registerAlias(reg, abi_size) };
31123255 }
31133256 }
31143257 }
31153258
3116 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, zcu));
3259 const frame_index = try self.allocFrameIndex(.initSpill(ty, zcu));
31173260 return .{ .load_frame = .{ .index = frame_index } };
31183261}
31193262
3120fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {
3263fn regClassForType(self: *Self, ty: Type) Register.Class {
31213264 const pt = self.pt;
31223265 const zcu = pt.zcu;
31233266 return switch (ty.zigTypeTag(zcu)) {
31243267 .float => switch (ty.floatBits(self.target.*)) {
3125 80 => abi.RegisterClass.x87,
3126 else => abi.RegisterClass.sse,
3268 80 => .x87,
3269 else => .sse,
31273270 },
31283271 .vector => switch (ty.childType(zcu).toIntern()) {
3129 .bool_type, .u1_type => abi.RegisterClass.gp,
3272 .bool_type, .u1_type => .general_purpose,
31303273 else => if (ty.isAbiInt(zcu) and ty.intInfo(zcu).bits == 1)
3131 abi.RegisterClass.gp
3274 .general_purpose
31323275 else
3133 abi.RegisterClass.sse,
3276 .sse,
31343277 },
3135 else => abi.RegisterClass.gp,
3278 else => .general_purpose,
31363279 };
31373280}
31383281
3282fn regSetForRegClass(rc: Register.Class) RegisterManager.RegisterBitSet {
3283 return switch (rc) {
3284 .general_purpose => abi.RegisterClass.gp,
3285 .segment, .ip => unreachable,
3286 .x87 => abi.RegisterClass.x87,
3287 .mmx => @panic("TODO"),
3288 .sse => abi.RegisterClass.sse,
3289 };
3290}
3291
3292fn regSetForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {
3293 return regSetForRegClass(self.regClassForType(ty));
3294}
3295
31393296const State = struct {
31403297 registers: RegisterManager.TrackedRegisters,
31413298 reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
......@@ -3292,7 +3449,7 @@ pub fn spillRegisters(self: *Self, comptime registers: []const Register) !void {
32923449/// allocated. A second call to `copyToTmpRegister` may return the same register.
32933450/// This can have a side effect of spilling instructions to the stack to free up a register.
32943451fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
3295 const reg = try self.register_manager.allocReg(null, self.regClassForType(ty));
3452 const reg = try self.register_manager.allocReg(null, self.regSetForType(ty));
32963453 try self.genSetReg(reg, ty, mcv, .{});
32973454 return reg;
32983455}
......@@ -3307,7 +3464,7 @@ fn copyToRegisterWithInstTracking(
33073464 ty: Type,
33083465 mcv: MCValue,
33093466) !MCValue {
3310 const reg: Register = try self.register_manager.allocReg(reg_owner, self.regClassForType(ty));
3467 const reg: Register = try self.register_manager.allocReg(reg_owner, self.regSetForType(ty));
33113468 try self.genSetReg(reg, ty, mcv, .{});
33123469 return MCValue{ .register = reg };
33133470}
......@@ -3910,7 +4067,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
39104067 .storage = .{ .repeated_elem = mask_val.ip_index },
39114068 } });
39124069
3913 const splat_mcv = try self.genTypedValue(Value.fromInterned(splat_val));
4070 const splat_mcv = try self.genTypedValue(.fromInterned(splat_val));
39144071 const splat_addr_mcv: MCValue = switch (splat_mcv) {
39154072 .memory, .indirect, .load_frame => splat_mcv.address(),
39164073 else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },
......@@ -3923,7 +4080,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
39234080 .{ .vp_, .@"and" },
39244081 dst_alias,
39254082 dst_alias,
3926 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
4083 try splat_addr_mcv.deref().mem(self, .fromSize(splat_abi_size)),
39274084 );
39284085 if (src_abi_size > 16) {
39294086 const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
......@@ -3947,7 +4104,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
39474104 try self.asmRegisterMemory(
39484105 .{ .p_, .@"and" },
39494106 dst_alias,
3950 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
4107 try splat_addr_mcv.deref().mem(self, .fromSize(splat_abi_size)),
39514108 );
39524109 try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
39534110 }
......@@ -3992,7 +4149,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
39924149 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
39934150
39944151 const slice_ty = self.typeOfIndex(inst);
3995 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));
4152 const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu));
39964153
39974154 const ptr_ty = self.typeOf(bin_op.lhs);
39984155 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }, .{});
......@@ -4129,7 +4286,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
41294286 state: State,
41304287 reloc: Mir.Inst.Index,
41314288 } = if (signed and tag == .div_floor) state: {
4132 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(Type.usize, zcu));
4289 const frame_index = try self.allocFrameIndex(.initType(Type.usize, zcu));
41334290 try self.asmMemoryImmediate(
41344291 .{ ._, .mov },
41354292 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
......@@ -4259,7 +4416,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
42594416 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
42604417
42614418 const tmp_regs =
4262 try self.register_manager.allocRegs(2, .{null} ** 2, abi.RegisterClass.gp);
4419 try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
42634420 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
42644421 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
42654422
......@@ -4698,8 +4855,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
46984855 else => {},
46994856 }
47004857
4701 const frame_index =
4702 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
4858 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
47034859 try self.genSetMem(
47044860 .{ .frame = frame_index },
47054861 @intCast(tuple_ty.structFieldOffset(1, zcu)),
......@@ -4717,8 +4873,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
47174873 break :result .{ .load_frame = .{ .index = frame_index } };
47184874 }
47194875
4720 const frame_index =
4721 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
4876 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
47224877 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
47234878 break :result .{ .load_frame = .{ .index = frame_index } };
47244879 },
......@@ -4776,8 +4931,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
47764931 else => {},
47774932 }
47784933
4779 const frame_index =
4780 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
4934 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
47814935 try self.genSetMem(
47824936 .{ .frame = frame_index },
47834937 @intCast(tuple_ty.structFieldOffset(1, zcu)),
......@@ -4796,7 +4950,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
47964950 }
47974951
47984952 const frame_index =
4799 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
4953 try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
48004954 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
48014955 break :result .{ .load_frame = .{ .index = frame_index } };
48024956 },
......@@ -4833,7 +4987,7 @@ fn genSetFrameTruncatedOverflowCompare(
48334987 const rest_ty = try pt.intType(.unsigned, int_info.bits - hi_bits);
48344988
48354989 const temp_regs =
4836 try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);
4990 try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp);
48374991 const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
48384992 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
48394993
......@@ -4916,7 +5070,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
49165070 const rhs_mcv = try self.resolveInst(bin_op.rhs);
49175071
49185072 const temp_regs =
4919 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
5073 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
49205074 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
49215075 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
49225076
......@@ -5080,7 +5234,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
50805234 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
50815235
50825236 const tmp_regs =
5083 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
5237 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
50845238 const tmp_locks = self.register_manager.lockRegsAssumeUnused(4, tmp_regs);
50855239 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
50865240
......@@ -5217,7 +5371,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
52175371 self.eflags_inst = inst;
52185372 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
52195373 } else {
5220 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
5374 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
52215375 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
52225376 break :result .{ .load_frame = .{ .index = frame_index } };
52235377 },
......@@ -5228,7 +5382,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
52285382 src_ty.fmt(pt), dst_ty.fmt(pt),
52295383 });
52305384
5231 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
5385 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
52325386 if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
52335387 try self.genSetMem(
52345388 .{ .frame = frame_index },
......@@ -5301,10 +5455,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
53015455 };
53025456 switch (mat_rhs) {
53035457 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),
5304 .memory, .indirect, .load_frame => try self.asmMemory(
5305 tag,
5306 try mat_rhs.mem(self, Memory.Size.fromSize(abi_size)),
5307 ),
5458 .memory, .indirect, .load_frame => try self.asmMemory(tag, try mat_rhs.mem(self, .fromSize(abi_size))),
53085459 else => unreachable,
53095460 }
53105461 if (tag[1] == .div and bit_size == 8) try self.asmRegisterRegister(.{ ._, .mov }, .dl, .ah);
......@@ -5575,7 +5726,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
55755726 defer self.register_manager.unlockReg(shift_lock);
55765727
55775728 const mask_ty = try pt.vectorType(.{ .len = 16, .child = .u8_type });
5578 const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
5729 const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
55795730 .ty = mask_ty.toIntern(),
55805731 .storage = .{ .elems = &([1]InternPool.Index{
55815732 (try rhs_ty.childType(zcu).maxIntScalar(pt, Type.u8)).toIntern(),
......@@ -5811,7 +5962,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
58115962 .{
58125963 .base = .{ .reg = src_reg },
58135964 .mod = .{ .rm = .{
5814 .size = Memory.Size.fromSize(err_abi_size),
5965 .size = .fromSize(err_abi_size),
58155966 .disp = err_off,
58165967 } },
58175968 },
......@@ -5853,7 +6004,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
58536004 .{
58546005 .base = .{ .reg = src_reg },
58556006 .mod = .{ .rm = .{
5856 .size = Memory.Size.fromSize(err_abi_size),
6007 .size = .fromSize(err_abi_size),
58576008 .disp = err_off,
58586009 } },
58596010 },
......@@ -5909,7 +6060,7 @@ fn genUnwrapErrUnionPayloadMir(
59096060 const eu_lock = self.register_manager.lockReg(reg);
59106061 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
59116062
5912 const payload_in_gp = self.regClassForType(payload_ty).supersetOf(abi.RegisterClass.gp);
6063 const payload_in_gp = self.regSetForType(payload_ty).supersetOf(abi.RegisterClass.gp);
59136064 const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)
59146065 try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)
59156066 else
......@@ -6043,7 +6194,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
60436194 const result: MCValue = result: {
60446195 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 };
60456196
6046 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
6197 const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu));
60476198 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
60486199 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
60496200 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{});
......@@ -6066,7 +6217,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
60666217 const result: MCValue = result: {
60676218 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand);
60686219
6069 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
6220 const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu));
60706221 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
60716222 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
60726223 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{});
......@@ -6297,7 +6448,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
62976448 index_reg.to64(),
62986449 ),
62996450 .sse => {
6300 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, zcu));
6451 const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu));
63016452 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});
63026453 try self.asmMemoryRegister(
63036454 .{ ._, .bt },
......@@ -6342,7 +6493,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
63426493
63436494 switch (array_mcv) {
63446495 .register => {
6345 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, zcu));
6496 const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu));
63466497 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});
63476498 try self.asmRegisterMemory(
63486499 .{ ._, .lea },
......@@ -7175,7 +7326,7 @@ fn genByteSwap(
71757326 const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
71767327
71777328 const temp_regs =
7178 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
7329 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
71797330 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
71807331 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
71817332
......@@ -7472,11 +7623,11 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
74727623 else => unreachable,
74737624 });
74747625 const sign_mem: Memory = if (sign_mcv.isMemory())
7475 try sign_mcv.mem(self, Memory.Size.fromSize(abi_size))
7626 try sign_mcv.mem(self, .fromSize(abi_size))
74767627 else
74777628 .{
74787629 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
7479 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } },
7630 .mod = .{ .rm = .{ .size = .fromSize(abi_size) } },
74807631 };
74817632
74827633 if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(
......@@ -7657,7 +7808,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
76577808 mir_tag,
76587809 dst_alias,
76597810 dst_alias,
7660 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7811 try src_mcv.mem(self, .fromSize(abi_size)),
76617812 .u(@as(u5, @bitCast(mode))),
76627813 ) else try self.asmRegisterRegisterRegisterImmediate(
76637814 mir_tag,
......@@ -7672,7 +7823,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
76727823 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
76737824 mir_tag,
76747825 dst_alias,
7675 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7826 try src_mcv.mem(self, .fromSize(abi_size)),
76767827 .u(@as(u5, @bitCast(mode))),
76777828 ) else try self.asmRegisterRegisterImmediate(
76787829 mir_tag,
......@@ -7714,7 +7865,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
77147865 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
77157866 .l,
77167867 registerAlias(dst_mcv.register, cmov_abi_size),
7717 try src_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
7868 try src_mcv.mem(self, .fromSize(cmov_abi_size)),
77187869 ),
77197870 else => {
77207871 const val_reg = try self.copyToTmpRegister(ty, src_mcv);
......@@ -7767,7 +7918,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
77677918 const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;
77687919
77697920 const tmp_regs =
7770 try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);
7921 try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp);
77717922 const tmp_locks = self.register_manager.lockRegsAssumeUnused(3, tmp_regs);
77727923 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
77737924
......@@ -7868,7 +8019,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
78688019 const dst_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
78698020 src_mcv.getReg().?
78708021 else
7871 try self.register_manager.allocReg(inst, self.regClassForType(ty));
8022 try self.register_manager.allocReg(inst, self.regSetForType(ty));
78728023 const dst_alias = registerAlias(dst_reg, abi_size);
78738024 if (src_mcv.isMemory()) try self.asmRegisterMemory(
78748025 mir_tag,
......@@ -7980,7 +8131,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
79808131 if (src_mcv.isMemory()) try self.asmRegisterMemory(
79818132 .{ .v_ps, .cvtph2 },
79828133 wide_reg,
7983 try src_mcv.mem(self, Memory.Size.fromSize(
8134 try src_mcv.mem(self, .fromSize(
79848135 @intCast(@divExact(wide_reg.bitSize(), 16)),
79858136 )),
79868137 ) else try self.asmRegisterRegister(
......@@ -8026,7 +8177,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
80268177 mir_tag,
80278178 dst_reg,
80288179 dst_reg,
8029 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
8180 try src_mcv.mem(self, .fromSize(abi_size)),
80308181 ) else try self.asmRegisterRegisterRegister(
80318182 mir_tag,
80328183 dst_reg,
......@@ -8039,7 +8190,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
80398190 else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
80408191 mir_tag,
80418192 dst_reg,
8042 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
8193 try src_mcv.mem(self, .fromSize(abi_size)),
80438194 ) else try self.asmRegisterRegister(
80448195 mir_tag,
80458196 dst_reg,
......@@ -8136,7 +8287,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
81368287 const zcu = pt.zcu;
81378288
81388289 const ptr_info = ptr_ty.ptrInfo(zcu);
8139 const val_ty = Type.fromInterned(ptr_info.child);
8290 const val_ty: Type = .fromInterned(ptr_info.child);
81408291 if (!val_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;
81418292 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));
81428293
......@@ -8204,7 +8355,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
82048355 try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
82058356 .base = .{ .reg = ptr_reg },
82068357 .mod = .{ .rm = .{
8207 .size = Memory.Size.fromSize(load_abi_size),
8358 .size = .fromSize(load_abi_size),
82088359 .disp = val_byte_off,
82098360 } },
82108361 });
......@@ -8220,14 +8371,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
82208371 try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{
82218372 .base = .{ .reg = ptr_reg },
82228373 .mod = .{ .rm = .{
8223 .size = Memory.Size.fromSize(val_abi_size),
8374 .size = .fromSize(val_abi_size),
82248375 .disp = val_byte_off,
82258376 } },
82268377 });
82278378 try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
82288379 .base = .{ .reg = ptr_reg },
82298380 .mod = .{ .rm = .{
8230 .size = Memory.Size.fromSize(val_abi_size),
8381 .size = .fromSize(val_abi_size),
82318382 .disp = val_byte_off + limb_abi_size,
82328383 } },
82338384 });
......@@ -8297,8 +8448,8 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
82978448 const ptr_ty = self.typeOf(ty_op.operand);
82988449 const elem_size = elem_ty.abiSize(zcu);
82998450
8300 const elem_rc = self.regClassForType(elem_ty);
8301 const ptr_rc = self.regClassForType(ptr_ty);
8451 const elem_rc = self.regSetForType(elem_ty);
8452 const ptr_rc = self.regSetForType(ptr_ty);
83028453
83038454 const ptr_mcv = try self.resolveInst(ty_op.operand);
83048455 const dst_mcv = if (elem_size <= 8 and elem_rc.supersetOf(ptr_rc) and
......@@ -8345,7 +8496,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
83458496 const pt = self.pt;
83468497 const zcu = pt.zcu;
83478498 const ptr_info = ptr_ty.ptrInfo(zcu);
8348 const src_ty = Type.fromInterned(ptr_info.child);
8499 const src_ty: Type = .fromInterned(ptr_info.child);
83498500 if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;
83508501
83518502 const limb_abi_size: u16 = @min(ptr_info.packed_offset.host_size, 8);
......@@ -8373,7 +8524,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
83738524 const limb_mem: Memory = .{
83748525 .base = .{ .reg = ptr_reg },
83758526 .mod = .{ .rm = .{
8376 .size = Memory.Size.fromSize(limb_abi_size),
8527 .size = .fromSize(limb_abi_size),
83778528 .disp = src_byte_off + limb_i * limb_abi_size,
83788529 } },
83798530 };
......@@ -8563,10 +8714,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
85638714 const index = extra.field_index;
85648715
85658716 const container_ty = self.typeOf(operand);
8566 const container_rc = self.regClassForType(container_ty);
8717 const container_rc = self.regSetForType(container_ty);
85678718 const field_ty = container_ty.fieldType(index, zcu);
85688719 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
8569 const field_rc = self.regClassForType(field_ty);
8720 const field_rc = self.regSetForType(field_ty);
85708721 const field_is_gp = field_rc.supersetOf(abi.RegisterClass.gp);
85718722
85728723 const src_mcv = try self.resolveInst(operand);
......@@ -8624,7 +8775,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
86248775 const dst_regs: [2]Register = if (field_rc.supersetOf(container_rc) and
86258776 self.reuseOperand(inst, operand, 0, src_mcv)) src_regs else dst: {
86268777 const dst_regs =
8627 try self.register_manager.allocRegs(2, .{null} ** 2, field_rc);
8778 try self.register_manager.allocRegs(2, @splat(null), field_rc);
86288779 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
86298780 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
86308781
......@@ -8786,7 +8937,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
87868937 try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
87878938 .base = .{ .frame = frame_addr.index },
87888939 .mod = .{ .rm = .{
8789 .size = Memory.Size.fromSize(load_abi_size),
8940 .size = .fromSize(load_abi_size),
87908941 .disp = frame_addr.off + field_byte_off,
87918942 } },
87928943 });
......@@ -8807,7 +8958,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
88078958 .{
88088959 .base = .{ .frame = frame_addr.index },
88098960 .mod = .{ .rm = .{
8810 .size = Memory.Size.fromSize(field_abi_size),
8961 .size = .fromSize(field_abi_size),
88118962 .disp = frame_addr.off + field_byte_off,
88128963 } },
88138964 },
......@@ -8815,7 +8966,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
88158966 try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
88168967 .base = .{ .frame = frame_addr.index },
88178968 .mod = .{ .rm = .{
8818 .size = Memory.Size.fromSize(field_abi_size),
8969 .size = .fromSize(field_abi_size),
88198970 .disp = frame_addr.off + field_byte_off + limb_abi_size,
88208971 } },
88218972 });
......@@ -8977,12 +9128,12 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
89779128
89789129 try self.genSetReg(addr_reg, Type.usize, dst_mcv.address(), .{});
89799130 try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{
8980 .size = Memory.Size.fromSize(abi_size),
9131 .size = .fromSize(abi_size),
89819132 } } });
89829133 },
89839134 .indirect, .load_frame => try self.asmMemory(
89849135 mir_tag,
8985 try dst_mcv.mem(self, Memory.Size.fromSize(abi_size)),
9136 try dst_mcv.mem(self, .fromSize(abi_size)),
89869137 ),
89879138 }
89889139}
......@@ -9024,7 +9175,7 @@ fn genShiftBinOpMir(
90249175 };
90259176 defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
90269177
9027 const temp_regs = try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
9178 const temp_regs = try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
90289179 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
90299180 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
90309181
......@@ -9576,7 +9727,7 @@ fn genShiftBinOpMir(
95769727 .memory => |addr| .{
95779728 .base = .{ .reg = .ds },
95789729 .mod = .{ .rm = .{
9579 .size = Memory.Size.fromSize(abi_size),
9730 .size = .fromSize(abi_size),
95809731 .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
95819732 return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
95829733 @tagName(lhs_mcv),
......@@ -9587,14 +9738,14 @@ fn genShiftBinOpMir(
95879738 .indirect => |reg_off| .{
95889739 .base = .{ .reg = reg_off.reg },
95899740 .mod = .{ .rm = .{
9590 .size = Memory.Size.fromSize(abi_size),
9741 .size = .fromSize(abi_size),
95919742 .disp = reg_off.off,
95929743 } },
95939744 },
95949745 .load_frame => |frame_addr| .{
95959746 .base = .{ .frame = frame_addr.index },
95969747 .mod = .{ .rm = .{
9597 .size = Memory.Size.fromSize(abi_size),
9748 .size = .fromSize(abi_size),
95989749 .disp = frame_addr.off,
95999750 } },
96009751 },
......@@ -9792,7 +9943,7 @@ fn genMulDivBinOp(
97929943 );
97939944
97949945 const temp_regs =
9795 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
9946 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
97969947 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
97979948 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
97989949
......@@ -10208,7 +10359,7 @@ fn genBinOp(
1020810359 mir_tag,
1020910360 dst_reg,
1021010361 dst_reg,
10211 try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
10362 try src_mcv.mem(self, .fromBitSize(float_bits)),
1021210363 ) else try self.asmRegisterRegisterRegister(
1021310364 mir_tag,
1021410365 dst_reg,
......@@ -10227,7 +10378,7 @@ fn genBinOp(
1022710378 if (src_mcv.isMemory()) try self.asmRegisterMemory(
1022810379 mir_tag,
1022910380 dst_reg,
10230 try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
10381 try src_mcv.mem(self, .fromBitSize(float_bits)),
1023110382 ) else try self.asmRegisterRegister(
1023210383 mir_tag,
1023310384 dst_reg,
......@@ -10358,14 +10509,14 @@ fn genBinOp(
1035810509 const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs);
1035910510 break :locks .{ locks[0], locks[1] };
1036010511 },
10361 else => .{null} ** 2,
10512 else => @splat(null),
1036210513 };
1036310514 defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
1036410515
1036510516 const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) {
1036610517 .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },
1036710518 .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),
10368 else => .{null} ** 2,
10519 else => @splat(null),
1036910520 };
1037010521 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1037110522
......@@ -10398,7 +10549,7 @@ fn genBinOp(
1039810549 const dst_locks: [2]?RegisterLock = switch (dst_mcv) {
1039910550 .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null },
1040010551 .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs),
10401 else => .{null} ** 2,
10552 else => @splat(null),
1040210553 };
1040310554 defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);
1040410555
......@@ -10414,7 +10565,7 @@ fn genBinOp(
1041410565 const src_locks: [2]?RegisterLock = switch (src_mcv) {
1041510566 .register => |src_reg| .{ self.register_manager.lockReg(src_reg), null },
1041610567 .register_pair => |src_regs| self.register_manager.lockRegs(2, src_regs),
10417 else => .{null} ** 2,
10568 else => @splat(null),
1041810569 };
1041910570 defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock);
1042010571
......@@ -10472,11 +10623,7 @@ fn genBinOp(
1047210623 const dst_regs = switch (dst_mcv) {
1047310624 .register_pair => |dst_regs| dst_regs,
1047410625 else => dst: {
10475 const dst_regs = try self.register_manager.allocRegs(
10476 2,
10477 .{null} ** 2,
10478 abi.RegisterClass.gp,
10479 );
10626 const dst_regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
1048010627 const dst_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
1048110628 defer for (dst_regs_locks) |lock| self.register_manager.unlockReg(lock);
1048210629
......@@ -10624,21 +10771,21 @@ fn genBinOp(
1062410771 .memory => |addr| .{
1062510772 .base = .{ .reg = .ds },
1062610773 .mod = .{ .rm = .{
10627 .size = Memory.Size.fromSize(cmov_abi_size),
10774 .size = .fromSize(cmov_abi_size),
1062810775 .disp = @intCast(@as(i64, @bitCast(addr))),
1062910776 } },
1063010777 },
1063110778 .indirect => |reg_off| .{
1063210779 .base = .{ .reg = reg_off.reg },
1063310780 .mod = .{ .rm = .{
10634 .size = Memory.Size.fromSize(cmov_abi_size),
10781 .size = .fromSize(cmov_abi_size),
1063510782 .disp = reg_off.off,
1063610783 } },
1063710784 },
1063810785 .load_frame => |frame_addr| .{
1063910786 .base = .{ .frame = frame_addr.index },
1064010787 .mod = .{ .rm = .{
10641 .size = Memory.Size.fromSize(cmov_abi_size),
10788 .size = .fromSize(cmov_abi_size),
1064210789 .disp = frame_addr.off,
1064310790 } },
1064410791 },
......@@ -11433,8 +11580,8 @@ fn genBinOp(
1143311580 dst_reg,
1143411581 lhs_reg,
1143511582 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11436 else => Memory.Size.fromSize(abi_size),
11437 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
11583 else => .fromSize(abi_size),
11584 .vector => .fromBitSize(dst_reg.bitSize()),
1143811585 }),
1143911586 ) else try self.asmRegisterRegisterRegister(
1144011587 mir_tag,
......@@ -11451,8 +11598,8 @@ fn genBinOp(
1145111598 mir_tag,
1145211599 dst_reg,
1145311600 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11454 else => Memory.Size.fromSize(abi_size),
11455 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
11601 else => .fromSize(abi_size),
11602 .vector => .fromBitSize(dst_reg.bitSize()),
1145611603 }),
1145711604 ) else try self.asmRegisterRegister(
1145811605 mir_tag,
......@@ -11479,8 +11626,8 @@ fn genBinOp(
1147911626 dst_reg,
1148011627 lhs_reg,
1148111628 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11482 else => Memory.Size.fromSize(abi_size),
11483 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
11629 else => .fromSize(abi_size),
11630 .vector => .fromBitSize(dst_reg.bitSize()),
1148411631 }),
1148511632 imm,
1148611633 ) else try self.asmRegisterRegisterRegisterImmediate(
......@@ -11499,8 +11646,8 @@ fn genBinOp(
1149911646 mir_tag,
1150011647 dst_reg,
1150111648 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11502 else => Memory.Size.fromSize(abi_size),
11503 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
11649 else => .fromSize(abi_size),
11650 .vector => .fromBitSize(dst_reg.bitSize()),
1150411651 }),
1150511652 imm,
1150611653 ) else try self.asmRegisterRegisterImmediate(
......@@ -11707,11 +11854,11 @@ fn genBinOp(
1170711854 const unsigned_ty = try lhs_ty.toUnsigned(pt);
1170811855 const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty));
1170911856 const not_mem: Memory = if (not_mcv.isMemory())
11710 try not_mcv.mem(self, Memory.Size.fromSize(abi_size))
11857 try not_mcv.mem(self, .fromSize(abi_size))
1171111858 else
1171211859 .{ .base = .{
1171311860 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
11714 }, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };
11861 }, .mod = .{ .rm = .{ .size = .fromSize(abi_size) } } };
1171511862 switch (mir_tag[0]) {
1171611863 .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
1171711864 .{ .vp_, .xor },
......@@ -11891,21 +12038,21 @@ fn genBinOpMir(
1189112038 .memory => |addr| .{
1189212039 .base = .{ .reg = .ds },
1189312040 .mod = .{ .rm = .{
11894 .size = Memory.Size.fromSize(limb_abi_size),
12041 .size = .fromSize(limb_abi_size),
1189512042 .disp = std.math.cast(i32, addr + off) orelse break :direct,
1189612043 } },
1189712044 },
1189812045 .indirect => |reg_off| .{
1189912046 .base = .{ .reg = reg_off.reg },
1190012047 .mod = .{ .rm = .{
11901 .size = Memory.Size.fromSize(limb_abi_size),
12048 .size = .fromSize(limb_abi_size),
1190212049 .disp = reg_off.off + off,
1190312050 } },
1190412051 },
1190512052 .load_frame => |frame_addr| .{
1190612053 .base = .{ .frame = frame_addr.index },
1190712054 .mod = .{ .rm = .{
11908 .size = Memory.Size.fromSize(limb_abi_size),
12055 .size = .fromSize(limb_abi_size),
1190912056 .disp = frame_addr.off + off,
1191012057 } },
1191112058 },
......@@ -12054,21 +12201,21 @@ fn genBinOpMir(
1205412201 => .{
1205512202 .base = .{ .reg = dst_info.?.addr_reg },
1205612203 .mod = .{ .rm = .{
12057 .size = Memory.Size.fromSize(limb_abi_size),
12204 .size = .fromSize(limb_abi_size),
1205812205 .disp = off,
1205912206 } },
1206012207 },
1206112208 .indirect => |reg_off| .{
1206212209 .base = .{ .reg = reg_off.reg },
1206312210 .mod = .{ .rm = .{
12064 .size = Memory.Size.fromSize(limb_abi_size),
12211 .size = .fromSize(limb_abi_size),
1206512212 .disp = reg_off.off + off,
1206612213 } },
1206712214 },
1206812215 .load_frame => |frame_addr| .{
1206912216 .base = .{ .frame = frame_addr.index },
1207012217 .mod = .{ .rm = .{
12071 .size = Memory.Size.fromSize(limb_abi_size),
12218 .size = .fromSize(limb_abi_size),
1207212219 .disp = frame_addr.off + off,
1207312220 } },
1207412221 },
......@@ -12277,7 +12424,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
1227712424 .memory => |addr| .{
1227812425 .base = .{ .reg = .ds },
1227912426 .mod = .{ .rm = .{
12280 .size = Memory.Size.fromSize(abi_size),
12427 .size = .fromSize(abi_size),
1228112428 .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
1228212429 return self.asmRegisterRegister(
1228312430 .{ .i_, .mul },
......@@ -12292,14 +12439,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
1229212439 .indirect => |reg_off| .{
1229312440 .base = .{ .reg = reg_off.reg },
1229412441 .mod = .{ .rm = .{
12295 .size = Memory.Size.fromSize(abi_size),
12442 .size = .fromSize(abi_size),
1229612443 .disp = reg_off.off,
1229712444 } },
1229812445 },
1229912446 .load_frame => |frame_addr| .{
1230012447 .base = .{ .frame = frame_addr.index },
1230112448 .mod = .{ .rm = .{
12302 .size = Memory.Size.fromSize(abi_size),
12449 .size = .fromSize(abi_size),
1230312450 .disp = frame_addr.off,
1230412451 } },
1230512452 },
......@@ -12475,7 +12622,7 @@ fn genLocalDebugInfo(
1247512622 self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op.operand,
1247612623 ),
1247712624 };
12478 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, self.pt.zcu));
12625 const frame_index = try self.allocFrameIndex(.initSpill(ty, self.pt.zcu));
1247912626 try self.genSetMem(.{ .frame = frame_index }, 0, ty, mcv, .{});
1248012627 try self.asmAirMemory(.dbg_local, inst, .{
1248112628 .base = .{ .frame = frame_index },
......@@ -12602,7 +12749,7 @@ fn genCall(self: *Self, info: union(enum) {
1260212749 const frame_indices = try allocator.alloc(FrameIndex, args.len);
1260312750 defer allocator.free(frame_indices);
1260412751
12605 var reg_locks = std.ArrayList(?RegisterLock).init(allocator);
12752 var reg_locks: std.ArrayList(?RegisterLock) = .init(allocator);
1260612753 defer reg_locks.deinit();
1260712754 try reg_locks.ensureTotalCapacity(16);
1260812755 defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
......@@ -12612,7 +12759,7 @@ fn genCall(self: *Self, info: union(enum) {
1261212759
1261312760 // We need a properly aligned and sized call frame to be able to call this function.
1261412761 {
12615 const needed_call_frame = FrameAlloc.init(.{
12762 const needed_call_frame: FrameAlloc = .init(.{
1261612763 .size = call_info.stack_byte_count,
1261712764 .alignment = call_info.stack_align,
1261812765 });
......@@ -12647,7 +12794,7 @@ fn genCall(self: *Self, info: union(enum) {
1264712794 try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs));
1264812795 },
1264912796 .indirect => |reg_off| {
12650 frame_index.* = try self.allocFrameIndex(FrameAlloc.initType(arg_ty, zcu));
12797 frame_index.* = try self.allocFrameIndex(.initType(arg_ty, zcu));
1265112798 try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, .{});
1265212799 try self.register_manager.getReg(reg_off.reg, null);
1265312800 try reg_locks.append(self.register_manager.lockReg(reg_off.reg));
......@@ -12718,8 +12865,8 @@ fn genCall(self: *Self, info: union(enum) {
1271812865 switch (call_info.return_value.long) {
1271912866 .none, .unreach => {},
1272012867 .indirect => |reg_off| {
12721 const ret_ty = Type.fromInterned(fn_info.return_type);
12722 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ret_ty, zcu));
12868 const ret_ty: Type = .fromInterned(fn_info.return_type);
12869 const frame_index = try self.allocFrameIndex(.initSpill(ret_ty, zcu));
1272312870 try self.genSetReg(reg_off.reg, Type.usize, .{
1272412871 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
1272512872 }, .{});
......@@ -12936,7 +13083,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
1293613083 self.register_manager.lockRegAssumeUnused(lhs_ro.reg),
1293713084 null,
1293813085 },
12939 else => .{null} ** 2,
13086 else => @splat(null),
1294013087 };
1294113088 defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
1294213089
......@@ -12945,7 +13092,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
1294513092 .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },
1294613093 .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),
1294713094 .register_offset => |rhs_ro| .{ self.register_manager.lockReg(rhs_ro.reg), null },
12948 else => .{null} ** 2,
13095 else => @splat(null),
1294913096 };
1295013097 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1295113098
......@@ -13080,7 +13227,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
1308013227 if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
1308113228 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
1308213229
13083 break :result Condition.fromCompareOperator(
13230 break :result .fromCompareOperator(
1308413231 if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned,
1308513232 result_op: {
1308613233 const flipped_op = if (flipped) op.reverse() else op;
......@@ -13212,11 +13359,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
1321213359 defer if (src_info) |info|
1321313360 self.register_manager.unlockReg(info.addr_lock);
1321413361
13215 const regs = try self.register_manager.allocRegs(
13216 2,
13217 .{null} ** 2,
13218 abi.RegisterClass.gp,
13219 );
13362 const regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
1322013363 const acc_reg = regs[0].to64();
1322113364 const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
1322213365 defer for (locks) |lock| self.register_manager.unlockReg(lock);
......@@ -13367,7 +13510,7 @@ fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
1336713510 const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
1336813511 const dst_mcv = try self.genBinOp(
1336913512 inst,
13370 Air.Inst.Tag.fromCmpOp(extra.compareOperator(), false),
13513 .fromCmpOp(extra.compareOperator(), false),
1337113514 extra.lhs,
1337213515 extra.rhs,
1337313516 );
......@@ -13399,7 +13542,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
1339913542 registerAlias(dst_reg, op_abi_size),
1340013543 .{
1340113544 .base = .{ .reg = addr_reg },
13402 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(op_abi_size) } },
13545 .mod = .{ .rm = .{ .size = .fromSize(op_abi_size) } },
1340313546 },
1340413547 );
1340513548
......@@ -13682,7 +13825,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
1368213825 .{
1368313826 .base = .{ .reg = addr_reg },
1368413827 .mod = .{ .rm = .{
13685 .size = Memory.Size.fromSize(some_abi_size),
13828 .size = .fromSize(some_abi_size),
1368613829 .disp = some_info.off,
1368713830 } },
1368813831 },
......@@ -13699,14 +13842,14 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
1369913842 .indirect => |reg_off| .{
1370013843 .base = .{ .reg = reg_off.reg },
1370113844 .mod = .{ .rm = .{
13702 .size = Memory.Size.fromSize(some_abi_size),
13845 .size = .fromSize(some_abi_size),
1370313846 .disp = reg_off.off + some_info.off,
1370413847 } },
1370513848 },
1370613849 .load_frame => |frame_addr| .{
1370713850 .base = .{ .frame = frame_addr.index },
1370813851 .mod = .{ .rm = .{
13709 .size = Memory.Size.fromSize(some_abi_size),
13852 .size = .fromSize(some_abi_size),
1371013853 .disp = frame_addr.off + some_info.off,
1371113854 } },
1371213855 },
......@@ -13745,7 +13888,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
1374513888 .{
1374613889 .base = .{ .reg = ptr_reg },
1374713890 .mod = .{ .rm = .{
13748 .size = Memory.Size.fromSize(some_abi_size),
13891 .size = .fromSize(some_abi_size),
1374913892 .disp = some_info.off,
1375013893 } },
1375113894 },
......@@ -13968,7 +14111,7 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
1396814111fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
1396914112 // A block is a setup to be able to jump to the end.
1397014113 const inst_tracking_i = self.inst_tracking.count();
13971 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));
14114 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(.unreach));
1397214115
1397314116 self.scope_generation += 1;
1397414117 try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() });
......@@ -14140,7 +14283,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
1414014283 try self.genCopy(self.typeOf(switch_br.operand), mat_cond, condition, .{});
1414114284 break :mat_cond mat_cond;
1414214285 };
14143 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(mat_cond));
14286 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(mat_cond));
1414414287
1414514288 // If the condition dies here in this switch instruction, process
1414614289 // that death now instead of later as this has an effect on
......@@ -14265,7 +14408,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
1426514408 }
1426614409
1426714410 if (first_br) {
14268 block_tracking.* = InstTracking.init(block_result);
14411 block_tracking.* = .init(block_result);
1426914412 try self.saveRetroactiveState(&block_data.state);
1427014413 } else try self.restoreState(block_data.state, &.{}, .{
1427114414 .emit_instructions = true,
......@@ -14309,7 +14452,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1430914452 extra_i += inputs.len;
1431014453
1431114454 var result: MCValue = .none;
14312 var args = std.ArrayList(MCValue).init(self.gpa);
14455 var args: std.ArrayList(MCValue) = .init(self.gpa);
1431314456 try args.ensureTotalCapacity(outputs.len + inputs.len);
1431414457 defer {
1431514458 for (args.items) |arg| if (arg.getReg()) |reg| self.register_manager.unlockReg(.{
......@@ -14317,7 +14460,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1431714460 });
1431814461 args.deinit();
1431914462 }
14320 var arg_map = std.StringHashMap(u8).init(self.gpa);
14463 var arg_map: std.StringHashMap(u8) = .init(self.gpa);
1432114464 try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len));
1432214465 defer arg_map.deinit();
1432314466
......@@ -14645,14 +14788,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1464514788 });
1464614789 };
1464714790
14648 const Operand = union(enum) {
14649 none,
14650 reg: Register,
14651 mem: Memory,
14652 imm: Immediate,
14653 inst: Mir.Inst.Index,
14654 };
14655 var ops: [4]Operand = .{.none} ** 4;
14791 var ops: [4]Operand = @splat(.none);
14792 var ops_len: usize = 0;
1465614793
1465714794 var last_op = false;
1465814795 var op_it = std.mem.splitScalar(u8, mnem_it.rest(), ',');
......@@ -14847,8 +14984,12 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1484714984 try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len));
1484814985 op.* = .{ .inst = label_gop.value_ptr.target };
1484914986 } else return self.fail("invalid operand: '{s}'", .{op_str});
14987 ops_len += 1;
1485014988 } else if (op_it.next()) |op_str| return self.fail("extra operand: '{s}'", .{op_str});
1485114989
14990 // convert from att syntax to intel syntax
14991 std.mem.reverse(Operand, ops[0..ops_len]);
14992
1485214993 (if (prefix == .directive) switch (mnem_tag) {
1485314994 .@".cfi_def_cfa" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none)
1485414995 self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, ops[0].reg, ops[1].imm)
......@@ -14904,68 +15045,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1490415045 error.InvalidInstruction,
1490515046 .@".cfi_escape" => error.InvalidInstruction,
1490615047 else => unreachable,
14907 } else switch (ops[0]) {
14908 .none => self.asmOpOnly(mnem_fixed_tag),
14909 .reg => |reg0| switch (ops[1]) {
14910 .none => self.asmRegister(mnem_fixed_tag, reg0),
14911 .reg => |reg1| switch (ops[2]) {
14912 .none => self.asmRegisterRegister(mnem_fixed_tag, reg1, reg0),
14913 .reg => |reg2| switch (ops[3]) {
14914 .none => self.asmRegisterRegisterRegister(mnem_fixed_tag, reg2, reg1, reg0),
14915 else => error.InvalidInstruction,
14916 },
14917 .mem => |mem2| switch (ops[3]) {
14918 .none => self.asmMemoryRegisterRegister(mnem_fixed_tag, mem2, reg1, reg0),
14919 else => error.InvalidInstruction,
14920 },
14921 else => error.InvalidInstruction,
14922 },
14923 .mem => |mem1| switch (ops[2]) {
14924 .none => self.asmMemoryRegister(mnem_fixed_tag, mem1, reg0),
14925 else => error.InvalidInstruction,
14926 },
14927 else => error.InvalidInstruction,
14928 },
14929 .mem => |mem0| switch (ops[1]) {
14930 .none => self.asmMemory(mnem_fixed_tag, mem0),
14931 .reg => |reg1| switch (ops[2]) {
14932 .none => self.asmRegisterMemory(mnem_fixed_tag, reg1, mem0),
14933 else => error.InvalidInstruction,
14934 },
14935 else => error.InvalidInstruction,
14936 },
14937 .imm => |imm0| switch (ops[1]) {
14938 .none => self.asmImmediate(mnem_fixed_tag, imm0),
14939 .reg => |reg1| switch (ops[2]) {
14940 .none => self.asmRegisterImmediate(mnem_fixed_tag, reg1, imm0),
14941 .reg => |reg2| switch (ops[3]) {
14942 .none => self.asmRegisterRegisterImmediate(mnem_fixed_tag, reg2, reg1, imm0),
14943 .reg => |reg3| self.asmRegisterRegisterRegisterImmediate(
14944 mnem_fixed_tag,
14945 reg3,
14946 reg2,
14947 reg1,
14948 imm0,
14949 ),
14950 else => error.InvalidInstruction,
14951 },
14952 .mem => |mem2| switch (ops[3]) {
14953 .none => self.asmMemoryRegisterImmediate(mnem_fixed_tag, mem2, reg1, imm0),
14954 else => error.InvalidInstruction,
14955 },
14956 else => error.InvalidInstruction,
14957 },
14958 .mem => |mem1| switch (ops[2]) {
14959 .none => self.asmMemoryImmediate(mnem_fixed_tag, mem1, imm0),
14960 else => error.InvalidInstruction,
14961 },
14962 else => error.InvalidInstruction,
14963 },
14964 .inst => |inst0| switch (ops[1]) {
14965 .none => self.asmReloc(mnem_fixed_tag, inst0),
14966 else => error.InvalidInstruction,
14967 },
14968 }) catch |err| switch (err) {
15048 } else self.asmOps(mnem_fixed_tag, ops)) catch |err| switch (err) {
1496915049 error.InvalidInstruction => return self.fail(
1497015050 "invalid instruction: '{s} {s} {s} {s} {s}'",
1497115051 .{
......@@ -15000,7 +15080,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1500015080 }
1500115081
1500215082 simple: {
15003 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
15083 var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
1500415084 var buf_index: usize = 0;
1500515085 for (outputs) |output| {
1500615086 if (output == .none) continue;
......@@ -15583,7 +15663,7 @@ fn genSetReg(
1558315663 .load_frame => |frame_addr| try self.moveStrategy(
1558415664 ty,
1558515665 dst_reg.class(),
15586 self.getFrameAddrAlignment(frame_addr).compare(.gte, InternPool.Alignment.fromLog2Units(
15666 self.getFrameAddrAlignment(frame_addr).compare(.gte, .fromLog2Units(
1558715667 std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
1558815668 )),
1558915669 ),
......@@ -15742,7 +15822,7 @@ fn genSetMem(
1574215822 try self.asmMemoryImmediate(
1574315823 .{ ._, .mov },
1574415824 .{ .base = base, .mod = .{ .rm = .{
15745 .size = Memory.Size.fromSize(abi_size),
15825 .size = .fromSize(abi_size),
1574615826 .disp = disp,
1574715827 } } },
1574815828 immediate,
......@@ -15753,7 +15833,7 @@ fn genSetMem(
1575315833 try self.asmMemoryImmediate(
1575415834 .{ ._, .mov },
1575515835 .{ .base = base, .mod = .{ .rm = .{
15756 .size = Memory.Size.fromSize(abi_size),
15836 .size = .fromSize(abi_size),
1575715837 .disp = disp,
1575815838 } } },
1575915839 .s(small),
......@@ -15796,11 +15876,11 @@ fn genSetMem(
1579615876 .general_purpose, .segment, .x87, .ip => @divExact(src_alias.bitSize(), 8),
1579715877 .mmx, .sse => abi_size,
1579815878 });
15799 const src_align = InternPool.Alignment.fromNonzeroByteUnits(
15879 const src_align: InternPool.Alignment = .fromNonzeroByteUnits(
1580015880 std.math.ceilPowerOfTwoAssert(u32, src_size),
1580115881 );
1580215882 if (src_size > mem_size) {
15803 const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
15883 const frame_index = try self.allocFrameIndex(.init(.{
1580415884 .size = src_size,
1580515885 .alignment = src_align,
1580615886 }));
......@@ -15808,7 +15888,7 @@ fn genSetMem(
1580815888 try (try self.moveStrategy(ty, src_alias.class(), true)).write(
1580915889 self,
1581015890 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{
15811 .size = Memory.Size.fromSize(src_size),
15891 .size = .fromSize(src_size),
1581215892 } } },
1581315893 src_alias,
1581415894 );
......@@ -15828,7 +15908,7 @@ fn genSetMem(
1582815908 })).write(
1582915909 self,
1583015910 .{ .base = base, .mod = .{ .rm = .{
15831 .size = Memory.Size.fromBitSize(@min(self.memSize(ty).bitSize(), src_alias.bitSize())),
15911 .size = .fromBitSize(@min(self.memSize(ty).bitSize(), src_alias.bitSize())),
1583215912 .disp = disp,
1583315913 } } },
1583415914 src_alias,
......@@ -16037,7 +16117,7 @@ fn genLazySymbolRef(
1603716117 .mov => try self.asmRegisterMemory(
1603816118 .{ ._, tag },
1603916119 reg.to64(),
16040 Memory.initSib(.qword, .{ .base = .{ .reg = reg.to64() } }),
16120 .initSib(.qword, .{ .base = .{ .reg = reg.to64() } }),
1604116121 ),
1604216122 else => unreachable,
1604316123 }
......@@ -16108,8 +16188,8 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
1610816188 else => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv,
1610916189 };
1611016190
16111 const dst_rc = self.regClassForType(dst_ty);
16112 const src_rc = self.regClassForType(src_ty);
16191 const dst_rc = self.regSetForType(dst_ty);
16192 const src_rc = self.regSetForType(src_ty);
1611316193
1611416194 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
1611516195 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
......@@ -16170,7 +16250,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
1617016250 const array_ty = ptr_ty.childType(zcu);
1617116251 const array_len = array_ty.arrayLen(zcu);
1617216252
16173 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));
16253 const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu));
1617416254 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr, .{});
1617516255 try self.genSetMem(
1617616256 .{ .frame = frame_index },
......@@ -16236,7 +16316,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
1623616316
1623716317 if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg);
1623816318
16239 const dst_reg = try self.register_manager.allocReg(inst, self.regClassForType(dst_ty));
16319 const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty));
1624016320 const dst_mcv = MCValue{ .register = dst_reg };
1624116321 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1624216322 defer self.register_manager.unlockReg(dst_lock);
......@@ -16314,7 +16394,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
1631416394 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
1631516395 defer self.register_manager.unlockReg(src_lock);
1631616396
16317 const dst_reg = try self.register_manager.allocReg(inst, self.regClassForType(dst_ty));
16397 const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty));
1631816398 const dst_mcv = MCValue{ .register = dst_reg };
1631916399 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1632016400 defer self.register_manager.unlockReg(dst_lock);
......@@ -16381,7 +16461,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1638116461 defer if (new_lock) |lock| self.register_manager.unlockReg(lock);
1638216462
1638316463 const ptr_mcv = try self.resolveInst(extra.ptr);
16384 const mem_size = Memory.Size.fromSize(val_abi_size);
16464 const mem_size: Memory.Size = .fromSize(val_abi_size);
1638516465 const ptr_mem: Memory = switch (ptr_mcv) {
1638616466 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
1638716467 else => .{
......@@ -16448,7 +16528,7 @@ fn atomicOp(
1644816528 defer if (val_lock) |lock| self.register_manager.unlockReg(lock);
1644916529
1645016530 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));
16451 const mem_size = Memory.Size.fromSize(val_abi_size);
16531 const mem_size: Memory.Size = .fromSize(val_abi_size);
1645216532 const ptr_mem: Memory = switch (ptr_mcv) {
1645316533 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
1645416534 else => .{
......@@ -16637,7 +16717,7 @@ fn atomicOp(
1663716717 try self.asmCmovccRegisterMemory(
1663816718 cc,
1663916719 registerAlias(tmp_reg, cmov_abi_size),
16640 try val_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
16720 try val_mcv.mem(self, .fromSize(cmov_abi_size)),
1664116721 );
1664216722 },
1664316723 else => {
......@@ -17069,7 +17149,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
1706917149
1707017150 // We need a properly aligned and sized call frame to be able to call this function.
1707117151 {
17072 const needed_call_frame = FrameAlloc.init(.{
17152 const needed_call_frame: FrameAlloc = .init(.{
1707317153 .size = inst_ty.abiSize(zcu),
1707417154 .alignment = inst_ty.abiAlignment(zcu),
1707517155 });
......@@ -17207,7 +17287,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1720717287 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1720817288 const vector_ty = self.typeOfIndex(inst);
1720917289 const vector_len = vector_ty.vectorLen(zcu);
17210 const dst_rc = self.regClassForType(vector_ty);
17290 const dst_rc = self.regSetForType(vector_ty);
1721117291 const scalar_ty = self.typeOf(ty_op.operand);
1721217292
1721317293 const result: MCValue = result: {
......@@ -17675,7 +17755,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1767517755 .ty = mask_elem_ty.toIntern(),
1767617756 .storage = .{ .u64 = bit / elem_bits },
1767717757 } });
17678 const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
17758 const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
1767917759 .ty = mask_ty.toIntern(),
1768017760 .storage = .{ .elems = mask_elems[0..vec_len] },
1768117761 } })));
......@@ -17700,7 +17780,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1770017780 .ty = mask_elem_ty.toIntern(),
1770117781 .storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) },
1770217782 } });
17703 const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
17783 const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
1770417784 .ty = mask_ty.toIntern(),
1770517785 .storage = .{ .elems = mask_elems[0..vec_len] },
1770617786 } })));
......@@ -17853,7 +17933,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
1785317933 if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
1785417934 .{ mir_fixes, .andn },
1785517935 mask_alias,
17856 try rhs_mcv.mem(self, Memory.Size.fromSize(abi_size)),
17936 try rhs_mcv.mem(self, .fromSize(abi_size)),
1785717937 ) else try self.asmRegisterRegister(
1785817938 .{ mir_fixes, .andn },
1785917939 mask_alias,
......@@ -17934,7 +18014,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1793418014 else
1793518015 self.hasFeature(.avx2)) 32 else 16)) break :unpck;
1793618016
17937 var sources = [1]?u1{null} ** 2;
18017 var sources: [2]?u1 = @splat(null);
1793818018 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
1793918019 const mask_elem = maybe_mask_elem orelse continue;
1794018020 const mask_elem_index =
......@@ -17994,7 +18074,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1799418074 mir_tag,
1799518075 dst_alias,
1799618076 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17997 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
18077 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
1799818078 ) else try self.asmRegisterRegisterRegister(
1799918079 mir_tag,
1800018080 dst_alias,
......@@ -18006,7 +18086,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1800618086 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
1800718087 mir_tag,
1800818088 dst_alias,
18009 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
18089 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
1801018090 ) else try self.asmRegisterRegister(
1801118091 mir_tag,
1801218092 dst_alias,
......@@ -18023,7 +18103,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1802318103 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd;
1802418104
1802518105 var control: u8 = 0b00_00_00_00;
18026 var sources = [1]?u1{null} ** 1;
18106 var sources: [1]?u1 = @splat(null);
1802718107 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
1802818108 const mask_elem = maybe_mask_elem orelse continue;
1802918109 const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
......@@ -18055,7 +18135,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1805518135 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
1805618136 .{ if (has_avx) .vp_d else .p_d, .shuf },
1805718137 dst_alias,
18058 try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
18138 try src_mcv.mem(self, .fromSize(max_abi_size)),
1805918139 .u(control),
1806018140 ) else try self.asmRegisterRegisterImmediate(
1806118141 .{ if (has_avx) .vp_d else .p_d, .shuf },
......@@ -18074,7 +18154,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1807418154 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps;
1807518155
1807618156 var control: u8 = 0b00_00_00_00;
18077 var sources = [1]?u1{null} ** 2;
18157 var sources: [2]?u1 = @splat(null);
1807818158 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
1807918159 const mask_elem = maybe_mask_elem orelse continue;
1808018160 const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
......@@ -18112,7 +18192,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1811218192 .{ .v_ps, .shuf },
1811318193 dst_alias,
1811418194 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
18115 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
18195 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
1811618196 .u(control),
1811718197 ) else try self.asmRegisterRegisterRegisterImmediate(
1811818198 .{ .v_ps, .shuf },
......@@ -18126,7 +18206,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1812618206 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
1812718207 .{ ._ps, .shuf },
1812818208 dst_alias,
18129 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
18209 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
1813018210 .u(control),
1813118211 ) else try self.asmRegisterRegisterImmediate(
1813218212 .{ ._ps, .shuf },
......@@ -18145,7 +18225,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1814518225 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd;
1814618226
1814718227 var control: u4 = 0b0_0_0_0;
18148 var sources = [1]?u1{null} ** 2;
18228 var sources: [2]?u1 = @splat(null);
1814918229 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
1815018230 const mask_elem = maybe_mask_elem orelse continue;
1815118231 const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
......@@ -18179,7 +18259,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1817918259 .{ .v_pd, .shuf },
1818018260 dst_alias,
1818118261 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
18182 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
18262 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
1818318263 .u(control),
1818418264 ) else try self.asmRegisterRegisterRegisterImmediate(
1818518265 .{ .v_pd, .shuf },
......@@ -18193,7 +18273,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1819318273 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
1819418274 .{ ._pd, .shuf },
1819518275 dst_alias,
18196 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
18276 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
1819718277 .u(control),
1819818278 ) else try self.asmRegisterRegisterImmediate(
1819918279 .{ ._pd, .shuf },
......@@ -18249,7 +18329,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1824918329 .{ .vp_d, .blend },
1825018330 registerAlias(dst_reg, dst_abi_size),
1825118331 registerAlias(lhs_reg, dst_abi_size),
18252 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18332 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
1825318333 .u(expanded_control),
1825418334 ) else try self.asmRegisterRegisterRegisterImmediate(
1825518335 .{ .vp_d, .blend },
......@@ -18304,7 +18384,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1830418384 lhs_mcv.getReg().?
1830518385 else
1830618386 dst_reg, dst_abi_size),
18307 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18387 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
1830818388 .u(expanded_control),
1830918389 ) else try self.asmRegisterRegisterRegisterImmediate(
1831018390 .{ .vp_w, .blend },
......@@ -18321,7 +18401,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1832118401 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
1832218402 .{ .p_w, .blend },
1832318403 registerAlias(dst_reg, dst_abi_size),
18324 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18404 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
1832518405 .u(expanded_control),
1832618406 ) else try self.asmRegisterRegisterImmediate(
1832718407 .{ .p_w, .blend },
......@@ -18365,7 +18445,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1836518445 lhs_mcv.getReg().?
1836618446 else
1836718447 dst_reg, dst_abi_size),
18368 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18448 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
1836918449 .u(expanded_control),
1837018450 ) else try self.asmRegisterRegisterRegisterImmediate(
1837118451 switch (elem_abi_size) {
......@@ -18390,7 +18470,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1839018470 else => unreachable,
1839118471 },
1839218472 registerAlias(dst_reg, dst_abi_size),
18393 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18473 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
1839418474 .u(expanded_control),
1839518475 ) else try self.asmRegisterRegisterImmediate(
1839618476 switch (elem_abi_size) {
......@@ -18435,7 +18515,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1843518515 else
1843618516 try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern();
1843718517 }
18438 const select_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
18518 const select_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
1843918519 .ty = select_mask_ty.toIntern(),
1844018520 .storage = .{ .elems = select_mask_elems[0..mask_elems.len] },
1844118521 } })));
......@@ -18480,7 +18560,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1848018560 registerAlias(lhs_mcv.getReg().?, dst_abi_size)
1848118561 else
1848218562 dst_alias,
18483 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18563 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
1848418564 select_mask_alias,
1848518565 ) else try self.asmRegisterRegisterRegisterRegister(
1848618566 mir_tag,
......@@ -18497,7 +18577,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1849718577 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
1849818578 mir_tag,
1849918579 dst_alias,
18500 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18580 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
1850118581 select_mask_alias,
1850218582 ) else try self.asmRegisterRegisterRegister(
1850318583 mir_tag,
......@@ -18540,7 +18620,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1854018620 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
1854118621 .{ mir_fixes, .andn },
1854218622 mask_alias,
18543 try lhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18623 try lhs_mcv.mem(self, .fromSize(dst_abi_size)),
1854418624 ) else try self.asmRegisterRegister(
1854518625 .{ mir_fixes, .andn },
1854618626 mask_alias,
......@@ -18583,13 +18663,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1858318663 } });
1858418664 }
1858518665 const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
18586 const lhs_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
18666 const lhs_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
1858718667 .ty = lhs_mask_ty.toIntern(),
1858818668 .storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
1858918669 } })));
1859018670 const lhs_mask_mem: Memory = .{
1859118671 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },
18592 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
18672 .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
1859318673 };
1859418674 if (has_avx) try self.asmRegisterRegisterMemory(
1859518675 .{ .vp_b, .shuf },
......@@ -18617,13 +18697,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
1861718697 } });
1861818698 }
1861918699 const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
18620 const rhs_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
18700 const rhs_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
1862118701 .ty = rhs_mask_ty.toIntern(),
1862218702 .storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },
1862318703 } })));
1862418704 const rhs_mask_mem: Memory = .{
1862518705 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },
18626 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
18706 .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
1862718707 };
1862818708 if (has_avx) try self.asmRegisterRegisterMemory(
1862918709 .{ .vp_b, .shuf },
......@@ -18696,7 +18776,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
1869618776 .Or => {
1869718777 if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
1869818778 .{ ._, .@"test" },
18699 try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)),
18779 try operand_mcv.mem(self, .fromSize(abi_size)),
1870018780 .u(mask),
1870118781 ) else {
1870218782 const operand_reg = registerAlias(if (operand_mcv.isRegister())
......@@ -18745,7 +18825,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
1874518825 const result: MCValue = result: {
1874618826 switch (result_ty.zigTypeTag(zcu)) {
1874718827 .@"struct" => {
18748 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));
18828 const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
1874918829 if (result_ty.containerLayout(zcu) == .@"packed") {
1875018830 const struct_obj = zcu.typeToStruct(result_ty).?;
1875118831 try self.genInlineMemset(
......@@ -18885,7 +18965,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
1888518965 }
1888618966 break :result .{ .register = dst_reg };
1888718967 } else {
18888 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));
18968 const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
1888918969 const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
1889018970
1889118971 for (elements, 0..) |elem, elem_i| {
......@@ -18918,7 +18998,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
1891818998 };
1891918999
1892019000 if (elements.len <= Liveness.bpi - 1) {
18921 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
19001 var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
1892219002 @memcpy(buf[0..elements.len], elements);
1892319003 return self.finishAir(inst, result, buf);
1892419004 }
......@@ -18952,7 +19032,7 @@ fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
1895219032
1895319033 const union_obj = zcu.typeToUnion(union_ty).?;
1895419034 const field_name = union_obj.loadTagType(ip).names.get(ip)[extra.field_index];
18955 const tag_ty = Type.fromInterned(union_obj.enum_tag_ty);
19035 const tag_ty: Type = .fromInterned(union_obj.enum_tag_ty);
1895619036 const field_index = tag_ty.enumFieldIndex(field_name, zcu).?;
1895719037 const tag_val = try pt.enumValueFieldIndex(tag_ty, field_index);
1895819038 const tag_int_val = try tag_val.intFromEnum(tag_ty, pt);
......@@ -19010,10 +19090,10 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
1901019090 }
1901119091
1901219092 var mcvs: [3]MCValue = undefined;
19013 var locks = [1]?RegisterManager.RegisterLock{null} ** 3;
19093 var locks: [3]?RegisterManager.RegisterLock = @splat(null);
1901419094 defer for (locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
19015 var order = [1]u2{0} ** 3;
19016 var unused = std.StaticBitSet(3).initFull();
19095 var order: [3]u2 = @splat(0);
19096 var unused: std.StaticBitSet(3) = .initFull();
1901719097 for (ops, &mcvs, &locks, 0..) |op, *mcv, *lock, op_i| {
1901819098 const op_index: u2 = @intCast(op_i);
1901919099 mcv.* = try self.resolveInst(op);
......@@ -19139,7 +19219,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
1913919219 mir_tag,
1914019220 mop1_reg,
1914119221 mop2_reg,
19142 try mops[2].mem(self, Memory.Size.fromSize(abi_size)),
19222 try mops[2].mem(self, .fromSize(abi_size)),
1914319223 );
1914419224 break :result mops[0];
1914519225 };
......@@ -19158,7 +19238,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
1915819238 )) {
1915919239 .x86_64_sysv => result: {
1916019240 const info = self.va_info.sysv;
19161 const dst_fi = try self.allocFrameIndex(FrameAlloc.initSpill(va_list_ty, zcu));
19241 const dst_fi = try self.allocFrameIndex(.initSpill(va_list_ty, zcu));
1916219242 var field_off: u31 = 0;
1916319243 // gp_offset: c_uint,
1916419244 try self.genSetMem(
......@@ -19221,7 +19301,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
1922119301 try self.spillEflagsIfOccupied();
1922219302
1922319303 const tmp_regs =
19224 try self.register_manager.allocRegs(2, .{null} ** 2, abi.RegisterClass.gp);
19304 try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
1922519305 const offset_reg = tmp_regs[0].to32();
1922619306 const addr_reg = tmp_regs[1].to64();
1922719307 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
......@@ -19426,8 +19506,8 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
1942619506 } else mcv: {
1942719507 const ip_index = ref.toInterned().?;
1942819508 const gop = try self.const_tracking.getOrPut(self.gpa, ip_index);
19429 if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(init: {
19430 const const_mcv = try self.genTypedValue(Value.fromInterned(ip_index));
19509 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
19510 const const_mcv = try self.genTypedValue(.fromInterned(ip_index));
1943119511 switch (const_mcv) {
1943219512 .lea_tlv => |tlv_sym| switch (self.bin_file.tag) {
1943319513 .elf, .macho => {
......@@ -19436,7 +19516,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
1943619516 } else {
1943719517 try self.spillRegisters(&.{.rax});
1943819518 }
19439 const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
19519 const frame_index = try self.allocFrameIndex(.init(.{
1944019520 .size = 8,
1944119521 .alignment = .@"8",
1944219522 }));
......@@ -19539,9 +19619,8 @@ fn resolveCallingConventionValues(
1953919619 const param_types = try self.gpa.alloc(Type, fn_info.param_types.len + var_args.len);
1954019620 defer self.gpa.free(param_types);
1954119621
19542 for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*dest, src| {
19543 dest.* = Type.fromInterned(src);
19544 }
19622 for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*dest, src|
19623 dest.* = .fromInterned(src);
1954519624 for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty|
1954619625 param_ty.* = self.promoteVarArg(arg_ty);
1954719626
......@@ -19556,13 +19635,13 @@ fn resolveCallingConventionValues(
1955619635 };
1955719636 errdefer self.gpa.free(result.args);
1955819637
19559 const ret_ty = Type.fromInterned(fn_info.return_type);
19638 const ret_ty: Type = .fromInterned(fn_info.return_type);
1956019639
1956119640 const resolved_cc = abi.resolveCallingConvention(cc, self.target.*);
1956219641 switch (cc) {
1956319642 .naked => {
1956419643 assert(result.args.len == 0);
19565 result.return_value = InstTracking.init(.unreach);
19644 result.return_value = .init(.unreach);
1956619645 result.stack_align = .@"8";
1956719646 },
1956819647 .x86_64_sysv, .x86_64_win => |cc_opts| {
......@@ -19583,10 +19662,10 @@ fn resolveCallingConventionValues(
1958319662
1958419663 // Return values
1958519664 if (ret_ty.zigTypeTag(zcu) == .noreturn) {
19586 result.return_value = InstTracking.init(.unreach);
19665 result.return_value = .init(.unreach);
1958719666 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
1958819667 // TODO: is this even possible for C calling convention?
19589 result.return_value = InstTracking.init(.none);
19668 result.return_value = .init(.none);
1959019669 } else {
1959119670 var ret_tracking: [2]InstTracking = undefined;
1959219671 var ret_tracking_i: usize = 0;
......@@ -19604,7 +19683,7 @@ fn resolveCallingConventionValues(
1960419683 );
1960519684 ret_int_reg_i += 1;
1960619685
19607 ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_int_reg });
19686 ret_tracking[ret_tracking_i] = .init(.{ .register = ret_int_reg });
1960819687 ret_tracking_i += 1;
1960919688 },
1961019689 .sse, .float, .float_combine, .win_i128 => {
......@@ -19614,18 +19693,17 @@ fn resolveCallingConventionValues(
1961419693 );
1961519694 ret_sse_reg_i += 1;
1961619695
19617 ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_sse_reg });
19696 ret_tracking[ret_tracking_i] = .init(.{ .register = ret_sse_reg });
1961819697 ret_tracking_i += 1;
1961919698 },
1962019699 .sseup => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .sse),
1962119700 .x87 => {
19622 ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = .st0 });
19701 ret_tracking[ret_tracking_i] = .init(.{ .register = .st0 });
1962319702 ret_tracking_i += 1;
1962419703 },
1962519704 .x87up => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .x87),
1962619705 .complex_x87 => {
19627 ret_tracking[ret_tracking_i] =
19628 InstTracking.init(.{ .register_pair = .{ .st0, .st1 } });
19706 ret_tracking[ret_tracking_i] = .init(.{ .register_pair = .{ .st0, .st1 } });
1962919707 ret_tracking_i += 1;
1963019708 },
1963119709 .memory => {
......@@ -19645,8 +19723,9 @@ fn resolveCallingConventionValues(
1964519723 result.return_value = switch (ret_tracking_i) {
1964619724 else => unreachable,
1964719725 1 => ret_tracking[0],
19648 2 => InstTracking.init(.{ .register_pair = .{
19649 ret_tracking[0].short.register, ret_tracking[1].short.register,
19726 2 => .init(.{ .register_pair = .{
19727 ret_tracking[0].short.register,
19728 ret_tracking[1].short.register,
1965019729 } }),
1965119730 };
1965219731 }
......@@ -19769,9 +19848,9 @@ fn resolveCallingConventionValues(
1976919848
1977019849 // Return values
1977119850 if (ret_ty.zigTypeTag(zcu) == .noreturn) {
19772 result.return_value = InstTracking.init(.unreach);
19851 result.return_value = .init(.unreach);
1977319852 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
19774 result.return_value = InstTracking.init(.none);
19853 result.return_value = .init(.none);
1977519854 } else {
1977619855 const ret_reg = abi.getCAbiIntReturnRegs(resolved_cc)[0];
1977719856 const ret_ty_size: u31 = @intCast(ret_ty.abiSize(zcu));
......@@ -19884,8 +19963,8 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
1988419963fn memSize(self: *Self, ty: Type) Memory.Size {
1988519964 const zcu = self.pt.zcu;
1988619965 return switch (ty.zigTypeTag(zcu)) {
19887 .float => Memory.Size.fromBitSize(ty.floatBits(self.target.*)),
19888 else => Memory.Size.fromSize(@intCast(ty.abiSize(zcu))),
19966 .float => .fromBitSize(ty.floatBits(self.target.*)),
19967 else => .fromSize(@intCast(ty.abiSize(zcu))),
1988919968 };
1989019969}
1989119970
......@@ -20140,7 +20219,7 @@ const Temp = struct {
2014020219 .register => |reg| {
2014120220 const new_reg =
2014220221 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20143 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
20222 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
2014420223 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
2014520224 .base = .{ .reg = reg.to64() },
2014620225 .mod = .{ .rm = .{
......@@ -20152,7 +20231,7 @@ const Temp = struct {
2015220231 .register_offset => |reg_off| {
2015320232 const new_reg =
2015420233 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20155 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
20234 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
2015620235 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
2015720236 .base = .{ .reg = reg_off.reg.to64() },
2015820237 .mod = .{ .rm = .{
......@@ -20161,14 +20240,14 @@ const Temp = struct {
2016120240 } },
2016220241 });
2016320242 },
20164 .lea_symbol => |sym_off| new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_symbol = .{
20243 .lea_symbol => |sym_off| new_temp_index.tracking(self).* = .init(.{ .lea_symbol = .{
2016520244 .sym_index = sym_off.sym_index,
2016620245 .off = sym_off.off + off,
2016720246 } }),
2016820247 .load_frame => |frame_addr| {
2016920248 const new_reg =
2017020249 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20171 new_temp_index.tracking(self).* = InstTracking.init(.{ .register_offset = .{
20250 new_temp_index.tracking(self).* = .init(.{ .register_offset = .{
2017220251 .reg = new_reg,
2017320252 .off = off,
2017420253 } });
......@@ -20180,7 +20259,7 @@ const Temp = struct {
2018020259 } },
2018120260 });
2018220261 },
20183 .lea_frame => |frame_addr| new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_frame = .{
20262 .lea_frame => |frame_addr| new_temp_index.tracking(self).* = .init(.{ .lea_frame = .{
2018420263 .index = frame_addr.index,
2018520264 .off = frame_addr.off + off,
2018620265 } }),
......@@ -20198,7 +20277,7 @@ const Temp = struct {
2019820277 else => {},
2019920278 .register => |reg| {
2020020279 try self.freeValue(temp_tracking.long);
20201 temp_tracking.* = InstTracking.init(.{ .register_offset = .{
20280 temp_tracking.* = .init(.{ .register_offset = .{
2020220281 .reg = reg,
2020320282 .off = off,
2020420283 } });
......@@ -20206,7 +20285,7 @@ const Temp = struct {
2020620285 },
2020720286 .register_offset => |reg_off| {
2020820287 try self.freeValue(temp_tracking.long);
20209 temp_tracking.* = InstTracking.init(.{ .register_offset = .{
20288 temp_tracking.* = .init(.{ .register_offset = .{
2021020289 .reg = reg_off.reg,
2021120290 .off = reg_off.off + off,
2021220291 } });
......@@ -20214,7 +20293,7 @@ const Temp = struct {
2021420293 },
2021520294 .lea_symbol => |sym_off| {
2021620295 assert(std.meta.eql(temp_tracking.long.lea_symbol, sym_off));
20217 temp_tracking.* = InstTracking.init(.{ .lea_symbol = .{
20296 temp_tracking.* = .init(.{ .lea_symbol = .{
2021820297 .sym_index = sym_off.sym_index,
2021920298 .off = sym_off.off + off,
2022020299 } });
......@@ -20222,7 +20301,7 @@ const Temp = struct {
2022220301 },
2022320302 .lea_frame => |frame_addr| {
2022420303 assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr));
20225 temp_tracking.* = InstTracking.init(.{ .lea_frame = .{
20304 temp_tracking.* = .init(.{ .lea_frame = .{
2022620305 .index = frame_addr.index,
2022720306 .off = frame_addr.off + off,
2022820307 } });
......@@ -20243,26 +20322,26 @@ const Temp = struct {
2024320322 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
2024420323 .immediate => |imm| {
2024520324 assert(limb_index == 0);
20246 new_temp_index.tracking(self).* = InstTracking.init(.{ .immediate = imm });
20325 new_temp_index.tracking(self).* = .init(.{ .immediate = imm });
2024720326 },
2024820327 .register => |reg| {
2024920328 assert(limb_index == 0);
2025020329 const new_reg =
2025120330 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20252 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
20331 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
2025320332 try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());
2025420333 },
2025520334 .register_pair => |regs| {
2025620335 const new_reg =
2025720336 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20258 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
20337 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
2025920338 try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());
2026020339 },
2026120340 .register_offset => |reg_off| {
2026220341 assert(limb_index == 0);
2026320342 const new_reg =
2026420343 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20265 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
20344 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
2026620345 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
2026720346 .base = .{ .reg = reg_off.reg.to64() },
2026820347 .mod = .{ .rm = .{
......@@ -20274,7 +20353,7 @@ const Temp = struct {
2027420353 .load_symbol => |sym_off| {
2027520354 const new_reg =
2027620355 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20277 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
20356 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
2027820357 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
2027920358 .base = .{ .reloc = sym_off.sym_index },
2028020359 .mod = .{ .rm = .{
......@@ -20285,12 +20364,12 @@ const Temp = struct {
2028520364 },
2028620365 .lea_symbol => |sym_off| {
2028720366 assert(limb_index == 0);
20288 new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_symbol = sym_off });
20367 new_temp_index.tracking(self).* = .init(.{ .lea_symbol = sym_off });
2028920368 },
2029020369 .load_frame => |frame_addr| {
2029120370 const new_reg =
2029220371 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20293 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
20372 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
2029420373 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
2029520374 .base = .{ .frame = frame_addr.index },
2029620375 .mod = .{ .rm = .{
......@@ -20301,7 +20380,7 @@ const Temp = struct {
2030120380 },
2030220381 .lea_frame => |frame_addr| {
2030320382 assert(limb_index == 0);
20304 new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_frame = frame_addr });
20383 new_temp_index.tracking(self).* = .init(.{ .lea_frame = frame_addr });
2030520384 },
2030620385 }
2030720386 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
......@@ -20328,13 +20407,13 @@ const Temp = struct {
2032820407 }
2032920408 for (regs, 0..) |reg, reg_index| if (reg_index != limb_index)
2033020409 self.register_manager.freeReg(reg);
20331 temp_tracking.* = InstTracking.init(.{ .register = regs[limb_index] });
20410 temp_tracking.* = .init(.{ .register = regs[limb_index] });
2033220411 self.temp_type[@intFromEnum(temp_index)] = Type.usize;
2033320412 return;
2033420413 },
2033520414 .load_symbol => |sym_off| {
2033620415 assert(std.meta.eql(temp_tracking.long.load_symbol, sym_off));
20337 temp_tracking.* = InstTracking.init(.{ .load_symbol = .{
20416 temp_tracking.* = .init(.{ .load_symbol = .{
2033820417 .sym_index = sym_off.sym_index,
2033920418 .off = sym_off.off + @as(u31, limb_index) * 8,
2034020419 } });
......@@ -20343,7 +20422,7 @@ const Temp = struct {
2034320422 },
2034420423 .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) {
2034520424 assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr));
20346 temp_tracking.* = InstTracking.init(.{ .load_frame = .{
20425 temp_tracking.* = .init(.{ .load_frame = .{
2034720426 .index = frame_addr.index,
2034820427 .off = frame_addr.off + @as(u31, limb_index) * 8,
2034920428 } });
......@@ -20371,7 +20450,7 @@ const Temp = struct {
2037120450 const new_temp_index = self.next_temp_index;
2037220451 self.temp_type[@intFromEnum(new_temp_index)] = ty;
2037320452 try self.genSetReg(new_reg, ty, val, .{});
20374 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
20453 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
2037520454 try temp.die(self);
2037620455 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
2037720456 temp.* = .{ .index = new_temp_index.toIndex() };
......@@ -20390,9 +20469,32 @@ const Temp = struct {
2039020469 const new_temp_index = self.next_temp_index;
2039120470 self.temp_type[@intFromEnum(new_temp_index)] = ty;
2039220471 const new_reg =
20393 try self.register_manager.allocReg(new_temp_index.toIndex(), self.regClassForType(ty));
20472 try self.register_manager.allocReg(new_temp_index.toIndex(), self.regSetForType(ty));
2039420473 try self.genSetReg(new_reg, ty, val, .{});
20395 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
20474 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20475 try temp.die(self);
20476 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20477 temp.* = .{ .index = new_temp_index.toIndex() };
20478 return true;
20479 }
20480
20481 fn toRegClass(temp: *Temp, rc: Register.Class, self: *Self) !bool {
20482 const val, const ty = switch (temp.unwrap(self)) {
20483 .ref => |ref| .{ temp.tracking(self).short, self.typeOf(ref) },
20484 .temp => |temp_index| val: {
20485 const temp_tracking = temp_index.tracking(self);
20486 switch (temp_tracking.short) {
20487 else => {},
20488 .register => |reg| if (reg.class() == rc) return false,
20489 }
20490 break :val .{ temp_tracking.short, temp_index.typeOf(self) };
20491 },
20492 };
20493 const new_temp_index = self.next_temp_index;
20494 self.temp_type[@intFromEnum(new_temp_index)] = ty;
20495 const new_reg = try self.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc));
20496 try self.genSetReg(new_reg, ty, val, .{});
20497 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
2039620498 try temp.die(self);
2039720499 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
2039820500 temp.* = .{ .index = new_temp_index.toIndex() };
......@@ -20414,7 +20516,7 @@ const Temp = struct {
2041420516 assert(self.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
2041520517 assert(self.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
2041620518 self.temp_type[@intFromEnum(result_temp_index)] = Type.slice_const_u8;
20417 result_temp_index.tracking(self).* = InstTracking.init(result);
20519 result_temp_index.tracking(self).* = .init(result);
2041820520 first_temp.* = result_temp;
2041920521 }
2042020522
......@@ -20520,13 +20622,43 @@ const Temp = struct {
2052020622 try self.asmOpOnly(.{ .@"rep _sb", .mov });
2052120623 }
2052220624
20625 // i, m, r
20626 fn add(lhs: *Temp, rhs: *Temp, self: *Self) !Temp {
20627 const res_index = self.next_temp_index;
20628 var res: Temp = .{ .index = res_index.toIndex() };
20629 try self.select(&.{ &res, lhs, rhs }, .{ ._, .add }, &.{
20630 .{ .ops = &.{ .{ .match = 1 }, .r, .i } },
20631 .{ .ops = &.{ .{ .match = 1 }, .m, .i } },
20632 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },
20633 .{ .ops = &.{ .{ .match = 1 }, .m, .r } },
20634 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },
20635 });
20636 self.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
20637 self.temp_type[@intFromEnum(res_index)] = lhs.typeOf(self);
20638 return res;
20639 }
20640
20641 fn mul(lhs: *Temp, rhs: *Temp, self: *Self) !Temp {
20642 const res_index = self.next_temp_index;
20643 var res: Temp = .{ .index = self.next_temp_index.toIndex() };
20644 try self.select(&.{ &res, lhs, rhs }, .{ .i_, .mul }, &.{
20645 .{ .ops = &.{ .r, .m, .i } },
20646 .{ .ops = &.{ .r, .r, .i } },
20647 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },
20648 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },
20649 });
20650 self.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
20651 self.temp_type[@intFromEnum(res_index)] = lhs.typeOf(self);
20652 return res;
20653 }
20654
2052320655 fn moveTo(temp: Temp, inst: Air.Inst.Index, self: *Self) !void {
2052420656 if (self.liveness.isUnused(inst)) try temp.die(self) else switch (temp.unwrap(self)) {
2052520657 .ref => {
2052620658 const result = try self.allocRegOrMem(inst, true);
2052720659 try self.genCopy(self.typeOfIndex(inst), result, temp.tracking(self).short, .{});
2052820660 tracking_log.debug("{} => {} (birth)", .{ inst, result });
20529 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
20661 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));
2053020662 },
2053120663 .temp => |temp_index| {
2053220664 const temp_tracking = temp_index.tracking(self);
......@@ -20548,7 +20680,7 @@ const Temp = struct {
2054820680 _,
2054920681
2055020682 fn toIndex(index: Index) Air.Inst.Index {
20551 return Air.Inst.Index.fromTargetIndex(@intFromEnum(index));
20683 return .fromTargetIndex(@intFromEnum(index));
2055220684 }
2055320685
2055420686 fn fromIndex(index: Air.Inst.Index) Index {
......@@ -20629,7 +20761,19 @@ fn reuseTemp(
2062920761
2063020762fn tempAlloc(self: *Self, ty: Type) !Temp {
2063120763 const temp_index = self.next_temp_index;
20632 temp_index.tracking(self).* = InstTracking.init(try self.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true));
20764 temp_index.tracking(self).* = .init(
20765 try self.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true),
20766 );
20767 self.temp_type[@intFromEnum(temp_index)] = ty;
20768 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
20769 return .{ .index = temp_index.toIndex() };
20770}
20771
20772fn tempAllocReg(self: *Self, ty: Type, rc: RegisterManager.RegisterBitSet) !Temp {
20773 const temp_index = self.next_temp_index;
20774 temp_index.tracking(self).* = .init(
20775 .{ .register = try self.register_manager.allocReg(temp_index.toIndex(), rc) },
20776 );
2063320777 self.temp_type[@intFromEnum(temp_index)] = ty;
2063420778 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
2063520779 return .{ .index = temp_index.toIndex() };
......@@ -20637,7 +20781,7 @@ fn tempAlloc(self: *Self, ty: Type) !Temp {
2063720781
2063820782fn tempFromValue(self: *Self, ty: Type, value: MCValue) !Temp {
2063920783 const temp_index = self.next_temp_index;
20640 temp_index.tracking(self).* = InstTracking.init(value);
20784 temp_index.tracking(self).* = .init(value);
2064120785 self.temp_type[@intFromEnum(temp_index)] = ty;
2064220786 try self.getValue(value, temp_index.toIndex());
2064320787 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
......@@ -20657,8 +20801,8 @@ fn tempFromOperand(
2065720801 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };
2065820802 const val = op_ref.toInterned().?;
2065920803 const gop = try self.const_tracking.getOrPut(self.gpa, val);
20660 if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(init: {
20661 const const_mcv = try self.genTypedValue(Value.fromInterned(val));
20804 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
20805 const const_mcv = try self.genTypedValue(.fromInterned(val));
2066220806 switch (const_mcv) {
2066320807 .lea_tlv => |tlv_sym| switch (self.bin_file.tag) {
2066420808 .elf, .macho => {
......@@ -20667,7 +20811,7 @@ fn tempFromOperand(
2066720811 } else {
2066820812 try self.spillRegisters(&.{.rax});
2066920813 }
20670 const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
20814 const frame_index = try self.allocFrameIndex(.init(.{
2067120815 .size = 8,
2067220816 .alignment = .@"8",
2067320817 }));
......@@ -20685,7 +20829,7 @@ fn tempFromOperand(
2068520829 else => break :init const_mcv,
2068620830 }
2068720831 });
20688 return self.tempFromValue(Type.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
20832 return self.tempFromValue(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
2068920833 }
2069020834
2069120835 const temp_index = self.next_temp_index;
......@@ -20706,3 +20850,193 @@ inline fn tempsFromOperands(self: *Self, inst: Air.Inst.Index, op_refs: anytype)
2070620850 }
2070720851 return temps;
2070820852}
20853
20854const Operand = union(enum) {
20855 none,
20856 reg: Register,
20857 mem: Memory,
20858 imm: Immediate,
20859 inst: Mir.Inst.Index,
20860};
20861
20862const Pattern = struct {
20863 tag: Mir.Inst.FixedTag,
20864 ops: []const Op,
20865 commute: struct { u8, u8 } = .{ 0, 0 },
20866 features: []const std.Target.x86.Feature = &.{},
20867
20868 const Op = union(enum) {
20869 /// match another operand
20870 match: u8,
20871 /// any general purpose register
20872 gpr,
20873 /// any 64-bit mmx register
20874 mm,
20875 /// any 128-bit sse register
20876 xmm,
20877 /// any 256-bit sse register
20878 ymm,
20879 /// any memory
20880 mem,
20881 /// specific immediate
20882 imm: i8,
20883 /// any immediate signed extended from 32 bits
20884 simm32,
20885
20886 fn matches(op: Op, is_mut: bool, temp: Temp, self: *Self) bool {
20887 return switch (op) {
20888 .match => unreachable,
20889 .gpr => switch (temp.tracking(self).short) {
20890 .register => |reg| reg.class() == .general_purpose,
20891 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and
20892 reg_off.off == 0,
20893 else => self.regClassForType(temp.typeOf(self)) == .general_purpose,
20894 },
20895 .mm => switch (temp.tracking(self).short) {
20896 .register => |reg| reg.class() == .mmx,
20897 .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0,
20898 else => self.regClassForType(temp.typeOf(self)) == .mmx,
20899 },
20900 .xmm => switch (temp.tracking(self).short) {
20901 .register => |reg| reg.class() == .sse,
20902 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
20903 else => self.regClassForType(temp.typeOf(self)) == .sse,
20904 },
20905 .ymm => switch (temp.tracking(self).short) {
20906 .register => |reg| reg.class() == .sse,
20907 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
20908 else => self.regClassForType(temp.typeOf(self)) == .sse,
20909 } and temp.typeOf(self).abiSize(self.pt.zcu) > 16,
20910 .mem => (!is_mut or temp.isMut(self)) and temp.tracking(self).short.isMemory(),
20911 .imm => |specific_imm| if (is_mut) unreachable else switch (temp.tracking(self).short) {
20912 .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm,
20913 else => false,
20914 },
20915 .simm32 => if (is_mut) unreachable else switch (temp.tracking(self).short) {
20916 .immediate => |imm| temp.typeOf(self).abiSize(self.pt.zcu) <= 4 or
20917 std.math.cast(i32, @as(i64, @bitCast(imm))) != null,
20918 else => false,
20919 },
20920 };
20921 }
20922 };
20923};
20924fn select(self: *Self, dst_temps: []Temp, src_temps: []const *Temp, patterns: []const Pattern) !void {
20925 patterns: for (patterns) |pattern| {
20926 for (pattern.features) |feature| if (!self.hasFeature(feature)) continue :patterns;
20927 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| if (!switch (src_op) {
20928 .match => |match_index| pattern.ops[match_index],
20929 else => src_op,
20930 }.matches(src_op == .match, src_temp.*, self)) continue :patterns;
20931 while (true) for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
20932 if (switch (switch (src_op) {
20933 .match => |match_index| pattern.ops[match_index],
20934 else => src_op,
20935 }) {
20936 .match => unreachable,
20937 .gpr => try src_temp.toRegClass(.general_purpose, self),
20938 .mm => try src_temp.toRegClass(.mmx, self),
20939 .xmm, .ymm => try src_temp.toRegClass(.sse, self),
20940 .mem, .imm, .simm32 => false,
20941 }) break;
20942 } else break;
20943 var mir_ops: [4]Operand = @splat(.none);
20944 var mir_ops_len = dst_temps.len;
20945 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
20946 const mir_op, const matched_src_op = op: switch (src_op) {
20947 .match => |match_index| {
20948 dst_temps[match_index] = src_temp.*;
20949 break :op .{ &mir_ops[match_index], pattern.ops[match_index] };
20950 },
20951 else => {
20952 defer mir_ops_len += 1;
20953 break :op .{ &mir_ops[mir_ops_len], src_op };
20954 },
20955 };
20956 const src_mcv = src_temp.tracking(self).short;
20957 mir_op.* = switch (matched_src_op) {
20958 .match => unreachable,
20959 .gpr => .{ .reg = registerAlias(
20960 src_mcv.register,
20961 @intCast(src_temp.typeOf(self).abiSize(self.pt.zcu)),
20962 ) },
20963 .mm => .{ .reg = src_mcv.register },
20964 .xmm => .{ .reg = src_mcv.register.to128() },
20965 .ymm => .{ .reg = src_mcv.register.to256() },
20966 .mem => .{ .mem = try src_mcv.mem(self, self.memSize(src_temp.typeOf(self))) },
20967 .imm => |imm| .{ .imm = .s(imm) },
20968 .simm32 => switch (src_temp.typeOf(self).abiSize(self.pt.zcu)) {
20969 else => unreachable,
20970 1 => .{ .imm = if (std.math.cast(i8, @as(i64, @bitCast(src_mcv.immediate)))) |small|
20971 .s(small)
20972 else
20973 .u(@as(u8, @intCast(src_mcv.immediate))) },
20974 2 => .{ .imm = if (std.math.cast(i16, @as(i64, @bitCast(src_mcv.immediate)))) |small|
20975 .s(small)
20976 else
20977 .u(@as(u16, @intCast(src_mcv.immediate))) },
20978 3...8 => .{ .imm = if (std.math.cast(i32, @as(i64, @bitCast(src_mcv.immediate)))) |small|
20979 .s(small)
20980 else
20981 .u(@as(u32, @intCast(src_mcv.immediate))) },
20982 },
20983 };
20984 }
20985 for (
20986 pattern.ops[0..dst_temps.len],
20987 dst_temps,
20988 mir_ops[0..dst_temps.len],
20989 ) |dst_op, *dst_temp, *mir_op| {
20990 if (mir_op.* != .none) continue;
20991 const ty = src_temps[0].typeOf(self);
20992 switch (dst_op) {
20993 .match => |match_index| {
20994 dst_temp.* = dst_temps[match_index];
20995 mir_op.* = mir_ops[match_index];
20996 },
20997 .gpr => {
20998 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.gp);
20999 mir_op.* = .{ .reg = registerAlias(
21000 dst_temp.tracking(self).short.register,
21001 @intCast(ty.abiSize(self.pt.zcu)),
21002 ) };
21003 },
21004 .mm => @panic("TODO"),
21005 .xmm => {
21006 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.sse);
21007 mir_op.* = .{ .reg = dst_temp.tracking(self).short.register.to128() };
21008 },
21009 .ymm => {
21010 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.sse);
21011 mir_op.* = .{ .reg = dst_temp.tracking(self).short.register.to256() };
21012 },
21013 .mem => @panic("TODO"),
21014 .imm, .simm32 => unreachable, // unmodifiable destination
21015 }
21016 }
21017 std.mem.swap(Operand, &mir_ops[pattern.commute[0]], &mir_ops[pattern.commute[1]]);
21018 self.asmOps(pattern.tag, mir_ops) catch |err| switch (err) {
21019 error.InvalidInstruction => {
21020 const fixes = @tagName(pattern.tag[0]);
21021 const fixes_replace = std.mem.indexOfScalar(u8, fixes, '_').?;
21022 return self.fail(
21023 "invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'",
21024 .{
21025 fixes[0..fixes_replace],
21026 @tagName(pattern.tag[1]),
21027 fixes[fixes_replace + 1 ..],
21028 @tagName(mir_ops[0]),
21029 @tagName(mir_ops[1]),
21030 @tagName(mir_ops[2]),
21031 @tagName(mir_ops[3]),
21032 },
21033 );
21034 },
21035 else => |e| return e,
21036 };
21037 return;
21038 }
21039 log.err("failed to select:", .{});
21040 for (src_temps) |src_temp| log.err("{}", .{src_temp.tracking(self)});
21041 return self.fail("failed to select", .{});
21042}
src/arch/x86_64/Mir.zig+1-1
......@@ -22,7 +22,7 @@ pub const Inst = struct {
2222 /// ___
2323 @"_",
2424
25 /// Integer __
25 /// Integer ___
2626 i_,
2727
2828 /// ___ Left
src/register_manager.zig+11-11
......@@ -41,12 +41,12 @@ pub fn RegisterManager(
4141 registers: TrackedRegisters = undefined,
4242 /// Tracks which registers are free (in which case the
4343 /// corresponding bit is set to 1)
44 free_registers: RegisterBitSet = RegisterBitSet.initFull(),
44 free_registers: RegisterBitSet = .initFull(),
4545 /// Tracks all registers allocated in the course of this
4646 /// function
47 allocated_registers: RegisterBitSet = RegisterBitSet.initEmpty(),
47 allocated_registers: RegisterBitSet = .initEmpty(),
4848 /// Tracks registers which are locked from being allocated
49 locked_registers: RegisterBitSet = RegisterBitSet.initEmpty(),
49 locked_registers: RegisterBitSet = .initEmpty(),
5050
5151 const Self = @This();
5252
......@@ -420,8 +420,8 @@ const MockRegister1 = enum(u2) {
420420 &MockRegister1.allocatable_registers,
421421 );
422422
423 const gp: RM.RegisterBitSet = blk: {
424 var set = RM.RegisterBitSet.initEmpty();
423 const gp = blk: {
424 var set: RM.RegisterBitSet = .initEmpty();
425425 set.setRangeValue(.{
426426 .start = 0,
427427 .end = allocatable_registers.len,
......@@ -448,8 +448,8 @@ const MockRegister2 = enum(u2) {
448448 &MockRegister2.allocatable_registers,
449449 );
450450
451 const gp: RM.RegisterBitSet = blk: {
452 var set = RM.RegisterBitSet.initEmpty();
451 const gp = blk: {
452 var set: RM.RegisterBitSet = .initEmpty();
453453 set.setRangeValue(.{
454454 .start = 0,
455455 .end = allocatable_registers.len,
......@@ -489,16 +489,16 @@ const MockRegister3 = enum(u3) {
489489 &MockRegister3.allocatable_registers,
490490 );
491491
492 const gp: RM.RegisterBitSet = blk: {
493 var set = RM.RegisterBitSet.initEmpty();
492 const gp = blk: {
493 var set: RM.RegisterBitSet = .initEmpty();
494494 set.setRangeValue(.{
495495 .start = 0,
496496 .end = gp_regs.len,
497497 }, true);
498498 break :blk set;
499499 };
500 const ext: RM.RegisterBitSet = blk: {
501 var set = RM.RegisterBitSet.initEmpty();
500 const ext = blk: {
501 var set: RM.RegisterBitSet = .initEmpty();
502502 set.setRangeValue(.{
503503 .start = gp_regs.len,
504504 .end = allocatable_registers.len,