authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-11-08 08:32:00-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-16 20:42:08-05:00
logbeadf702b8d0421cc84c47ebc9644ce07e22c306
tree2a9db54d61ab71db479248d06350dfa1f3f99d2a
parentaf1191ea8ba55a6eab53e0b561355bb116bdbf2d

x86_64: rewrite arithmetic

4 files changed, 693 insertions(+), 359 deletions(-)

lib/std/mem.zig+2-2
...@@ -883,8 +883,8 @@ fn SliceTo(comptime T: type, comptime end: std.meta.Elem(T)) type {...@@ -883,8 +883,8 @@ fn SliceTo(comptime T: type, comptime end: std.meta.Elem(T)) type {
883 @compileError("invalid type given to std.mem.sliceTo: " ++ @typeName(T));883 @compileError("invalid type given to std.mem.sliceTo: " ++ @typeName(T));
884}884}
885885
886/// Takes an array, a pointer to an array, a sentinel-terminated pointer, or a slice and886/// Takes a pointer to an array, a sentinel-terminated pointer, or a slice and iterates searching for
887/// iterates searching for the first occurrence of `end`, returning the scanned slice.887/// the first occurrence of `end`, returning the scanned slice.
888/// If `end` is not found, the full length of the array/slice/sentinel terminated pointer is returned.888/// If `end` is not found, the full length of the array/slice/sentinel terminated pointer is returned.
889/// If the pointer type is sentinel terminated and `end` matches that terminator, the889/// If the pointer type is sentinel terminated and `end` matches that terminator, the
890/// resulting slice is also sentinel terminated.890/// resulting slice is also sentinel terminated.
src/arch/x86_64/CodeGen.zig+679-345
...@@ -767,7 +767,7 @@ const FrameAlloc = struct {...@@ -767,7 +767,7 @@ const FrameAlloc = struct {
767 .size = spill_size,767 .size = spill_size,
768 .pad = @intCast(spill_size - abi_size),768 .pad = @intCast(spill_size - abi_size),
769 .alignment = ty.abiAlignment(zcu).maxStrict(769 .alignment = ty.abiAlignment(zcu).maxStrict(
770 InternPool.Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),770 .fromNonzeroByteUnits(@min(spill_size, 8)),
771 ),771 ),
772 });772 });
773 }773 }
...@@ -806,7 +806,7 @@ pub fn generate(...@@ -806,7 +806,7 @@ pub fn generate(
806 const gpa = zcu.gpa;806 const gpa = zcu.gpa;
807 const ip = &zcu.intern_pool;807 const ip = &zcu.intern_pool;
808 const func = zcu.funcInfo(func_index);808 const func = zcu.funcInfo(func_index);
809 const fn_type = Type.fromInterned(func.ty);809 const fn_type: Type = .fromInterned(func.ty);
810 const mod = zcu.navFileScope(func.owner_nav).mod;810 const mod = zcu.navFileScope(func.owner_nav).mod;
811811
812 var function: Self = .{812 var function: Self = .{
...@@ -846,7 +846,7 @@ pub fn generate(...@@ -846,7 +846,7 @@ pub fn generate(
846 try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max);846 try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max);
847 for (0..Temp.Index.max) |temp_index| {847 for (0..Temp.Index.max) |temp_index| {
848 const temp: Temp.Index = @enumFromInt(temp_index);848 const temp: Temp.Index = @enumFromInt(temp_index);
849 function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), InstTracking.init(.none));849 function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), .init(.none));
850 }850 }
851851
852 wip_mir_log.debug("{}:", .{fmtNav(func.owner_nav, ip)});852 wip_mir_log.debug("{}:", .{fmtNav(func.owner_nav, ip)});
...@@ -854,11 +854,11 @@ pub fn generate(...@@ -854,11 +854,11 @@ pub fn generate(
854 try function.frame_allocs.resize(gpa, FrameIndex.named_count);854 try function.frame_allocs.resize(gpa, FrameIndex.named_count);
855 function.frame_allocs.set(855 function.frame_allocs.set(
856 @intFromEnum(FrameIndex.stack_frame),856 @intFromEnum(FrameIndex.stack_frame),
857 FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),857 .init(.{ .size = 0, .alignment = .@"1" }),
858 );858 );
859 function.frame_allocs.set(859 function.frame_allocs.set(
860 @intFromEnum(FrameIndex.call_frame),860 @intFromEnum(FrameIndex.call_frame),
861 FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),861 .init(.{ .size = 0, .alignment = .@"1" }),
862 );862 );
863863
864 const fn_info = zcu.typeToFunc(fn_type).?;864 const fn_info = zcu.typeToFunc(fn_type).?;
...@@ -871,20 +871,19 @@ pub fn generate(...@@ -871,20 +871,19 @@ pub fn generate(
871871
872 function.args = call_info.args;872 function.args = call_info.args;
873 function.ret_mcv = call_info.return_value;873 function.ret_mcv = call_info.return_value;
874 function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), FrameAlloc.init(.{874 function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), .init(.{
875 .size = Type.usize.abiSize(zcu),875 .size = Type.usize.abiSize(zcu),
876 .alignment = Type.usize.abiAlignment(zcu).min(call_info.stack_align),876 .alignment = Type.usize.abiAlignment(zcu).min(call_info.stack_align),
877 }));877 }));
878 function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{878 function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), .init(.{
879 .size = Type.usize.abiSize(zcu),879 .size = Type.usize.abiSize(zcu),
880 .alignment = InternPool.Alignment.min(880 .alignment = call_info.stack_align.min(
881 call_info.stack_align,881 .fromNonzeroByteUnits(function.target.stackAlignment()),
882 InternPool.Alignment.fromNonzeroByteUnits(function.target.stackAlignment()),
883 ),882 ),
884 }));883 }));
885 function.frame_allocs.set(884 function.frame_allocs.set(
886 @intFromEnum(FrameIndex.args_frame),885 @intFromEnum(FrameIndex.args_frame),
887 FrameAlloc.init(.{886 .init(.{
888 .size = call_info.stack_byte_count,887 .size = call_info.stack_byte_count,
889 .alignment = call_info.stack_align,888 .alignment = call_info.stack_align,
890 }),889 }),
...@@ -1242,6 +1241,80 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {...@@ -1242,6 +1241,80 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1242 return result;1241 return result;
1243}1242}
12441243
1244fn asmOps(self: *Self, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {
1245 return switch (ops[0]) {
1246 .none => self.asmOpOnly(tag),
1247 .reg => |reg0| switch (ops[1]) {
1248 .none => self.asmRegister(tag, reg0),
1249 .reg => |reg1| switch (ops[2]) {
1250 .none => self.asmRegisterRegister(tag, reg0, reg1),
1251 .reg => |reg2| switch (ops[3]) {
1252 .none => self.asmRegisterRegisterRegister(tag, reg0, reg1, reg2),
1253 .reg => |reg3| self.asmRegisterRegisterRegisterRegister(tag, reg0, reg1, reg2, reg3),
1254 .imm => |imm3| self.asmRegisterRegisterRegisterImmediate(tag, reg0, reg1, reg2, imm3),
1255 else => error.InvalidInstruction,
1256 },
1257 .mem => |mem2| switch (ops[3]) {
1258 .none => self.asmRegisterRegisterMemory(tag, reg0, reg1, mem2),
1259 .reg => |reg3| self.asmRegisterRegisterMemoryRegister(tag, reg0, reg1, mem2, reg3),
1260 .imm => |imm3| self.asmRegisterRegisterMemoryImmediate(tag, reg0, reg1, mem2, imm3),
1261 else => error.InvalidInstruction,
1262 },
1263 .imm => |imm2| switch (ops[3]) {
1264 .none => self.asmRegisterRegisterImmediate(tag, reg0, reg1, imm2),
1265 else => error.InvalidInstruction,
1266 },
1267 else => error.InvalidInstruction,
1268 },
1269 .mem => |mem1| switch (ops[2]) {
1270 .none => self.asmRegisterMemory(tag, reg0, mem1),
1271 .reg => |reg2| switch (ops[3]) {
1272 .none => self.asmRegisterMemoryRegister(tag, reg0, mem1, reg2),
1273 else => error.InvalidInstruction,
1274 },
1275 .imm => |imm2| switch (ops[3]) {
1276 .none => self.asmRegisterMemoryImmediate(tag, reg0, mem1, imm2),
1277 else => error.InvalidInstruction,
1278 },
1279 else => error.InvalidInstruction,
1280 },
1281 .imm => |imm1| switch (ops[2]) {
1282 .none => self.asmRegisterImmediate(tag, reg0, imm1),
1283 else => error.InvalidInstruction,
1284 },
1285 else => error.InvalidInstruction,
1286 },
1287 .mem => |mem0| switch (ops[1]) {
1288 .none => self.asmMemory(tag, mem0),
1289 .reg => |reg1| switch (ops[2]) {
1290 .none => self.asmMemoryRegister(tag, mem0, reg1),
1291 .reg => |reg2| switch (ops[3]) {
1292 .none => self.asmMemoryRegisterRegister(tag, mem0, reg1, reg2),
1293 else => error.InvalidInstruction,
1294 },
1295 .imm => |imm2| switch (ops[3]) {
1296 .none => self.asmMemoryRegisterImmediate(tag, mem0, reg1, imm2),
1297 else => error.InvalidInstruction,
1298 },
1299 else => error.InvalidInstruction,
1300 },
1301 .imm => |imm1| switch (ops[2]) {
1302 .none => self.asmMemoryImmediate(tag, mem0, imm1),
1303 else => error.InvalidInstruction,
1304 },
1305 else => error.InvalidInstruction,
1306 },
1307 .imm => |imm0| switch (ops[1]) {
1308 .none => self.asmImmediate(tag, imm0),
1309 else => error.InvalidInstruction,
1310 },
1311 .inst => |inst0| switch (ops[1]) {
1312 .none => self.asmReloc(tag, inst0),
1313 else => error.InvalidInstruction,
1314 },
1315 };
1316}
1317
1245/// A `cc` of `.z_and_np` clobbers `reg2`!1318/// A `cc` of `.z_and_np` clobbers `reg2`!
1246fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: Register) !void {1319fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: Register) !void {
1247 _ = try self.addInst(.{1320 _ = try self.addInst(.{
...@@ -1256,7 +1329,7 @@ fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: R...@@ -1256,7 +1329,7 @@ fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: R
1256 },1329 },
1257 .data = .{ .rr = .{1330 .data = .{ .rr = .{
1258 .fixes = switch (cc) {1331 .fixes = switch (cc) {
1259 else => Mir.Inst.Fixes.fromCondition(cc),1332 else => .fromCondition(cc),
1260 .z_and_np, .nz_or_p => ._,1333 .z_and_np, .nz_or_p => ._,
1261 },1334 },
1262 .r1 = reg1,1335 .r1 = reg1,
...@@ -1280,7 +1353,7 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)...@@ -1280,7 +1353,7 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)
1280 },1353 },
1281 .data = .{ .rx = .{1354 .data = .{ .rx = .{
1282 .fixes = switch (cc) {1355 .fixes = switch (cc) {
1283 else => Mir.Inst.Fixes.fromCondition(cc),1356 else => .fromCondition(cc),
1284 .z_and_np => unreachable,1357 .z_and_np => unreachable,
1285 .nz_or_p => ._,1358 .nz_or_p => ._,
1286 },1359 },
...@@ -1303,7 +1376,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {...@@ -1303,7 +1376,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
1303 },1376 },
1304 .data = switch (cc) {1377 .data = switch (cc) {
1305 else => .{ .r = .{1378 else => .{ .r = .{
1306 .fixes = Mir.Inst.Fixes.fromCondition(cc),1379 .fixes = .fromCondition(cc),
1307 .r1 = reg,1380 .r1 = reg,
1308 } },1381 } },
1309 .z_and_np, .nz_or_p => .{ .rr = .{1382 .z_and_np, .nz_or_p => .{ .rr = .{
...@@ -1328,7 +1401,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {...@@ -1328,7 +1401,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
1328 },1401 },
1329 .data = switch (cc) {1402 .data = switch (cc) {
1330 else => .{ .x = .{1403 else => .{ .x = .{
1331 .fixes = Mir.Inst.Fixes.fromCondition(cc),1404 .fixes = .fromCondition(cc),
1332 .payload = payload,1405 .payload = payload,
1333 } },1406 } },
1334 .z_and_np, .nz_or_p => .{ .rx = .{1407 .z_and_np, .nz_or_p => .{ .rx = .{
...@@ -1362,7 +1435,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind...@@ -1362,7 +1435,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind
1362 },1435 },
1363 .data = .{ .inst = .{1436 .data = .{ .inst = .{
1364 .fixes = switch (cc) {1437 .fixes = switch (cc) {
1365 else => Mir.Inst.Fixes.fromCondition(cc),1438 else => .fromCondition(cc),
1366 .z_and_np, .nz_or_p => ._,1439 .z_and_np, .nz_or_p => ._,
1367 },1440 },
1368 .inst = target,1441 .inst = target,
...@@ -1969,7 +2042,7 @@ fn gen(self: *Self) InnerError!void {...@@ -1969,7 +2042,7 @@ fn gen(self: *Self) InnerError!void {
1969 // The address where to store the return value for the caller is in a2042 // The address where to store the return value for the caller is in a
1970 // register which the callee is free to clobber. Therefore, we purposely2043 // register which the callee is free to clobber. Therefore, we purposely
1971 // spill it to stack immediately.2044 // spill it to stack immediately.
1972 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(Type.usize, zcu));2045 const frame_index = try self.allocFrameIndex(.initSpill(Type.usize, zcu));
1973 try self.genSetMem(2046 try self.genSetMem(
1974 .{ .frame = frame_index },2047 .{ .frame = frame_index },
1975 0,2048 0,
...@@ -1986,7 +2059,7 @@ fn gen(self: *Self) InnerError!void {...@@ -1986,7 +2059,7 @@ fn gen(self: *Self) InnerError!void {
1986 if (fn_info.is_var_args) switch (cc) {2059 if (fn_info.is_var_args) switch (cc) {
1987 .x86_64_sysv => {2060 .x86_64_sysv => {
1988 const info = &self.va_info.sysv;2061 const info = &self.va_info.sysv;
1989 const reg_save_area_fi = try self.allocFrameIndex(FrameAlloc.init(.{2062 const reg_save_area_fi = try self.allocFrameIndex(.init(.{
1990 .size = abi.SysV.c_abi_int_param_regs.len * 8 +2063 .size = abi.SysV.c_abi_int_param_regs.len * 8 +
1991 abi.SysV.c_abi_sse_param_regs.len * 16,2064 abi.SysV.c_abi_sse_param_regs.len * 16,
1992 .alignment = .@"16",2065 .alignment = .@"16",
...@@ -2192,7 +2265,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2192,7 +2265,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2192 wip_mir_log.debug("{}", .{self.fmtAir(inst)});2265 wip_mir_log.debug("{}", .{self.fmtAir(inst)});
2193 verbose_tracking_log.debug("{}", .{self.fmtTracking()});2266 verbose_tracking_log.debug("{}", .{self.fmtTracking()});
21942267
2195 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();2268 self.reused_operands = .initEmpty();
2196 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);2269 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
21972270
2198 try self.airArg(inst);2271 try self.airArg(inst);
...@@ -2210,7 +2283,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2210,7 +2283,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2210 wip_mir_log.debug("{}", .{self.fmtAir(inst)});2283 wip_mir_log.debug("{}", .{self.fmtAir(inst)});
2211 verbose_tracking_log.debug("{}", .{self.fmtTracking()});2284 verbose_tracking_log.debug("{}", .{self.fmtTracking()});
22122285
2213 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();2286 self.reused_operands = .initEmpty();
2214 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);2287 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
2215 switch (air_tags[@intFromEnum(inst)]) {2288 switch (air_tags[@intFromEnum(inst)]) {
2216 // zig fmt: off2289 // zig fmt: off
...@@ -2223,9 +2296,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2223,9 +2296,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2223 .sub_wrap,2296 .sub_wrap,
2224 .bool_and,2297 .bool_and,
2225 .bool_or,2298 .bool_or,
2226 .bit_and,
2227 .bit_or,
2228 .xor,
2229 .min,2299 .min,
2230 .max,2300 .max,
2231 => |tag| try self.airBinOp(inst, tag),2301 => |tag| try self.airBinOp(inst, tag),
...@@ -2391,6 +2461,55 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2391,6 +2461,55 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2391 try slot.moveTo(inst, self);2461 try slot.moveTo(inst, self);
2392 },2462 },
2393 .assembly => try self.airAsm(inst),2463 .assembly => try self.airAsm(inst),
2464 inline .bit_and, .bit_or, .xor => |tag| if (use_old or self.typeOfIndex(inst).abiSize(zcu) > @as(
2465 u64,
2466 if (!self.typeOfIndex(inst).isVector(zcu))
2467 8
2468 else if (!self.hasFeature(.avx2))
2469 16
2470 else
2471 32,
2472 )) try self.airBinOp(inst, tag) else {
2473 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
2474 var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
2475 try self.spillEflagsIfOccupied();
2476 const mir_tag: Mir.Inst.Tag = switch (tag) {
2477 else => unreachable,
2478 .bit_and => .@"and",
2479 .bit_or => .@"or",
2480 .xor => .xor,
2481 };
2482 var res: [1]Temp = undefined;
2483 try self.select(&res, &.{ &ops[0], &ops[1] }, &.{
2484 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .ymm, .mem }, .features = &.{.avx2} },
2485 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .mem, .ymm }, .commute = .{ 1, 2 }, .features = &.{.avx2} },
2486 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .ymm, .ymm }, .features = &.{.avx2} },
2487 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .xmm, .mem }, .features = &.{.avx} },
2488 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .mem, .xmm }, .commute = .{ 1, 2 }, .features = &.{.avx} },
2489 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .xmm, .xmm }, .features = &.{.avx} },
2490 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .{ .match = 0 }, .mem }, .features = &.{.sse2} },
2491 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .mem, .{ .match = 0 } }, .features = &.{.sse2} },
2492 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .{ .match = 0 }, .xmm }, .features = &.{.sse2} },
2493 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .xmm, .{ .match = 0 } }, .features = &.{.sse2} },
2494 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .{ .match = 0 }, .mem }, .features = &.{.mmx} },
2495 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .mem, .{ .match = 0 } }, .features = &.{.mmx} },
2496 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .{ .match = 0 }, .mm }, .features = &.{.mmx} },
2497 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .mm, .{ .match = 0 } }, .features = &.{.mmx} },
2498 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .{ .match = 0 }, .simm32 } },
2499 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .simm32, .{ .match = 0 } } },
2500 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .{ .match = 0 }, .gpr } },
2501 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .gpr, .{ .match = 0 } } },
2502 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .simm32 } },
2503 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .simm32, .{ .match = 0 } } },
2504 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .mem } },
2505 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .mem, .{ .match = 0 } } },
2506 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .gpr } },
2507 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .gpr, .{ .match = 0 } } },
2508 });
2509 if (ops[0].index != res[0].index) try ops[0].die(self);
2510 if (ops[1].index != res[0].index) try ops[1].die(self);
2511 try res[0].moveTo(inst, self);
2512 },
2394 .block => if (use_old) try self.airBlock(inst) else {2513 .block => if (use_old) try self.airBlock(inst) else {
2395 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;2514 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
2396 const extra = self.air.extraData(Air.Block, ty_pl.payload);2515 const extra = self.air.extraData(Air.Block, ty_pl.payload);
...@@ -2506,7 +2625,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2506,7 +2625,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2506 else if (opt_child_ty.isSlice(zcu))2625 else if (opt_child_ty.isSlice(zcu))
2507 .qword2626 .qword
2508 else2627 else
2509 Memory.Size.fromSize(opt_child_abi_size)),2628 .fromSize(opt_child_abi_size)),
2510 .u(0),2629 .u(0),
2511 );2630 );
2512 var is_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });2631 var is_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });
...@@ -2529,7 +2648,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2529,7 +2648,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2529 else if (opt_child_ty.isSlice(zcu))2648 else if (opt_child_ty.isSlice(zcu))
2530 .qword2649 .qword
2531 else2650 else
2532 Memory.Size.fromSize(opt_child_abi_size)),2651 .fromSize(opt_child_abi_size)),
2533 .u(0),2652 .u(0),
2534 );2653 );
2535 var is_non_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });2654 var is_non_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });
...@@ -2646,31 +2765,51 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2646,31 +2765,51 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2646 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;2765 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
2647 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;2766 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
2648 var ops = try self.tempsFromOperands(inst, .{extra.struct_operand});2767 var ops = try self.tempsFromOperands(inst, .{extra.struct_operand});
2649 try ops[0].toOffset(self.fieldOffset(self.typeOf(extra.struct_operand), self.typeOfIndex(inst), extra.field_index), self);2768 try ops[0].toOffset(self.fieldOffset(
2769 self.typeOf(extra.struct_operand),
2770 self.typeOfIndex(inst),
2771 extra.field_index,
2772 ), self);
2650 try ops[0].moveTo(inst, self);2773 try ops[0].moveTo(inst, self);
2651 },2774 },
2652 .struct_field_ptr_index_0 => if (use_old) try self.airStructFieldPtrIndex(inst, 0) else {2775 .struct_field_ptr_index_0 => if (use_old) try self.airStructFieldPtrIndex(inst, 0) else {
2653 const ty_op = air_datas[@intFromEnum(inst)].ty_op;2776 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
2654 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});2777 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
2655 try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 0), self);2778 try ops[0].toOffset(self.fieldOffset(
2779 self.typeOf(ty_op.operand),
2780 self.typeOfIndex(inst),
2781 0,
2782 ), self);
2656 try ops[0].moveTo(inst, self);2783 try ops[0].moveTo(inst, self);
2657 },2784 },
2658 .struct_field_ptr_index_1 => if (use_old) try self.airStructFieldPtrIndex(inst, 1) else {2785 .struct_field_ptr_index_1 => if (use_old) try self.airStructFieldPtrIndex(inst, 1) else {
2659 const ty_op = air_datas[@intFromEnum(inst)].ty_op;2786 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
2660 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});2787 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
2661 try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 1), self);2788 try ops[0].toOffset(self.fieldOffset(
2789 self.typeOf(ty_op.operand),
2790 self.typeOfIndex(inst),
2791 1,
2792 ), self);
2662 try ops[0].moveTo(inst, self);2793 try ops[0].moveTo(inst, self);
2663 },2794 },
2664 .struct_field_ptr_index_2 => if (use_old) try self.airStructFieldPtrIndex(inst, 2) else {2795 .struct_field_ptr_index_2 => if (use_old) try self.airStructFieldPtrIndex(inst, 2) else {
2665 const ty_op = air_datas[@intFromEnum(inst)].ty_op;2796 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
2666 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});2797 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
2667 try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 2), self);2798 try ops[0].toOffset(self.fieldOffset(
2799 self.typeOf(ty_op.operand),
2800 self.typeOfIndex(inst),
2801 2,
2802 ), self);
2668 try ops[0].moveTo(inst, self);2803 try ops[0].moveTo(inst, self);
2669 },2804 },
2670 .struct_field_ptr_index_3 => if (use_old) try self.airStructFieldPtrIndex(inst, 3) else {2805 .struct_field_ptr_index_3 => if (use_old) try self.airStructFieldPtrIndex(inst, 3) else {
2671 const ty_op = air_datas[@intFromEnum(inst)].ty_op;2806 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
2672 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});2807 var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
2673 try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 3), self);2808 try ops[0].toOffset(self.fieldOffset(
2809 self.typeOf(ty_op.operand),
2810 self.typeOfIndex(inst),
2811 3,
2812 ), self);
2674 try ops[0].moveTo(inst, self);2813 try ops[0].moveTo(inst, self);
2675 },2814 },
2676 .slice => if (use_old) try self.airSlice(inst) else {2815 .slice => if (use_old) try self.airSlice(inst) else {
...@@ -2775,7 +2914,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2775,7 +2914,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2775 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;2914 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
2776 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;2915 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
2777 var ops = try self.tempsFromOperands(inst, .{extra.field_ptr});2916 var ops = try self.tempsFromOperands(inst, .{extra.field_ptr});
2778 try ops[0].toOffset(-self.fieldOffset(self.typeOfIndex(inst), self.typeOf(extra.field_ptr), extra.field_index), self);2917 try ops[0].toOffset(-self.fieldOffset(
2918 self.typeOfIndex(inst),
2919 self.typeOf(extra.field_ptr),
2920 extra.field_index,
2921 ), self);
2779 try ops[0].moveTo(inst, self);2922 try ops[0].moveTo(inst, self);
2780 },2923 },
27812924
...@@ -2813,7 +2956,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {...@@ -2813,7 +2956,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2813 const ip = &zcu.intern_pool;2956 const ip = &zcu.intern_pool;
2814 switch (Type.fromInterned(lazy_sym.ty).zigTypeTag(zcu)) {2957 switch (Type.fromInterned(lazy_sym.ty).zigTypeTag(zcu)) {
2815 .@"enum" => {2958 .@"enum" => {
2816 const enum_ty = Type.fromInterned(lazy_sym.ty);2959 const enum_ty: Type = .fromInterned(lazy_sym.ty);
2817 wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)});2960 wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)});
28182961
2819 const resolved_cc = abi.resolveCallingConvention(.auto, self.target.*);2962 const resolved_cc = abi.resolveCallingConvention(.auto, self.target.*);
...@@ -2919,7 +3062,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {...@@ -2919,7 +3062,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
2919 else => unreachable, // Why didn't the result die?3062 else => unreachable, // Why didn't the result die?
2920 } else {3063 } else {
2921 tracking_log.debug("{} => {} (birth)", .{ inst, result });3064 tracking_log.debug("{} => {} (birth)", .{ inst, result });
2922 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));3065 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));
2923 // In some cases, an operand may be reused as the result.3066 // In some cases, an operand may be reused as the result.
2924 // If that operand died and was a register, it was freed by3067 // If that operand died and was a register, it was freed by
2925 // processDeath, so we have to "re-allocate" the register.3068 // processDeath, so we have to "re-allocate" the register.
...@@ -3067,7 +3210,7 @@ fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {...@@ -3067,7 +3210,7 @@ fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
3067 const zcu = pt.zcu;3210 const zcu = pt.zcu;
3068 const ptr_ty = self.typeOfIndex(inst);3211 const ptr_ty = self.typeOfIndex(inst);
3069 const val_ty = ptr_ty.childType(zcu);3212 const val_ty = ptr_ty.childType(zcu);
3070 return self.allocFrameIndex(FrameAlloc.init(.{3213 return self.allocFrameIndex(.init(.{
3071 .size = std.math.cast(u32, val_ty.abiSize(zcu)) orelse {3214 .size = std.math.cast(u32, val_ty.abiSize(zcu)) orelse {
3072 return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(pt)});3215 return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(pt)});
3073 },3216 },
...@@ -3107,35 +3250,49 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b...@@ -3107,35 +3250,49 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
3107 },3250 },
3108 else => 8,3251 else => 8,
3109 })) {3252 })) {
3110 if (self.register_manager.tryAllocReg(inst, self.regClassForType(ty))) |reg| {3253 if (self.register_manager.tryAllocReg(inst, self.regSetForType(ty))) |reg| {
3111 return MCValue{ .register = registerAlias(reg, abi_size) };3254 return MCValue{ .register = registerAlias(reg, abi_size) };
3112 }3255 }
3113 }3256 }
3114 }3257 }
31153258
3116 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, zcu));3259 const frame_index = try self.allocFrameIndex(.initSpill(ty, zcu));
3117 return .{ .load_frame = .{ .index = frame_index } };3260 return .{ .load_frame = .{ .index = frame_index } };
3118}3261}
31193262
3120fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {3263fn regClassForType(self: *Self, ty: Type) Register.Class {
3121 const pt = self.pt;3264 const pt = self.pt;
3122 const zcu = pt.zcu;3265 const zcu = pt.zcu;
3123 return switch (ty.zigTypeTag(zcu)) {3266 return switch (ty.zigTypeTag(zcu)) {
3124 .float => switch (ty.floatBits(self.target.*)) {3267 .float => switch (ty.floatBits(self.target.*)) {
3125 80 => abi.RegisterClass.x87,3268 80 => .x87,
3126 else => abi.RegisterClass.sse,3269 else => .sse,
3127 },3270 },
3128 .vector => switch (ty.childType(zcu).toIntern()) {3271 .vector => switch (ty.childType(zcu).toIntern()) {
3129 .bool_type, .u1_type => abi.RegisterClass.gp,3272 .bool_type, .u1_type => .general_purpose,
3130 else => if (ty.isAbiInt(zcu) and ty.intInfo(zcu).bits == 1)3273 else => if (ty.isAbiInt(zcu) and ty.intInfo(zcu).bits == 1)
3131 abi.RegisterClass.gp3274 .general_purpose
3132 else3275 else
3133 abi.RegisterClass.sse,3276 .sse,
3134 },3277 },
3135 else => abi.RegisterClass.gp,3278 else => .general_purpose,
3136 };3279 };
3137}3280}
31383281
3282fn regSetForRegClass(rc: Register.Class) RegisterManager.RegisterBitSet {
3283 return switch (rc) {
3284 .general_purpose => abi.RegisterClass.gp,
3285 .segment, .ip => unreachable,
3286 .x87 => abi.RegisterClass.x87,
3287 .mmx => @panic("TODO"),
3288 .sse => abi.RegisterClass.sse,
3289 };
3290}
3291
3292fn regSetForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {
3293 return regSetForRegClass(self.regClassForType(ty));
3294}
3295
3139const State = struct {3296const State = struct {
3140 registers: RegisterManager.TrackedRegisters,3297 registers: RegisterManager.TrackedRegisters,
3141 reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,3298 reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
...@@ -3292,7 +3449,7 @@ pub fn spillRegisters(self: *Self, comptime registers: []const Register) !void {...@@ -3292,7 +3449,7 @@ pub fn spillRegisters(self: *Self, comptime registers: []const Register) !void {
3292/// allocated. A second call to `copyToTmpRegister` may return the same register.3449/// allocated. A second call to `copyToTmpRegister` may return the same register.
3293/// This can have a side effect of spilling instructions to the stack to free up a register.3450/// This can have a side effect of spilling instructions to the stack to free up a register.
3294fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {3451fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
3295 const reg = try self.register_manager.allocReg(null, self.regClassForType(ty));3452 const reg = try self.register_manager.allocReg(null, self.regSetForType(ty));
3296 try self.genSetReg(reg, ty, mcv, .{});3453 try self.genSetReg(reg, ty, mcv, .{});
3297 return reg;3454 return reg;
3298}3455}
...@@ -3307,7 +3464,7 @@ fn copyToRegisterWithInstTracking(...@@ -3307,7 +3464,7 @@ fn copyToRegisterWithInstTracking(
3307 ty: Type,3464 ty: Type,
3308 mcv: MCValue,3465 mcv: MCValue,
3309) !MCValue {3466) !MCValue {
3310 const reg: Register = try self.register_manager.allocReg(reg_owner, self.regClassForType(ty));3467 const reg: Register = try self.register_manager.allocReg(reg_owner, self.regSetForType(ty));
3311 try self.genSetReg(reg, ty, mcv, .{});3468 try self.genSetReg(reg, ty, mcv, .{});
3312 return MCValue{ .register = reg };3469 return MCValue{ .register = reg };
3313}3470}
...@@ -3910,7 +4067,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -3910,7 +4067,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3910 .storage = .{ .repeated_elem = mask_val.ip_index },4067 .storage = .{ .repeated_elem = mask_val.ip_index },
3911 } });4068 } });
39124069
3913 const splat_mcv = try self.genTypedValue(Value.fromInterned(splat_val));4070 const splat_mcv = try self.genTypedValue(.fromInterned(splat_val));
3914 const splat_addr_mcv: MCValue = switch (splat_mcv) {4071 const splat_addr_mcv: MCValue = switch (splat_mcv) {
3915 .memory, .indirect, .load_frame => splat_mcv.address(),4072 .memory, .indirect, .load_frame => splat_mcv.address(),
3916 else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },4073 else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },
...@@ -3923,7 +4080,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -3923,7 +4080,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3923 .{ .vp_, .@"and" },4080 .{ .vp_, .@"and" },
3924 dst_alias,4081 dst_alias,
3925 dst_alias,4082 dst_alias,
3926 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),4083 try splat_addr_mcv.deref().mem(self, .fromSize(splat_abi_size)),
3927 );4084 );
3928 if (src_abi_size > 16) {4085 if (src_abi_size > 16) {
3929 const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);4086 const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
...@@ -3947,7 +4104,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -3947,7 +4104,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3947 try self.asmRegisterMemory(4104 try self.asmRegisterMemory(
3948 .{ .p_, .@"and" },4105 .{ .p_, .@"and" },
3949 dst_alias,4106 dst_alias,
3950 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),4107 try splat_addr_mcv.deref().mem(self, .fromSize(splat_abi_size)),
3951 );4108 );
3952 try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);4109 try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
3953 }4110 }
...@@ -3992,7 +4149,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {...@@ -3992,7 +4149,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
3992 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;4149 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
39934150
3994 const slice_ty = self.typeOfIndex(inst);4151 const slice_ty = self.typeOfIndex(inst);
3995 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));4152 const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu));
39964153
3997 const ptr_ty = self.typeOf(bin_op.lhs);4154 const ptr_ty = self.typeOf(bin_op.lhs);
3998 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }, .{});4155 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }, .{});
...@@ -4129,7 +4286,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -4129,7 +4286,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
4129 state: State,4286 state: State,
4130 reloc: Mir.Inst.Index,4287 reloc: Mir.Inst.Index,
4131 } = if (signed and tag == .div_floor) state: {4288 } = if (signed and tag == .div_floor) state: {
4132 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(Type.usize, zcu));4289 const frame_index = try self.allocFrameIndex(.initType(Type.usize, zcu));
4133 try self.asmMemoryImmediate(4290 try self.asmMemoryImmediate(
4134 .{ ._, .mov },4291 .{ ._, .mov },
4135 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },4292 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
...@@ -4259,7 +4416,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -4259,7 +4416,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
4259 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);4416 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
42604417
4261 const tmp_regs =4418 const tmp_regs =
4262 try self.register_manager.allocRegs(2, .{null} ** 2, abi.RegisterClass.gp);4419 try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
4263 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);4420 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
4264 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);4421 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
42654422
...@@ -4698,8 +4855,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4698,8 +4855,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4698 else => {},4855 else => {},
4699 }4856 }
47004857
4701 const frame_index =4858 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
4702 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
4703 try self.genSetMem(4859 try self.genSetMem(
4704 .{ .frame = frame_index },4860 .{ .frame = frame_index },
4705 @intCast(tuple_ty.structFieldOffset(1, zcu)),4861 @intCast(tuple_ty.structFieldOffset(1, zcu)),
...@@ -4717,8 +4873,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4717,8 +4873,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4717 break :result .{ .load_frame = .{ .index = frame_index } };4873 break :result .{ .load_frame = .{ .index = frame_index } };
4718 }4874 }
47194875
4720 const frame_index =4876 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
4721 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
4722 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);4877 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
4723 break :result .{ .load_frame = .{ .index = frame_index } };4878 break :result .{ .load_frame = .{ .index = frame_index } };
4724 },4879 },
...@@ -4776,8 +4931,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4776,8 +4931,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4776 else => {},4931 else => {},
4777 }4932 }
47784933
4779 const frame_index =4934 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
4780 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
4781 try self.genSetMem(4935 try self.genSetMem(
4782 .{ .frame = frame_index },4936 .{ .frame = frame_index },
4783 @intCast(tuple_ty.structFieldOffset(1, zcu)),4937 @intCast(tuple_ty.structFieldOffset(1, zcu)),
...@@ -4796,7 +4950,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4796,7 +4950,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4796 }4950 }
47974951
4798 const frame_index =4952 const frame_index =
4799 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));4953 try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
4800 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);4954 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
4801 break :result .{ .load_frame = .{ .index = frame_index } };4955 break :result .{ .load_frame = .{ .index = frame_index } };
4802 },4956 },
...@@ -4833,7 +4987,7 @@ fn genSetFrameTruncatedOverflowCompare(...@@ -4833,7 +4987,7 @@ fn genSetFrameTruncatedOverflowCompare(
4833 const rest_ty = try pt.intType(.unsigned, int_info.bits - hi_bits);4987 const rest_ty = try pt.intType(.unsigned, int_info.bits - hi_bits);
48344988
4835 const temp_regs =4989 const temp_regs =
4836 try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);4990 try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp);
4837 const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);4991 const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
4838 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);4992 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
48394993
...@@ -4916,7 +5070,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4916,7 +5070,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4916 const rhs_mcv = try self.resolveInst(bin_op.rhs);5070 const rhs_mcv = try self.resolveInst(bin_op.rhs);
49175071
4918 const temp_regs =5072 const temp_regs =
4919 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);5073 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
4920 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);5074 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
4921 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);5075 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
49225076
...@@ -5080,7 +5234,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -5080,7 +5234,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
5080 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);5234 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
50815235
5082 const tmp_regs =5236 const tmp_regs =
5083 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);5237 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
5084 const tmp_locks = self.register_manager.lockRegsAssumeUnused(4, tmp_regs);5238 const tmp_locks = self.register_manager.lockRegsAssumeUnused(4, tmp_regs);
5085 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);5239 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
50865240
...@@ -5217,7 +5371,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -5217,7 +5371,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
5217 self.eflags_inst = inst;5371 self.eflags_inst = inst;
5218 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };5372 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
5219 } else {5373 } else {
5220 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));5374 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
5221 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);5375 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
5222 break :result .{ .load_frame = .{ .index = frame_index } };5376 break :result .{ .load_frame = .{ .index = frame_index } };
5223 },5377 },
...@@ -5228,7 +5382,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -5228,7 +5382,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
5228 src_ty.fmt(pt), dst_ty.fmt(pt),5382 src_ty.fmt(pt), dst_ty.fmt(pt),
5229 });5383 });
52305384
5231 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));5385 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
5232 if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {5386 if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
5233 try self.genSetMem(5387 try self.genSetMem(
5234 .{ .frame = frame_index },5388 .{ .frame = frame_index },
...@@ -5301,10 +5455,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue...@@ -5301,10 +5455,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
5301 };5455 };
5302 switch (mat_rhs) {5456 switch (mat_rhs) {
5303 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),5457 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),
5304 .memory, .indirect, .load_frame => try self.asmMemory(5458 .memory, .indirect, .load_frame => try self.asmMemory(tag, try mat_rhs.mem(self, .fromSize(abi_size))),
5305 tag,
5306 try mat_rhs.mem(self, Memory.Size.fromSize(abi_size)),
5307 ),
5308 else => unreachable,5459 else => unreachable,
5309 }5460 }
5310 if (tag[1] == .div and bit_size == 8) try self.asmRegisterRegister(.{ ._, .mov }, .dl, .ah);5461 if (tag[1] == .div and bit_size == 8) try self.asmRegisterRegister(.{ ._, .mov }, .dl, .ah);
...@@ -5575,7 +5726,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -5575,7 +5726,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
5575 defer self.register_manager.unlockReg(shift_lock);5726 defer self.register_manager.unlockReg(shift_lock);
55765727
5577 const mask_ty = try pt.vectorType(.{ .len = 16, .child = .u8_type });5728 const mask_ty = try pt.vectorType(.{ .len = 16, .child = .u8_type });
5578 const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{5729 const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
5579 .ty = mask_ty.toIntern(),5730 .ty = mask_ty.toIntern(),
5580 .storage = .{ .elems = &([1]InternPool.Index{5731 .storage = .{ .elems = &([1]InternPool.Index{
5581 (try rhs_ty.childType(zcu).maxIntScalar(pt, Type.u8)).toIntern(),5732 (try rhs_ty.childType(zcu).maxIntScalar(pt, Type.u8)).toIntern(),
...@@ -5811,7 +5962,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -5811,7 +5962,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
5811 .{5962 .{
5812 .base = .{ .reg = src_reg },5963 .base = .{ .reg = src_reg },
5813 .mod = .{ .rm = .{5964 .mod = .{ .rm = .{
5814 .size = Memory.Size.fromSize(err_abi_size),5965 .size = .fromSize(err_abi_size),
5815 .disp = err_off,5966 .disp = err_off,
5816 } },5967 } },
5817 },5968 },
...@@ -5853,7 +6004,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {...@@ -5853,7 +6004,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5853 .{6004 .{
5854 .base = .{ .reg = src_reg },6005 .base = .{ .reg = src_reg },
5855 .mod = .{ .rm = .{6006 .mod = .{ .rm = .{
5856 .size = Memory.Size.fromSize(err_abi_size),6007 .size = .fromSize(err_abi_size),
5857 .disp = err_off,6008 .disp = err_off,
5858 } },6009 } },
5859 },6010 },
...@@ -5909,7 +6060,7 @@ fn genUnwrapErrUnionPayloadMir(...@@ -5909,7 +6060,7 @@ fn genUnwrapErrUnionPayloadMir(
5909 const eu_lock = self.register_manager.lockReg(reg);6060 const eu_lock = self.register_manager.lockReg(reg);
5910 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);6061 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
59116062
5912 const payload_in_gp = self.regClassForType(payload_ty).supersetOf(abi.RegisterClass.gp);6063 const payload_in_gp = self.regSetForType(payload_ty).supersetOf(abi.RegisterClass.gp);
5913 const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)6064 const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)
5914 try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)6065 try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)
5915 else6066 else
...@@ -6043,7 +6194,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -6043,7 +6194,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
6043 const result: MCValue = result: {6194 const result: MCValue = result: {
6044 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 };6195 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 };
60456196
6046 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));6197 const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu));
6047 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));6198 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
6048 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));6199 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
6049 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{});6200 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{});
...@@ -6066,7 +6217,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6066,7 +6217,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
6066 const result: MCValue = result: {6217 const result: MCValue = result: {
6067 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand);6218 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand);
60686219
6069 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));6220 const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu));
6070 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));6221 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
6071 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));6222 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
6072 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{});6223 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{});
...@@ -6297,7 +6448,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6297,7 +6448,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
6297 index_reg.to64(),6448 index_reg.to64(),
6298 ),6449 ),
6299 .sse => {6450 .sse => {
6300 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, zcu));6451 const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu));
6301 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});6452 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});
6302 try self.asmMemoryRegister(6453 try self.asmMemoryRegister(
6303 .{ ._, .bt },6454 .{ ._, .bt },
...@@ -6342,7 +6493,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6342,7 +6493,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
63426493
6343 switch (array_mcv) {6494 switch (array_mcv) {
6344 .register => {6495 .register => {
6345 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, zcu));6496 const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu));
6346 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});6497 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});
6347 try self.asmRegisterMemory(6498 try self.asmRegisterMemory(
6348 .{ ._, .lea },6499 .{ ._, .lea },
...@@ -7175,7 +7326,7 @@ fn genByteSwap(...@@ -7175,7 +7326,7 @@ fn genByteSwap(
7175 const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;7326 const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
71767327
7177 const temp_regs =7328 const temp_regs =
7178 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);7329 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
7179 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);7330 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
7180 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);7331 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
71817332
...@@ -7472,11 +7623,11 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)...@@ -7472,11 +7623,11 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
7472 else => unreachable,7623 else => unreachable,
7473 });7624 });
7474 const sign_mem: Memory = if (sign_mcv.isMemory())7625 const sign_mem: Memory = if (sign_mcv.isMemory())
7475 try sign_mcv.mem(self, Memory.Size.fromSize(abi_size))7626 try sign_mcv.mem(self, .fromSize(abi_size))
7476 else7627 else
7477 .{7628 .{
7478 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },7629 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
7479 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } },7630 .mod = .{ .rm = .{ .size = .fromSize(abi_size) } },
7480 };7631 };
74817632
7482 if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(7633 if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(
...@@ -7657,7 +7808,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro...@@ -7657,7 +7808,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7657 mir_tag,7808 mir_tag,
7658 dst_alias,7809 dst_alias,
7659 dst_alias,7810 dst_alias,
7660 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),7811 try src_mcv.mem(self, .fromSize(abi_size)),
7661 .u(@as(u5, @bitCast(mode))),7812 .u(@as(u5, @bitCast(mode))),
7662 ) else try self.asmRegisterRegisterRegisterImmediate(7813 ) else try self.asmRegisterRegisterRegisterImmediate(
7663 mir_tag,7814 mir_tag,
...@@ -7672,7 +7823,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro...@@ -7672,7 +7823,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7672 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(7823 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
7673 mir_tag,7824 mir_tag,
7674 dst_alias,7825 dst_alias,
7675 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),7826 try src_mcv.mem(self, .fromSize(abi_size)),
7676 .u(@as(u5, @bitCast(mode))),7827 .u(@as(u5, @bitCast(mode))),
7677 ) else try self.asmRegisterRegisterImmediate(7828 ) else try self.asmRegisterRegisterImmediate(
7678 mir_tag,7829 mir_tag,
...@@ -7714,7 +7865,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -7714,7 +7865,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7714 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(7865 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
7715 .l,7866 .l,
7716 registerAlias(dst_mcv.register, cmov_abi_size),7867 registerAlias(dst_mcv.register, cmov_abi_size),
7717 try src_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),7868 try src_mcv.mem(self, .fromSize(cmov_abi_size)),
7718 ),7869 ),
7719 else => {7870 else => {
7720 const val_reg = try self.copyToTmpRegister(ty, src_mcv);7871 const val_reg = try self.copyToTmpRegister(ty, src_mcv);
...@@ -7767,7 +7918,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -7767,7 +7918,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7767 const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;7918 const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;
77687919
7769 const tmp_regs =7920 const tmp_regs =
7770 try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);7921 try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp);
7771 const tmp_locks = self.register_manager.lockRegsAssumeUnused(3, tmp_regs);7922 const tmp_locks = self.register_manager.lockRegsAssumeUnused(3, tmp_regs);
7772 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);7923 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
77737924
...@@ -7868,7 +8019,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -7868,7 +8019,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7868 const dst_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))8019 const dst_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
7869 src_mcv.getReg().?8020 src_mcv.getReg().?
7870 else8021 else
7871 try self.register_manager.allocReg(inst, self.regClassForType(ty));8022 try self.register_manager.allocReg(inst, self.regSetForType(ty));
7872 const dst_alias = registerAlias(dst_reg, abi_size);8023 const dst_alias = registerAlias(dst_reg, abi_size);
7873 if (src_mcv.isMemory()) try self.asmRegisterMemory(8024 if (src_mcv.isMemory()) try self.asmRegisterMemory(
7874 mir_tag,8025 mir_tag,
...@@ -7980,7 +8131,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -7980,7 +8131,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7980 if (src_mcv.isMemory()) try self.asmRegisterMemory(8131 if (src_mcv.isMemory()) try self.asmRegisterMemory(
7981 .{ .v_ps, .cvtph2 },8132 .{ .v_ps, .cvtph2 },
7982 wide_reg,8133 wide_reg,
7983 try src_mcv.mem(self, Memory.Size.fromSize(8134 try src_mcv.mem(self, .fromSize(
7984 @intCast(@divExact(wide_reg.bitSize(), 16)),8135 @intCast(@divExact(wide_reg.bitSize(), 16)),
7985 )),8136 )),
7986 ) else try self.asmRegisterRegister(8137 ) else try self.asmRegisterRegister(
...@@ -8026,7 +8177,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -8026,7 +8177,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
8026 mir_tag,8177 mir_tag,
8027 dst_reg,8178 dst_reg,
8028 dst_reg,8179 dst_reg,
8029 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),8180 try src_mcv.mem(self, .fromSize(abi_size)),
8030 ) else try self.asmRegisterRegisterRegister(8181 ) else try self.asmRegisterRegisterRegister(
8031 mir_tag,8182 mir_tag,
8032 dst_reg,8183 dst_reg,
...@@ -8039,7 +8190,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -8039,7 +8190,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
8039 else => if (src_mcv.isMemory()) try self.asmRegisterMemory(8190 else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
8040 mir_tag,8191 mir_tag,
8041 dst_reg,8192 dst_reg,
8042 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),8193 try src_mcv.mem(self, .fromSize(abi_size)),
8043 ) else try self.asmRegisterRegister(8194 ) else try self.asmRegisterRegister(
8044 mir_tag,8195 mir_tag,
8045 dst_reg,8196 dst_reg,
...@@ -8136,7 +8287,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn...@@ -8136,7 +8287,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
8136 const zcu = pt.zcu;8287 const zcu = pt.zcu;
81378288
8138 const ptr_info = ptr_ty.ptrInfo(zcu);8289 const ptr_info = ptr_ty.ptrInfo(zcu);
8139 const val_ty = Type.fromInterned(ptr_info.child);8290 const val_ty: Type = .fromInterned(ptr_info.child);
8140 if (!val_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;8291 if (!val_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;
8141 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));8292 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));
81428293
...@@ -8204,7 +8355,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn...@@ -8204,7 +8355,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
8204 try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{8355 try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
8205 .base = .{ .reg = ptr_reg },8356 .base = .{ .reg = ptr_reg },
8206 .mod = .{ .rm = .{8357 .mod = .{ .rm = .{
8207 .size = Memory.Size.fromSize(load_abi_size),8358 .size = .fromSize(load_abi_size),
8208 .disp = val_byte_off,8359 .disp = val_byte_off,
8209 } },8360 } },
8210 });8361 });
...@@ -8220,14 +8371,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn...@@ -8220,14 +8371,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
8220 try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{8371 try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{
8221 .base = .{ .reg = ptr_reg },8372 .base = .{ .reg = ptr_reg },
8222 .mod = .{ .rm = .{8373 .mod = .{ .rm = .{
8223 .size = Memory.Size.fromSize(val_abi_size),8374 .size = .fromSize(val_abi_size),
8224 .disp = val_byte_off,8375 .disp = val_byte_off,
8225 } },8376 } },
8226 });8377 });
8227 try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{8378 try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
8228 .base = .{ .reg = ptr_reg },8379 .base = .{ .reg = ptr_reg },
8229 .mod = .{ .rm = .{8380 .mod = .{ .rm = .{
8230 .size = Memory.Size.fromSize(val_abi_size),8381 .size = .fromSize(val_abi_size),
8231 .disp = val_byte_off + limb_abi_size,8382 .disp = val_byte_off + limb_abi_size,
8232 } },8383 } },
8233 });8384 });
...@@ -8297,8 +8448,8 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -8297,8 +8448,8 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
8297 const ptr_ty = self.typeOf(ty_op.operand);8448 const ptr_ty = self.typeOf(ty_op.operand);
8298 const elem_size = elem_ty.abiSize(zcu);8449 const elem_size = elem_ty.abiSize(zcu);
82998450
8300 const elem_rc = self.regClassForType(elem_ty);8451 const elem_rc = self.regSetForType(elem_ty);
8301 const ptr_rc = self.regClassForType(ptr_ty);8452 const ptr_rc = self.regSetForType(ptr_ty);
83028453
8303 const ptr_mcv = try self.resolveInst(ty_op.operand);8454 const ptr_mcv = try self.resolveInst(ty_op.operand);
8304 const dst_mcv = if (elem_size <= 8 and elem_rc.supersetOf(ptr_rc) and8455 const dst_mcv = if (elem_size <= 8 and elem_rc.supersetOf(ptr_rc) and
...@@ -8345,7 +8496,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In...@@ -8345,7 +8496,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
8345 const pt = self.pt;8496 const pt = self.pt;
8346 const zcu = pt.zcu;8497 const zcu = pt.zcu;
8347 const ptr_info = ptr_ty.ptrInfo(zcu);8498 const ptr_info = ptr_ty.ptrInfo(zcu);
8348 const src_ty = Type.fromInterned(ptr_info.child);8499 const src_ty: Type = .fromInterned(ptr_info.child);
8349 if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;8500 if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;
83508501
8351 const limb_abi_size: u16 = @min(ptr_info.packed_offset.host_size, 8);8502 const limb_abi_size: u16 = @min(ptr_info.packed_offset.host_size, 8);
...@@ -8373,7 +8524,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In...@@ -8373,7 +8524,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
8373 const limb_mem: Memory = .{8524 const limb_mem: Memory = .{
8374 .base = .{ .reg = ptr_reg },8525 .base = .{ .reg = ptr_reg },
8375 .mod = .{ .rm = .{8526 .mod = .{ .rm = .{
8376 .size = Memory.Size.fromSize(limb_abi_size),8527 .size = .fromSize(limb_abi_size),
8377 .disp = src_byte_off + limb_i * limb_abi_size,8528 .disp = src_byte_off + limb_i * limb_abi_size,
8378 } },8529 } },
8379 };8530 };
...@@ -8563,10 +8714,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -8563,10 +8714,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8563 const index = extra.field_index;8714 const index = extra.field_index;
85648715
8565 const container_ty = self.typeOf(operand);8716 const container_ty = self.typeOf(operand);
8566 const container_rc = self.regClassForType(container_ty);8717 const container_rc = self.regSetForType(container_ty);
8567 const field_ty = container_ty.fieldType(index, zcu);8718 const field_ty = container_ty.fieldType(index, zcu);
8568 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;8719 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
8569 const field_rc = self.regClassForType(field_ty);8720 const field_rc = self.regSetForType(field_ty);
8570 const field_is_gp = field_rc.supersetOf(abi.RegisterClass.gp);8721 const field_is_gp = field_rc.supersetOf(abi.RegisterClass.gp);
85718722
8572 const src_mcv = try self.resolveInst(operand);8723 const src_mcv = try self.resolveInst(operand);
...@@ -8624,7 +8775,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -8624,7 +8775,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8624 const dst_regs: [2]Register = if (field_rc.supersetOf(container_rc) and8775 const dst_regs: [2]Register = if (field_rc.supersetOf(container_rc) and
8625 self.reuseOperand(inst, operand, 0, src_mcv)) src_regs else dst: {8776 self.reuseOperand(inst, operand, 0, src_mcv)) src_regs else dst: {
8626 const dst_regs =8777 const dst_regs =
8627 try self.register_manager.allocRegs(2, .{null} ** 2, field_rc);8778 try self.register_manager.allocRegs(2, @splat(null), field_rc);
8628 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);8779 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
8629 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);8780 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
86308781
...@@ -8786,7 +8937,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -8786,7 +8937,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8786 try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{8937 try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
8787 .base = .{ .frame = frame_addr.index },8938 .base = .{ .frame = frame_addr.index },
8788 .mod = .{ .rm = .{8939 .mod = .{ .rm = .{
8789 .size = Memory.Size.fromSize(load_abi_size),8940 .size = .fromSize(load_abi_size),
8790 .disp = frame_addr.off + field_byte_off,8941 .disp = frame_addr.off + field_byte_off,
8791 } },8942 } },
8792 });8943 });
...@@ -8807,7 +8958,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -8807,7 +8958,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8807 .{8958 .{
8808 .base = .{ .frame = frame_addr.index },8959 .base = .{ .frame = frame_addr.index },
8809 .mod = .{ .rm = .{8960 .mod = .{ .rm = .{
8810 .size = Memory.Size.fromSize(field_abi_size),8961 .size = .fromSize(field_abi_size),
8811 .disp = frame_addr.off + field_byte_off,8962 .disp = frame_addr.off + field_byte_off,
8812 } },8963 } },
8813 },8964 },
...@@ -8815,7 +8966,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -8815,7 +8966,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8815 try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{8966 try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
8816 .base = .{ .frame = frame_addr.index },8967 .base = .{ .frame = frame_addr.index },
8817 .mod = .{ .rm = .{8968 .mod = .{ .rm = .{
8818 .size = Memory.Size.fromSize(field_abi_size),8969 .size = .fromSize(field_abi_size),
8819 .disp = frame_addr.off + field_byte_off + limb_abi_size,8970 .disp = frame_addr.off + field_byte_off + limb_abi_size,
8820 } },8971 } },
8821 });8972 });
...@@ -8977,12 +9128,12 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC...@@ -8977,12 +9128,12 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
89779128
8978 try self.genSetReg(addr_reg, Type.usize, dst_mcv.address(), .{});9129 try self.genSetReg(addr_reg, Type.usize, dst_mcv.address(), .{});
8979 try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{9130 try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{
8980 .size = Memory.Size.fromSize(abi_size),9131 .size = .fromSize(abi_size),
8981 } } });9132 } } });
8982 },9133 },
8983 .indirect, .load_frame => try self.asmMemory(9134 .indirect, .load_frame => try self.asmMemory(
8984 mir_tag,9135 mir_tag,
8985 try dst_mcv.mem(self, Memory.Size.fromSize(abi_size)),9136 try dst_mcv.mem(self, .fromSize(abi_size)),
8986 ),9137 ),
8987 }9138 }
8988}9139}
...@@ -9024,7 +9175,7 @@ fn genShiftBinOpMir(...@@ -9024,7 +9175,7 @@ fn genShiftBinOpMir(
9024 };9175 };
9025 defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);9176 defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
90269177
9027 const temp_regs = try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);9178 const temp_regs = try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
9028 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);9179 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
9029 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);9180 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
90309181
...@@ -9576,7 +9727,7 @@ fn genShiftBinOpMir(...@@ -9576,7 +9727,7 @@ fn genShiftBinOpMir(
9576 .memory => |addr| .{9727 .memory => |addr| .{
9577 .base = .{ .reg = .ds },9728 .base = .{ .reg = .ds },
9578 .mod = .{ .rm = .{9729 .mod = .{ .rm = .{
9579 .size = Memory.Size.fromSize(abi_size),9730 .size = .fromSize(abi_size),
9580 .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse9731 .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
9581 return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{9732 return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
9582 @tagName(lhs_mcv),9733 @tagName(lhs_mcv),
...@@ -9587,14 +9738,14 @@ fn genShiftBinOpMir(...@@ -9587,14 +9738,14 @@ fn genShiftBinOpMir(
9587 .indirect => |reg_off| .{9738 .indirect => |reg_off| .{
9588 .base = .{ .reg = reg_off.reg },9739 .base = .{ .reg = reg_off.reg },
9589 .mod = .{ .rm = .{9740 .mod = .{ .rm = .{
9590 .size = Memory.Size.fromSize(abi_size),9741 .size = .fromSize(abi_size),
9591 .disp = reg_off.off,9742 .disp = reg_off.off,
9592 } },9743 } },
9593 },9744 },
9594 .load_frame => |frame_addr| .{9745 .load_frame => |frame_addr| .{
9595 .base = .{ .frame = frame_addr.index },9746 .base = .{ .frame = frame_addr.index },
9596 .mod = .{ .rm = .{9747 .mod = .{ .rm = .{
9597 .size = Memory.Size.fromSize(abi_size),9748 .size = .fromSize(abi_size),
9598 .disp = frame_addr.off,9749 .disp = frame_addr.off,
9599 } },9750 } },
9600 },9751 },
...@@ -9792,7 +9943,7 @@ fn genMulDivBinOp(...@@ -9792,7 +9943,7 @@ fn genMulDivBinOp(
9792 );9943 );
97939944
9794 const temp_regs =9945 const temp_regs =
9795 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);9946 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
9796 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);9947 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
9797 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);9948 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
97989949
...@@ -10208,7 +10359,7 @@ fn genBinOp(...@@ -10208,7 +10359,7 @@ fn genBinOp(
10208 mir_tag,10359 mir_tag,
10209 dst_reg,10360 dst_reg,
10210 dst_reg,10361 dst_reg,
10211 try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),10362 try src_mcv.mem(self, .fromBitSize(float_bits)),
10212 ) else try self.asmRegisterRegisterRegister(10363 ) else try self.asmRegisterRegisterRegister(
10213 mir_tag,10364 mir_tag,
10214 dst_reg,10365 dst_reg,
...@@ -10227,7 +10378,7 @@ fn genBinOp(...@@ -10227,7 +10378,7 @@ fn genBinOp(
10227 if (src_mcv.isMemory()) try self.asmRegisterMemory(10378 if (src_mcv.isMemory()) try self.asmRegisterMemory(
10228 mir_tag,10379 mir_tag,
10229 dst_reg,10380 dst_reg,
10230 try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),10381 try src_mcv.mem(self, .fromBitSize(float_bits)),
10231 ) else try self.asmRegisterRegister(10382 ) else try self.asmRegisterRegister(
10232 mir_tag,10383 mir_tag,
10233 dst_reg,10384 dst_reg,
...@@ -10358,14 +10509,14 @@ fn genBinOp(...@@ -10358,14 +10509,14 @@ fn genBinOp(
10358 const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs);10509 const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs);
10359 break :locks .{ locks[0], locks[1] };10510 break :locks .{ locks[0], locks[1] };
10360 },10511 },
10361 else => .{null} ** 2,10512 else => @splat(null),
10362 };10513 };
10363 defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);10514 defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
1036410515
10365 const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) {10516 const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) {
10366 .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },10517 .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },
10367 .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),10518 .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),
10368 else => .{null} ** 2,10519 else => @splat(null),
10369 };10520 };
10370 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);10521 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1037110522
...@@ -10398,7 +10549,7 @@ fn genBinOp(...@@ -10398,7 +10549,7 @@ fn genBinOp(
10398 const dst_locks: [2]?RegisterLock = switch (dst_mcv) {10549 const dst_locks: [2]?RegisterLock = switch (dst_mcv) {
10399 .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null },10550 .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null },
10400 .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs),10551 .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs),
10401 else => .{null} ** 2,10552 else => @splat(null),
10402 };10553 };
10403 defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);10554 defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);
1040410555
...@@ -10414,7 +10565,7 @@ fn genBinOp(...@@ -10414,7 +10565,7 @@ fn genBinOp(
10414 const src_locks: [2]?RegisterLock = switch (src_mcv) {10565 const src_locks: [2]?RegisterLock = switch (src_mcv) {
10415 .register => |src_reg| .{ self.register_manager.lockReg(src_reg), null },10566 .register => |src_reg| .{ self.register_manager.lockReg(src_reg), null },
10416 .register_pair => |src_regs| self.register_manager.lockRegs(2, src_regs),10567 .register_pair => |src_regs| self.register_manager.lockRegs(2, src_regs),
10417 else => .{null} ** 2,10568 else => @splat(null),
10418 };10569 };
10419 defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock);10570 defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock);
1042010571
...@@ -10472,11 +10623,7 @@ fn genBinOp(...@@ -10472,11 +10623,7 @@ fn genBinOp(
10472 const dst_regs = switch (dst_mcv) {10623 const dst_regs = switch (dst_mcv) {
10473 .register_pair => |dst_regs| dst_regs,10624 .register_pair => |dst_regs| dst_regs,
10474 else => dst: {10625 else => dst: {
10475 const dst_regs = try self.register_manager.allocRegs(10626 const dst_regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
10476 2,
10477 .{null} ** 2,
10478 abi.RegisterClass.gp,
10479 );
10480 const dst_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);10627 const dst_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
10481 defer for (dst_regs_locks) |lock| self.register_manager.unlockReg(lock);10628 defer for (dst_regs_locks) |lock| self.register_manager.unlockReg(lock);
1048210629
...@@ -10624,21 +10771,21 @@ fn genBinOp(...@@ -10624,21 +10771,21 @@ fn genBinOp(
10624 .memory => |addr| .{10771 .memory => |addr| .{
10625 .base = .{ .reg = .ds },10772 .base = .{ .reg = .ds },
10626 .mod = .{ .rm = .{10773 .mod = .{ .rm = .{
10627 .size = Memory.Size.fromSize(cmov_abi_size),10774 .size = .fromSize(cmov_abi_size),
10628 .disp = @intCast(@as(i64, @bitCast(addr))),10775 .disp = @intCast(@as(i64, @bitCast(addr))),
10629 } },10776 } },
10630 },10777 },
10631 .indirect => |reg_off| .{10778 .indirect => |reg_off| .{
10632 .base = .{ .reg = reg_off.reg },10779 .base = .{ .reg = reg_off.reg },
10633 .mod = .{ .rm = .{10780 .mod = .{ .rm = .{
10634 .size = Memory.Size.fromSize(cmov_abi_size),10781 .size = .fromSize(cmov_abi_size),
10635 .disp = reg_off.off,10782 .disp = reg_off.off,
10636 } },10783 } },
10637 },10784 },
10638 .load_frame => |frame_addr| .{10785 .load_frame => |frame_addr| .{
10639 .base = .{ .frame = frame_addr.index },10786 .base = .{ .frame = frame_addr.index },
10640 .mod = .{ .rm = .{10787 .mod = .{ .rm = .{
10641 .size = Memory.Size.fromSize(cmov_abi_size),10788 .size = .fromSize(cmov_abi_size),
10642 .disp = frame_addr.off,10789 .disp = frame_addr.off,
10643 } },10790 } },
10644 },10791 },
...@@ -11433,8 +11580,8 @@ fn genBinOp(...@@ -11433,8 +11580,8 @@ fn genBinOp(
11433 dst_reg,11580 dst_reg,
11434 lhs_reg,11581 lhs_reg,
11435 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {11582 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11436 else => Memory.Size.fromSize(abi_size),11583 else => .fromSize(abi_size),
11437 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),11584 .vector => .fromBitSize(dst_reg.bitSize()),
11438 }),11585 }),
11439 ) else try self.asmRegisterRegisterRegister(11586 ) else try self.asmRegisterRegisterRegister(
11440 mir_tag,11587 mir_tag,
...@@ -11451,8 +11598,8 @@ fn genBinOp(...@@ -11451,8 +11598,8 @@ fn genBinOp(
11451 mir_tag,11598 mir_tag,
11452 dst_reg,11599 dst_reg,
11453 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {11600 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11454 else => Memory.Size.fromSize(abi_size),11601 else => .fromSize(abi_size),
11455 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),11602 .vector => .fromBitSize(dst_reg.bitSize()),
11456 }),11603 }),
11457 ) else try self.asmRegisterRegister(11604 ) else try self.asmRegisterRegister(
11458 mir_tag,11605 mir_tag,
...@@ -11479,8 +11626,8 @@ fn genBinOp(...@@ -11479,8 +11626,8 @@ fn genBinOp(
11479 dst_reg,11626 dst_reg,
11480 lhs_reg,11627 lhs_reg,
11481 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {11628 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11482 else => Memory.Size.fromSize(abi_size),11629 else => .fromSize(abi_size),
11483 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),11630 .vector => .fromBitSize(dst_reg.bitSize()),
11484 }),11631 }),
11485 imm,11632 imm,
11486 ) else try self.asmRegisterRegisterRegisterImmediate(11633 ) else try self.asmRegisterRegisterRegisterImmediate(
...@@ -11499,8 +11646,8 @@ fn genBinOp(...@@ -11499,8 +11646,8 @@ fn genBinOp(
11499 mir_tag,11646 mir_tag,
11500 dst_reg,11647 dst_reg,
11501 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {11648 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {
11502 else => Memory.Size.fromSize(abi_size),11649 else => .fromSize(abi_size),
11503 .vector => Memory.Size.fromBitSize(dst_reg.bitSize()),11650 .vector => .fromBitSize(dst_reg.bitSize()),
11504 }),11651 }),
11505 imm,11652 imm,
11506 ) else try self.asmRegisterRegisterImmediate(11653 ) else try self.asmRegisterRegisterImmediate(
...@@ -11707,11 +11854,11 @@ fn genBinOp(...@@ -11707,11 +11854,11 @@ fn genBinOp(
11707 const unsigned_ty = try lhs_ty.toUnsigned(pt);11854 const unsigned_ty = try lhs_ty.toUnsigned(pt);
11708 const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty));11855 const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty));
11709 const not_mem: Memory = if (not_mcv.isMemory())11856 const not_mem: Memory = if (not_mcv.isMemory())
11710 try not_mcv.mem(self, Memory.Size.fromSize(abi_size))11857 try not_mcv.mem(self, .fromSize(abi_size))
11711 else11858 else
11712 .{ .base = .{11859 .{ .base = .{
11713 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),11860 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
11714 }, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };11861 }, .mod = .{ .rm = .{ .size = .fromSize(abi_size) } } };
11715 switch (mir_tag[0]) {11862 switch (mir_tag[0]) {
11716 .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(11863 .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
11717 .{ .vp_, .xor },11864 .{ .vp_, .xor },
...@@ -11891,21 +12038,21 @@ fn genBinOpMir(...@@ -11891,21 +12038,21 @@ fn genBinOpMir(
11891 .memory => |addr| .{12038 .memory => |addr| .{
11892 .base = .{ .reg = .ds },12039 .base = .{ .reg = .ds },
11893 .mod = .{ .rm = .{12040 .mod = .{ .rm = .{
11894 .size = Memory.Size.fromSize(limb_abi_size),12041 .size = .fromSize(limb_abi_size),
11895 .disp = std.math.cast(i32, addr + off) orelse break :direct,12042 .disp = std.math.cast(i32, addr + off) orelse break :direct,
11896 } },12043 } },
11897 },12044 },
11898 .indirect => |reg_off| .{12045 .indirect => |reg_off| .{
11899 .base = .{ .reg = reg_off.reg },12046 .base = .{ .reg = reg_off.reg },
11900 .mod = .{ .rm = .{12047 .mod = .{ .rm = .{
11901 .size = Memory.Size.fromSize(limb_abi_size),12048 .size = .fromSize(limb_abi_size),
11902 .disp = reg_off.off + off,12049 .disp = reg_off.off + off,
11903 } },12050 } },
11904 },12051 },
11905 .load_frame => |frame_addr| .{12052 .load_frame => |frame_addr| .{
11906 .base = .{ .frame = frame_addr.index },12053 .base = .{ .frame = frame_addr.index },
11907 .mod = .{ .rm = .{12054 .mod = .{ .rm = .{
11908 .size = Memory.Size.fromSize(limb_abi_size),12055 .size = .fromSize(limb_abi_size),
11909 .disp = frame_addr.off + off,12056 .disp = frame_addr.off + off,
11910 } },12057 } },
11911 },12058 },
...@@ -12054,21 +12201,21 @@ fn genBinOpMir(...@@ -12054,21 +12201,21 @@ fn genBinOpMir(
12054 => .{12201 => .{
12055 .base = .{ .reg = dst_info.?.addr_reg },12202 .base = .{ .reg = dst_info.?.addr_reg },
12056 .mod = .{ .rm = .{12203 .mod = .{ .rm = .{
12057 .size = Memory.Size.fromSize(limb_abi_size),12204 .size = .fromSize(limb_abi_size),
12058 .disp = off,12205 .disp = off,
12059 } },12206 } },
12060 },12207 },
12061 .indirect => |reg_off| .{12208 .indirect => |reg_off| .{
12062 .base = .{ .reg = reg_off.reg },12209 .base = .{ .reg = reg_off.reg },
12063 .mod = .{ .rm = .{12210 .mod = .{ .rm = .{
12064 .size = Memory.Size.fromSize(limb_abi_size),12211 .size = .fromSize(limb_abi_size),
12065 .disp = reg_off.off + off,12212 .disp = reg_off.off + off,
12066 } },12213 } },
12067 },12214 },
12068 .load_frame => |frame_addr| .{12215 .load_frame => |frame_addr| .{
12069 .base = .{ .frame = frame_addr.index },12216 .base = .{ .frame = frame_addr.index },
12070 .mod = .{ .rm = .{12217 .mod = .{ .rm = .{
12071 .size = Memory.Size.fromSize(limb_abi_size),12218 .size = .fromSize(limb_abi_size),
12072 .disp = frame_addr.off + off,12219 .disp = frame_addr.off + off,
12073 } },12220 } },
12074 },12221 },
...@@ -12277,7 +12424,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M...@@ -12277,7 +12424,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
12277 .memory => |addr| .{12424 .memory => |addr| .{
12278 .base = .{ .reg = .ds },12425 .base = .{ .reg = .ds },
12279 .mod = .{ .rm = .{12426 .mod = .{ .rm = .{
12280 .size = Memory.Size.fromSize(abi_size),12427 .size = .fromSize(abi_size),
12281 .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse12428 .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
12282 return self.asmRegisterRegister(12429 return self.asmRegisterRegister(
12283 .{ .i_, .mul },12430 .{ .i_, .mul },
...@@ -12292,14 +12439,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M...@@ -12292,14 +12439,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
12292 .indirect => |reg_off| .{12439 .indirect => |reg_off| .{
12293 .base = .{ .reg = reg_off.reg },12440 .base = .{ .reg = reg_off.reg },
12294 .mod = .{ .rm = .{12441 .mod = .{ .rm = .{
12295 .size = Memory.Size.fromSize(abi_size),12442 .size = .fromSize(abi_size),
12296 .disp = reg_off.off,12443 .disp = reg_off.off,
12297 } },12444 } },
12298 },12445 },
12299 .load_frame => |frame_addr| .{12446 .load_frame => |frame_addr| .{
12300 .base = .{ .frame = frame_addr.index },12447 .base = .{ .frame = frame_addr.index },
12301 .mod = .{ .rm = .{12448 .mod = .{ .rm = .{
12302 .size = Memory.Size.fromSize(abi_size),12449 .size = .fromSize(abi_size),
12303 .disp = frame_addr.off,12450 .disp = frame_addr.off,
12304 } },12451 } },
12305 },12452 },
...@@ -12475,7 +12622,7 @@ fn genLocalDebugInfo(...@@ -12475,7 +12622,7 @@ fn genLocalDebugInfo(
12475 self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op.operand,12622 self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op.operand,
12476 ),12623 ),
12477 };12624 };
12478 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, self.pt.zcu));12625 const frame_index = try self.allocFrameIndex(.initSpill(ty, self.pt.zcu));
12479 try self.genSetMem(.{ .frame = frame_index }, 0, ty, mcv, .{});12626 try self.genSetMem(.{ .frame = frame_index }, 0, ty, mcv, .{});
12480 try self.asmAirMemory(.dbg_local, inst, .{12627 try self.asmAirMemory(.dbg_local, inst, .{
12481 .base = .{ .frame = frame_index },12628 .base = .{ .frame = frame_index },
...@@ -12602,7 +12749,7 @@ fn genCall(self: *Self, info: union(enum) {...@@ -12602,7 +12749,7 @@ fn genCall(self: *Self, info: union(enum) {
12602 const frame_indices = try allocator.alloc(FrameIndex, args.len);12749 const frame_indices = try allocator.alloc(FrameIndex, args.len);
12603 defer allocator.free(frame_indices);12750 defer allocator.free(frame_indices);
1260412751
12605 var reg_locks = std.ArrayList(?RegisterLock).init(allocator);12752 var reg_locks: std.ArrayList(?RegisterLock) = .init(allocator);
12606 defer reg_locks.deinit();12753 defer reg_locks.deinit();
12607 try reg_locks.ensureTotalCapacity(16);12754 try reg_locks.ensureTotalCapacity(16);
12608 defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);12755 defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
...@@ -12612,7 +12759,7 @@ fn genCall(self: *Self, info: union(enum) {...@@ -12612,7 +12759,7 @@ fn genCall(self: *Self, info: union(enum) {
1261212759
12613 // We need a properly aligned and sized call frame to be able to call this function.12760 // We need a properly aligned and sized call frame to be able to call this function.
12614 {12761 {
12615 const needed_call_frame = FrameAlloc.init(.{12762 const needed_call_frame: FrameAlloc = .init(.{
12616 .size = call_info.stack_byte_count,12763 .size = call_info.stack_byte_count,
12617 .alignment = call_info.stack_align,12764 .alignment = call_info.stack_align,
12618 });12765 });
...@@ -12647,7 +12794,7 @@ fn genCall(self: *Self, info: union(enum) {...@@ -12647,7 +12794,7 @@ fn genCall(self: *Self, info: union(enum) {
12647 try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs));12794 try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs));
12648 },12795 },
12649 .indirect => |reg_off| {12796 .indirect => |reg_off| {
12650 frame_index.* = try self.allocFrameIndex(FrameAlloc.initType(arg_ty, zcu));12797 frame_index.* = try self.allocFrameIndex(.initType(arg_ty, zcu));
12651 try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, .{});12798 try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, .{});
12652 try self.register_manager.getReg(reg_off.reg, null);12799 try self.register_manager.getReg(reg_off.reg, null);
12653 try reg_locks.append(self.register_manager.lockReg(reg_off.reg));12800 try reg_locks.append(self.register_manager.lockReg(reg_off.reg));
...@@ -12718,8 +12865,8 @@ fn genCall(self: *Self, info: union(enum) {...@@ -12718,8 +12865,8 @@ fn genCall(self: *Self, info: union(enum) {
12718 switch (call_info.return_value.long) {12865 switch (call_info.return_value.long) {
12719 .none, .unreach => {},12866 .none, .unreach => {},
12720 .indirect => |reg_off| {12867 .indirect => |reg_off| {
12721 const ret_ty = Type.fromInterned(fn_info.return_type);12868 const ret_ty: Type = .fromInterned(fn_info.return_type);
12722 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ret_ty, zcu));12869 const frame_index = try self.allocFrameIndex(.initSpill(ret_ty, zcu));
12723 try self.genSetReg(reg_off.reg, Type.usize, .{12870 try self.genSetReg(reg_off.reg, Type.usize, .{
12724 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },12871 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
12725 }, .{});12872 }, .{});
...@@ -12936,7 +13083,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void...@@ -12936,7 +13083,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
12936 self.register_manager.lockRegAssumeUnused(lhs_ro.reg),13083 self.register_manager.lockRegAssumeUnused(lhs_ro.reg),
12937 null,13084 null,
12938 },13085 },
12939 else => .{null} ** 2,13086 else => @splat(null),
12940 };13087 };
12941 defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);13088 defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
1294213089
...@@ -12945,7 +13092,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void...@@ -12945,7 +13092,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
12945 .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },13092 .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },
12946 .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),13093 .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),
12947 .register_offset => |rhs_ro| .{ self.register_manager.lockReg(rhs_ro.reg), null },13094 .register_offset => |rhs_ro| .{ self.register_manager.lockReg(rhs_ro.reg), null },
12948 else => .{null} ** 2,13095 else => @splat(null),
12949 };13096 };
12950 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);13097 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1295113098
...@@ -13080,7 +13227,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void...@@ -13080,7 +13227,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
13080 if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;13227 if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
13081 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);13228 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
1308213229
13083 break :result Condition.fromCompareOperator(13230 break :result .fromCompareOperator(
13084 if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned,13231 if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned,
13085 result_op: {13232 result_op: {
13086 const flipped_op = if (flipped) op.reverse() else op;13233 const flipped_op = if (flipped) op.reverse() else op;
...@@ -13212,11 +13359,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void...@@ -13212,11 +13359,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
13212 defer if (src_info) |info|13359 defer if (src_info) |info|
13213 self.register_manager.unlockReg(info.addr_lock);13360 self.register_manager.unlockReg(info.addr_lock);
1321413361
13215 const regs = try self.register_manager.allocRegs(13362 const regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
13216 2,
13217 .{null} ** 2,
13218 abi.RegisterClass.gp,
13219 );
13220 const acc_reg = regs[0].to64();13363 const acc_reg = regs[0].to64();
13221 const locks = self.register_manager.lockRegsAssumeUnused(2, regs);13364 const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
13222 defer for (locks) |lock| self.register_manager.unlockReg(lock);13365 defer for (locks) |lock| self.register_manager.unlockReg(lock);
...@@ -13367,7 +13510,7 @@ fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {...@@ -13367,7 +13510,7 @@ fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
13367 const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;13510 const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
13368 const dst_mcv = try self.genBinOp(13511 const dst_mcv = try self.genBinOp(
13369 inst,13512 inst,
13370 Air.Inst.Tag.fromCmpOp(extra.compareOperator(), false),13513 .fromCmpOp(extra.compareOperator(), false),
13371 extra.lhs,13514 extra.lhs,
13372 extra.rhs,13515 extra.rhs,
13373 );13516 );
...@@ -13399,7 +13542,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {...@@ -13399,7 +13542,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
13399 registerAlias(dst_reg, op_abi_size),13542 registerAlias(dst_reg, op_abi_size),
13400 .{13543 .{
13401 .base = .{ .reg = addr_reg },13544 .base = .{ .reg = addr_reg },
13402 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(op_abi_size) } },13545 .mod = .{ .rm = .{ .size = .fromSize(op_abi_size) } },
13403 },13546 },
13404 );13547 );
1340513548
...@@ -13682,7 +13825,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC...@@ -13682,7 +13825,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13682 .{13825 .{
13683 .base = .{ .reg = addr_reg },13826 .base = .{ .reg = addr_reg },
13684 .mod = .{ .rm = .{13827 .mod = .{ .rm = .{
13685 .size = Memory.Size.fromSize(some_abi_size),13828 .size = .fromSize(some_abi_size),
13686 .disp = some_info.off,13829 .disp = some_info.off,
13687 } },13830 } },
13688 },13831 },
...@@ -13699,14 +13842,14 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC...@@ -13699,14 +13842,14 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13699 .indirect => |reg_off| .{13842 .indirect => |reg_off| .{
13700 .base = .{ .reg = reg_off.reg },13843 .base = .{ .reg = reg_off.reg },
13701 .mod = .{ .rm = .{13844 .mod = .{ .rm = .{
13702 .size = Memory.Size.fromSize(some_abi_size),13845 .size = .fromSize(some_abi_size),
13703 .disp = reg_off.off + some_info.off,13846 .disp = reg_off.off + some_info.off,
13704 } },13847 } },
13705 },13848 },
13706 .load_frame => |frame_addr| .{13849 .load_frame => |frame_addr| .{
13707 .base = .{ .frame = frame_addr.index },13850 .base = .{ .frame = frame_addr.index },
13708 .mod = .{ .rm = .{13851 .mod = .{ .rm = .{
13709 .size = Memory.Size.fromSize(some_abi_size),13852 .size = .fromSize(some_abi_size),
13710 .disp = frame_addr.off + some_info.off,13853 .disp = frame_addr.off + some_info.off,
13711 } },13854 } },
13712 },13855 },
...@@ -13745,7 +13888,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)...@@ -13745,7 +13888,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
13745 .{13888 .{
13746 .base = .{ .reg = ptr_reg },13889 .base = .{ .reg = ptr_reg },
13747 .mod = .{ .rm = .{13890 .mod = .{ .rm = .{
13748 .size = Memory.Size.fromSize(some_abi_size),13891 .size = .fromSize(some_abi_size),
13749 .disp = some_info.off,13892 .disp = some_info.off,
13750 } },13893 } },
13751 },13894 },
...@@ -13968,7 +14111,7 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {...@@ -13968,7 +14111,7 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
13968fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {14111fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
13969 // A block is a setup to be able to jump to the end.14112 // A block is a setup to be able to jump to the end.
13970 const inst_tracking_i = self.inst_tracking.count();14113 const inst_tracking_i = self.inst_tracking.count();
13971 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));14114 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(.unreach));
1397214115
13973 self.scope_generation += 1;14116 self.scope_generation += 1;
13974 try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() });14117 try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() });
...@@ -14140,7 +14283,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -14140,7 +14283,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
14140 try self.genCopy(self.typeOf(switch_br.operand), mat_cond, condition, .{});14283 try self.genCopy(self.typeOf(switch_br.operand), mat_cond, condition, .{});
14141 break :mat_cond mat_cond;14284 break :mat_cond mat_cond;
14142 };14285 };
14143 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(mat_cond));14286 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(mat_cond));
1414414287
14145 // If the condition dies here in this switch instruction, process14288 // If the condition dies here in this switch instruction, process
14146 // that death now instead of later as this has an effect on14289 // that death now instead of later as this has an effect on
...@@ -14265,7 +14408,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -14265,7 +14408,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
14265 }14408 }
1426614409
14267 if (first_br) {14410 if (first_br) {
14268 block_tracking.* = InstTracking.init(block_result);14411 block_tracking.* = .init(block_result);
14269 try self.saveRetroactiveState(&block_data.state);14412 try self.saveRetroactiveState(&block_data.state);
14270 } else try self.restoreState(block_data.state, &.{}, .{14413 } else try self.restoreState(block_data.state, &.{}, .{
14271 .emit_instructions = true,14414 .emit_instructions = true,
...@@ -14309,7 +14452,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -14309,7 +14452,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14309 extra_i += inputs.len;14452 extra_i += inputs.len;
1431014453
14311 var result: MCValue = .none;14454 var result: MCValue = .none;
14312 var args = std.ArrayList(MCValue).init(self.gpa);14455 var args: std.ArrayList(MCValue) = .init(self.gpa);
14313 try args.ensureTotalCapacity(outputs.len + inputs.len);14456 try args.ensureTotalCapacity(outputs.len + inputs.len);
14314 defer {14457 defer {
14315 for (args.items) |arg| if (arg.getReg()) |reg| self.register_manager.unlockReg(.{14458 for (args.items) |arg| if (arg.getReg()) |reg| self.register_manager.unlockReg(.{
...@@ -14317,7 +14460,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -14317,7 +14460,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14317 });14460 });
14318 args.deinit();14461 args.deinit();
14319 }14462 }
14320 var arg_map = std.StringHashMap(u8).init(self.gpa);14463 var arg_map: std.StringHashMap(u8) = .init(self.gpa);
14321 try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len));14464 try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len));
14322 defer arg_map.deinit();14465 defer arg_map.deinit();
1432314466
...@@ -14645,14 +14788,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -14645,14 +14788,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14645 });14788 });
14646 };14789 };
1464714790
14648 const Operand = union(enum) {14791 var ops: [4]Operand = @splat(.none);
14649 none,14792 var ops_len: usize = 0;
14650 reg: Register,
14651 mem: Memory,
14652 imm: Immediate,
14653 inst: Mir.Inst.Index,
14654 };
14655 var ops: [4]Operand = .{.none} ** 4;
1465614793
14657 var last_op = false;14794 var last_op = false;
14658 var op_it = std.mem.splitScalar(u8, mnem_it.rest(), ',');14795 var op_it = std.mem.splitScalar(u8, mnem_it.rest(), ',');
...@@ -14847,8 +14984,12 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -14847,8 +14984,12 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14847 try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len));14984 try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len));
14848 op.* = .{ .inst = label_gop.value_ptr.target };14985 op.* = .{ .inst = label_gop.value_ptr.target };
14849 } else return self.fail("invalid operand: '{s}'", .{op_str});14986 } else return self.fail("invalid operand: '{s}'", .{op_str});
14987 ops_len += 1;
14850 } else if (op_it.next()) |op_str| return self.fail("extra operand: '{s}'", .{op_str});14988 } else if (op_it.next()) |op_str| return self.fail("extra operand: '{s}'", .{op_str});
1485114989
14990 // convert from att syntax to intel syntax
14991 std.mem.reverse(Operand, ops[0..ops_len]);
14992
14852 (if (prefix == .directive) switch (mnem_tag) {14993 (if (prefix == .directive) switch (mnem_tag) {
14853 .@".cfi_def_cfa" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none)14994 .@".cfi_def_cfa" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none)
14854 self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, ops[0].reg, ops[1].imm)14995 self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, ops[0].reg, ops[1].imm)
...@@ -14904,68 +15045,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -14904,68 +15045,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14904 error.InvalidInstruction,15045 error.InvalidInstruction,
14905 .@".cfi_escape" => error.InvalidInstruction,15046 .@".cfi_escape" => error.InvalidInstruction,
14906 else => unreachable,15047 else => unreachable,
14907 } else switch (ops[0]) {15048 } else self.asmOps(mnem_fixed_tag, ops)) catch |err| switch (err) {
14908 .none => self.asmOpOnly(mnem_fixed_tag),
14909 .reg => |reg0| switch (ops[1]) {
14910 .none => self.asmRegister(mnem_fixed_tag, reg0),
14911 .reg => |reg1| switch (ops[2]) {
14912 .none => self.asmRegisterRegister(mnem_fixed_tag, reg1, reg0),
14913 .reg => |reg2| switch (ops[3]) {
14914 .none => self.asmRegisterRegisterRegister(mnem_fixed_tag, reg2, reg1, reg0),
14915 else => error.InvalidInstruction,
14916 },
14917 .mem => |mem2| switch (ops[3]) {
14918 .none => self.asmMemoryRegisterRegister(mnem_fixed_tag, mem2, reg1, reg0),
14919 else => error.InvalidInstruction,
14920 },
14921 else => error.InvalidInstruction,
14922 },
14923 .mem => |mem1| switch (ops[2]) {
14924 .none => self.asmMemoryRegister(mnem_fixed_tag, mem1, reg0),
14925 else => error.InvalidInstruction,
14926 },
14927 else => error.InvalidInstruction,
14928 },
14929 .mem => |mem0| switch (ops[1]) {
14930 .none => self.asmMemory(mnem_fixed_tag, mem0),
14931 .reg => |reg1| switch (ops[2]) {
14932 .none => self.asmRegisterMemory(mnem_fixed_tag, reg1, mem0),
14933 else => error.InvalidInstruction,
14934 },
14935 else => error.InvalidInstruction,
14936 },
14937 .imm => |imm0| switch (ops[1]) {
14938 .none => self.asmImmediate(mnem_fixed_tag, imm0),
14939 .reg => |reg1| switch (ops[2]) {
14940 .none => self.asmRegisterImmediate(mnem_fixed_tag, reg1, imm0),
14941 .reg => |reg2| switch (ops[3]) {
14942 .none => self.asmRegisterRegisterImmediate(mnem_fixed_tag, reg2, reg1, imm0),
14943 .reg => |reg3| self.asmRegisterRegisterRegisterImmediate(
14944 mnem_fixed_tag,
14945 reg3,
14946 reg2,
14947 reg1,
14948 imm0,
14949 ),
14950 else => error.InvalidInstruction,
14951 },
14952 .mem => |mem2| switch (ops[3]) {
14953 .none => self.asmMemoryRegisterImmediate(mnem_fixed_tag, mem2, reg1, imm0),
14954 else => error.InvalidInstruction,
14955 },
14956 else => error.InvalidInstruction,
14957 },
14958 .mem => |mem1| switch (ops[2]) {
14959 .none => self.asmMemoryImmediate(mnem_fixed_tag, mem1, imm0),
14960 else => error.InvalidInstruction,
14961 },
14962 else => error.InvalidInstruction,
14963 },
14964 .inst => |inst0| switch (ops[1]) {
14965 .none => self.asmReloc(mnem_fixed_tag, inst0),
14966 else => error.InvalidInstruction,
14967 },
14968 }) catch |err| switch (err) {
14969 error.InvalidInstruction => return self.fail(15049 error.InvalidInstruction => return self.fail(
14970 "invalid instruction: '{s} {s} {s} {s} {s}'",15050 "invalid instruction: '{s} {s} {s} {s} {s}'",
14971 .{15051 .{
...@@ -15000,7 +15080,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -15000,7 +15080,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
15000 }15080 }
1500115081
15002 simple: {15082 simple: {
15003 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);15083 var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
15004 var buf_index: usize = 0;15084 var buf_index: usize = 0;
15005 for (outputs) |output| {15085 for (outputs) |output| {
15006 if (output == .none) continue;15086 if (output == .none) continue;
...@@ -15583,7 +15663,7 @@ fn genSetReg(...@@ -15583,7 +15663,7 @@ fn genSetReg(
15583 .load_frame => |frame_addr| try self.moveStrategy(15663 .load_frame => |frame_addr| try self.moveStrategy(
15584 ty,15664 ty,
15585 dst_reg.class(),15665 dst_reg.class(),
15586 self.getFrameAddrAlignment(frame_addr).compare(.gte, InternPool.Alignment.fromLog2Units(15666 self.getFrameAddrAlignment(frame_addr).compare(.gte, .fromLog2Units(
15587 std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),15667 std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
15588 )),15668 )),
15589 ),15669 ),
...@@ -15742,7 +15822,7 @@ fn genSetMem(...@@ -15742,7 +15822,7 @@ fn genSetMem(
15742 try self.asmMemoryImmediate(15822 try self.asmMemoryImmediate(
15743 .{ ._, .mov },15823 .{ ._, .mov },
15744 .{ .base = base, .mod = .{ .rm = .{15824 .{ .base = base, .mod = .{ .rm = .{
15745 .size = Memory.Size.fromSize(abi_size),15825 .size = .fromSize(abi_size),
15746 .disp = disp,15826 .disp = disp,
15747 } } },15827 } } },
15748 immediate,15828 immediate,
...@@ -15753,7 +15833,7 @@ fn genSetMem(...@@ -15753,7 +15833,7 @@ fn genSetMem(
15753 try self.asmMemoryImmediate(15833 try self.asmMemoryImmediate(
15754 .{ ._, .mov },15834 .{ ._, .mov },
15755 .{ .base = base, .mod = .{ .rm = .{15835 .{ .base = base, .mod = .{ .rm = .{
15756 .size = Memory.Size.fromSize(abi_size),15836 .size = .fromSize(abi_size),
15757 .disp = disp,15837 .disp = disp,
15758 } } },15838 } } },
15759 .s(small),15839 .s(small),
...@@ -15796,11 +15876,11 @@ fn genSetMem(...@@ -15796,11 +15876,11 @@ fn genSetMem(
15796 .general_purpose, .segment, .x87, .ip => @divExact(src_alias.bitSize(), 8),15876 .general_purpose, .segment, .x87, .ip => @divExact(src_alias.bitSize(), 8),
15797 .mmx, .sse => abi_size,15877 .mmx, .sse => abi_size,
15798 });15878 });
15799 const src_align = InternPool.Alignment.fromNonzeroByteUnits(15879 const src_align: InternPool.Alignment = .fromNonzeroByteUnits(
15800 std.math.ceilPowerOfTwoAssert(u32, src_size),15880 std.math.ceilPowerOfTwoAssert(u32, src_size),
15801 );15881 );
15802 if (src_size > mem_size) {15882 if (src_size > mem_size) {
15803 const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{15883 const frame_index = try self.allocFrameIndex(.init(.{
15804 .size = src_size,15884 .size = src_size,
15805 .alignment = src_align,15885 .alignment = src_align,
15806 }));15886 }));
...@@ -15808,7 +15888,7 @@ fn genSetMem(...@@ -15808,7 +15888,7 @@ fn genSetMem(
15808 try (try self.moveStrategy(ty, src_alias.class(), true)).write(15888 try (try self.moveStrategy(ty, src_alias.class(), true)).write(
15809 self,15889 self,
15810 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{15890 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{
15811 .size = Memory.Size.fromSize(src_size),15891 .size = .fromSize(src_size),
15812 } } },15892 } } },
15813 src_alias,15893 src_alias,
15814 );15894 );
...@@ -15828,7 +15908,7 @@ fn genSetMem(...@@ -15828,7 +15908,7 @@ fn genSetMem(
15828 })).write(15908 })).write(
15829 self,15909 self,
15830 .{ .base = base, .mod = .{ .rm = .{15910 .{ .base = base, .mod = .{ .rm = .{
15831 .size = Memory.Size.fromBitSize(@min(self.memSize(ty).bitSize(), src_alias.bitSize())),15911 .size = .fromBitSize(@min(self.memSize(ty).bitSize(), src_alias.bitSize())),
15832 .disp = disp,15912 .disp = disp,
15833 } } },15913 } } },
15834 src_alias,15914 src_alias,
...@@ -16037,7 +16117,7 @@ fn genLazySymbolRef(...@@ -16037,7 +16117,7 @@ fn genLazySymbolRef(
16037 .mov => try self.asmRegisterMemory(16117 .mov => try self.asmRegisterMemory(
16038 .{ ._, tag },16118 .{ ._, tag },
16039 reg.to64(),16119 reg.to64(),
16040 Memory.initSib(.qword, .{ .base = .{ .reg = reg.to64() } }),16120 .initSib(.qword, .{ .base = .{ .reg = reg.to64() } }),
16041 ),16121 ),
16042 else => unreachable,16122 else => unreachable,
16043 }16123 }
...@@ -16108,8 +16188,8 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -16108,8 +16188,8 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
16108 else => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv,16188 else => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv,
16109 };16189 };
1611016190
16111 const dst_rc = self.regClassForType(dst_ty);16191 const dst_rc = self.regSetForType(dst_ty);
16112 const src_rc = self.regClassForType(src_ty);16192 const src_rc = self.regSetForType(src_ty);
1611316193
16114 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;16194 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
16115 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);16195 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
...@@ -16170,7 +16250,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {...@@ -16170,7 +16250,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
16170 const array_ty = ptr_ty.childType(zcu);16250 const array_ty = ptr_ty.childType(zcu);
16171 const array_len = array_ty.arrayLen(zcu);16251 const array_len = array_ty.arrayLen(zcu);
1617216252
16173 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));16253 const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu));
16174 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr, .{});16254 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr, .{});
16175 try self.genSetMem(16255 try self.genSetMem(
16176 .{ .frame = frame_index },16256 .{ .frame = frame_index },
...@@ -16236,7 +16316,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {...@@ -16236,7 +16316,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
1623616316
16237 if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg);16317 if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg);
1623816318
16239 const dst_reg = try self.register_manager.allocReg(inst, self.regClassForType(dst_ty));16319 const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty));
16240 const dst_mcv = MCValue{ .register = dst_reg };16320 const dst_mcv = MCValue{ .register = dst_reg };
16241 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);16321 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
16242 defer self.register_manager.unlockReg(dst_lock);16322 defer self.register_manager.unlockReg(dst_lock);
...@@ -16314,7 +16394,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {...@@ -16314,7 +16394,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
16314 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);16394 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
16315 defer self.register_manager.unlockReg(src_lock);16395 defer self.register_manager.unlockReg(src_lock);
1631616396
16317 const dst_reg = try self.register_manager.allocReg(inst, self.regClassForType(dst_ty));16397 const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty));
16318 const dst_mcv = MCValue{ .register = dst_reg };16398 const dst_mcv = MCValue{ .register = dst_reg };
16319 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);16399 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
16320 defer self.register_manager.unlockReg(dst_lock);16400 defer self.register_manager.unlockReg(dst_lock);
...@@ -16381,7 +16461,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -16381,7 +16461,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
16381 defer if (new_lock) |lock| self.register_manager.unlockReg(lock);16461 defer if (new_lock) |lock| self.register_manager.unlockReg(lock);
1638216462
16383 const ptr_mcv = try self.resolveInst(extra.ptr);16463 const ptr_mcv = try self.resolveInst(extra.ptr);
16384 const mem_size = Memory.Size.fromSize(val_abi_size);16464 const mem_size: Memory.Size = .fromSize(val_abi_size);
16385 const ptr_mem: Memory = switch (ptr_mcv) {16465 const ptr_mem: Memory = switch (ptr_mcv) {
16386 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),16466 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
16387 else => .{16467 else => .{
...@@ -16448,7 +16528,7 @@ fn atomicOp(...@@ -16448,7 +16528,7 @@ fn atomicOp(
16448 defer if (val_lock) |lock| self.register_manager.unlockReg(lock);16528 defer if (val_lock) |lock| self.register_manager.unlockReg(lock);
1644916529
16450 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));16530 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));
16451 const mem_size = Memory.Size.fromSize(val_abi_size);16531 const mem_size: Memory.Size = .fromSize(val_abi_size);
16452 const ptr_mem: Memory = switch (ptr_mcv) {16532 const ptr_mem: Memory = switch (ptr_mcv) {
16453 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),16533 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
16454 else => .{16534 else => .{
...@@ -16637,7 +16717,7 @@ fn atomicOp(...@@ -16637,7 +16717,7 @@ fn atomicOp(
16637 try self.asmCmovccRegisterMemory(16717 try self.asmCmovccRegisterMemory(
16638 cc,16718 cc,
16639 registerAlias(tmp_reg, cmov_abi_size),16719 registerAlias(tmp_reg, cmov_abi_size),
16640 try val_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),16720 try val_mcv.mem(self, .fromSize(cmov_abi_size)),
16641 );16721 );
16642 },16722 },
16643 else => {16723 else => {
...@@ -17069,7 +17149,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {...@@ -17069,7 +17149,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
1706917149
17070 // We need a properly aligned and sized call frame to be able to call this function.17150 // We need a properly aligned and sized call frame to be able to call this function.
17071 {17151 {
17072 const needed_call_frame = FrameAlloc.init(.{17152 const needed_call_frame: FrameAlloc = .init(.{
17073 .size = inst_ty.abiSize(zcu),17153 .size = inst_ty.abiSize(zcu),
17074 .alignment = inst_ty.abiAlignment(zcu),17154 .alignment = inst_ty.abiAlignment(zcu),
17075 });17155 });
...@@ -17207,7 +17287,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -17207,7 +17287,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17207 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;17287 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
17208 const vector_ty = self.typeOfIndex(inst);17288 const vector_ty = self.typeOfIndex(inst);
17209 const vector_len = vector_ty.vectorLen(zcu);17289 const vector_len = vector_ty.vectorLen(zcu);
17210 const dst_rc = self.regClassForType(vector_ty);17290 const dst_rc = self.regSetForType(vector_ty);
17211 const scalar_ty = self.typeOf(ty_op.operand);17291 const scalar_ty = self.typeOf(ty_op.operand);
1721217292
17213 const result: MCValue = result: {17293 const result: MCValue = result: {
...@@ -17675,7 +17755,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -17675,7 +17755,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17675 .ty = mask_elem_ty.toIntern(),17755 .ty = mask_elem_ty.toIntern(),
17676 .storage = .{ .u64 = bit / elem_bits },17756 .storage = .{ .u64 = bit / elem_bits },
17677 } });17757 } });
17678 const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{17758 const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
17679 .ty = mask_ty.toIntern(),17759 .ty = mask_ty.toIntern(),
17680 .storage = .{ .elems = mask_elems[0..vec_len] },17760 .storage = .{ .elems = mask_elems[0..vec_len] },
17681 } })));17761 } })));
...@@ -17700,7 +17780,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -17700,7 +17780,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17700 .ty = mask_elem_ty.toIntern(),17780 .ty = mask_elem_ty.toIntern(),
17701 .storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) },17781 .storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) },
17702 } });17782 } });
17703 const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{17783 const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
17704 .ty = mask_ty.toIntern(),17784 .ty = mask_ty.toIntern(),
17705 .storage = .{ .elems = mask_elems[0..vec_len] },17785 .storage = .{ .elems = mask_elems[0..vec_len] },
17706 } })));17786 } })));
...@@ -17853,7 +17933,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -17853,7 +17933,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17853 if (rhs_mcv.isMemory()) try self.asmRegisterMemory(17933 if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
17854 .{ mir_fixes, .andn },17934 .{ mir_fixes, .andn },
17855 mask_alias,17935 mask_alias,
17856 try rhs_mcv.mem(self, Memory.Size.fromSize(abi_size)),17936 try rhs_mcv.mem(self, .fromSize(abi_size)),
17857 ) else try self.asmRegisterRegister(17937 ) else try self.asmRegisterRegister(
17858 .{ mir_fixes, .andn },17938 .{ mir_fixes, .andn },
17859 mask_alias,17939 mask_alias,
...@@ -17934,7 +18014,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -17934,7 +18014,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17934 else18014 else
17935 self.hasFeature(.avx2)) 32 else 16)) break :unpck;18015 self.hasFeature(.avx2)) 32 else 16)) break :unpck;
1793618016
17937 var sources = [1]?u1{null} ** 2;18017 var sources: [2]?u1 = @splat(null);
17938 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {18018 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
17939 const mask_elem = maybe_mask_elem orelse continue;18019 const mask_elem = maybe_mask_elem orelse continue;
17940 const mask_elem_index =18020 const mask_elem_index =
...@@ -17994,7 +18074,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -17994,7 +18074,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17994 mir_tag,18074 mir_tag,
17995 dst_alias,18075 dst_alias,
17996 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),18076 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17997 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),18077 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
17998 ) else try self.asmRegisterRegisterRegister(18078 ) else try self.asmRegisterRegisterRegister(
17999 mir_tag,18079 mir_tag,
18000 dst_alias,18080 dst_alias,
...@@ -18006,7 +18086,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18006,7 +18086,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18006 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(18086 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
18007 mir_tag,18087 mir_tag,
18008 dst_alias,18088 dst_alias,
18009 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),18089 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
18010 ) else try self.asmRegisterRegister(18090 ) else try self.asmRegisterRegister(
18011 mir_tag,18091 mir_tag,
18012 dst_alias,18092 dst_alias,
...@@ -18023,7 +18103,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18023,7 +18103,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18023 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd;18103 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd;
1802418104
18025 var control: u8 = 0b00_00_00_00;18105 var control: u8 = 0b00_00_00_00;
18026 var sources = [1]?u1{null} ** 1;18106 var sources: [1]?u1 = @splat(null);
18027 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {18107 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
18028 const mask_elem = maybe_mask_elem orelse continue;18108 const mask_elem = maybe_mask_elem orelse continue;
18029 const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);18109 const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
...@@ -18055,7 +18135,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18055,7 +18135,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18055 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(18135 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18056 .{ if (has_avx) .vp_d else .p_d, .shuf },18136 .{ if (has_avx) .vp_d else .p_d, .shuf },
18057 dst_alias,18137 dst_alias,
18058 try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),18138 try src_mcv.mem(self, .fromSize(max_abi_size)),
18059 .u(control),18139 .u(control),
18060 ) else try self.asmRegisterRegisterImmediate(18140 ) else try self.asmRegisterRegisterImmediate(
18061 .{ if (has_avx) .vp_d else .p_d, .shuf },18141 .{ if (has_avx) .vp_d else .p_d, .shuf },
...@@ -18074,7 +18154,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18074,7 +18154,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18074 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps;18154 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps;
1807518155
18076 var control: u8 = 0b00_00_00_00;18156 var control: u8 = 0b00_00_00_00;
18077 var sources = [1]?u1{null} ** 2;18157 var sources: [2]?u1 = @splat(null);
18078 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {18158 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
18079 const mask_elem = maybe_mask_elem orelse continue;18159 const mask_elem = maybe_mask_elem orelse continue;
18080 const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);18160 const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
...@@ -18112,7 +18192,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18112,7 +18192,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18112 .{ .v_ps, .shuf },18192 .{ .v_ps, .shuf },
18113 dst_alias,18193 dst_alias,
18114 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),18194 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
18115 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),18195 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
18116 .u(control),18196 .u(control),
18117 ) else try self.asmRegisterRegisterRegisterImmediate(18197 ) else try self.asmRegisterRegisterRegisterImmediate(
18118 .{ .v_ps, .shuf },18198 .{ .v_ps, .shuf },
...@@ -18126,7 +18206,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18126,7 +18206,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18126 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(18206 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18127 .{ ._ps, .shuf },18207 .{ ._ps, .shuf },
18128 dst_alias,18208 dst_alias,
18129 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),18209 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
18130 .u(control),18210 .u(control),
18131 ) else try self.asmRegisterRegisterImmediate(18211 ) else try self.asmRegisterRegisterImmediate(
18132 .{ ._ps, .shuf },18212 .{ ._ps, .shuf },
...@@ -18145,7 +18225,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18145,7 +18225,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18145 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd;18225 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd;
1814618226
18147 var control: u4 = 0b0_0_0_0;18227 var control: u4 = 0b0_0_0_0;
18148 var sources = [1]?u1{null} ** 2;18228 var sources: [2]?u1 = @splat(null);
18149 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {18229 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
18150 const mask_elem = maybe_mask_elem orelse continue;18230 const mask_elem = maybe_mask_elem orelse continue;
18151 const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);18231 const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
...@@ -18179,7 +18259,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18179,7 +18259,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18179 .{ .v_pd, .shuf },18259 .{ .v_pd, .shuf },
18180 dst_alias,18260 dst_alias,
18181 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),18261 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
18182 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),18262 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
18183 .u(control),18263 .u(control),
18184 ) else try self.asmRegisterRegisterRegisterImmediate(18264 ) else try self.asmRegisterRegisterRegisterImmediate(
18185 .{ .v_pd, .shuf },18265 .{ .v_pd, .shuf },
...@@ -18193,7 +18273,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18193,7 +18273,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18193 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(18273 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18194 .{ ._pd, .shuf },18274 .{ ._pd, .shuf },
18195 dst_alias,18275 dst_alias,
18196 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),18276 try rhs_mcv.mem(self, .fromSize(max_abi_size)),
18197 .u(control),18277 .u(control),
18198 ) else try self.asmRegisterRegisterImmediate(18278 ) else try self.asmRegisterRegisterImmediate(
18199 .{ ._pd, .shuf },18279 .{ ._pd, .shuf },
...@@ -18249,7 +18329,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18249,7 +18329,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18249 .{ .vp_d, .blend },18329 .{ .vp_d, .blend },
18250 registerAlias(dst_reg, dst_abi_size),18330 registerAlias(dst_reg, dst_abi_size),
18251 registerAlias(lhs_reg, dst_abi_size),18331 registerAlias(lhs_reg, dst_abi_size),
18252 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),18332 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18253 .u(expanded_control),18333 .u(expanded_control),
18254 ) else try self.asmRegisterRegisterRegisterImmediate(18334 ) else try self.asmRegisterRegisterRegisterImmediate(
18255 .{ .vp_d, .blend },18335 .{ .vp_d, .blend },
...@@ -18304,7 +18384,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18304,7 +18384,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18304 lhs_mcv.getReg().?18384 lhs_mcv.getReg().?
18305 else18385 else
18306 dst_reg, dst_abi_size),18386 dst_reg, dst_abi_size),
18307 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),18387 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18308 .u(expanded_control),18388 .u(expanded_control),
18309 ) else try self.asmRegisterRegisterRegisterImmediate(18389 ) else try self.asmRegisterRegisterRegisterImmediate(
18310 .{ .vp_w, .blend },18390 .{ .vp_w, .blend },
...@@ -18321,7 +18401,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18321,7 +18401,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18321 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(18401 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18322 .{ .p_w, .blend },18402 .{ .p_w, .blend },
18323 registerAlias(dst_reg, dst_abi_size),18403 registerAlias(dst_reg, dst_abi_size),
18324 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),18404 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18325 .u(expanded_control),18405 .u(expanded_control),
18326 ) else try self.asmRegisterRegisterImmediate(18406 ) else try self.asmRegisterRegisterImmediate(
18327 .{ .p_w, .blend },18407 .{ .p_w, .blend },
...@@ -18365,7 +18445,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18365,7 +18445,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18365 lhs_mcv.getReg().?18445 lhs_mcv.getReg().?
18366 else18446 else
18367 dst_reg, dst_abi_size),18447 dst_reg, dst_abi_size),
18368 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),18448 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18369 .u(expanded_control),18449 .u(expanded_control),
18370 ) else try self.asmRegisterRegisterRegisterImmediate(18450 ) else try self.asmRegisterRegisterRegisterImmediate(
18371 switch (elem_abi_size) {18451 switch (elem_abi_size) {
...@@ -18390,7 +18470,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18390,7 +18470,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18390 else => unreachable,18470 else => unreachable,
18391 },18471 },
18392 registerAlias(dst_reg, dst_abi_size),18472 registerAlias(dst_reg, dst_abi_size),
18393 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),18473 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18394 .u(expanded_control),18474 .u(expanded_control),
18395 ) else try self.asmRegisterRegisterImmediate(18475 ) else try self.asmRegisterRegisterImmediate(
18396 switch (elem_abi_size) {18476 switch (elem_abi_size) {
...@@ -18435,7 +18515,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18435,7 +18515,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18435 else18515 else
18436 try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern();18516 try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern();
18437 }18517 }
18438 const select_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{18518 const select_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
18439 .ty = select_mask_ty.toIntern(),18519 .ty = select_mask_ty.toIntern(),
18440 .storage = .{ .elems = select_mask_elems[0..mask_elems.len] },18520 .storage = .{ .elems = select_mask_elems[0..mask_elems.len] },
18441 } })));18521 } })));
...@@ -18480,7 +18560,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18480,7 +18560,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18480 registerAlias(lhs_mcv.getReg().?, dst_abi_size)18560 registerAlias(lhs_mcv.getReg().?, dst_abi_size)
18481 else18561 else
18482 dst_alias,18562 dst_alias,
18483 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),18563 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18484 select_mask_alias,18564 select_mask_alias,
18485 ) else try self.asmRegisterRegisterRegisterRegister(18565 ) else try self.asmRegisterRegisterRegisterRegister(
18486 mir_tag,18566 mir_tag,
...@@ -18497,7 +18577,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18497,7 +18577,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18497 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(18577 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
18498 mir_tag,18578 mir_tag,
18499 dst_alias,18579 dst_alias,
18500 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),18580 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),
18501 select_mask_alias,18581 select_mask_alias,
18502 ) else try self.asmRegisterRegisterRegister(18582 ) else try self.asmRegisterRegisterRegister(
18503 mir_tag,18583 mir_tag,
...@@ -18540,7 +18620,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18540,7 +18620,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18540 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(18620 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
18541 .{ mir_fixes, .andn },18621 .{ mir_fixes, .andn },
18542 mask_alias,18622 mask_alias,
18543 try lhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),18623 try lhs_mcv.mem(self, .fromSize(dst_abi_size)),
18544 ) else try self.asmRegisterRegister(18624 ) else try self.asmRegisterRegister(
18545 .{ mir_fixes, .andn },18625 .{ mir_fixes, .andn },
18546 mask_alias,18626 mask_alias,
...@@ -18583,13 +18663,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18583,13 +18663,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18583 } });18663 } });
18584 }18664 }
18585 const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });18665 const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
18586 const lhs_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{18666 const lhs_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
18587 .ty = lhs_mask_ty.toIntern(),18667 .ty = lhs_mask_ty.toIntern(),
18588 .storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },18668 .storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
18589 } })));18669 } })));
18590 const lhs_mask_mem: Memory = .{18670 const lhs_mask_mem: Memory = .{
18591 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },18671 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },
18592 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },18672 .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
18593 };18673 };
18594 if (has_avx) try self.asmRegisterRegisterMemory(18674 if (has_avx) try self.asmRegisterRegisterMemory(
18595 .{ .vp_b, .shuf },18675 .{ .vp_b, .shuf },
...@@ -18617,13 +18697,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18617,13 +18697,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18617 } });18697 } });
18618 }18698 }
18619 const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });18699 const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
18620 const rhs_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{18700 const rhs_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
18621 .ty = rhs_mask_ty.toIntern(),18701 .ty = rhs_mask_ty.toIntern(),
18622 .storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },18702 .storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },
18623 } })));18703 } })));
18624 const rhs_mask_mem: Memory = .{18704 const rhs_mask_mem: Memory = .{
18625 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },18705 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },
18626 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },18706 .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
18627 };18707 };
18628 if (has_avx) try self.asmRegisterRegisterMemory(18708 if (has_avx) try self.asmRegisterRegisterMemory(
18629 .{ .vp_b, .shuf },18709 .{ .vp_b, .shuf },
...@@ -18696,7 +18776,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {...@@ -18696,7 +18776,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18696 .Or => {18776 .Or => {
18697 if (operand_mcv.isMemory()) try self.asmMemoryImmediate(18777 if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
18698 .{ ._, .@"test" },18778 .{ ._, .@"test" },
18699 try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)),18779 try operand_mcv.mem(self, .fromSize(abi_size)),
18700 .u(mask),18780 .u(mask),
18701 ) else {18781 ) else {
18702 const operand_reg = registerAlias(if (operand_mcv.isRegister())18782 const operand_reg = registerAlias(if (operand_mcv.isRegister())
...@@ -18745,7 +18825,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -18745,7 +18825,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
18745 const result: MCValue = result: {18825 const result: MCValue = result: {
18746 switch (result_ty.zigTypeTag(zcu)) {18826 switch (result_ty.zigTypeTag(zcu)) {
18747 .@"struct" => {18827 .@"struct" => {
18748 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));18828 const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
18749 if (result_ty.containerLayout(zcu) == .@"packed") {18829 if (result_ty.containerLayout(zcu) == .@"packed") {
18750 const struct_obj = zcu.typeToStruct(result_ty).?;18830 const struct_obj = zcu.typeToStruct(result_ty).?;
18751 try self.genInlineMemset(18831 try self.genInlineMemset(
...@@ -18885,7 +18965,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -18885,7 +18965,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
18885 }18965 }
18886 break :result .{ .register = dst_reg };18966 break :result .{ .register = dst_reg };
18887 } else {18967 } else {
18888 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));18968 const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
18889 const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));18969 const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
1889018970
18891 for (elements, 0..) |elem, elem_i| {18971 for (elements, 0..) |elem, elem_i| {
...@@ -18918,7 +18998,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -18918,7 +18998,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
18918 };18998 };
1891918999
18920 if (elements.len <= Liveness.bpi - 1) {19000 if (elements.len <= Liveness.bpi - 1) {
18921 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);19001 var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
18922 @memcpy(buf[0..elements.len], elements);19002 @memcpy(buf[0..elements.len], elements);
18923 return self.finishAir(inst, result, buf);19003 return self.finishAir(inst, result, buf);
18924 }19004 }
...@@ -18952,7 +19032,7 @@ fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -18952,7 +19032,7 @@ fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
1895219032
18953 const union_obj = zcu.typeToUnion(union_ty).?;19033 const union_obj = zcu.typeToUnion(union_ty).?;
18954 const field_name = union_obj.loadTagType(ip).names.get(ip)[extra.field_index];19034 const field_name = union_obj.loadTagType(ip).names.get(ip)[extra.field_index];
18955 const tag_ty = Type.fromInterned(union_obj.enum_tag_ty);19035 const tag_ty: Type = .fromInterned(union_obj.enum_tag_ty);
18956 const field_index = tag_ty.enumFieldIndex(field_name, zcu).?;19036 const field_index = tag_ty.enumFieldIndex(field_name, zcu).?;
18957 const tag_val = try pt.enumValueFieldIndex(tag_ty, field_index);19037 const tag_val = try pt.enumValueFieldIndex(tag_ty, field_index);
18958 const tag_int_val = try tag_val.intFromEnum(tag_ty, pt);19038 const tag_int_val = try tag_val.intFromEnum(tag_ty, pt);
...@@ -19010,10 +19090,10 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {...@@ -19010,10 +19090,10 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
19010 }19090 }
1901119091
19012 var mcvs: [3]MCValue = undefined;19092 var mcvs: [3]MCValue = undefined;
19013 var locks = [1]?RegisterManager.RegisterLock{null} ** 3;19093 var locks: [3]?RegisterManager.RegisterLock = @splat(null);
19014 defer for (locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);19094 defer for (locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
19015 var order = [1]u2{0} ** 3;19095 var order: [3]u2 = @splat(0);
19016 var unused = std.StaticBitSet(3).initFull();19096 var unused: std.StaticBitSet(3) = .initFull();
19017 for (ops, &mcvs, &locks, 0..) |op, *mcv, *lock, op_i| {19097 for (ops, &mcvs, &locks, 0..) |op, *mcv, *lock, op_i| {
19018 const op_index: u2 = @intCast(op_i);19098 const op_index: u2 = @intCast(op_i);
19019 mcv.* = try self.resolveInst(op);19099 mcv.* = try self.resolveInst(op);
...@@ -19139,7 +19219,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {...@@ -19139,7 +19219,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
19139 mir_tag,19219 mir_tag,
19140 mop1_reg,19220 mop1_reg,
19141 mop2_reg,19221 mop2_reg,
19142 try mops[2].mem(self, Memory.Size.fromSize(abi_size)),19222 try mops[2].mem(self, .fromSize(abi_size)),
19143 );19223 );
19144 break :result mops[0];19224 break :result mops[0];
19145 };19225 };
...@@ -19158,7 +19238,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {...@@ -19158,7 +19238,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
19158 )) {19238 )) {
19159 .x86_64_sysv => result: {19239 .x86_64_sysv => result: {
19160 const info = self.va_info.sysv;19240 const info = self.va_info.sysv;
19161 const dst_fi = try self.allocFrameIndex(FrameAlloc.initSpill(va_list_ty, zcu));19241 const dst_fi = try self.allocFrameIndex(.initSpill(va_list_ty, zcu));
19162 var field_off: u31 = 0;19242 var field_off: u31 = 0;
19163 // gp_offset: c_uint,19243 // gp_offset: c_uint,
19164 try self.genSetMem(19244 try self.genSetMem(
...@@ -19221,7 +19301,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -19221,7 +19301,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
19221 try self.spillEflagsIfOccupied();19301 try self.spillEflagsIfOccupied();
1922219302
19223 const tmp_regs =19303 const tmp_regs =
19224 try self.register_manager.allocRegs(2, .{null} ** 2, abi.RegisterClass.gp);19304 try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
19225 const offset_reg = tmp_regs[0].to32();19305 const offset_reg = tmp_regs[0].to32();
19226 const addr_reg = tmp_regs[1].to64();19306 const addr_reg = tmp_regs[1].to64();
19227 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);19307 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
...@@ -19426,8 +19506,8 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {...@@ -19426,8 +19506,8 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
19426 } else mcv: {19506 } else mcv: {
19427 const ip_index = ref.toInterned().?;19507 const ip_index = ref.toInterned().?;
19428 const gop = try self.const_tracking.getOrPut(self.gpa, ip_index);19508 const gop = try self.const_tracking.getOrPut(self.gpa, ip_index);
19429 if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(init: {19509 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
19430 const const_mcv = try self.genTypedValue(Value.fromInterned(ip_index));19510 const const_mcv = try self.genTypedValue(.fromInterned(ip_index));
19431 switch (const_mcv) {19511 switch (const_mcv) {
19432 .lea_tlv => |tlv_sym| switch (self.bin_file.tag) {19512 .lea_tlv => |tlv_sym| switch (self.bin_file.tag) {
19433 .elf, .macho => {19513 .elf, .macho => {
...@@ -19436,7 +19516,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {...@@ -19436,7 +19516,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
19436 } else {19516 } else {
19437 try self.spillRegisters(&.{.rax});19517 try self.spillRegisters(&.{.rax});
19438 }19518 }
19439 const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{19519 const frame_index = try self.allocFrameIndex(.init(.{
19440 .size = 8,19520 .size = 8,
19441 .alignment = .@"8",19521 .alignment = .@"8",
19442 }));19522 }));
...@@ -19539,9 +19619,8 @@ fn resolveCallingConventionValues(...@@ -19539,9 +19619,8 @@ fn resolveCallingConventionValues(
19539 const param_types = try self.gpa.alloc(Type, fn_info.param_types.len + var_args.len);19619 const param_types = try self.gpa.alloc(Type, fn_info.param_types.len + var_args.len);
19540 defer self.gpa.free(param_types);19620 defer self.gpa.free(param_types);
1954119621
19542 for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*dest, src| {19622 for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*dest, src|
19543 dest.* = Type.fromInterned(src);19623 dest.* = .fromInterned(src);
19544 }
19545 for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty|19624 for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty|
19546 param_ty.* = self.promoteVarArg(arg_ty);19625 param_ty.* = self.promoteVarArg(arg_ty);
1954719626
...@@ -19556,13 +19635,13 @@ fn resolveCallingConventionValues(...@@ -19556,13 +19635,13 @@ fn resolveCallingConventionValues(
19556 };19635 };
19557 errdefer self.gpa.free(result.args);19636 errdefer self.gpa.free(result.args);
1955819637
19559 const ret_ty = Type.fromInterned(fn_info.return_type);19638 const ret_ty: Type = .fromInterned(fn_info.return_type);
1956019639
19561 const resolved_cc = abi.resolveCallingConvention(cc, self.target.*);19640 const resolved_cc = abi.resolveCallingConvention(cc, self.target.*);
19562 switch (cc) {19641 switch (cc) {
19563 .naked => {19642 .naked => {
19564 assert(result.args.len == 0);19643 assert(result.args.len == 0);
19565 result.return_value = InstTracking.init(.unreach);19644 result.return_value = .init(.unreach);
19566 result.stack_align = .@"8";19645 result.stack_align = .@"8";
19567 },19646 },
19568 .x86_64_sysv, .x86_64_win => |cc_opts| {19647 .x86_64_sysv, .x86_64_win => |cc_opts| {
...@@ -19583,10 +19662,10 @@ fn resolveCallingConventionValues(...@@ -19583,10 +19662,10 @@ fn resolveCallingConventionValues(
1958319662
19584 // Return values19663 // Return values
19585 if (ret_ty.zigTypeTag(zcu) == .noreturn) {19664 if (ret_ty.zigTypeTag(zcu) == .noreturn) {
19586 result.return_value = InstTracking.init(.unreach);19665 result.return_value = .init(.unreach);
19587 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {19666 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
19588 // TODO: is this even possible for C calling convention?19667 // TODO: is this even possible for C calling convention?
19589 result.return_value = InstTracking.init(.none);19668 result.return_value = .init(.none);
19590 } else {19669 } else {
19591 var ret_tracking: [2]InstTracking = undefined;19670 var ret_tracking: [2]InstTracking = undefined;
19592 var ret_tracking_i: usize = 0;19671 var ret_tracking_i: usize = 0;
...@@ -19604,7 +19683,7 @@ fn resolveCallingConventionValues(...@@ -19604,7 +19683,7 @@ fn resolveCallingConventionValues(
19604 );19683 );
19605 ret_int_reg_i += 1;19684 ret_int_reg_i += 1;
1960619685
19607 ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_int_reg });19686 ret_tracking[ret_tracking_i] = .init(.{ .register = ret_int_reg });
19608 ret_tracking_i += 1;19687 ret_tracking_i += 1;
19609 },19688 },
19610 .sse, .float, .float_combine, .win_i128 => {19689 .sse, .float, .float_combine, .win_i128 => {
...@@ -19614,18 +19693,17 @@ fn resolveCallingConventionValues(...@@ -19614,18 +19693,17 @@ fn resolveCallingConventionValues(
19614 );19693 );
19615 ret_sse_reg_i += 1;19694 ret_sse_reg_i += 1;
1961619695
19617 ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_sse_reg });19696 ret_tracking[ret_tracking_i] = .init(.{ .register = ret_sse_reg });
19618 ret_tracking_i += 1;19697 ret_tracking_i += 1;
19619 },19698 },
19620 .sseup => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .sse),19699 .sseup => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .sse),
19621 .x87 => {19700 .x87 => {
19622 ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = .st0 });19701 ret_tracking[ret_tracking_i] = .init(.{ .register = .st0 });
19623 ret_tracking_i += 1;19702 ret_tracking_i += 1;
19624 },19703 },
19625 .x87up => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .x87),19704 .x87up => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .x87),
19626 .complex_x87 => {19705 .complex_x87 => {
19627 ret_tracking[ret_tracking_i] =19706 ret_tracking[ret_tracking_i] = .init(.{ .register_pair = .{ .st0, .st1 } });
19628 InstTracking.init(.{ .register_pair = .{ .st0, .st1 } });
19629 ret_tracking_i += 1;19707 ret_tracking_i += 1;
19630 },19708 },
19631 .memory => {19709 .memory => {
...@@ -19645,8 +19723,9 @@ fn resolveCallingConventionValues(...@@ -19645,8 +19723,9 @@ fn resolveCallingConventionValues(
19645 result.return_value = switch (ret_tracking_i) {19723 result.return_value = switch (ret_tracking_i) {
19646 else => unreachable,19724 else => unreachable,
19647 1 => ret_tracking[0],19725 1 => ret_tracking[0],
19648 2 => InstTracking.init(.{ .register_pair = .{19726 2 => .init(.{ .register_pair = .{
19649 ret_tracking[0].short.register, ret_tracking[1].short.register,19727 ret_tracking[0].short.register,
19728 ret_tracking[1].short.register,
19650 } }),19729 } }),
19651 };19730 };
19652 }19731 }
...@@ -19769,9 +19848,9 @@ fn resolveCallingConventionValues(...@@ -19769,9 +19848,9 @@ fn resolveCallingConventionValues(
1976919848
19770 // Return values19849 // Return values
19771 if (ret_ty.zigTypeTag(zcu) == .noreturn) {19850 if (ret_ty.zigTypeTag(zcu) == .noreturn) {
19772 result.return_value = InstTracking.init(.unreach);19851 result.return_value = .init(.unreach);
19773 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {19852 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
19774 result.return_value = InstTracking.init(.none);19853 result.return_value = .init(.none);
19775 } else {19854 } else {
19776 const ret_reg = abi.getCAbiIntReturnRegs(resolved_cc)[0];19855 const ret_reg = abi.getCAbiIntReturnRegs(resolved_cc)[0];
19777 const ret_ty_size: u31 = @intCast(ret_ty.abiSize(zcu));19856 const ret_ty_size: u31 = @intCast(ret_ty.abiSize(zcu));
...@@ -19884,8 +19963,8 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {...@@ -19884,8 +19963,8 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
19884fn memSize(self: *Self, ty: Type) Memory.Size {19963fn memSize(self: *Self, ty: Type) Memory.Size {
19885 const zcu = self.pt.zcu;19964 const zcu = self.pt.zcu;
19886 return switch (ty.zigTypeTag(zcu)) {19965 return switch (ty.zigTypeTag(zcu)) {
19887 .float => Memory.Size.fromBitSize(ty.floatBits(self.target.*)),19966 .float => .fromBitSize(ty.floatBits(self.target.*)),
19888 else => Memory.Size.fromSize(@intCast(ty.abiSize(zcu))),19967 else => .fromSize(@intCast(ty.abiSize(zcu))),
19889 };19968 };
19890}19969}
1989119970
...@@ -20140,7 +20219,7 @@ const Temp = struct {...@@ -20140,7 +20219,7 @@ const Temp = struct {
20140 .register => |reg| {20219 .register => |reg| {
20141 const new_reg =20220 const new_reg =
20142 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20221 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20143 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });20222 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20144 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{20223 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
20145 .base = .{ .reg = reg.to64() },20224 .base = .{ .reg = reg.to64() },
20146 .mod = .{ .rm = .{20225 .mod = .{ .rm = .{
...@@ -20152,7 +20231,7 @@ const Temp = struct {...@@ -20152,7 +20231,7 @@ const Temp = struct {
20152 .register_offset => |reg_off| {20231 .register_offset => |reg_off| {
20153 const new_reg =20232 const new_reg =
20154 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20233 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20155 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });20234 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20156 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{20235 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
20157 .base = .{ .reg = reg_off.reg.to64() },20236 .base = .{ .reg = reg_off.reg.to64() },
20158 .mod = .{ .rm = .{20237 .mod = .{ .rm = .{
...@@ -20161,14 +20240,14 @@ const Temp = struct {...@@ -20161,14 +20240,14 @@ const Temp = struct {
20161 } },20240 } },
20162 });20241 });
20163 },20242 },
20164 .lea_symbol => |sym_off| new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_symbol = .{20243 .lea_symbol => |sym_off| new_temp_index.tracking(self).* = .init(.{ .lea_symbol = .{
20165 .sym_index = sym_off.sym_index,20244 .sym_index = sym_off.sym_index,
20166 .off = sym_off.off + off,20245 .off = sym_off.off + off,
20167 } }),20246 } }),
20168 .load_frame => |frame_addr| {20247 .load_frame => |frame_addr| {
20169 const new_reg =20248 const new_reg =
20170 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20249 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20171 new_temp_index.tracking(self).* = InstTracking.init(.{ .register_offset = .{20250 new_temp_index.tracking(self).* = .init(.{ .register_offset = .{
20172 .reg = new_reg,20251 .reg = new_reg,
20173 .off = off,20252 .off = off,
20174 } });20253 } });
...@@ -20180,7 +20259,7 @@ const Temp = struct {...@@ -20180,7 +20259,7 @@ const Temp = struct {
20180 } },20259 } },
20181 });20260 });
20182 },20261 },
20183 .lea_frame => |frame_addr| new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_frame = .{20262 .lea_frame => |frame_addr| new_temp_index.tracking(self).* = .init(.{ .lea_frame = .{
20184 .index = frame_addr.index,20263 .index = frame_addr.index,
20185 .off = frame_addr.off + off,20264 .off = frame_addr.off + off,
20186 } }),20265 } }),
...@@ -20198,7 +20277,7 @@ const Temp = struct {...@@ -20198,7 +20277,7 @@ const Temp = struct {
20198 else => {},20277 else => {},
20199 .register => |reg| {20278 .register => |reg| {
20200 try self.freeValue(temp_tracking.long);20279 try self.freeValue(temp_tracking.long);
20201 temp_tracking.* = InstTracking.init(.{ .register_offset = .{20280 temp_tracking.* = .init(.{ .register_offset = .{
20202 .reg = reg,20281 .reg = reg,
20203 .off = off,20282 .off = off,
20204 } });20283 } });
...@@ -20206,7 +20285,7 @@ const Temp = struct {...@@ -20206,7 +20285,7 @@ const Temp = struct {
20206 },20285 },
20207 .register_offset => |reg_off| {20286 .register_offset => |reg_off| {
20208 try self.freeValue(temp_tracking.long);20287 try self.freeValue(temp_tracking.long);
20209 temp_tracking.* = InstTracking.init(.{ .register_offset = .{20288 temp_tracking.* = .init(.{ .register_offset = .{
20210 .reg = reg_off.reg,20289 .reg = reg_off.reg,
20211 .off = reg_off.off + off,20290 .off = reg_off.off + off,
20212 } });20291 } });
...@@ -20214,7 +20293,7 @@ const Temp = struct {...@@ -20214,7 +20293,7 @@ const Temp = struct {
20214 },20293 },
20215 .lea_symbol => |sym_off| {20294 .lea_symbol => |sym_off| {
20216 assert(std.meta.eql(temp_tracking.long.lea_symbol, sym_off));20295 assert(std.meta.eql(temp_tracking.long.lea_symbol, sym_off));
20217 temp_tracking.* = InstTracking.init(.{ .lea_symbol = .{20296 temp_tracking.* = .init(.{ .lea_symbol = .{
20218 .sym_index = sym_off.sym_index,20297 .sym_index = sym_off.sym_index,
20219 .off = sym_off.off + off,20298 .off = sym_off.off + off,
20220 } });20299 } });
...@@ -20222,7 +20301,7 @@ const Temp = struct {...@@ -20222,7 +20301,7 @@ const Temp = struct {
20222 },20301 },
20223 .lea_frame => |frame_addr| {20302 .lea_frame => |frame_addr| {
20224 assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr));20303 assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr));
20225 temp_tracking.* = InstTracking.init(.{ .lea_frame = .{20304 temp_tracking.* = .init(.{ .lea_frame = .{
20226 .index = frame_addr.index,20305 .index = frame_addr.index,
20227 .off = frame_addr.off + off,20306 .off = frame_addr.off + off,
20228 } });20307 } });
...@@ -20243,26 +20322,26 @@ const Temp = struct {...@@ -20243,26 +20322,26 @@ const Temp = struct {
20243 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),20322 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
20244 .immediate => |imm| {20323 .immediate => |imm| {
20245 assert(limb_index == 0);20324 assert(limb_index == 0);
20246 new_temp_index.tracking(self).* = InstTracking.init(.{ .immediate = imm });20325 new_temp_index.tracking(self).* = .init(.{ .immediate = imm });
20247 },20326 },
20248 .register => |reg| {20327 .register => |reg| {
20249 assert(limb_index == 0);20328 assert(limb_index == 0);
20250 const new_reg =20329 const new_reg =
20251 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20330 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20252 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });20331 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20253 try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());20332 try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());
20254 },20333 },
20255 .register_pair => |regs| {20334 .register_pair => |regs| {
20256 const new_reg =20335 const new_reg =
20257 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20336 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20258 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });20337 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20259 try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());20338 try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());
20260 },20339 },
20261 .register_offset => |reg_off| {20340 .register_offset => |reg_off| {
20262 assert(limb_index == 0);20341 assert(limb_index == 0);
20263 const new_reg =20342 const new_reg =
20264 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20343 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20265 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });20344 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20266 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{20345 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
20267 .base = .{ .reg = reg_off.reg.to64() },20346 .base = .{ .reg = reg_off.reg.to64() },
20268 .mod = .{ .rm = .{20347 .mod = .{ .rm = .{
...@@ -20274,7 +20353,7 @@ const Temp = struct {...@@ -20274,7 +20353,7 @@ const Temp = struct {
20274 .load_symbol => |sym_off| {20353 .load_symbol => |sym_off| {
20275 const new_reg =20354 const new_reg =
20276 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20355 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20277 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });20356 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20278 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{20357 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
20279 .base = .{ .reloc = sym_off.sym_index },20358 .base = .{ .reloc = sym_off.sym_index },
20280 .mod = .{ .rm = .{20359 .mod = .{ .rm = .{
...@@ -20285,12 +20364,12 @@ const Temp = struct {...@@ -20285,12 +20364,12 @@ const Temp = struct {
20285 },20364 },
20286 .lea_symbol => |sym_off| {20365 .lea_symbol => |sym_off| {
20287 assert(limb_index == 0);20366 assert(limb_index == 0);
20288 new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_symbol = sym_off });20367 new_temp_index.tracking(self).* = .init(.{ .lea_symbol = sym_off });
20289 },20368 },
20290 .load_frame => |frame_addr| {20369 .load_frame => |frame_addr| {
20291 const new_reg =20370 const new_reg =
20292 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20371 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20293 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });20372 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20294 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{20373 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
20295 .base = .{ .frame = frame_addr.index },20374 .base = .{ .frame = frame_addr.index },
20296 .mod = .{ .rm = .{20375 .mod = .{ .rm = .{
...@@ -20301,7 +20380,7 @@ const Temp = struct {...@@ -20301,7 +20380,7 @@ const Temp = struct {
20301 },20380 },
20302 .lea_frame => |frame_addr| {20381 .lea_frame => |frame_addr| {
20303 assert(limb_index == 0);20382 assert(limb_index == 0);
20304 new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_frame = frame_addr });20383 new_temp_index.tracking(self).* = .init(.{ .lea_frame = frame_addr });
20305 },20384 },
20306 }20385 }
20307 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);20386 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
...@@ -20328,13 +20407,13 @@ const Temp = struct {...@@ -20328,13 +20407,13 @@ const Temp = struct {
20328 }20407 }
20329 for (regs, 0..) |reg, reg_index| if (reg_index != limb_index)20408 for (regs, 0..) |reg, reg_index| if (reg_index != limb_index)
20330 self.register_manager.freeReg(reg);20409 self.register_manager.freeReg(reg);
20331 temp_tracking.* = InstTracking.init(.{ .register = regs[limb_index] });20410 temp_tracking.* = .init(.{ .register = regs[limb_index] });
20332 self.temp_type[@intFromEnum(temp_index)] = Type.usize;20411 self.temp_type[@intFromEnum(temp_index)] = Type.usize;
20333 return;20412 return;
20334 },20413 },
20335 .load_symbol => |sym_off| {20414 .load_symbol => |sym_off| {
20336 assert(std.meta.eql(temp_tracking.long.load_symbol, sym_off));20415 assert(std.meta.eql(temp_tracking.long.load_symbol, sym_off));
20337 temp_tracking.* = InstTracking.init(.{ .load_symbol = .{20416 temp_tracking.* = .init(.{ .load_symbol = .{
20338 .sym_index = sym_off.sym_index,20417 .sym_index = sym_off.sym_index,
20339 .off = sym_off.off + @as(u31, limb_index) * 8,20418 .off = sym_off.off + @as(u31, limb_index) * 8,
20340 } });20419 } });
...@@ -20343,7 +20422,7 @@ const Temp = struct {...@@ -20343,7 +20422,7 @@ const Temp = struct {
20343 },20422 },
20344 .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) {20423 .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) {
20345 assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr));20424 assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr));
20346 temp_tracking.* = InstTracking.init(.{ .load_frame = .{20425 temp_tracking.* = .init(.{ .load_frame = .{
20347 .index = frame_addr.index,20426 .index = frame_addr.index,
20348 .off = frame_addr.off + @as(u31, limb_index) * 8,20427 .off = frame_addr.off + @as(u31, limb_index) * 8,
20349 } });20428 } });
...@@ -20371,7 +20450,7 @@ const Temp = struct {...@@ -20371,7 +20450,7 @@ const Temp = struct {
20371 const new_temp_index = self.next_temp_index;20450 const new_temp_index = self.next_temp_index;
20372 self.temp_type[@intFromEnum(new_temp_index)] = ty;20451 self.temp_type[@intFromEnum(new_temp_index)] = ty;
20373 try self.genSetReg(new_reg, ty, val, .{});20452 try self.genSetReg(new_reg, ty, val, .{});
20374 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });20453 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20375 try temp.die(self);20454 try temp.die(self);
20376 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);20455 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20377 temp.* = .{ .index = new_temp_index.toIndex() };20456 temp.* = .{ .index = new_temp_index.toIndex() };
...@@ -20390,9 +20469,32 @@ const Temp = struct {...@@ -20390,9 +20469,32 @@ const Temp = struct {
20390 const new_temp_index = self.next_temp_index;20469 const new_temp_index = self.next_temp_index;
20391 self.temp_type[@intFromEnum(new_temp_index)] = ty;20470 self.temp_type[@intFromEnum(new_temp_index)] = ty;
20392 const new_reg =20471 const new_reg =
20393 try self.register_manager.allocReg(new_temp_index.toIndex(), self.regClassForType(ty));20472 try self.register_manager.allocReg(new_temp_index.toIndex(), self.regSetForType(ty));
20394 try self.genSetReg(new_reg, ty, val, .{});20473 try self.genSetReg(new_reg, ty, val, .{});
20395 new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });20474 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20475 try temp.die(self);
20476 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20477 temp.* = .{ .index = new_temp_index.toIndex() };
20478 return true;
20479 }
20480
20481 fn toRegClass(temp: *Temp, rc: Register.Class, self: *Self) !bool {
20482 const val, const ty = switch (temp.unwrap(self)) {
20483 .ref => |ref| .{ temp.tracking(self).short, self.typeOf(ref) },
20484 .temp => |temp_index| val: {
20485 const temp_tracking = temp_index.tracking(self);
20486 switch (temp_tracking.short) {
20487 else => {},
20488 .register => |reg| if (reg.class() == rc) return false,
20489 }
20490 break :val .{ temp_tracking.short, temp_index.typeOf(self) };
20491 },
20492 };
20493 const new_temp_index = self.next_temp_index;
20494 self.temp_type[@intFromEnum(new_temp_index)] = ty;
20495 const new_reg = try self.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc));
20496 try self.genSetReg(new_reg, ty, val, .{});
20497 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });
20396 try temp.die(self);20498 try temp.die(self);
20397 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);20499 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20398 temp.* = .{ .index = new_temp_index.toIndex() };20500 temp.* = .{ .index = new_temp_index.toIndex() };
...@@ -20414,7 +20516,7 @@ const Temp = struct {...@@ -20414,7 +20516,7 @@ const Temp = struct {
20414 assert(self.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));20516 assert(self.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
20415 assert(self.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));20517 assert(self.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
20416 self.temp_type[@intFromEnum(result_temp_index)] = Type.slice_const_u8;20518 self.temp_type[@intFromEnum(result_temp_index)] = Type.slice_const_u8;
20417 result_temp_index.tracking(self).* = InstTracking.init(result);20519 result_temp_index.tracking(self).* = .init(result);
20418 first_temp.* = result_temp;20520 first_temp.* = result_temp;
20419 }20521 }
2042020522
...@@ -20520,13 +20622,43 @@ const Temp = struct {...@@ -20520,13 +20622,43 @@ const Temp = struct {
20520 try self.asmOpOnly(.{ .@"rep _sb", .mov });20622 try self.asmOpOnly(.{ .@"rep _sb", .mov });
20521 }20623 }
2052220624
20625 // i, m, r
20626 fn add(lhs: *Temp, rhs: *Temp, self: *Self) !Temp {
20627 const res_index = self.next_temp_index;
20628 var res: Temp = .{ .index = res_index.toIndex() };
20629 try self.select(&.{ &res, lhs, rhs }, .{ ._, .add }, &.{
20630 .{ .ops = &.{ .{ .match = 1 }, .r, .i } },
20631 .{ .ops = &.{ .{ .match = 1 }, .m, .i } },
20632 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },
20633 .{ .ops = &.{ .{ .match = 1 }, .m, .r } },
20634 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },
20635 });
20636 self.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
20637 self.temp_type[@intFromEnum(res_index)] = lhs.typeOf(self);
20638 return res;
20639 }
20640
20641 fn mul(lhs: *Temp, rhs: *Temp, self: *Self) !Temp {
20642 const res_index = self.next_temp_index;
20643 var res: Temp = .{ .index = self.next_temp_index.toIndex() };
20644 try self.select(&.{ &res, lhs, rhs }, .{ .i_, .mul }, &.{
20645 .{ .ops = &.{ .r, .m, .i } },
20646 .{ .ops = &.{ .r, .r, .i } },
20647 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },
20648 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },
20649 });
20650 self.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
20651 self.temp_type[@intFromEnum(res_index)] = lhs.typeOf(self);
20652 return res;
20653 }
20654
20523 fn moveTo(temp: Temp, inst: Air.Inst.Index, self: *Self) !void {20655 fn moveTo(temp: Temp, inst: Air.Inst.Index, self: *Self) !void {
20524 if (self.liveness.isUnused(inst)) try temp.die(self) else switch (temp.unwrap(self)) {20656 if (self.liveness.isUnused(inst)) try temp.die(self) else switch (temp.unwrap(self)) {
20525 .ref => {20657 .ref => {
20526 const result = try self.allocRegOrMem(inst, true);20658 const result = try self.allocRegOrMem(inst, true);
20527 try self.genCopy(self.typeOfIndex(inst), result, temp.tracking(self).short, .{});20659 try self.genCopy(self.typeOfIndex(inst), result, temp.tracking(self).short, .{});
20528 tracking_log.debug("{} => {} (birth)", .{ inst, result });20660 tracking_log.debug("{} => {} (birth)", .{ inst, result });
20529 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));20661 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));
20530 },20662 },
20531 .temp => |temp_index| {20663 .temp => |temp_index| {
20532 const temp_tracking = temp_index.tracking(self);20664 const temp_tracking = temp_index.tracking(self);
...@@ -20548,7 +20680,7 @@ const Temp = struct {...@@ -20548,7 +20680,7 @@ const Temp = struct {
20548 _,20680 _,
2054920681
20550 fn toIndex(index: Index) Air.Inst.Index {20682 fn toIndex(index: Index) Air.Inst.Index {
20551 return Air.Inst.Index.fromTargetIndex(@intFromEnum(index));20683 return .fromTargetIndex(@intFromEnum(index));
20552 }20684 }
2055320685
20554 fn fromIndex(index: Air.Inst.Index) Index {20686 fn fromIndex(index: Air.Inst.Index) Index {
...@@ -20629,7 +20761,19 @@ fn reuseTemp(...@@ -20629,7 +20761,19 @@ fn reuseTemp(
2062920761
20630fn tempAlloc(self: *Self, ty: Type) !Temp {20762fn tempAlloc(self: *Self, ty: Type) !Temp {
20631 const temp_index = self.next_temp_index;20763 const temp_index = self.next_temp_index;
20632 temp_index.tracking(self).* = InstTracking.init(try self.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true));20764 temp_index.tracking(self).* = .init(
20765 try self.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true),
20766 );
20767 self.temp_type[@intFromEnum(temp_index)] = ty;
20768 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
20769 return .{ .index = temp_index.toIndex() };
20770}
20771
20772fn tempAllocReg(self: *Self, ty: Type, rc: RegisterManager.RegisterBitSet) !Temp {
20773 const temp_index = self.next_temp_index;
20774 temp_index.tracking(self).* = .init(
20775 .{ .register = try self.register_manager.allocReg(temp_index.toIndex(), rc) },
20776 );
20633 self.temp_type[@intFromEnum(temp_index)] = ty;20777 self.temp_type[@intFromEnum(temp_index)] = ty;
20634 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);20778 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
20635 return .{ .index = temp_index.toIndex() };20779 return .{ .index = temp_index.toIndex() };
...@@ -20637,7 +20781,7 @@ fn tempAlloc(self: *Self, ty: Type) !Temp {...@@ -20637,7 +20781,7 @@ fn tempAlloc(self: *Self, ty: Type) !Temp {
2063720781
20638fn tempFromValue(self: *Self, ty: Type, value: MCValue) !Temp {20782fn tempFromValue(self: *Self, ty: Type, value: MCValue) !Temp {
20639 const temp_index = self.next_temp_index;20783 const temp_index = self.next_temp_index;
20640 temp_index.tracking(self).* = InstTracking.init(value);20784 temp_index.tracking(self).* = .init(value);
20641 self.temp_type[@intFromEnum(temp_index)] = ty;20785 self.temp_type[@intFromEnum(temp_index)] = ty;
20642 try self.getValue(value, temp_index.toIndex());20786 try self.getValue(value, temp_index.toIndex());
20643 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);20787 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
...@@ -20657,8 +20801,8 @@ fn tempFromOperand(...@@ -20657,8 +20801,8 @@ fn tempFromOperand(
20657 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };20801 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };
20658 const val = op_ref.toInterned().?;20802 const val = op_ref.toInterned().?;
20659 const gop = try self.const_tracking.getOrPut(self.gpa, val);20803 const gop = try self.const_tracking.getOrPut(self.gpa, val);
20660 if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(init: {20804 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
20661 const const_mcv = try self.genTypedValue(Value.fromInterned(val));20805 const const_mcv = try self.genTypedValue(.fromInterned(val));
20662 switch (const_mcv) {20806 switch (const_mcv) {
20663 .lea_tlv => |tlv_sym| switch (self.bin_file.tag) {20807 .lea_tlv => |tlv_sym| switch (self.bin_file.tag) {
20664 .elf, .macho => {20808 .elf, .macho => {
...@@ -20667,7 +20811,7 @@ fn tempFromOperand(...@@ -20667,7 +20811,7 @@ fn tempFromOperand(
20667 } else {20811 } else {
20668 try self.spillRegisters(&.{.rax});20812 try self.spillRegisters(&.{.rax});
20669 }20813 }
20670 const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{20814 const frame_index = try self.allocFrameIndex(.init(.{
20671 .size = 8,20815 .size = 8,
20672 .alignment = .@"8",20816 .alignment = .@"8",
20673 }));20817 }));
...@@ -20685,7 +20829,7 @@ fn tempFromOperand(...@@ -20685,7 +20829,7 @@ fn tempFromOperand(
20685 else => break :init const_mcv,20829 else => break :init const_mcv,
20686 }20830 }
20687 });20831 });
20688 return self.tempFromValue(Type.fromInterned(ip.typeOf(val)), gop.value_ptr.short);20832 return self.tempFromValue(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
20689 }20833 }
2069020834
20691 const temp_index = self.next_temp_index;20835 const temp_index = self.next_temp_index;
...@@ -20706,3 +20850,193 @@ inline fn tempsFromOperands(self: *Self, inst: Air.Inst.Index, op_refs: anytype)...@@ -20706,3 +20850,193 @@ inline fn tempsFromOperands(self: *Self, inst: Air.Inst.Index, op_refs: anytype)
20706 }20850 }
20707 return temps;20851 return temps;
20708}20852}
20853
20854const Operand = union(enum) {
20855 none,
20856 reg: Register,
20857 mem: Memory,
20858 imm: Immediate,
20859 inst: Mir.Inst.Index,
20860};
20861
20862const Pattern = struct {
20863 tag: Mir.Inst.FixedTag,
20864 ops: []const Op,
20865 commute: struct { u8, u8 } = .{ 0, 0 },
20866 features: []const std.Target.x86.Feature = &.{},
20867
20868 const Op = union(enum) {
20869 /// match another operand
20870 match: u8,
20871 /// any general purpose register
20872 gpr,
20873 /// any 64-bit mmx register
20874 mm,
20875 /// any 128-bit sse register
20876 xmm,
20877 /// any 256-bit sse register
20878 ymm,
20879 /// any memory
20880 mem,
20881 /// specific immediate
20882 imm: i8,
20883 /// any immediate signed extended from 32 bits
20884 simm32,
20885
20886 fn matches(op: Op, is_mut: bool, temp: Temp, self: *Self) bool {
20887 return switch (op) {
20888 .match => unreachable,
20889 .gpr => switch (temp.tracking(self).short) {
20890 .register => |reg| reg.class() == .general_purpose,
20891 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and
20892 reg_off.off == 0,
20893 else => self.regClassForType(temp.typeOf(self)) == .general_purpose,
20894 },
20895 .mm => switch (temp.tracking(self).short) {
20896 .register => |reg| reg.class() == .mmx,
20897 .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0,
20898 else => self.regClassForType(temp.typeOf(self)) == .mmx,
20899 },
20900 .xmm => switch (temp.tracking(self).short) {
20901 .register => |reg| reg.class() == .sse,
20902 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
20903 else => self.regClassForType(temp.typeOf(self)) == .sse,
20904 },
20905 .ymm => switch (temp.tracking(self).short) {
20906 .register => |reg| reg.class() == .sse,
20907 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
20908 else => self.regClassForType(temp.typeOf(self)) == .sse,
20909 } and temp.typeOf(self).abiSize(self.pt.zcu) > 16,
20910 .mem => (!is_mut or temp.isMut(self)) and temp.tracking(self).short.isMemory(),
20911 .imm => |specific_imm| if (is_mut) unreachable else switch (temp.tracking(self).short) {
20912 .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm,
20913 else => false,
20914 },
20915 .simm32 => if (is_mut) unreachable else switch (temp.tracking(self).short) {
20916 .immediate => |imm| temp.typeOf(self).abiSize(self.pt.zcu) <= 4 or
20917 std.math.cast(i32, @as(i64, @bitCast(imm))) != null,
20918 else => false,
20919 },
20920 };
20921 }
20922 };
20923};
20924fn select(self: *Self, dst_temps: []Temp, src_temps: []const *Temp, patterns: []const Pattern) !void {
20925 patterns: for (patterns) |pattern| {
20926 for (pattern.features) |feature| if (!self.hasFeature(feature)) continue :patterns;
20927 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| if (!switch (src_op) {
20928 .match => |match_index| pattern.ops[match_index],
20929 else => src_op,
20930 }.matches(src_op == .match, src_temp.*, self)) continue :patterns;
20931 while (true) for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
20932 if (switch (switch (src_op) {
20933 .match => |match_index| pattern.ops[match_index],
20934 else => src_op,
20935 }) {
20936 .match => unreachable,
20937 .gpr => try src_temp.toRegClass(.general_purpose, self),
20938 .mm => try src_temp.toRegClass(.mmx, self),
20939 .xmm, .ymm => try src_temp.toRegClass(.sse, self),
20940 .mem, .imm, .simm32 => false,
20941 }) break;
20942 } else break;
20943 var mir_ops: [4]Operand = @splat(.none);
20944 var mir_ops_len = dst_temps.len;
20945 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
20946 const mir_op, const matched_src_op = op: switch (src_op) {
20947 .match => |match_index| {
20948 dst_temps[match_index] = src_temp.*;
20949 break :op .{ &mir_ops[match_index], pattern.ops[match_index] };
20950 },
20951 else => {
20952 defer mir_ops_len += 1;
20953 break :op .{ &mir_ops[mir_ops_len], src_op };
20954 },
20955 };
20956 const src_mcv = src_temp.tracking(self).short;
20957 mir_op.* = switch (matched_src_op) {
20958 .match => unreachable,
20959 .gpr => .{ .reg = registerAlias(
20960 src_mcv.register,
20961 @intCast(src_temp.typeOf(self).abiSize(self.pt.zcu)),
20962 ) },
20963 .mm => .{ .reg = src_mcv.register },
20964 .xmm => .{ .reg = src_mcv.register.to128() },
20965 .ymm => .{ .reg = src_mcv.register.to256() },
20966 .mem => .{ .mem = try src_mcv.mem(self, self.memSize(src_temp.typeOf(self))) },
20967 .imm => |imm| .{ .imm = .s(imm) },
20968 .simm32 => switch (src_temp.typeOf(self).abiSize(self.pt.zcu)) {
20969 else => unreachable,
20970 1 => .{ .imm = if (std.math.cast(i8, @as(i64, @bitCast(src_mcv.immediate)))) |small|
20971 .s(small)
20972 else
20973 .u(@as(u8, @intCast(src_mcv.immediate))) },
20974 2 => .{ .imm = if (std.math.cast(i16, @as(i64, @bitCast(src_mcv.immediate)))) |small|
20975 .s(small)
20976 else
20977 .u(@as(u16, @intCast(src_mcv.immediate))) },
20978 3...8 => .{ .imm = if (std.math.cast(i32, @as(i64, @bitCast(src_mcv.immediate)))) |small|
20979 .s(small)
20980 else
20981 .u(@as(u32, @intCast(src_mcv.immediate))) },
20982 },
20983 };
20984 }
20985 for (
20986 pattern.ops[0..dst_temps.len],
20987 dst_temps,
20988 mir_ops[0..dst_temps.len],
20989 ) |dst_op, *dst_temp, *mir_op| {
20990 if (mir_op.* != .none) continue;
20991 const ty = src_temps[0].typeOf(self);
20992 switch (dst_op) {
20993 .match => |match_index| {
20994 dst_temp.* = dst_temps[match_index];
20995 mir_op.* = mir_ops[match_index];
20996 },
20997 .gpr => {
20998 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.gp);
20999 mir_op.* = .{ .reg = registerAlias(
21000 dst_temp.tracking(self).short.register,
21001 @intCast(ty.abiSize(self.pt.zcu)),
21002 ) };
21003 },
21004 .mm => @panic("TODO"),
21005 .xmm => {
21006 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.sse);
21007 mir_op.* = .{ .reg = dst_temp.tracking(self).short.register.to128() };
21008 },
21009 .ymm => {
21010 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.sse);
21011 mir_op.* = .{ .reg = dst_temp.tracking(self).short.register.to256() };
21012 },
21013 .mem => @panic("TODO"),
21014 .imm, .simm32 => unreachable, // unmodifiable destination
21015 }
21016 }
21017 std.mem.swap(Operand, &mir_ops[pattern.commute[0]], &mir_ops[pattern.commute[1]]);
21018 self.asmOps(pattern.tag, mir_ops) catch |err| switch (err) {
21019 error.InvalidInstruction => {
21020 const fixes = @tagName(pattern.tag[0]);
21021 const fixes_replace = std.mem.indexOfScalar(u8, fixes, '_').?;
21022 return self.fail(
21023 "invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'",
21024 .{
21025 fixes[0..fixes_replace],
21026 @tagName(pattern.tag[1]),
21027 fixes[fixes_replace + 1 ..],
21028 @tagName(mir_ops[0]),
21029 @tagName(mir_ops[1]),
21030 @tagName(mir_ops[2]),
21031 @tagName(mir_ops[3]),
21032 },
21033 );
21034 },
21035 else => |e| return e,
21036 };
21037 return;
21038 }
21039 log.err("failed to select:", .{});
21040 for (src_temps) |src_temp| log.err("{}", .{src_temp.tracking(self)});
21041 return self.fail("failed to select", .{});
21042}
src/arch/x86_64/Mir.zig+1-1
...@@ -22,7 +22,7 @@ pub const Inst = struct {...@@ -22,7 +22,7 @@ pub const Inst = struct {
22 /// ___22 /// ___
23 @"_",23 @"_",
2424
25 /// Integer __25 /// Integer ___
26 i_,26 i_,
2727
28 /// ___ Left28 /// ___ Left
src/register_manager.zig+11-11
...@@ -41,12 +41,12 @@ pub fn RegisterManager(...@@ -41,12 +41,12 @@ pub fn RegisterManager(
41 registers: TrackedRegisters = undefined,41 registers: TrackedRegisters = undefined,
42 /// Tracks which registers are free (in which case the42 /// Tracks which registers are free (in which case the
43 /// corresponding bit is set to 1)43 /// corresponding bit is set to 1)
44 free_registers: RegisterBitSet = RegisterBitSet.initFull(),44 free_registers: RegisterBitSet = .initFull(),
45 /// Tracks all registers allocated in the course of this45 /// Tracks all registers allocated in the course of this
46 /// function46 /// function
47 allocated_registers: RegisterBitSet = RegisterBitSet.initEmpty(),47 allocated_registers: RegisterBitSet = .initEmpty(),
48 /// Tracks registers which are locked from being allocated48 /// Tracks registers which are locked from being allocated
49 locked_registers: RegisterBitSet = RegisterBitSet.initEmpty(),49 locked_registers: RegisterBitSet = .initEmpty(),
5050
51 const Self = @This();51 const Self = @This();
5252
...@@ -420,8 +420,8 @@ const MockRegister1 = enum(u2) {...@@ -420,8 +420,8 @@ const MockRegister1 = enum(u2) {
420 &MockRegister1.allocatable_registers,420 &MockRegister1.allocatable_registers,
421 );421 );
422422
423 const gp: RM.RegisterBitSet = blk: {423 const gp = blk: {
424 var set = RM.RegisterBitSet.initEmpty();424 var set: RM.RegisterBitSet = .initEmpty();
425 set.setRangeValue(.{425 set.setRangeValue(.{
426 .start = 0,426 .start = 0,
427 .end = allocatable_registers.len,427 .end = allocatable_registers.len,
...@@ -448,8 +448,8 @@ const MockRegister2 = enum(u2) {...@@ -448,8 +448,8 @@ const MockRegister2 = enum(u2) {
448 &MockRegister2.allocatable_registers,448 &MockRegister2.allocatable_registers,
449 );449 );
450450
451 const gp: RM.RegisterBitSet = blk: {451 const gp = blk: {
452 var set = RM.RegisterBitSet.initEmpty();452 var set: RM.RegisterBitSet = .initEmpty();
453 set.setRangeValue(.{453 set.setRangeValue(.{
454 .start = 0,454 .start = 0,
455 .end = allocatable_registers.len,455 .end = allocatable_registers.len,
...@@ -489,16 +489,16 @@ const MockRegister3 = enum(u3) {...@@ -489,16 +489,16 @@ const MockRegister3 = enum(u3) {
489 &MockRegister3.allocatable_registers,489 &MockRegister3.allocatable_registers,
490 );490 );
491491
492 const gp: RM.RegisterBitSet = blk: {492 const gp = blk: {
493 var set = RM.RegisterBitSet.initEmpty();493 var set: RM.RegisterBitSet = .initEmpty();
494 set.setRangeValue(.{494 set.setRangeValue(.{
495 .start = 0,495 .start = 0,
496 .end = gp_regs.len,496 .end = gp_regs.len,
497 }, true);497 }, true);
498 break :blk set;498 break :blk set;
499 };499 };
500 const ext: RM.RegisterBitSet = blk: {500 const ext = blk: {
501 var set = RM.RegisterBitSet.initEmpty();501 var set: RM.RegisterBitSet = .initEmpty();
502 set.setRangeValue(.{502 set.setRangeValue(.{
503 .start = gp_regs.len,503 .start = gp_regs.len,
504 .end = allocatable_registers.len,504 .end = allocatable_registers.len,