authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-08 06:51:05-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-08 07:36:20-04:00
logecb5feaf94bf49dc4c180f09c170223d6c1898b3
tree9b168cfd880fb316f86fba14b3773297745e3ba3
parent6c14eb2863c7c00f809c5e447ceb8186b55f2eef

x86_64: continue to optimize mir tag usage

Migrate mnemonic literals to tuples that represent the compressed storage. 225 tags left in use, many tags left to compress.

3 files changed, 870 insertions(+), 708 deletions(-)

src/arch/x86_64/CodeGen.zig+802-667
...@@ -1167,11 +1167,13 @@ fn asmPlaceholder(self: *Self) !Mir.Inst.Index {...@@ -1167,11 +1167,13 @@ fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
1167 });1167 });
1168}1168}
11691169
1170fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {1170fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {
1171 _ = try self.addInst(.{1171 _ = try self.addInst(.{
1172 .tag = tag,1172 .tag = tag[1],
1173 .ops = .none,1173 .ops = .none,
1174 .data = undefined,1174 .data = .{ .none = .{
1175 .fixes = tag[0],
1176 } },
1175 });1177 });
1176}1178}
11771179
...@@ -1183,22 +1185,26 @@ fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {...@@ -1183,22 +1185,26 @@ fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
1183 });1185 });
1184}1186}
11851187
1186fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {1188fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {
1187 _ = try self.addInst(.{1189 _ = try self.addInst(.{
1188 .tag = tag,1190 .tag = tag[1],
1189 .ops = .r,1191 .ops = .r,
1190 .data = .{ .r = .{ .r1 = reg } },1192 .data = .{ .r = .{
1193 .fixes = tag[0],
1194 .r1 = reg,
1195 } },
1191 });1196 });
1192}1197}
11931198
1194fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {1199fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
1195 _ = try self.addInst(.{1200 _ = try self.addInst(.{
1196 .tag = tag,1201 .tag = tag[1],
1197 .ops = switch (imm) {1202 .ops = switch (imm) {
1198 .signed => .i_s,1203 .signed => .i_s,
1199 .unsigned => .i_u,1204 .unsigned => .i_u,
1200 },1205 },
1201 .data = .{ .i = .{1206 .data = .{ .i = .{
1207 .fixes = tag[0],
1202 .i = switch (imm) {1208 .i = switch (imm) {
1203 .signed => |s| @bitCast(u32, s),1209 .signed => |s| @bitCast(u32, s),
1204 .unsigned => |u| @intCast(u32, u),1210 .unsigned => |u| @intCast(u32, u),
...@@ -1207,24 +1213,29 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {...@@ -1207,24 +1213,29 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
1207 });1213 });
1208}1214}
12091215
1210fn asmRegisterRegister(self: *Self, tag: Mir.Inst.Tag, reg1: Register, reg2: Register) !void {1216fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {
1211 _ = try self.addInst(.{1217 _ = try self.addInst(.{
1212 .tag = tag,1218 .tag = tag[1],
1213 .ops = .rr,1219 .ops = .rr,
1214 .data = .{ .rr = .{ .r1 = reg1, .r2 = reg2 } },1220 .data = .{ .rr = .{
1221 .fixes = tag[0],
1222 .r1 = reg1,
1223 .r2 = reg2,
1224 } },
1215 });1225 });
1216}1226}
12171227
1218fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Immediate) !void {1228fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
1219 const ops: Mir.Inst.Ops = switch (imm) {1229 const ops: Mir.Inst.Ops = switch (imm) {
1220 .signed => .ri_s,1230 .signed => .ri_s,
1221 .unsigned => |u| if (math.cast(u32, u)) |_| .ri_u else .ri64,1231 .unsigned => |u| if (math.cast(u32, u)) |_| .ri_u else .ri64,
1222 };1232 };
1223 _ = try self.addInst(.{1233 _ = try self.addInst(.{
1224 .tag = tag,1234 .tag = tag[1],
1225 .ops = ops,1235 .ops = ops,
1226 .data = switch (ops) {1236 .data = switch (ops) {
1227 .ri_s, .ri_u => .{ .ri = .{1237 .ri_s, .ri_u => .{ .ri = .{
1238 .fixes = tag[0],
1228 .r1 = reg,1239 .r1 = reg,
1229 .i = switch (imm) {1240 .i = switch (imm) {
1230 .signed => |s| @bitCast(u32, s),1241 .signed => |s| @bitCast(u32, s),
...@@ -1232,6 +1243,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme...@@ -1232,6 +1243,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
1232 },1243 },
1233 } },1244 } },
1234 .ri64 => .{ .rx = .{1245 .ri64 => .{ .rx = .{
1246 .fixes = tag[0],
1235 .r1 = reg,1247 .r1 = reg,
1236 .payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),1248 .payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
1237 } },1249 } },
...@@ -1242,47 +1254,59 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme...@@ -1242,47 +1254,59 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
12421254
1243fn asmRegisterRegisterRegister(1255fn asmRegisterRegisterRegister(
1244 self: *Self,1256 self: *Self,
1245 tag: Mir.Inst.Tag,1257 tag: Mir.Inst.FixedTag,
1246 reg1: Register,1258 reg1: Register,
1247 reg2: Register,1259 reg2: Register,
1248 reg3: Register,1260 reg3: Register,
1249) !void {1261) !void {
1250 _ = try self.addInst(.{1262 _ = try self.addInst(.{
1251 .tag = tag,1263 .tag = tag[1],
1252 .ops = .rrr,1264 .ops = .rrr,
1253 .data = .{ .rrr = .{ .r1 = reg1, .r2 = reg2, .r3 = reg3 } },1265 .data = .{ .rrr = .{
1266 .fixes = tag[0],
1267 .r1 = reg1,
1268 .r2 = reg2,
1269 .r3 = reg3,
1270 } },
1254 });1271 });
1255}1272}
12561273
1257fn asmRegisterRegisterRegisterImmediate(1274fn asmRegisterRegisterRegisterImmediate(
1258 self: *Self,1275 self: *Self,
1259 tag: Mir.Inst.Tag,1276 tag: Mir.Inst.FixedTag,
1260 reg1: Register,1277 reg1: Register,
1261 reg2: Register,1278 reg2: Register,
1262 reg3: Register,1279 reg3: Register,
1263 imm: Immediate,1280 imm: Immediate,
1264) !void {1281) !void {
1265 _ = try self.addInst(.{1282 _ = try self.addInst(.{
1266 .tag = tag,1283 .tag = tag[1],
1267 .ops = .rrri,1284 .ops = .rrri,
1268 .data = .{ .rrri = .{ .r1 = reg1, .r2 = reg2, .r3 = reg3, .i = @intCast(u8, imm.unsigned) } },1285 .data = .{ .rrri = .{
1286 .fixes = tag[0],
1287 .r1 = reg1,
1288 .r2 = reg2,
1289 .r3 = reg3,
1290 .i = @intCast(u8, imm.unsigned),
1291 } },
1269 });1292 });
1270}1293}
12711294
1272fn asmRegisterRegisterImmediate(1295fn asmRegisterRegisterImmediate(
1273 self: *Self,1296 self: *Self,
1274 tag: Mir.Inst.Tag,1297 tag: Mir.Inst.FixedTag,
1275 reg1: Register,1298 reg1: Register,
1276 reg2: Register,1299 reg2: Register,
1277 imm: Immediate,1300 imm: Immediate,
1278) !void {1301) !void {
1279 _ = try self.addInst(.{1302 _ = try self.addInst(.{
1280 .tag = tag,1303 .tag = tag[1],
1281 .ops = switch (imm) {1304 .ops = switch (imm) {
1282 .signed => .rri_s,1305 .signed => .rri_s,
1283 .unsigned => .rri_u,1306 .unsigned => .rri_u,
1284 },1307 },
1285 .data = .{ .rri = .{1308 .data = .{ .rri = .{
1309 .fixes = tag[0],
1286 .r1 = reg1,1310 .r1 = reg1,
1287 .r2 = reg2,1311 .r2 = reg2,
1288 .i = switch (imm) {1312 .i = switch (imm) {
...@@ -1295,19 +1319,20 @@ fn asmRegisterRegisterImmediate(...@@ -1295,19 +1319,20 @@ fn asmRegisterRegisterImmediate(
12951319
1296fn asmRegisterRegisterMemory(1320fn asmRegisterRegisterMemory(
1297 self: *Self,1321 self: *Self,
1298 tag: Mir.Inst.Tag,1322 tag: Mir.Inst.FixedTag,
1299 reg1: Register,1323 reg1: Register,
1300 reg2: Register,1324 reg2: Register,
1301 m: Memory,1325 m: Memory,
1302) !void {1326) !void {
1303 _ = try self.addInst(.{1327 _ = try self.addInst(.{
1304 .tag = tag,1328 .tag = tag[1],
1305 .ops = switch (m) {1329 .ops = switch (m) {
1306 .sib => .rrm_sib,1330 .sib => .rrm_sib,
1307 .rip => .rrm_rip,1331 .rip => .rrm_rip,
1308 else => unreachable,1332 else => unreachable,
1309 },1333 },
1310 .data = .{ .rrx = .{1334 .data = .{ .rrx = .{
1335 .fixes = tag[0],
1311 .r1 = reg1,1336 .r1 = reg1,
1312 .r2 = reg2,1337 .r2 = reg2,
1313 .payload = switch (m) {1338 .payload = switch (m) {
...@@ -1319,15 +1344,16 @@ fn asmRegisterRegisterMemory(...@@ -1319,15 +1344,16 @@ fn asmRegisterRegisterMemory(
1319 });1344 });
1320}1345}
13211346
1322fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {1347fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1323 _ = try self.addInst(.{1348 _ = try self.addInst(.{
1324 .tag = tag,1349 .tag = tag[1],
1325 .ops = switch (m) {1350 .ops = switch (m) {
1326 .sib => .m_sib,1351 .sib => .m_sib,
1327 .rip => .m_rip,1352 .rip => .m_rip,
1328 else => unreachable,1353 else => unreachable,
1329 },1354 },
1330 .data = .{ .x = .{1355 .data = .{ .x = .{
1356 .fixes = tag[0],
1331 .payload = switch (m) {1357 .payload = switch (m) {
1332 .sib => try self.addExtra(Mir.MemorySib.encode(m)),1358 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
1333 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),1359 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
...@@ -1337,15 +1363,16 @@ fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {...@@ -1337,15 +1363,16 @@ fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
1337 });1363 });
1338}1364}
13391365
1340fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !void {1366fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
1341 _ = try self.addInst(.{1367 _ = try self.addInst(.{
1342 .tag = tag,1368 .tag = tag[1],
1343 .ops = switch (m) {1369 .ops = switch (m) {
1344 .sib => .rm_sib,1370 .sib => .rm_sib,
1345 .rip => .rm_rip,1371 .rip => .rm_rip,
1346 else => unreachable,1372 else => unreachable,
1347 },1373 },
1348 .data = .{ .rx = .{1374 .data = .{ .rx = .{
1375 .fixes = tag[0],
1349 .r1 = reg,1376 .r1 = reg,
1350 .payload = switch (m) {1377 .payload = switch (m) {
1351 .sib => try self.addExtra(Mir.MemorySib.encode(m)),1378 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
...@@ -1358,19 +1385,20 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !...@@ -1358,19 +1385,20 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !
13581385
1359fn asmRegisterMemoryImmediate(1386fn asmRegisterMemoryImmediate(
1360 self: *Self,1387 self: *Self,
1361 tag: Mir.Inst.Tag,1388 tag: Mir.Inst.FixedTag,
1362 reg: Register,1389 reg: Register,
1363 m: Memory,1390 m: Memory,
1364 imm: Immediate,1391 imm: Immediate,
1365) !void {1392) !void {
1366 _ = try self.addInst(.{1393 _ = try self.addInst(.{
1367 .tag = tag,1394 .tag = tag[1],
1368 .ops = switch (m) {1395 .ops = switch (m) {
1369 .sib => .rmi_sib,1396 .sib => .rmi_sib,
1370 .rip => .rmi_rip,1397 .rip => .rmi_rip,
1371 else => unreachable,1398 else => unreachable,
1372 },1399 },
1373 .data = .{ .rix = .{1400 .data = .{ .rix = .{
1401 .fixes = tag[0],
1374 .r1 = reg,1402 .r1 = reg,
1375 .i = @intCast(u8, imm.unsigned),1403 .i = @intCast(u8, imm.unsigned),
1376 .payload = switch (m) {1404 .payload = switch (m) {
...@@ -1384,20 +1412,21 @@ fn asmRegisterMemoryImmediate(...@@ -1384,20 +1412,21 @@ fn asmRegisterMemoryImmediate(
13841412
1385fn asmRegisterRegisterMemoryImmediate(1413fn asmRegisterRegisterMemoryImmediate(
1386 self: *Self,1414 self: *Self,
1387 tag: Mir.Inst.Tag,1415 tag: Mir.Inst.FixedTag,
1388 reg1: Register,1416 reg1: Register,
1389 reg2: Register,1417 reg2: Register,
1390 m: Memory,1418 m: Memory,
1391 imm: Immediate,1419 imm: Immediate,
1392) !void {1420) !void {
1393 _ = try self.addInst(.{1421 _ = try self.addInst(.{
1394 .tag = tag,1422 .tag = tag[1],
1395 .ops = switch (m) {1423 .ops = switch (m) {
1396 .sib => .rrmi_sib,1424 .sib => .rrmi_sib,
1397 .rip => .rrmi_rip,1425 .rip => .rrmi_rip,
1398 else => unreachable,1426 else => unreachable,
1399 },1427 },
1400 .data = .{ .rrix = .{1428 .data = .{ .rrix = .{
1429 .fixes = tag[0],
1401 .r1 = reg1,1430 .r1 = reg1,
1402 .r2 = reg2,1431 .r2 = reg2,
1403 .i = @intCast(u8, imm.unsigned),1432 .i = @intCast(u8, imm.unsigned),
...@@ -1410,15 +1439,16 @@ fn asmRegisterRegisterMemoryImmediate(...@@ -1410,15 +1439,16 @@ fn asmRegisterRegisterMemoryImmediate(
1410 });1439 });
1411}1440}
14121441
1413fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {1442fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
1414 _ = try self.addInst(.{1443 _ = try self.addInst(.{
1415 .tag = tag,1444 .tag = tag[1],
1416 .ops = switch (m) {1445 .ops = switch (m) {
1417 .sib => .mr_sib,1446 .sib => .mr_sib,
1418 .rip => .mr_rip,1447 .rip => .mr_rip,
1419 else => unreachable,1448 else => unreachable,
1420 },1449 },
1421 .data = .{ .rx = .{1450 .data = .{ .rx = .{
1451 .fixes = tag[0],
1422 .r1 = reg,1452 .r1 = reg,
1423 .payload = switch (m) {1453 .payload = switch (m) {
1424 .sib => try self.addExtra(Mir.MemorySib.encode(m)),1454 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
...@@ -1429,9 +1459,9 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !...@@ -1429,9 +1459,9 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !
1429 });1459 });
1430}1460}
14311461
1432fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate) !void {1462fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
1433 _ = try self.addInst(.{1463 _ = try self.addInst(.{
1434 .tag = tag,1464 .tag = tag[1],
1435 .ops = switch (m) {1465 .ops = switch (m) {
1436 .sib => switch (imm) {1466 .sib => switch (imm) {
1437 .signed => .mi_sib_s,1467 .signed => .mi_sib_s,
...@@ -1443,57 +1473,64 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate)...@@ -1443,57 +1473,64 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate)
1443 },1473 },
1444 else => unreachable,1474 else => unreachable,
1445 },1475 },
1446 .data = .{ .ix = .{1476 .data = .{ .x = .{
1447 .i = switch (imm) {1477 .fixes = tag[0],
1478 .payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
1448 .signed => |s| @bitCast(u32, s),1479 .signed => |s| @bitCast(u32, s),
1449 .unsigned => |u| @intCast(u32, u),1480 .unsigned => |u| @intCast(u32, u),
1450 },1481 } }),
1451 .payload = switch (m) {
1452 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
1453 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1454 else => unreachable,
1455 },
1456 } },1482 } },
1457 });1483 });
1484 _ = switch (m) {
1485 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
1486 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1487 else => unreachable,
1488 };
1458}1489}
14591490
1460fn asmMemoryRegisterRegister(1491fn asmMemoryRegisterRegister(
1461 self: *Self,1492 self: *Self,
1462 tag: Mir.Inst.Tag,1493 tag: Mir.Inst.FixedTag,
1463 m: Memory,1494 m: Memory,
1464 reg1: Register,1495 reg1: Register,
1465 reg2: Register,1496 reg2: Register,
1466) !void {1497) !void {
1467 _ = try self.addInst(.{1498 _ = try self.addInst(.{
1468 .tag = tag,1499 .tag = tag[1],
1469 .ops = switch (m) {1500 .ops = switch (m) {
1470 .sib => .mrr_sib,1501 .sib => .mrr_sib,
1471 .rip => .mrr_rip,1502 .rip => .mrr_rip,
1472 else => unreachable,1503 else => unreachable,
1473 },1504 },
1474 .data = .{ .rrx = .{ .r1 = reg1, .r2 = reg2, .payload = switch (m) {1505 .data = .{ .rrx = .{
1475 .sib => try self.addExtra(Mir.MemorySib.encode(m)),1506 .fixes = tag[0],
1476 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),1507 .r1 = reg1,
1477 else => unreachable,1508 .r2 = reg2,
1478 } } },1509 .payload = switch (m) {
1510 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
1511 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1512 else => unreachable,
1513 },
1514 } },
1479 });1515 });
1480}1516}
14811517
1482fn asmMemoryRegisterImmediate(1518fn asmMemoryRegisterImmediate(
1483 self: *Self,1519 self: *Self,
1484 tag: Mir.Inst.Tag,1520 tag: Mir.Inst.FixedTag,
1485 m: Memory,1521 m: Memory,
1486 reg: Register,1522 reg: Register,
1487 imm: Immediate,1523 imm: Immediate,
1488) !void {1524) !void {
1489 _ = try self.addInst(.{1525 _ = try self.addInst(.{
1490 .tag = tag,1526 .tag = tag[1],
1491 .ops = switch (m) {1527 .ops = switch (m) {
1492 .sib => .mri_sib,1528 .sib => .mri_sib,
1493 .rip => .mri_rip,1529 .rip => .mri_rip,
1494 else => unreachable,1530 else => unreachable,
1495 },1531 },
1496 .data = .{ .rix = .{1532 .data = .{ .rix = .{
1533 .fixes = tag[0],
1497 .r1 = reg,1534 .r1 = reg,
1498 .i = @intCast(u8, imm.unsigned),1535 .i = @intCast(u8, imm.unsigned),
1499 .payload = switch (m) {1536 .payload = switch (m) {
...@@ -1508,9 +1545,9 @@ fn asmMemoryRegisterImmediate(...@@ -1508,9 +1545,9 @@ fn asmMemoryRegisterImmediate(
1508fn gen(self: *Self) InnerError!void {1545fn gen(self: *Self) InnerError!void {
1509 const cc = self.fn_type.fnCallingConvention();1546 const cc = self.fn_type.fnCallingConvention();
1510 if (cc != .Naked) {1547 if (cc != .Naked) {
1511 try self.asmRegister(.push, .rbp);1548 try self.asmRegister(.{ ._, .push }, .rbp);
1512 const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();1549 const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
1513 try self.asmRegisterRegister(.mov, .rbp, .rsp);1550 try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
1514 const backpatch_frame_align = try self.asmPlaceholder();1551 const backpatch_frame_align = try self.asmPlaceholder();
1515 const backpatch_stack_alloc = try self.asmPlaceholder();1552 const backpatch_stack_alloc = try self.asmPlaceholder();
15161553
...@@ -1553,8 +1590,8 @@ fn gen(self: *Self) InnerError!void {...@@ -1553,8 +1590,8 @@ fn gen(self: *Self) InnerError!void {
1553 try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);1590 try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
1554 const backpatch_stack_dealloc = try self.asmPlaceholder();1591 const backpatch_stack_dealloc = try self.asmPlaceholder();
1555 const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();1592 const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
1556 try self.asmRegister(.pop, .rbp);1593 try self.asmRegister(.{ ._, .pop }, .rbp);
1557 try self.asmOpOnly(.ret);1594 try self.asmOpOnly(.{ ._, .ret });
15581595
1559 const frame_layout = try self.computeFrameLayout();1596 const frame_layout = try self.computeFrameLayout();
1560 const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);1597 const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
...@@ -1927,7 +1964,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {...@@ -1927,7 +1964,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
1927 };1964 };
1928 const tag_val = Value.initPayload(&tag_pl.base);1965 const tag_val = Value.initPayload(&tag_pl.base);
1929 const tag_mcv = try self.genTypedValue(.{ .ty = enum_ty, .val = tag_val });1966 const tag_mcv = try self.genTypedValue(.{ .ty = enum_ty, .val = tag_val });
1930 try self.genBinOpMir(.cmp, enum_ty, enum_mcv, tag_mcv);1967 try self.genBinOpMir(.{ ._, .cmp }, enum_ty, enum_mcv, tag_mcv);
1931 const skip_reloc = try self.asmJccReloc(undefined, .ne);1968 const skip_reloc = try self.asmJccReloc(undefined, .ne);
19321969
1933 try self.genSetMem(1970 try self.genSetMem(
...@@ -1947,7 +1984,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {...@@ -1947,7 +1984,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
1947 try self.airTrap();1984 try self.airTrap();
19481985
1949 for (exitlude_jump_relocs) |reloc| try self.performReloc(reloc);1986 for (exitlude_jump_relocs) |reloc| try self.performReloc(reloc);
1950 try self.asmOpOnly(.ret);1987 try self.asmOpOnly(.{ ._, .ret });
1951 },1988 },
1952 else => return self.fail(1989 else => return self.fail(
1953 "TODO implement {s} for {}",1990 "TODO implement {s} for {}",
...@@ -2406,7 +2443,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -2406,7 +2443,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2406 else2443 else
2407 try self.copyToTmpRegister(src_ty, src_mcv);2444 try self.copyToTmpRegister(src_ty, src_mcv);
2408 try self.asmRegisterRegisterImmediate(2445 try self.asmRegisterRegisterImmediate(
2409 .vcvtps2ph,2446 .{ ._, .vcvtps2ph },
2410 dst_reg,2447 dst_reg,
2411 mat_src_reg.to128(),2448 mat_src_reg.to128(),
2412 Immediate.u(0b1_00),2449 Immediate.u(0b1_00),
...@@ -2418,12 +2455,12 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -2418,12 +2455,12 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2418 }2455 }
2419 } else if (src_bits == 64 and dst_bits == 32) {2456 } else if (src_bits == 64 and dst_bits == 32) {
2420 if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(2457 if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
2421 .vcvtsd2ss,2458 .{ ._, .vcvtsd2ss },
2422 dst_reg,2459 dst_reg,
2423 dst_reg,2460 dst_reg,
2424 src_mcv.mem(.qword),2461 src_mcv.mem(.qword),
2425 ) else try self.asmRegisterRegisterRegister(2462 ) else try self.asmRegisterRegisterRegister(
2426 .vcvtsd2ss,2463 .{ ._, .vcvtsd2ss },
2427 dst_reg,2464 dst_reg,
2428 dst_reg,2465 dst_reg,
2429 (if (src_mcv.isRegister())2466 (if (src_mcv.isRegister())
...@@ -2431,11 +2468,11 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -2431,11 +2468,11 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2431 else2468 else
2432 try self.copyToTmpRegister(src_ty, src_mcv)).to128(),2469 try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
2433 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(2470 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2434 .cvtsd2ss,2471 .{ ._, .cvtsd2ss },
2435 dst_reg,2472 dst_reg,
2436 src_mcv.mem(.qword),2473 src_mcv.mem(.qword),
2437 ) else try self.asmRegisterRegister(2474 ) else try self.asmRegisterRegister(
2438 .cvtsd2ss,2475 .{ ._, .cvtsd2ss },
2439 dst_reg,2476 dst_reg,
2440 (if (src_mcv.isRegister())2477 (if (src_mcv.isRegister())
2441 src_mcv.getReg().?2478 src_mcv.getReg().?
...@@ -2469,22 +2506,22 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -2469,22 +2506,22 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2469 src_mcv.getReg().?2506 src_mcv.getReg().?
2470 else2507 else
2471 try self.copyToTmpRegister(src_ty, src_mcv);2508 try self.copyToTmpRegister(src_ty, src_mcv);
2472 try self.asmRegisterRegister(.vcvtph2ps, dst_reg, mat_src_reg.to128());2509 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, mat_src_reg.to128());
2473 switch (dst_bits) {2510 switch (dst_bits) {
2474 32 => {},2511 32 => {},
2475 64 => try self.asmRegisterRegisterRegister(.vcvtss2sd, dst_reg, dst_reg, dst_reg),2512 64 => try self.asmRegisterRegisterRegister(.{ ._, .vcvtss2sd }, dst_reg, dst_reg, dst_reg),
2476 else => return self.fail("TODO implement airFpext from {} to {}", .{2513 else => return self.fail("TODO implement airFpext from {} to {}", .{
2477 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),2514 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
2478 }),2515 }),
2479 }2516 }
2480 } else if (src_bits == 32 and dst_bits == 64) {2517 } else if (src_bits == 32 and dst_bits == 64) {
2481 if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(2518 if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
2482 .vcvtss2sd,2519 .{ ._, .vcvtss2sd },
2483 dst_reg,2520 dst_reg,
2484 dst_reg,2521 dst_reg,
2485 src_mcv.mem(.dword),2522 src_mcv.mem(.dword),
2486 ) else try self.asmRegisterRegisterRegister(2523 ) else try self.asmRegisterRegisterRegister(
2487 .vcvtss2sd,2524 .{ ._, .vcvtss2sd },
2488 dst_reg,2525 dst_reg,
2489 dst_reg,2526 dst_reg,
2490 (if (src_mcv.isRegister())2527 (if (src_mcv.isRegister())
...@@ -2492,11 +2529,11 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -2492,11 +2529,11 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2492 else2529 else
2493 try self.copyToTmpRegister(src_ty, src_mcv)).to128(),2530 try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
2494 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(2531 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2495 .cvtss2sd,2532 .{ ._, .cvtss2sd },
2496 dst_reg,2533 dst_reg,
2497 src_mcv.mem(.dword),2534 src_mcv.mem(.dword),
2498 ) else try self.asmRegisterRegister(2535 ) else try self.asmRegisterRegister(
2499 .cvtss2sd,2536 .{ ._, .cvtss2sd },
2500 dst_reg,2537 dst_reg,
2501 (if (src_mcv.isRegister())2538 (if (src_mcv.isRegister())
2502 src_mcv.getReg().?2539 src_mcv.getReg().?
...@@ -2537,12 +2574,12 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -2537,12 +2574,12 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2537 switch (dst_mcv) {2574 switch (dst_mcv) {
2538 .register => |dst_reg| {2575 .register => |dst_reg| {
2539 const min_abi_size = @min(dst_abi_size, src_abi_size);2576 const min_abi_size = @min(dst_abi_size, src_abi_size);
2540 const tag: Mir.Inst.Tag = switch (signedness) {2577 const tag: Mir.Inst.FixedTag = switch (signedness) {
2541 .signed => if (min_abi_size >= 4) .movsxd else .movsx,2578 .signed => if (min_abi_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },
2542 .unsigned => if (min_abi_size >= 4) .mov else .movzx,2579 .unsigned => if (min_abi_size >= 4) .{ ._, .mov } else .{ ._, .movzx },
2543 };2580 };
2544 const dst_alias = switch (tag) {2581 const dst_alias = switch (tag[1]) {
2545 .movsx, .movsxd => dst_reg.to64(),2582 .movsx => dst_reg.to64(),
2546 .mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),2583 .mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),
2547 else => unreachable,2584 else => unreachable,
2548 };2585 };
...@@ -2570,14 +2607,24 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -2570,14 +2607,24 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2570 try self.genCopy(min_ty, dst_mcv, src_mcv);2607 try self.genCopy(min_ty, dst_mcv, src_mcv);
2571 const extra = dst_abi_size * 8 - dst_int_info.bits;2608 const extra = dst_abi_size * 8 - dst_int_info.bits;
2572 if (extra > 0) {2609 if (extra > 0) {
2573 try self.genShiftBinOpMir(switch (signedness) {2610 try self.genShiftBinOpMir(
2574 .signed => .sal,2611 switch (signedness) {
2575 .unsigned => .shl,2612 .signed => .{ ._l, .sa },
2576 }, dst_ty, dst_mcv, .{ .immediate = extra });2613 .unsigned => .{ ._l, .sh },
2577 try self.genShiftBinOpMir(switch (signedness) {2614 },
2578 .signed => .sar,2615 dst_ty,
2579 .unsigned => .shr,2616 dst_mcv,
2580 }, dst_ty, dst_mcv, .{ .immediate = extra });2617 .{ .immediate = extra },
2618 );
2619 try self.genShiftBinOpMir(
2620 switch (signedness) {
2621 .signed => .{ ._r, .sa },
2622 .unsigned => .{ ._r, .sh },
2623 },
2624 dst_ty,
2625 dst_mcv,
2626 .{ .immediate = extra },
2627 );
2581 }2628 }
2582 },2629 },
2583 }2630 }
...@@ -2762,8 +2809,8 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -2762,8 +2809,8 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2762 const reg_bits = self.regBitSize(ty);2809 const reg_bits = self.regBitSize(ty);
2763 const cc: Condition = if (ty.isSignedInt()) cc: {2810 const cc: Condition = if (ty.isSignedInt()) cc: {
2764 try self.genSetReg(limit_reg, ty, dst_mcv);2811 try self.genSetReg(limit_reg, ty, dst_mcv);
2765 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });2812 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2766 try self.genBinOpMir(.xor, ty, limit_mcv, .{2813 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2767 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,2814 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2768 });2815 });
2769 break :cc .o;2816 break :cc .o;
...@@ -2773,7 +2820,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -2773,7 +2820,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2773 });2820 });
2774 break :cc .c;2821 break :cc .c;
2775 };2822 };
2776 try self.genBinOpMir(.add, ty, dst_mcv, rhs_mcv);2823 try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
27772824
2778 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);2825 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
2779 try self.asmCmovccRegisterRegister(2826 try self.asmCmovccRegisterRegister(
...@@ -2813,8 +2860,8 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -2813,8 +2860,8 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
2813 const reg_bits = self.regBitSize(ty);2860 const reg_bits = self.regBitSize(ty);
2814 const cc: Condition = if (ty.isSignedInt()) cc: {2861 const cc: Condition = if (ty.isSignedInt()) cc: {
2815 try self.genSetReg(limit_reg, ty, dst_mcv);2862 try self.genSetReg(limit_reg, ty, dst_mcv);
2816 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });2863 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2817 try self.genBinOpMir(.xor, ty, limit_mcv, .{2864 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2818 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,2865 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2819 });2866 });
2820 break :cc .o;2867 break :cc .o;
...@@ -2822,7 +2869,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -2822,7 +2869,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
2822 try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });2869 try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });
2823 break :cc .c;2870 break :cc .c;
2824 };2871 };
2825 try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);2872 try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
28262873
2827 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);2874 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
2828 try self.asmCmovccRegisterRegister(2875 try self.asmCmovccRegisterRegister(
...@@ -2864,9 +2911,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -2864,9 +2911,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
2864 const reg_bits = self.regBitSize(ty);2911 const reg_bits = self.regBitSize(ty);
2865 const cc: Condition = if (ty.isSignedInt()) cc: {2912 const cc: Condition = if (ty.isSignedInt()) cc: {
2866 try self.genSetReg(limit_reg, ty, lhs_mcv);2913 try self.genSetReg(limit_reg, ty, lhs_mcv);
2867 try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);2914 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
2868 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });2915 try self.genShiftBinOpMir(.{ ._, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2869 try self.genBinOpMir(.xor, ty, limit_mcv, .{2916 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2870 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,2917 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2871 });2918 });
2872 break :cc .o;2919 break :cc .o;
...@@ -2979,7 +3026,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2979,7 +3026,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2979 };3026 };
2980 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);3027 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
29813028
2982 try self.genBinOpMir(.cmp, lhs_ty, tmp_mcv, lhs);3029 try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, tmp_mcv, lhs);
2983 const cc = Condition.ne;3030 const cc = Condition.ne;
29843031
2985 const tuple_ty = self.air.typeOfIndex(inst);3032 const tuple_ty = self.air.typeOfIndex(inst);
...@@ -3066,12 +3113,17 @@ fn genSetFrameTruncatedOverflowCompare(...@@ -3066,12 +3113,17 @@ fn genSetFrameTruncatedOverflowCompare(
3066 src_mcv;3113 src_mcv;
3067 try self.genSetReg(scratch_reg, hi_limb_ty, hi_limb_mcv);3114 try self.genSetReg(scratch_reg, hi_limb_ty, hi_limb_mcv);
3068 try self.truncateRegister(hi_limb_ty, scratch_reg);3115 try self.truncateRegister(hi_limb_ty, scratch_reg);
3069 try self.genBinOpMir(.cmp, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);3116 try self.genBinOpMir(.{ ._, .cmp }, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
30703117
3071 const eq_reg = temp_regs[2];3118 const eq_reg = temp_regs[2];
3072 if (overflow_cc) |_| {3119 if (overflow_cc) |_| {
3073 try self.asmSetccRegister(eq_reg.to8(), .ne);3120 try self.asmSetccRegister(eq_reg.to8(), .ne);
3074 try self.genBinOpMir(.@"or", Type.u8, .{ .register = overflow_reg }, .{ .register = eq_reg });3121 try self.genBinOpMir(
3122 .{ ._, .@"or" },
3123 Type.u8,
3124 .{ .register = overflow_reg },
3125 .{ .register = eq_reg },
3126 );
3075 }3127 }
30763128
3077 const payload_off = @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*));3129 const payload_off = @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*));
...@@ -3200,28 +3252,25 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3200,28 +3252,25 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3200/// Generates signed or unsigned integer multiplication/division.3252/// Generates signed or unsigned integer multiplication/division.
3201/// Clobbers .rax and .rdx registers.3253/// Clobbers .rax and .rdx registers.
3202/// Quotient is saved in .rax and remainder in .rdx.3254/// Quotient is saved in .rax and remainder in .rdx.
3203fn genIntMulDivOpMir(3255fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
3204 self: *Self,
3205 tag: Mir.Inst.Tag,
3206 ty: Type,
3207 lhs: MCValue,
3208 rhs: MCValue,
3209) !void {
3210 const abi_size = @intCast(u32, ty.abiSize(self.target.*));3256 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
3211 if (abi_size > 8) {3257 if (abi_size > 8) {
3212 return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});3258 return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
3213 }3259 }
32143260
3215 try self.genSetReg(.rax, ty, lhs);3261 try self.genSetReg(.rax, ty, lhs);
3216 switch (tag) {3262 switch (tag[1]) {
3217 else => unreachable,3263 else => unreachable,
3218 .mul, .imul => {},3264 .mul => {},
3219 .div => try self.asmRegisterRegister(.xor, .edx, .edx),3265 .div => switch (tag[0]) {
3220 .idiv => switch (self.regBitSize(ty)) {3266 ._ => try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx),
3221 8 => try self.asmOpOnly(.cbw),3267 .i_ => switch (self.regBitSize(ty)) {
3222 16 => try self.asmOpOnly(.cwd),3268 8 => try self.asmOpOnly(.{ ._, .cbw }),
3223 32 => try self.asmOpOnly(.cdq),3269 16 => try self.asmOpOnly(.{ ._, .cwd }),
3224 64 => try self.asmOpOnly(.cqo),3270 32 => try self.asmOpOnly(.{ ._, .cdq }),
3271 64 => try self.asmOpOnly(.{ ._, .cqo }),
3272 else => unreachable,
3273 },
3225 else => unreachable,3274 else => unreachable,
3226 },3275 },
3227 }3276 }
...@@ -3259,23 +3308,28 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa...@@ -3259,23 +3308,28 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
3259 const divisor_lock = self.register_manager.lockReg(divisor);3308 const divisor_lock = self.register_manager.lockReg(divisor);
3260 defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock);3309 defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock);
32613310
3262 try self.genIntMulDivOpMir(switch (int_info.signedness) {3311 try self.genIntMulDivOpMir(
3263 .signed => .idiv,3312 switch (int_info.signedness) {
3264 .unsigned => .div,3313 .signed => .{ .i_, .div },
3265 }, ty, .{ .register = dividend }, .{ .register = divisor });3314 .unsigned => .{ ._, .div },
3315 },
3316 ty,
3317 .{ .register = dividend },
3318 .{ .register = divisor },
3319 );
32663320
3267 try self.asmRegisterRegister(3321 try self.asmRegisterRegister(
3268 .xor,3322 .{ ._, .xor },
3269 registerAlias(divisor, abi_size),3323 registerAlias(divisor, abi_size),
3270 registerAlias(dividend, abi_size),3324 registerAlias(dividend, abi_size),
3271 );3325 );
3272 try self.asmRegisterImmediate(3326 try self.asmRegisterImmediate(
3273 .sar,3327 .{ ._r, .sa },
3274 registerAlias(divisor, abi_size),3328 registerAlias(divisor, abi_size),
3275 Immediate.u(int_info.bits - 1),3329 Immediate.u(int_info.bits - 1),
3276 );3330 );
3277 try self.asmRegisterRegister(3331 try self.asmRegisterRegister(
3278 .@"test",3332 .{ ._, .@"test" },
3279 registerAlias(.rdx, abi_size),3333 registerAlias(.rdx, abi_size),
3280 registerAlias(.rdx, abi_size),3334 registerAlias(.rdx, abi_size),
3281 );3335 );
...@@ -3284,7 +3338,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa...@@ -3284,7 +3338,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
3284 registerAlias(.rdx, abi_size),3338 registerAlias(.rdx, abi_size),
3285 .z,3339 .z,
3286 );3340 );
3287 try self.genBinOpMir(.add, ty, .{ .register = divisor }, .{ .register = .rax });3341 try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });
3288 return MCValue{ .register = divisor };3342 return MCValue{ .register = divisor };
3289}3343}
32903344
...@@ -3406,7 +3460,12 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -3406,7 +3460,12 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
3406 const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);3460 const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
3407 if (err_off > 0) {3461 if (err_off > 0) {
3408 const shift = @intCast(u6, err_off * 8);3462 const shift = @intCast(u6, err_off * 8);
3409 try self.genShiftBinOpMir(.shr, err_union_ty, result, .{ .immediate = shift });3463 try self.genShiftBinOpMir(
3464 .{ ._r, .sh },
3465 err_union_ty,
3466 result,
3467 .{ .immediate = shift },
3468 );
3410 } else {3469 } else {
3411 try self.truncateRegister(Type.anyerror, result.register);3470 try self.truncateRegister(Type.anyerror, result.register);
3412 }3471 }
...@@ -3458,7 +3517,12 @@ fn genUnwrapErrorUnionPayloadMir(...@@ -3458,7 +3517,12 @@ fn genUnwrapErrorUnionPayloadMir(
3458 .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };3517 .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
3459 if (payload_off > 0) {3518 if (payload_off > 0) {
3460 const shift = @intCast(u6, payload_off * 8);3519 const shift = @intCast(u6, payload_off * 8);
3461 try self.genShiftBinOpMir(.shr, err_union_ty, result_mcv, .{ .immediate = shift });3520 try self.genShiftBinOpMir(
3521 .{ ._r, .sh },
3522 err_union_ty,
3523 result_mcv,
3524 .{ .immediate = shift },
3525 );
3462 } else {3526 } else {
3463 try self.truncateRegister(payload_ty, result_mcv.register);3527 try self.truncateRegister(payload_ty, result_mcv.register);
3464 }3528 }
...@@ -3495,7 +3559,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -3495,7 +3559,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
3495 const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));3559 const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3496 const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));3560 const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
3497 try self.asmRegisterMemory(3561 try self.asmRegisterMemory(
3498 .mov,3562 .{ ._, .mov },
3499 registerAlias(dst_reg, err_abi_size),3563 registerAlias(dst_reg, err_abi_size),
3500 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{3564 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
3501 .base = .{ .reg = src_reg },3565 .base = .{ .reg = src_reg },
...@@ -3533,7 +3597,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -3533,7 +3597,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
3533 const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));3597 const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3534 const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));3598 const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3535 try self.asmRegisterMemory(3599 try self.asmRegisterMemory(
3536 .lea,3600 .{ ._, .lea },
3537 registerAlias(dst_reg, dst_abi_size),3601 registerAlias(dst_reg, dst_abi_size),
3538 Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),3602 Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
3539 );3603 );
...@@ -3559,7 +3623,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {...@@ -3559,7 +3623,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3559 const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));3623 const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3560 const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));3624 const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
3561 try self.asmMemoryImmediate(3625 try self.asmMemoryImmediate(
3562 .mov,3626 .{ ._, .mov },
3563 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{3627 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
3564 .base = .{ .reg = src_reg },3628 .base = .{ .reg = src_reg },
3565 .disp = err_off,3629 .disp = err_off,
...@@ -3580,7 +3644,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {...@@ -3580,7 +3644,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3580 const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));3644 const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3581 const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));3645 const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3582 try self.asmRegisterMemory(3646 try self.asmRegisterMemory(
3583 .lea,3647 .{ ._, .lea },
3584 registerAlias(dst_reg, dst_abi_size),3648 registerAlias(dst_reg, dst_abi_size),
3585 Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),3649 Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
3586 );3650 );
...@@ -3631,13 +3695,13 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {...@@ -3631,13 +3695,13 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
3631 else => unreachable,3695 else => unreachable,
36323696
3633 .register => |opt_reg| try self.asmRegisterImmediate(3697 .register => |opt_reg| try self.asmRegisterImmediate(
3634 .bts,3698 .{ ._s, .bt },
3635 opt_reg,3699 opt_reg,
3636 Immediate.u(@intCast(u6, pl_abi_size * 8)),3700 Immediate.u(@intCast(u6, pl_abi_size * 8)),
3637 ),3701 ),
36383702
3639 .load_frame => |frame_addr| try self.asmMemoryImmediate(3703 .load_frame => |frame_addr| try self.asmMemoryImmediate(
3640 .mov,3704 .{ ._, .mov },
3641 Memory.sib(.byte, .{3705 Memory.sib(.byte, .{
3642 .base = .{ .frame = frame_addr.index },3706 .base = .{ .frame = frame_addr.index },
3643 .disp = frame_addr.off + pl_abi_size,3707 .disp = frame_addr.off + pl_abi_size,
...@@ -3749,7 +3813,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -3749,7 +3813,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
37493813
3750 const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));3814 const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3751 try self.asmRegisterMemory(3815 try self.asmRegisterMemory(
3752 .lea,3816 .{ ._, .lea },
3753 registerAlias(dst_reg, dst_abi_size),3817 registerAlias(dst_reg, dst_abi_size),
3754 Memory.sib(.qword, .{3818 Memory.sib(.qword, .{
3755 .base = .{ .reg = src_reg },3819 .base = .{ .reg = src_reg },
...@@ -3823,7 +3887,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {...@@ -3823,7 +3887,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
3823 try self.genSetReg(addr_reg, Type.usize, slice_mcv);3887 try self.genSetReg(addr_reg, Type.usize, slice_mcv);
3824 // TODO we could allocate register here, but need to expect addr register and potentially3888 // TODO we could allocate register here, but need to expect addr register and potentially
3825 // offset register.3889 // offset register.
3826 try self.genBinOpMir(.add, slice_ptr_field_type, .{ .register = addr_reg }, .{3890 try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{
3827 .register = offset_reg,3891 .register = offset_reg,
3828 });3892 });
3829 return MCValue{ .register = addr_reg.to64() };3893 return MCValue{ .register = addr_reg.to64() };
...@@ -3881,13 +3945,13 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -3881,13 +3945,13 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
3881 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, self.target.*));3945 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, self.target.*));
3882 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array);3946 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array);
3883 try self.asmRegisterMemory(3947 try self.asmRegisterMemory(
3884 .lea,3948 .{ ._, .lea },
3885 addr_reg,3949 addr_reg,
3886 Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),3950 Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
3887 );3951 );
3888 },3952 },
3889 .load_frame => |frame_addr| try self.asmRegisterMemory(3953 .load_frame => |frame_addr| try self.asmRegisterMemory(
3890 .lea,3954 .{ ._, .lea },
3891 addr_reg,3955 addr_reg,
3892 Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),3956 Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),
3893 ),3957 ),
...@@ -3903,7 +3967,12 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -3903,7 +3967,12 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
3903 // TODO we could allocate register here, but need to expect addr register and potentially3967 // TODO we could allocate register here, but need to expect addr register and potentially
3904 // offset register.3968 // offset register.
3905 const dst_mcv = try self.allocRegOrMem(inst, false);3969 const dst_mcv = try self.allocRegOrMem(inst, false);
3906 try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });3970 try self.genBinOpMir(
3971 .{ ._, .add },
3972 Type.usize,
3973 .{ .register = addr_reg },
3974 .{ .register = offset_reg },
3975 );
3907 try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });3976 try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
39083977
3909 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });3978 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
...@@ -3937,7 +4006,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -3937,7 +4006,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
3937 try self.copyToTmpRegister(ptr_ty, ptr_mcv);4006 try self.copyToTmpRegister(ptr_ty, ptr_mcv);
3938 const elem_ptr_lock = self.register_manager.lockRegAssumeUnused(elem_ptr_reg);4007 const elem_ptr_lock = self.register_manager.lockRegAssumeUnused(elem_ptr_reg);
3939 defer self.register_manager.unlockReg(elem_ptr_lock);4008 defer self.register_manager.unlockReg(elem_ptr_lock);
3940 try self.asmRegisterRegister(.add, elem_ptr_reg, offset_reg);4009 try self.asmRegisterRegister(
4010 .{ ._, .add },
4011 elem_ptr_reg,
4012 offset_reg,
4013 );
39414014
3942 const dst_mcv = try self.allocRegOrMem(inst, true);4015 const dst_mcv = try self.allocRegOrMem(inst, true);
3943 const dst_lock = switch (dst_mcv) {4016 const dst_lock = switch (dst_mcv) {
...@@ -3977,7 +4050,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -3977,7 +4050,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
3977 defer self.register_manager.unlockReg(offset_reg_lock);4050 defer self.register_manager.unlockReg(offset_reg_lock);
39784051
3979 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);4052 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
3980 try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });4053 try self.genBinOpMir(.{ ._, .add }, ptr_ty, dst_mcv, .{ .register = offset_reg });
39814054
3982 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });4055 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
3983}4056}
...@@ -4010,7 +4083,12 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {...@@ -4010,7 +4083,12 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
4010 const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {4083 const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
4011 // TODO reusing the operand4084 // TODO reusing the operand
4012 const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);4085 const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);
4013 try self.genBinOpMir(.add, ptr_union_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });4086 try self.genBinOpMir(
4087 .{ ._, .add },
4088 ptr_union_ty,
4089 .{ .register = reg },
4090 .{ .immediate = layout.payload_size },
4091 );
4014 break :blk MCValue{ .register = reg };4092 break :blk MCValue{ .register = reg };
4015 } else ptr;4093 } else ptr;
40164094
...@@ -4063,7 +4141,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {...@@ -4063,7 +4141,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
4063 else4141 else
4064 0;4142 0;
4065 const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);4143 const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
4066 try self.genShiftBinOpMir(.shr, Type.usize, result, .{ .immediate = shift });4144 try self.genShiftBinOpMir(.{ ._r, .sh }, Type.usize, result, .{ .immediate = shift });
4067 break :blk MCValue{4145 break :blk MCValue{
4068 .register = registerAlias(result.register, @intCast(u32, layout.tag_size)),4146 .register = registerAlias(result.register, @intCast(u32, layout.tag_size)),
4069 };4147 };
...@@ -4100,11 +4178,11 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -4100,11 +4178,11 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4100 const src_bits = src_ty.bitSize(self.target.*);4178 const src_bits = src_ty.bitSize(self.target.*);
4101 if (self.hasFeature(.lzcnt)) {4179 if (self.hasFeature(.lzcnt)) {
4102 if (src_bits <= 64) {4180 if (src_bits <= 64) {
4103 try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);4181 try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);
41044182
4105 const extra_bits = self.regExtraBits(src_ty);4183 const extra_bits = self.regExtraBits(src_ty);
4106 if (extra_bits > 0) {4184 if (extra_bits > 0) {
4107 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });4185 try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits });
4108 }4186 }
4109 } else if (src_bits <= 128) {4187 } else if (src_bits <= 128) {
4110 const tmp_reg = try self.register_manager.allocReg(null, gp);4188 const tmp_reg = try self.register_manager.allocReg(null, gp);
...@@ -4112,13 +4190,23 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -4112,13 +4190,23 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4112 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);4190 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
4113 defer self.register_manager.unlockReg(tmp_lock);4191 defer self.register_manager.unlockReg(tmp_lock);
41144192
4115 try self.genBinOpMir(.lzcnt, Type.u64, dst_mcv, mat_src_mcv);4193 try self.genBinOpMir(.{ ._, .lzcnt }, Type.u64, dst_mcv, mat_src_mcv);
4116 try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });4194 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
4117 try self.genBinOpMir(.lzcnt, Type.u64, tmp_mcv, mat_src_mcv.address().offset(8).deref());4195 try self.genBinOpMir(
4196 .{ ._, .lzcnt },
4197 Type.u64,
4198 tmp_mcv,
4199 mat_src_mcv.address().offset(8).deref(),
4200 );
4118 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);4201 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
41194202
4120 if (src_bits < 128) {4203 if (src_bits < 128) {
4121 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = 128 - src_bits });4204 try self.genBinOpMir(
4205 .{ ._, .sub },
4206 dst_ty,
4207 dst_mcv,
4208 .{ .immediate = 128 - src_bits },
4209 );
4122 }4210 }
4123 } else return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});4211 } else return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
4124 break :result dst_mcv;4212 break :result dst_mcv;
...@@ -4130,7 +4218,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -4130,7 +4218,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4130 const imm_reg = try self.copyToTmpRegister(dst_ty, .{4218 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
4131 .immediate = src_bits ^ (src_bits - 1),4219 .immediate = src_bits ^ (src_bits - 1),
4132 });4220 });
4133 try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);4221 try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
41344222
4135 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);4223 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
4136 try self.asmCmovccRegisterRegister(4224 try self.asmCmovccRegisterRegister(
...@@ -4139,12 +4227,12 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -4139,12 +4227,12 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4139 .z,4227 .z,
4140 );4228 );
41414229
4142 try self.genBinOpMir(.xor, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });4230 try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
4143 } else {4231 } else {
4144 const imm_reg = try self.copyToTmpRegister(dst_ty, .{4232 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
4145 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - self.regBitSize(dst_ty)),4233 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - self.regBitSize(dst_ty)),
4146 });4234 });
4147 try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);4235 try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
41484236
4149 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);4237 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
4150 try self.asmCmovccRegisterRegister(4238 try self.asmCmovccRegisterRegister(
...@@ -4154,7 +4242,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -4154,7 +4242,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4154 );4242 );
41554243
4156 try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });4244 try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });
4157 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });4245 try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .register = imm_reg });
4158 }4246 }
4159 break :result dst_mcv;4247 break :result dst_mcv;
4160 };4248 };
...@@ -4195,7 +4283,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -4195,7 +4283,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4195 break :tmp dst_mcv;4283 break :tmp dst_mcv;
4196 };4284 };
4197 try self.genBinOpMir(4285 try self.genBinOpMir(
4198 .@"or",4286 .{ ._, .@"or" },
4199 src_ty,4287 src_ty,
4200 tmp_mcv,4288 tmp_mcv,
4201 .{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - extra_bits)) <<4289 .{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - extra_bits)) <<
...@@ -4203,7 +4291,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -4203,7 +4291,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4203 );4291 );
4204 break :masked tmp_mcv;4292 break :masked tmp_mcv;
4205 } else mat_src_mcv;4293 } else mat_src_mcv;
4206 try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);4294 try self.genBinOpMir(.{ ._, .tzcnt }, src_ty, dst_mcv, masked_mcv);
4207 } else if (src_bits <= 128) {4295 } else if (src_bits <= 128) {
4208 const tmp_reg = try self.register_manager.allocReg(null, gp);4296 const tmp_reg = try self.register_manager.allocReg(null, gp);
4209 const tmp_mcv = MCValue{ .register = tmp_reg };4297 const tmp_mcv = MCValue{ .register = tmp_reg };
...@@ -4213,16 +4301,16 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -4213,16 +4301,16 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4213 const masked_mcv = if (src_bits < 128) masked: {4301 const masked_mcv = if (src_bits < 128) masked: {
4214 try self.genCopy(Type.u64, dst_mcv, mat_src_mcv.address().offset(8).deref());4302 try self.genCopy(Type.u64, dst_mcv, mat_src_mcv.address().offset(8).deref());
4215 try self.genBinOpMir(4303 try self.genBinOpMir(
4216 .@"or",4304 .{ ._, .@"or" },
4217 Type.u64,4305 Type.u64,
4218 dst_mcv,4306 dst_mcv,
4219 .{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(u6, src_bits - 64) },4307 .{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(u6, src_bits - 64) },
4220 );4308 );
4221 break :masked dst_mcv;4309 break :masked dst_mcv;
4222 } else mat_src_mcv.address().offset(8).deref();4310 } else mat_src_mcv.address().offset(8).deref();
4223 try self.genBinOpMir(.tzcnt, Type.u64, dst_mcv, masked_mcv);4311 try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, dst_mcv, masked_mcv);
4224 try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });4312 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
4225 try self.genBinOpMir(.tzcnt, Type.u64, tmp_mcv, mat_src_mcv);4313 try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, mat_src_mcv);
4226 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);4314 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
4227 } else return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});4315 } else return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
4228 break :result dst_mcv;4316 break :result dst_mcv;
...@@ -4232,7 +4320,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -4232,7 +4320,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4232 return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});4320 return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
42334321
4234 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });4322 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
4235 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);4323 try self.genBinOpMir(.{ ._, .bsf }, src_ty, dst_mcv, mat_src_mcv);
42364324
4237 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);4325 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
4238 try self.asmCmovccRegisterRegister(4326 try self.asmCmovccRegisterRegister(
...@@ -4270,7 +4358,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {...@@ -4270,7 +4358,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
4270 .{ .register = try self.register_manager.allocReg(inst, gp) };4358 .{ .register = try self.register_manager.allocReg(inst, gp) };
42714359
4272 const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16;4360 const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16;
4273 try self.genBinOpMir(.popcnt, popcnt_ty, dst_mcv, mat_src_mcv);4361 try self.genBinOpMir(.{ ._, .popcnt }, popcnt_ty, dst_mcv, mat_src_mcv);
4274 break :result dst_mcv;4362 break :result dst_mcv;
4275 }4363 }
42764364
...@@ -4301,54 +4389,54 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {...@@ -4301,54 +4389,54 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
4301 undefined;4389 undefined;
43024390
4303 // dst = operand4391 // dst = operand
4304 try self.asmRegisterRegister(.mov, tmp, dst);4392 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4305 // tmp = operand4393 // tmp = operand
4306 try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));4394 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
4307 // tmp = operand >> 14395 // tmp = operand >> 1
4308 if (src_abi_size > 4) {4396 if (src_abi_size > 4) {
4309 try self.asmRegisterImmediate(.mov, imm, imm_0_1);4397 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
4310 try self.asmRegisterRegister(.@"and", tmp, imm);4398 try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
4311 } else try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);4399 } else try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
4312 // tmp = (operand >> 1) & 0x55...554400 // tmp = (operand >> 1) & 0x55...55
4313 try self.asmRegisterRegister(.sub, dst, tmp);4401 try self.asmRegisterRegister(.{ ._, .sub }, dst, tmp);
4314 // dst = temp1 = operand - ((operand >> 1) & 0x55...55)4402 // dst = temp1 = operand - ((operand >> 1) & 0x55...55)
4315 try self.asmRegisterRegister(.mov, tmp, dst);4403 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4316 // tmp = temp14404 // tmp = temp1
4317 try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));4405 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
4318 // dst = temp1 >> 24406 // dst = temp1 >> 2
4319 if (src_abi_size > 4) {4407 if (src_abi_size > 4) {
4320 try self.asmRegisterImmediate(.mov, imm, imm_00_11);4408 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
4321 try self.asmRegisterRegister(.@"and", tmp, imm);4409 try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
4322 try self.asmRegisterRegister(.@"and", dst, imm);4410 try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4323 } else {4411 } else {
4324 try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);4412 try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
4325 try self.asmRegisterImmediate(.@"and", dst, imm_00_11);4413 try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
4326 }4414 }
4327 // tmp = temp1 & 0x33...334415 // tmp = temp1 & 0x33...33
4328 // dst = (temp1 >> 2) & 0x33...334416 // dst = (temp1 >> 2) & 0x33...33
4329 try self.asmRegisterRegister(.add, tmp, dst);4417 try self.asmRegisterRegister(.{ ._, .add }, tmp, dst);
4330 // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)4418 // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
4331 try self.asmRegisterRegister(.mov, dst, tmp);4419 try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
4332 // dst = temp24420 // dst = temp2
4333 try self.asmRegisterImmediate(.shr, tmp, Immediate.u(4));4421 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(4));
4334 // tmp = temp2 >> 44422 // tmp = temp2 >> 4
4335 try self.asmRegisterRegister(.add, dst, tmp);4423 try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);
4336 // dst = temp2 + (temp2 >> 4)4424 // dst = temp2 + (temp2 >> 4)
4337 if (src_abi_size > 4) {4425 if (src_abi_size > 4) {
4338 try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);4426 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
4339 try self.asmRegisterImmediate(.mov, tmp, imm_0000_0001);4427 try self.asmRegisterImmediate(.{ ._, .mov }, tmp, imm_0000_0001);
4340 try self.asmRegisterRegister(.@"and", dst, imm);4428 try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4341 try self.asmRegisterRegister(.imul, dst, tmp);4429 try self.asmRegisterRegister(.{ .i_, .mul }, dst, tmp);
4342 } else {4430 } else {
4343 try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);4431 try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
4344 if (src_abi_size > 1) {4432 if (src_abi_size > 1) {
4345 try self.asmRegisterRegisterImmediate(.imul, dst, dst, imm_0000_0001);4433 try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst, dst, imm_0000_0001);
4346 }4434 }
4347 }4435 }
4348 // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f4436 // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
4349 // dst = temp3 * 0x01...014437 // dst = temp3 * 0x01...01
4350 if (src_abi_size > 1) {4438 if (src_abi_size > 1) {
4351 try self.asmRegisterImmediate(.shr, dst, Immediate.u((src_abi_size - 1) * 8));4439 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u((src_abi_size - 1) * 8));
4352 }4440 }
4353 // dst = (temp3 * 0x01...01) >> (bits - 8)4441 // dst = (temp3 * 0x01...01) >> (bits - 8)
4354 }4442 }
...@@ -4377,11 +4465,11 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m...@@ -4377,11 +4465,11 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4377 16 => if ((mem_ok or src_mcv.isRegister()) and4465 16 => if ((mem_ok or src_mcv.isRegister()) and
4378 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))4466 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
4379 {4467 {
4380 try self.genBinOpMir(.rol, src_ty, src_mcv, .{ .immediate = 8 });4468 try self.genBinOpMir(.{ ._l, .ro }, src_ty, src_mcv, .{ .immediate = 8 });
4381 return src_mcv;4469 return src_mcv;
4382 },4470 },
4383 32, 64 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {4471 32, 64 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
4384 try self.genUnOpMir(.bswap, src_ty, src_mcv);4472 try self.genUnOpMir(.{ ._, .bswap }, src_ty, src_mcv);
4385 return src_mcv;4473 return src_mcv;
4386 },4474 },
4387 }4475 }
...@@ -4398,10 +4486,10 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m...@@ -4398,10 +4486,10 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4398 try self.genSetReg(dst_mcv.register, src_ty, src_mcv);4486 try self.genSetReg(dst_mcv.register, src_ty, src_mcv);
4399 switch (src_bits) {4487 switch (src_bits) {
4400 else => unreachable,4488 else => unreachable,
4401 16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),4489 16 => try self.genBinOpMir(.{ ._l, .ro }, src_ty, dst_mcv, .{ .immediate = 8 }),
4402 32, 64 => try self.genUnOpMir(.bswap, src_ty, dst_mcv),4490 32, 64 => try self.genUnOpMir(.{ ._, .bswap }, src_ty, dst_mcv),
4403 }4491 }
4404 } else try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);4492 } else try self.genBinOpMir(.{ ._, .movbe }, src_ty, dst_mcv, src_mcv);
4405 return dst_mcv;4493 return dst_mcv;
4406 }4494 }
44074495
...@@ -4410,7 +4498,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m...@@ -4410,7 +4498,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4410 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);4498 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
4411 defer self.register_manager.unlockReg(dst_lock);4499 defer self.register_manager.unlockReg(dst_lock);
44124500
4413 try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);4501 try self.genBinOpMir(.{ ._, .movbe }, src_ty, dst_mcv, src_mcv);
4414 return dst_mcv;4502 return dst_mcv;
4415}4503}
44164504
...@@ -4424,7 +4512,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {...@@ -4424,7 +4512,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
4424 switch (self.regExtraBits(src_ty)) {4512 switch (self.regExtraBits(src_ty)) {
4425 0 => {},4513 0 => {},
4426 else => |extra| try self.genBinOpMir(4514 else => |extra| try self.genBinOpMir(
4427 if (src_ty.isSignedInt()) .sar else .shr,4515 if (src_ty.isSignedInt()) .{ ._r, .sa } else .{ ._r, .sh },
4428 src_ty,4516 src_ty,
4429 dst_mcv,4517 dst_mcv,
4430 .{ .immediate = extra },4518 .{ .immediate = extra },
...@@ -4464,40 +4552,40 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {...@@ -4464,40 +4552,40 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4464 const imm_0_1 = Immediate.u(mask / 0b1_1);4552 const imm_0_1 = Immediate.u(mask / 0b1_1);
44654553
4466 // dst = temp1 = bswap(operand)4554 // dst = temp1 = bswap(operand)
4467 try self.asmRegisterRegister(.mov, tmp, dst);4555 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4468 // tmp = temp14556 // tmp = temp1
4469 try self.asmRegisterImmediate(.shr, dst, Immediate.u(4));4557 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(4));
4470 // dst = temp1 >> 44558 // dst = temp1 >> 4
4471 if (src_abi_size > 4) {4559 if (src_abi_size > 4) {
4472 try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);4560 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
4473 try self.asmRegisterRegister(.@"and", tmp, imm);4561 try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
4474 try self.asmRegisterRegister(.@"and", dst, imm);4562 try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4475 } else {4563 } else {
4476 try self.asmRegisterImmediate(.@"and", tmp, imm_0000_1111);4564 try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0000_1111);
4477 try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);4565 try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
4478 }4566 }
4479 // tmp = temp1 & 0x0F...0F4567 // tmp = temp1 & 0x0F...0F
4480 // dst = (temp1 >> 4) & 0x0F...0F4568 // dst = (temp1 >> 4) & 0x0F...0F
4481 try self.asmRegisterImmediate(.shl, tmp, Immediate.u(4));4569 try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, Immediate.u(4));
4482 // tmp = (temp1 & 0x0F...0F) << 44570 // tmp = (temp1 & 0x0F...0F) << 4
4483 try self.asmRegisterRegister(.@"or", dst, tmp);4571 try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);
4484 // dst = temp2 = ((temp1 >> 4) & 0x0F...0F) | ((temp1 & 0x0F...0F) << 4)4572 // dst = temp2 = ((temp1 >> 4) & 0x0F...0F) | ((temp1 & 0x0F...0F) << 4)
4485 try self.asmRegisterRegister(.mov, tmp, dst);4573 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4486 // tmp = temp24574 // tmp = temp2
4487 try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));4575 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
4488 // dst = temp2 >> 24576 // dst = temp2 >> 2
4489 if (src_abi_size > 4) {4577 if (src_abi_size > 4) {
4490 try self.asmRegisterImmediate(.mov, imm, imm_00_11);4578 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
4491 try self.asmRegisterRegister(.@"and", tmp, imm);4579 try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
4492 try self.asmRegisterRegister(.@"and", dst, imm);4580 try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4493 } else {4581 } else {
4494 try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);4582 try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
4495 try self.asmRegisterImmediate(.@"and", dst, imm_00_11);4583 try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
4496 }4584 }
4497 // tmp = temp2 & 0x33...334585 // tmp = temp2 & 0x33...33
4498 // dst = (temp2 >> 2) & 0x33...334586 // dst = (temp2 >> 2) & 0x33...33
4499 try self.asmRegisterMemory(4587 try self.asmRegisterMemory(
4500 .lea,4588 .{ ._, .lea },
4501 if (src_abi_size > 4) tmp.to64() else tmp.to32(),4589 if (src_abi_size > 4) tmp.to64() else tmp.to32(),
4502 Memory.sib(.qword, .{4590 Memory.sib(.qword, .{
4503 .base = .{ .reg = dst.to64() },4591 .base = .{ .reg = dst.to64() },
...@@ -4505,22 +4593,22 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {...@@ -4505,22 +4593,22 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4505 }),4593 }),
4506 );4594 );
4507 // tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)4595 // tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
4508 try self.asmRegisterRegister(.mov, dst, tmp);4596 try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
4509 // dst = temp34597 // dst = temp3
4510 try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));4598 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
4511 // tmp = temp3 >> 14599 // tmp = temp3 >> 1
4512 if (src_abi_size > 4) {4600 if (src_abi_size > 4) {
4513 try self.asmRegisterImmediate(.mov, imm, imm_0_1);4601 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
4514 try self.asmRegisterRegister(.@"and", dst, imm);4602 try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4515 try self.asmRegisterRegister(.@"and", tmp, imm);4603 try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
4516 } else {4604 } else {
4517 try self.asmRegisterImmediate(.@"and", dst, imm_0_1);4605 try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0_1);
4518 try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);4606 try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
4519 }4607 }
4520 // dst = temp3 & 0x55...554608 // dst = temp3 & 0x55...55
4521 // tmp = (temp3 >> 1) & 0x55...554609 // tmp = (temp3 >> 1) & 0x55...55
4522 try self.asmRegisterMemory(4610 try self.asmRegisterMemory(
4523 .lea,4611 .{ ._, .lea },
4524 if (src_abi_size > 4) dst.to64() else dst.to32(),4612 if (src_abi_size > 4) dst.to64() else dst.to32(),
4525 Memory.sib(.qword, .{4613 Memory.sib(.qword, .{
4526 .base = .{ .reg = tmp.to64() },4614 .base = .{ .reg = tmp.to64() },
...@@ -4533,7 +4621,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {...@@ -4533,7 +4621,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4533 switch (self.regExtraBits(src_ty)) {4621 switch (self.regExtraBits(src_ty)) {
4534 0 => {},4622 0 => {},
4535 else => |extra| try self.genBinOpMir(4623 else => |extra| try self.genBinOpMir(
4536 if (src_ty.isSignedInt()) .sar else .shr,4624 if (src_ty.isSignedInt()) .{ ._r, .sa } else .{ ._r, .sh },
4537 src_ty,4625 src_ty,
4538 dst_mcv,4626 dst_mcv,
4539 .{ .immediate = extra },4627 .{ .immediate = extra },
...@@ -4590,8 +4678,8 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {...@@ -4590,8 +4678,8 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
4590 try self.genBinOpMir(switch (ty_bits) {4678 try self.genBinOpMir(switch (ty_bits) {
4591 // No point using an extra prefix byte for *pd which performs the same operation.4679 // No point using an extra prefix byte for *pd which performs the same operation.
4592 16, 32, 64, 128 => switch (tag) {4680 16, 32, 64, 128 => switch (tag) {
4593 .neg => .xorps,4681 .neg => .{ ._, .xorps },
4594 .fabs => .andnps,4682 .fabs => .{ ._, .andnps },
4595 else => unreachable,4683 else => unreachable,
4596 },4684 },
4597 80 => return self.fail("TODO implement airFloatSign for {}", .{4685 80 => return self.fail("TODO implement airFloatSign for {}", .{
...@@ -4622,25 +4710,25 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4...@@ -4622,25 +4710,25 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
4622 if (!self.hasFeature(.sse4_1))4710 if (!self.hasFeature(.sse4_1))
4623 return self.fail("TODO implement genRound without sse4_1 feature", .{});4711 return self.fail("TODO implement genRound without sse4_1 feature", .{});
46244712
4625 const mir_tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {4713 const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
4626 .Float => switch (ty.floatBits(self.target.*)) {4714 .Float => switch (ty.floatBits(self.target.*)) {
4627 32 => if (self.hasFeature(.avx)) .vroundss else .roundss,4715 32 => if (self.hasFeature(.avx)) .{ ._, .vroundss } else .{ ._, .roundss },
4628 64 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,4716 64 => if (self.hasFeature(.avx)) .{ ._, .vroundsd } else .{ ._, .roundsd },
4629 16, 80, 128 => null,4717 16, 80, 128 => null,
4630 else => unreachable,4718 else => unreachable,
4631 },4719 },
4632 .Vector => switch (ty.childType().zigTypeTag()) {4720 .Vector => switch (ty.childType().zigTypeTag()) {
4633 .Float => switch (ty.childType().floatBits(self.target.*)) {4721 .Float => switch (ty.childType().floatBits(self.target.*)) {
4634 32 => switch (ty.vectorLen()) {4722 32 => switch (ty.vectorLen()) {
4635 1 => if (self.hasFeature(.avx)) .vroundss else .roundss,4723 1 => if (self.hasFeature(.avx)) .{ ._, .vroundss } else .{ ._, .roundss },
4636 2...4 => if (self.hasFeature(.avx)) .vroundps else .roundps,4724 2...4 => if (self.hasFeature(.avx)) .{ ._, .vroundps } else .{ ._, .roundps },
4637 5...8 => if (self.hasFeature(.avx)) .vroundps else null,4725 5...8 => if (self.hasFeature(.avx)) .{ ._, .vroundps } else null,
4638 else => null,4726 else => null,
4639 },4727 },
4640 64 => switch (ty.vectorLen()) {4728 64 => switch (ty.vectorLen()) {
4641 1 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,4729 1 => if (self.hasFeature(.avx)) .{ ._, .vroundsd } else .{ ._, .roundsd },
4642 2 => if (self.hasFeature(.avx)) .vroundpd else .roundpd,4730 2 => if (self.hasFeature(.avx)) .{ ._, .vroundpd } else .{ ._, .roundpd },
4643 3...4 => if (self.hasFeature(.avx)) .vroundpd else null,4731 3...4 => if (self.hasFeature(.avx)) .{ ._, .vroundpd } else null,
4644 else => null,4732 else => null,
4645 },4733 },
4646 16, 80, 128 => null,4734 16, 80, 128 => null,
...@@ -4655,7 +4743,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4...@@ -4655,7 +4743,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
46554743
4656 const abi_size = @intCast(u32, ty.abiSize(self.target.*));4744 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4657 const dst_alias = registerAlias(dst_reg, abi_size);4745 const dst_alias = registerAlias(dst_reg, abi_size);
4658 switch (mir_tag) {4746 switch (mir_tag[1]) {
4659 .vroundss, .vroundsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(4747 .vroundss, .vroundsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
4660 mir_tag,4748 mir_tag,
4661 dst_alias,4749 dst_alias,
...@@ -4704,25 +4792,25 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -4704,25 +4792,25 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4704 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);4792 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
47054793
4706 const result: MCValue = result: {4794 const result: MCValue = result: {
4707 const mir_tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {4795 const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
4708 .Float => switch (ty.floatBits(self.target.*)) {4796 .Float => switch (ty.floatBits(self.target.*)) {
4709 16 => if (self.hasFeature(.f16c)) {4797 16 => if (self.hasFeature(.f16c)) {
4710 const mat_src_reg = if (src_mcv.isRegister())4798 const mat_src_reg = if (src_mcv.isRegister())
4711 src_mcv.getReg().?4799 src_mcv.getReg().?
4712 else4800 else
4713 try self.copyToTmpRegister(ty, src_mcv);4801 try self.copyToTmpRegister(ty, src_mcv);
4714 try self.asmRegisterRegister(.vcvtph2ps, dst_reg, mat_src_reg.to128());4802 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, mat_src_reg.to128());
4715 try self.asmRegisterRegisterRegister(.vsqrtss, dst_reg, dst_reg, dst_reg);4803 try self.asmRegisterRegisterRegister(.{ ._, .vsqrtss }, dst_reg, dst_reg, dst_reg);
4716 try self.asmRegisterRegisterImmediate(4804 try self.asmRegisterRegisterImmediate(
4717 .vcvtps2ph,4805 .{ ._, .vcvtps2ph },
4718 dst_reg,4806 dst_reg,
4719 dst_reg,4807 dst_reg,
4720 Immediate.u(0b1_00),4808 Immediate.u(0b1_00),
4721 );4809 );
4722 break :result dst_mcv;4810 break :result dst_mcv;
4723 } else null,4811 } else null,
4724 32 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,4812 32 => if (self.hasFeature(.avx)) .{ ._, .vsqrtss } else .{ ._, .sqrtss },
4725 64 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,4813 64 => if (self.hasFeature(.avx)) .{ ._, .vsqrtsd } else .{ ._, .sqrtsd },
4726 80, 128 => null,4814 80, 128 => null,
4727 else => unreachable,4815 else => unreachable,
4728 },4816 },
...@@ -4731,16 +4819,21 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -4731,16 +4819,21 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4731 16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen()) {4819 16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen()) {
4732 1 => {4820 1 => {
4733 try self.asmRegisterRegister(4821 try self.asmRegisterRegister(
4734 .vcvtph2ps,4822 .{ ._, .vcvtph2ps },
4735 dst_reg,4823 dst_reg,
4736 (if (src_mcv.isRegister())4824 (if (src_mcv.isRegister())
4737 src_mcv.getReg().?4825 src_mcv.getReg().?
4738 else4826 else
4739 try self.copyToTmpRegister(ty, src_mcv)).to128(),4827 try self.copyToTmpRegister(ty, src_mcv)).to128(),
4740 );4828 );
4741 try self.asmRegisterRegisterRegister(.vsqrtss, dst_reg, dst_reg, dst_reg);4829 try self.asmRegisterRegisterRegister(
4830 .{ ._, .vsqrtss },
4831 dst_reg,
4832 dst_reg,
4833 dst_reg,
4834 );
4742 try self.asmRegisterRegisterImmediate(4835 try self.asmRegisterRegisterImmediate(
4743 .vcvtps2ph,4836 .{ ._, .vcvtps2ph },
4744 dst_reg,4837 dst_reg,
4745 dst_reg,4838 dst_reg,
4746 Immediate.u(0b1_00),4839 Immediate.u(0b1_00),
...@@ -4750,22 +4843,22 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -4750,22 +4843,22 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4750 2...8 => {4843 2...8 => {
4751 const wide_reg = registerAlias(dst_reg, abi_size * 2);4844 const wide_reg = registerAlias(dst_reg, abi_size * 2);
4752 if (src_mcv.isMemory()) try self.asmRegisterMemory(4845 if (src_mcv.isMemory()) try self.asmRegisterMemory(
4753 .vcvtph2ps,4846 .{ ._, .vcvtph2ps },
4754 wide_reg,4847 wide_reg,
4755 src_mcv.mem(Memory.PtrSize.fromSize(4848 src_mcv.mem(Memory.PtrSize.fromSize(
4756 @intCast(u32, @divExact(wide_reg.bitSize(), 16)),4849 @intCast(u32, @divExact(wide_reg.bitSize(), 16)),
4757 )),4850 )),
4758 ) else try self.asmRegisterRegister(4851 ) else try self.asmRegisterRegister(
4759 .vcvtph2ps,4852 .{ ._, .vcvtph2ps },
4760 wide_reg,4853 wide_reg,
4761 (if (src_mcv.isRegister())4854 (if (src_mcv.isRegister())
4762 src_mcv.getReg().?4855 src_mcv.getReg().?
4763 else4856 else
4764 try self.copyToTmpRegister(ty, src_mcv)).to128(),4857 try self.copyToTmpRegister(ty, src_mcv)).to128(),
4765 );4858 );
4766 try self.asmRegisterRegister(.vsqrtps, wide_reg, wide_reg);4859 try self.asmRegisterRegister(.{ ._, .vsqrtps }, wide_reg, wide_reg);
4767 try self.asmRegisterRegisterImmediate(4860 try self.asmRegisterRegisterImmediate(
4768 .vcvtps2ph,4861 .{ ._, .vcvtps2ph },
4769 dst_reg,4862 dst_reg,
4770 wide_reg,4863 wide_reg,
4771 Immediate.u(0b1_00),4864 Immediate.u(0b1_00),
...@@ -4775,15 +4868,15 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -4775,15 +4868,15 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4775 else => null,4868 else => null,
4776 } else null,4869 } else null,
4777 32 => switch (ty.vectorLen()) {4870 32 => switch (ty.vectorLen()) {
4778 1 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,4871 1 => if (self.hasFeature(.avx)) .{ ._, .vsqrtss } else .{ ._, .sqrtss },
4779 2...4 => if (self.hasFeature(.avx)) .vsqrtps else .sqrtps,4872 2...4 => if (self.hasFeature(.avx)) .{ ._, .vsqrtps } else .{ ._, .sqrtps },
4780 5...8 => if (self.hasFeature(.avx)) .vsqrtps else null,4873 5...8 => if (self.hasFeature(.avx)) .{ ._, .vsqrtps } else null,
4781 else => null,4874 else => null,
4782 },4875 },
4783 64 => switch (ty.vectorLen()) {4876 64 => switch (ty.vectorLen()) {
4784 1 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,4877 1 => if (self.hasFeature(.avx)) .{ ._, .vsqrtsd } else .{ ._, .sqrtsd },
4785 2 => if (self.hasFeature(.avx)) .vsqrtpd else .sqrtpd,4878 2 => if (self.hasFeature(.avx)) .{ ._, .vsqrtpd } else .{ ._, .sqrtpd },
4786 3...4 => if (self.hasFeature(.avx)) .vsqrtpd else null,4879 3...4 => if (self.hasFeature(.avx)) .{ ._, .vsqrtpd } else null,
4787 else => null,4880 else => null,
4788 },4881 },
4789 80, 128 => null,4882 80, 128 => null,
...@@ -4795,7 +4888,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -4795,7 +4888,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4795 })) |tag| tag else return self.fail("TODO implement airSqrt for {}", .{4888 })) |tag| tag else return self.fail("TODO implement airSqrt for {}", .{
4796 ty.fmt(self.bin_file.options.module.?),4889 ty.fmt(self.bin_file.options.module.?),
4797 });4890 });
4798 switch (mir_tag) {4891 switch (mir_tag[1]) {
4799 .vsqrtss, .vsqrtsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(4892 .vsqrtss, .vsqrtsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
4800 mir_tag,4893 mir_tag,
4801 dst_reg,4894 dst_reg,
...@@ -4911,14 +5004,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn...@@ -4911,14 +5004,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
4911 if (load_abi_size <= 8) {5004 if (load_abi_size <= 8) {
4912 const load_reg = registerAlias(dst_reg, load_abi_size);5005 const load_reg = registerAlias(dst_reg, load_abi_size);
4913 try self.asmRegisterMemory(5006 try self.asmRegisterMemory(
4914 .mov,5007 .{ ._, .mov },
4915 load_reg,5008 load_reg,
4916 Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{5009 Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
4917 .base = .{ .reg = ptr_reg },5010 .base = .{ .reg = ptr_reg },
4918 .disp = val_byte_off,5011 .disp = val_byte_off,
4919 }),5012 }),
4920 );5013 );
4921 try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));5014 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
4922 } else {5015 } else {
4923 const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);5016 const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);
4924 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);5017 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
...@@ -4926,7 +5019,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn...@@ -4926,7 +5019,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
49265019
4927 const dst_alias = registerAlias(dst_reg, val_abi_size);5020 const dst_alias = registerAlias(dst_reg, val_abi_size);
4928 try self.asmRegisterMemory(5021 try self.asmRegisterMemory(
4929 .mov,5022 .{ ._, .mov },
4930 dst_alias,5023 dst_alias,
4931 Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{5024 Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
4932 .base = .{ .reg = ptr_reg },5025 .base = .{ .reg = ptr_reg },
...@@ -4934,14 +5027,19 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn...@@ -4934,14 +5027,19 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
4934 }),5027 }),
4935 );5028 );
4936 try self.asmRegisterMemory(5029 try self.asmRegisterMemory(
4937 .mov,5030 .{ ._, .mov },
4938 tmp_reg,5031 tmp_reg,
4939 Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{5032 Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
4940 .base = .{ .reg = ptr_reg },5033 .base = .{ .reg = ptr_reg },
4941 .disp = val_byte_off + 1,5034 .disp = val_byte_off + 1,
4942 }),5035 }),
4943 );5036 );
4944 try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));5037 try self.asmRegisterRegisterImmediate(
5038 .{ ._rd, .sh },
5039 dst_alias,
5040 tmp_reg,
5041 Immediate.u(val_bit_off),
5042 );
4945 }5043 }
49465044
4947 if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);5045 if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
...@@ -5047,13 +5145,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In...@@ -5047,13 +5145,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
5047 const part_mask_not = part_mask ^5145 const part_mask_not = part_mask ^
5048 (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_abi_bits));5146 (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_abi_bits));
5049 if (limb_abi_size <= 4) {5147 if (limb_abi_size <= 4) {
5050 try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.u(part_mask_not));5148 try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.u(part_mask_not));
5051 } else if (math.cast(i32, @bitCast(i64, part_mask_not))) |small| {5149 } else if (math.cast(i32, @bitCast(i64, part_mask_not))) |small| {
5052 try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.s(small));5150 try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.s(small));
5053 } else {5151 } else {
5054 const part_mask_reg = try self.register_manager.allocReg(null, gp);5152 const part_mask_reg = try self.register_manager.allocReg(null, gp);
5055 try self.asmRegisterImmediate(.mov, part_mask_reg, Immediate.u(part_mask_not));5153 try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, Immediate.u(part_mask_not));
5056 try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);5154 try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);
5057 }5155 }
50585156
5059 if (src_bit_size <= 64) {5157 if (src_bit_size <= 64) {
...@@ -5064,14 +5162,26 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In...@@ -5064,14 +5162,26 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
50645162
5065 try self.genSetReg(tmp_reg, src_ty, src_mcv);5163 try self.genSetReg(tmp_reg, src_ty, src_mcv);
5066 switch (limb_i) {5164 switch (limb_i) {
5067 0 => try self.genShiftBinOpMir(.shl, src_ty, tmp_mcv, .{ .immediate = src_bit_off }),5165 0 => try self.genShiftBinOpMir(
5068 1 => try self.genShiftBinOpMir(.shr, src_ty, tmp_mcv, .{5166 .{ ._l, .sh },
5069 .immediate = limb_abi_bits - src_bit_off,5167 src_ty,
5070 }),5168 tmp_mcv,
5169 .{ .immediate = src_bit_off },
5170 ),
5171 1 => try self.genShiftBinOpMir(
5172 .{ ._r, .sh },
5173 src_ty,
5174 tmp_mcv,
5175 .{ .immediate = limb_abi_bits - src_bit_off },
5176 ),
5071 else => unreachable,5177 else => unreachable,
5072 }5178 }
5073 try self.genBinOpMir(.@"and", src_ty, tmp_mcv, .{ .immediate = part_mask });5179 try self.genBinOpMir(.{ ._, .@"and" }, src_ty, tmp_mcv, .{ .immediate = part_mask });
5074 try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));5180 try self.asmMemoryRegister(
5181 .{ ._, .@"or" },
5182 limb_mem,
5183 registerAlias(tmp_reg, limb_abi_size),
5184 );
5075 } else return self.fail("TODO: implement packed store of {}", .{5185 } else return self.fail("TODO: implement packed store of {}", .{
5076 src_ty.fmt(self.bin_file.options.module.?),5186 src_ty.fmt(self.bin_file.options.module.?),
5077 });5187 });
...@@ -5171,7 +5281,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32...@@ -5171,7 +5281,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
5171 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },5281 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
5172 else => mcv,5282 else => mcv,
5173 });5283 });
5174 try self.genBinOpMir(.add, Type.usize, dst_mcv, .{ .register = offset_reg });5284 try self.genBinOpMir(.{ ._, .add }, Type.usize, dst_mcv, .{ .register = offset_reg });
5175 break :result dst_mcv;5285 break :result dst_mcv;
5176 },5286 },
5177 .indirect => |reg_off| break :result .{ .indirect = .{5287 .indirect => |reg_off| break :result .{ .indirect = .{
...@@ -5255,14 +5365,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -5255,14 +5365,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5255 if (load_abi_size <= 8) {5365 if (load_abi_size <= 8) {
5256 const load_reg = registerAlias(dst_reg, load_abi_size);5366 const load_reg = registerAlias(dst_reg, load_abi_size);
5257 try self.asmRegisterMemory(5367 try self.asmRegisterMemory(
5258 .mov,5368 .{ ._, .mov },
5259 load_reg,5369 load_reg,
5260 Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{5370 Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
5261 .base = .{ .frame = frame_addr.index },5371 .base = .{ .frame = frame_addr.index },
5262 .disp = frame_addr.off + field_byte_off,5372 .disp = frame_addr.off + field_byte_off,
5263 }),5373 }),
5264 );5374 );
5265 try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(field_bit_off));5375 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
5266 } else {5376 } else {
5267 const tmp_reg = registerAlias(5377 const tmp_reg = registerAlias(
5268 try self.register_manager.allocReg(null, gp),5378 try self.register_manager.allocReg(null, gp),
...@@ -5273,7 +5383,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -5273,7 +5383,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
52735383
5274 const dst_alias = registerAlias(dst_reg, field_abi_size);5384 const dst_alias = registerAlias(dst_reg, field_abi_size);
5275 try self.asmRegisterMemory(5385 try self.asmRegisterMemory(
5276 .mov,5386 .{ ._, .mov },
5277 dst_alias,5387 dst_alias,
5278 Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{5388 Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
5279 .base = .{ .frame = frame_addr.index },5389 .base = .{ .frame = frame_addr.index },
...@@ -5281,7 +5391,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -5281,7 +5391,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5281 }),5391 }),
5282 );5392 );
5283 try self.asmRegisterMemory(5393 try self.asmRegisterMemory(
5284 .mov,5394 .{ ._, .mov },
5285 tmp_reg,5395 tmp_reg,
5286 Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{5396 Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
5287 .base = .{ .frame = frame_addr.index },5397 .base = .{ .frame = frame_addr.index },
...@@ -5289,7 +5399,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -5289,7 +5399,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5289 }),5399 }),
5290 );5400 );
5291 try self.asmRegisterRegisterImmediate(5401 try self.asmRegisterRegisterImmediate(
5292 .shrd,5402 .{ ._rd, .sh },
5293 dst_alias,5403 dst_alias,
5294 tmp_reg,5404 tmp_reg,
5295 Immediate.u(field_bit_off),5405 Immediate.u(field_bit_off),
...@@ -5325,21 +5435,26 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -5325,21 +5435,26 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5325 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);5435 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
53265436
5327 // Shift by struct_field_offset.5437 // Shift by struct_field_offset.
5328 try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv, .{ .immediate = field_off });5438 try self.genShiftBinOpMir(
5439 .{ ._r, .sh },
5440 Type.usize,
5441 dst_mcv,
5442 .{ .immediate = field_off },
5443 );
53295444
5330 // Mask to field_bit_size bits5445 // Mask to field_bit_size bits
5331 const field_bit_size = field_ty.bitSize(self.target.*);5446 const field_bit_size = field_ty.bitSize(self.target.*);
5332 const mask = ~@as(u64, 0) >> @intCast(u6, 64 - field_bit_size);5447 const mask = ~@as(u64, 0) >> @intCast(u6, 64 - field_bit_size);
53335448
5334 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });5449 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
5335 try self.genBinOpMir(.@"and", Type.usize, dst_mcv, .{ .register = tmp_reg });5450 try self.genBinOpMir(.{ ._, .@"and" }, Type.usize, dst_mcv, .{ .register = tmp_reg });
53365451
5337 const signedness =5452 const signedness =
5338 if (field_ty.isAbiInt()) field_ty.intInfo(self.target.*).signedness else .unsigned;5453 if (field_ty.isAbiInt()) field_ty.intInfo(self.target.*).signedness else .unsigned;
5339 const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));5454 const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));
5340 if (signedness == .signed and field_byte_size < 8) {5455 if (signedness == .signed and field_byte_size < 8) {
5341 try self.asmRegisterRegister(5456 try self.asmRegisterRegister(
5342 if (field_byte_size >= 4) .movsxd else .movsx,5457 if (field_byte_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },
5343 dst_mcv.register,5458 dst_mcv.register,
5344 registerAlias(dst_mcv.register, field_byte_size),5459 registerAlias(dst_mcv.register, field_byte_size),
5345 );5460 );
...@@ -5451,17 +5566,17 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:...@@ -5451,17 +5566,17 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
54515566
5452 if (limb_pl.base.tag == .int_unsigned and self.regExtraBits(limb_ty) > 0) {5567 if (limb_pl.base.tag == .int_unsigned and self.regExtraBits(limb_ty) > 0) {
5453 const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_pl.data);5568 const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_pl.data);
5454 try self.genBinOpMir(.xor, limb_ty, limb_mcv, .{ .immediate = mask });5569 try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });
5455 } else try self.genUnOpMir(.not, limb_ty, limb_mcv);5570 } else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
5456 }5571 }
5457 },5572 },
5458 .neg => try self.genUnOpMir(.neg, src_ty, dst_mcv),5573 .neg => try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv),
5459 else => unreachable,5574 else => unreachable,
5460 }5575 }
5461 return dst_mcv;5576 return dst_mcv;
5462}5577}
54635578
5464fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {5579fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
5465 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));5580 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
5466 if (abi_size > 8) return self.fail("TODO implement {} for {}", .{5581 if (abi_size > 8) return self.fail("TODO implement {} for {}", .{
5467 mir_tag,5582 mir_tag,
...@@ -5504,7 +5619,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue...@@ -5504,7 +5619,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue
5504/// Clobbers .rcx for non-immediate shift value.5619/// Clobbers .rcx for non-immediate shift value.
5505fn genShiftBinOpMir(5620fn genShiftBinOpMir(
5506 self: *Self,5621 self: *Self,
5507 tag: Mir.Inst.Tag,5622 tag: Mir.Inst.FixedTag,
5508 ty: Type,5623 ty: Type,
5509 lhs_mcv: MCValue,5624 lhs_mcv: MCValue,
5510 shift_mcv: MCValue,5625 shift_mcv: MCValue,
...@@ -5589,16 +5704,16 @@ fn genShiftBinOpMir(...@@ -5589,16 +5704,16 @@ fn genShiftBinOpMir(
5589 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);5704 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
5590 defer self.register_manager.unlockReg(tmp_lock);5705 defer self.register_manager.unlockReg(tmp_lock);
55915706
5592 const info: struct { offsets: [2]i32, double_tag: Mir.Inst.Tag } = switch (tag) {5707 const info: struct { offsets: [2]i32, double_tag: Mir.Inst.FixedTag } = switch (tag[0]) {
5593 .shl, .sal => .{ .offsets = .{ 0, 8 }, .double_tag = .shld },5708 ._l => .{ .offsets = .{ 0, 8 }, .double_tag = .{ ._ld, .sh } },
5594 .shr, .sar => .{ .offsets = .{ 8, 0 }, .double_tag = .shrd },5709 ._r => .{ .offsets = .{ 8, 0 }, .double_tag = .{ ._rd, .sh } },
5595 else => unreachable,5710 else => unreachable,
5596 };5711 };
5597 switch (lhs_mcv) {5712 switch (lhs_mcv) {
5598 .load_frame => |dst_frame_addr| switch (rhs_mcv) {5713 .load_frame => |dst_frame_addr| switch (rhs_mcv) {
5599 .immediate => |rhs_imm| if (rhs_imm == 0) {} else if (rhs_imm < 64) {5714 .immediate => |rhs_imm| if (rhs_imm == 0) {} else if (rhs_imm < 64) {
5600 try self.asmRegisterMemory(5715 try self.asmRegisterMemory(
5601 .mov,5716 .{ ._, .mov },
5602 tmp_reg,5717 tmp_reg,
5603 Memory.sib(.qword, .{5718 Memory.sib(.qword, .{
5604 .base = .{ .frame = dst_frame_addr.index },5719 .base = .{ .frame = dst_frame_addr.index },
...@@ -5625,7 +5740,7 @@ fn genShiftBinOpMir(...@@ -5625,7 +5740,7 @@ fn genShiftBinOpMir(
5625 } else {5740 } else {
5626 assert(rhs_imm < 128);5741 assert(rhs_imm < 128);
5627 try self.asmRegisterMemory(5742 try self.asmRegisterMemory(
5628 .mov,5743 .{ ._, .mov },
5629 tmp_reg,5744 tmp_reg,
5630 Memory.sib(.qword, .{5745 Memory.sib(.qword, .{
5631 .base = .{ .frame = dst_frame_addr.index },5746 .base = .{ .frame = dst_frame_addr.index },
...@@ -5636,34 +5751,30 @@ fn genShiftBinOpMir(...@@ -5636,34 +5751,30 @@ fn genShiftBinOpMir(
5636 try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));5751 try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));
5637 }5752 }
5638 try self.asmMemoryRegister(5753 try self.asmMemoryRegister(
5639 .mov,5754 .{ ._, .mov },
5640 Memory.sib(.qword, .{5755 Memory.sib(.qword, .{
5641 .base = .{ .frame = dst_frame_addr.index },5756 .base = .{ .frame = dst_frame_addr.index },
5642 .disp = dst_frame_addr.off + info.offsets[1],5757 .disp = dst_frame_addr.off + info.offsets[1],
5643 }),5758 }),
5644 tmp_reg,5759 tmp_reg,
5645 );5760 );
5646 switch (tag) {5761 if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryImmediate(
5647 .shl, .sal, .shr => {5762 tag,
5648 try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());5763 Memory.sib(.qword, .{
5649 try self.asmMemoryRegister(5764 .base = .{ .frame = dst_frame_addr.index },
5650 .mov,5765 .disp = dst_frame_addr.off + info.offsets[0],
5651 Memory.sib(.qword, .{5766 }),
5652 .base = .{ .frame = dst_frame_addr.index },5767 Immediate.u(63),
5653 .disp = dst_frame_addr.off + info.offsets[0],5768 ) else {
5654 }),5769 try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());
5655 tmp_reg,5770 try self.asmMemoryRegister(
5656 );5771 .{ ._, .mov },
5657 },
5658 .sar => try self.asmMemoryImmediate(
5659 tag,
5660 Memory.sib(.qword, .{5772 Memory.sib(.qword, .{
5661 .base = .{ .frame = dst_frame_addr.index },5773 .base = .{ .frame = dst_frame_addr.index },
5662 .disp = dst_frame_addr.off + info.offsets[0],5774 .disp = dst_frame_addr.off + info.offsets[0],
5663 }),5775 }),
5664 Immediate.u(63),5776 tmp_reg,
5665 ),5777 );
5666 else => unreachable,
5667 }5778 }
5668 },5779 },
5669 else => {5780 else => {
...@@ -5677,7 +5788,7 @@ fn genShiftBinOpMir(...@@ -5677,7 +5788,7 @@ fn genShiftBinOpMir(
56775788
5678 try self.genSetReg(.cl, Type.u8, rhs_mcv);5789 try self.genSetReg(.cl, Type.u8, rhs_mcv);
5679 try self.asmRegisterMemory(5790 try self.asmRegisterMemory(
5680 .mov,5791 .{ ._, .mov },
5681 first_reg,5792 first_reg,
5682 Memory.sib(.qword, .{5793 Memory.sib(.qword, .{
5683 .base = .{ .frame = dst_frame_addr.index },5794 .base = .{ .frame = dst_frame_addr.index },
...@@ -5685,32 +5796,28 @@ fn genShiftBinOpMir(...@@ -5685,32 +5796,28 @@ fn genShiftBinOpMir(
5685 }),5796 }),
5686 );5797 );
5687 try self.asmRegisterMemory(5798 try self.asmRegisterMemory(
5688 .mov,5799 .{ ._, .mov },
5689 second_reg,5800 second_reg,
5690 Memory.sib(.qword, .{5801 Memory.sib(.qword, .{
5691 .base = .{ .frame = dst_frame_addr.index },5802 .base = .{ .frame = dst_frame_addr.index },
5692 .disp = dst_frame_addr.off + info.offsets[1],5803 .disp = dst_frame_addr.off + info.offsets[1],
5693 }),5804 }),
5694 );5805 );
5695 switch (tag) {5806 if (tag[0] == ._r and tag[1] == .sa) {
5696 .shl, .sal, .shr => try self.asmRegisterRegister(5807 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);
5697 .xor,5808 try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
5698 tmp_reg.to32(),5809 } else try self.asmRegisterRegister(
5699 tmp_reg.to32(),5810 .{ ._, .xor },
5700 ),5811 tmp_reg.to32(),
5701 .sar => {5812 tmp_reg.to32(),
5702 try self.asmRegisterRegister(.mov, tmp_reg, first_reg);5813 );
5703 try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
5704 },
5705 else => unreachable,
5706 }
5707 try self.asmRegisterRegisterRegister(info.double_tag, second_reg, first_reg, .cl);5814 try self.asmRegisterRegisterRegister(info.double_tag, second_reg, first_reg, .cl);
5708 try self.asmRegisterRegister(tag, first_reg, .cl);5815 try self.asmRegisterRegister(tag, first_reg, .cl);
5709 try self.asmRegisterImmediate(.cmp, .cl, Immediate.u(64));5816 try self.asmRegisterImmediate(.{ ._, .cmp }, .cl, Immediate.u(64));
5710 try self.asmCmovccRegisterRegister(second_reg, first_reg, .ae);5817 try self.asmCmovccRegisterRegister(second_reg, first_reg, .ae);
5711 try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);5818 try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
5712 try self.asmMemoryRegister(5819 try self.asmMemoryRegister(
5713 .mov,5820 .{ ._, .mov },
5714 Memory.sib(.qword, .{5821 Memory.sib(.qword, .{
5715 .base = .{ .frame = dst_frame_addr.index },5822 .base = .{ .frame = dst_frame_addr.index },
5716 .disp = dst_frame_addr.off + info.offsets[1],5823 .disp = dst_frame_addr.off + info.offsets[1],
...@@ -5718,7 +5825,7 @@ fn genShiftBinOpMir(...@@ -5718,7 +5825,7 @@ fn genShiftBinOpMir(
5718 second_reg,5825 second_reg,
5719 );5826 );
5720 try self.asmMemoryRegister(5827 try self.asmMemoryRegister(
5721 .mov,5828 .{ ._, .mov },
5722 Memory.sib(.qword, .{5829 Memory.sib(.qword, .{
5723 .base = .{ .frame = dst_frame_addr.index },5830 .base = .{ .frame = dst_frame_addr.index },
5724 .disp = dst_frame_addr.off + info.offsets[0],5831 .disp = dst_frame_addr.off + info.offsets[0],
...@@ -5743,7 +5850,7 @@ fn genShiftBinOpMir(...@@ -5743,7 +5850,7 @@ fn genShiftBinOpMir(
5743/// Asserts .rcx is free.5850/// Asserts .rcx is free.
5744fn genShiftBinOp(5851fn genShiftBinOp(
5745 self: *Self,5852 self: *Self,
5746 tag: Air.Inst.Tag,5853 air_tag: Air.Inst.Tag,
5747 maybe_inst: ?Air.Inst.Index,5854 maybe_inst: ?Air.Inst.Index,
5748 lhs_mcv: MCValue,5855 lhs_mcv: MCValue,
5749 rhs_mcv: MCValue,5856 rhs_mcv: MCValue,
...@@ -5788,14 +5895,14 @@ fn genShiftBinOp(...@@ -5788,14 +5895,14 @@ fn genShiftBinOp(
5788 };5895 };
57895896
5790 const signedness = lhs_ty.intInfo(self.target.*).signedness;5897 const signedness = lhs_ty.intInfo(self.target.*).signedness;
5791 try self.genShiftBinOpMir(switch (tag) {5898 try self.genShiftBinOpMir(switch (air_tag) {
5792 .shl, .shl_exact => switch (signedness) {5899 .shl, .shl_exact => switch (signedness) {
5793 .signed => .sal,5900 .signed => .{ ._l, .sa },
5794 .unsigned => .shl,5901 .unsigned => .{ ._l, .sh },
5795 },5902 },
5796 .shr, .shr_exact => switch (signedness) {5903 .shr, .shr_exact => switch (signedness) {
5797 .signed => .sar,5904 .signed => .{ ._r, .sa },
5798 .unsigned => .shr,5905 .unsigned => .{ ._r, .sh },
5799 },5906 },
5800 else => unreachable,5907 else => unreachable,
5801 }, lhs_ty, dst_mcv, rhs_mcv);5908 }, lhs_ty, dst_mcv, rhs_mcv);
...@@ -5855,20 +5962,18 @@ fn genMulDivBinOp(...@@ -5855,20 +5962,18 @@ fn genMulDivBinOp(
5855 try self.register_manager.getReg(.rax, track_inst_rax);5962 try self.register_manager.getReg(.rax, track_inst_rax);
5856 try self.register_manager.getReg(.rdx, track_inst_rdx);5963 try self.register_manager.getReg(.rdx, track_inst_rdx);
58575964
5858 const mir_tag: Mir.Inst.Tag = switch (signedness) {5965 try self.genIntMulDivOpMir(switch (signedness) {
5859 .signed => switch (tag) {5966 .signed => switch (tag) {
5860 .mul, .mulwrap => .imul,5967 .mul, .mulwrap => .{ .i_, .mul },
5861 .div_trunc, .div_exact, .rem => .idiv,5968 .div_trunc, .div_exact, .rem => .{ .i_, .div },
5862 else => unreachable,5969 else => unreachable,
5863 },5970 },
5864 .unsigned => switch (tag) {5971 .unsigned => switch (tag) {
5865 .mul, .mulwrap => .mul,5972 .mul, .mulwrap => .{ ._, .mul },
5866 .div_trunc, .div_exact, .rem => .div,5973 .div_trunc, .div_exact, .rem => .{ ._, .div },
5867 else => unreachable,5974 else => unreachable,
5868 },5975 },
5869 };5976 }, ty, lhs, rhs);
5870
5871 try self.genIntMulDivOpMir(mir_tag, ty, lhs, rhs);
58725977
5873 if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) {5978 if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) {
5874 .mul, .mulwrap, .div_trunc, .div_exact => .rax,5979 .mul, .mulwrap, .div_trunc, .div_exact => .rax,
...@@ -5878,7 +5983,7 @@ fn genMulDivBinOp(...@@ -5878,7 +5983,7 @@ fn genMulDivBinOp(
58785983
5879 const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);5984 const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);
5880 try self.asmMemoryRegister(5985 try self.asmMemoryRegister(
5881 .mov,5986 .{ ._, .mov },
5882 Memory.sib(.qword, .{5987 Memory.sib(.qword, .{
5883 .base = .{ .frame = dst_mcv.load_frame.index },5988 .base = .{ .frame = dst_mcv.load_frame.index },
5884 .disp = dst_mcv.load_frame.off,5989 .disp = dst_mcv.load_frame.off,
...@@ -5886,7 +5991,7 @@ fn genMulDivBinOp(...@@ -5886,7 +5991,7 @@ fn genMulDivBinOp(
5886 .rax,5991 .rax,
5887 );5992 );
5888 try self.asmMemoryRegister(5993 try self.asmMemoryRegister(
5889 .mov,5994 .{ ._, .mov },
5890 Memory.sib(.qword, .{5995 Memory.sib(.qword, .{
5891 .base = .{ .frame = dst_mcv.load_frame.index },5996 .base = .{ .frame = dst_mcv.load_frame.index },
5892 .disp = dst_mcv.load_frame.off + 8,5997 .disp = dst_mcv.load_frame.off + 8,
...@@ -5927,12 +6032,12 @@ fn genMulDivBinOp(...@@ -5927,12 +6032,12 @@ fn genMulDivBinOp(
5927 try self.copyToRegisterWithInstTracking(inst, ty, lhs)6032 try self.copyToRegisterWithInstTracking(inst, ty, lhs)
5928 else6033 else
5929 .{ .register = try self.copyToTmpRegister(ty, lhs) };6034 .{ .register = try self.copyToTmpRegister(ty, lhs) };
5930 try self.genBinOpMir(.sub, ty, result, div_floor);6035 try self.genBinOpMir(.{ ._, .sub }, ty, result, div_floor);
59316036
5932 return result;6037 return result;
5933 },6038 },
5934 .unsigned => {6039 .unsigned => {
5935 try self.genIntMulDivOpMir(.div, ty, lhs, rhs);6040 try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, rhs);
5936 return .{ .register = registerAlias(.rdx, abi_size) };6041 return .{ .register = registerAlias(.rdx, abi_size) };
5937 },6042 },
5938 }6043 }
...@@ -5974,7 +6079,7 @@ fn genMulDivBinOp(...@@ -5974,7 +6079,7 @@ fn genMulDivBinOp(
5974 switch (signedness) {6079 switch (signedness) {
5975 .signed => return try self.genInlineIntDivFloor(ty, lhs, actual_rhs),6080 .signed => return try self.genInlineIntDivFloor(ty, lhs, actual_rhs),
5976 .unsigned => {6081 .unsigned => {
5977 try self.genIntMulDivOpMir(.div, ty, lhs, actual_rhs);6082 try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, actual_rhs);
5978 return .{ .register = registerAlias(.rax, abi_size) };6083 return .{ .register = registerAlias(.rax, abi_size) };
5979 },6084 },
5980 }6085 }
...@@ -6072,11 +6177,11 @@ fn genBinOp(...@@ -6072,11 +6177,11 @@ fn genBinOp(
6072 switch (air_tag) {6177 switch (air_tag) {
6073 .add,6178 .add,
6074 .addwrap,6179 .addwrap,
6075 => try self.genBinOpMir(.add, lhs_ty, dst_mcv, src_mcv),6180 => try self.genBinOpMir(.{ ._, .add }, lhs_ty, dst_mcv, src_mcv),
60766181
6077 .sub,6182 .sub,
6078 .subwrap,6183 .subwrap,
6079 => try self.genBinOpMir(.sub, lhs_ty, dst_mcv, src_mcv),6184 => try self.genBinOpMir(.{ ._, .sub }, lhs_ty, dst_mcv, src_mcv),
60806185
6081 .ptr_add,6186 .ptr_add,
6082 .ptr_sub,6187 .ptr_sub,
...@@ -6088,22 +6193,27 @@ fn genBinOp(...@@ -6088,22 +6193,27 @@ fn genBinOp(
60886193
6089 const elem_size = lhs_ty.elemType2().abiSize(self.target.*);6194 const elem_size = lhs_ty.elemType2().abiSize(self.target.*);
6090 try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size });6195 try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size });
6091 try self.genBinOpMir(switch (air_tag) {6196 try self.genBinOpMir(
6092 .ptr_add => .add,6197 switch (air_tag) {
6093 .ptr_sub => .sub,6198 .ptr_add => .{ ._, .add },
6094 else => unreachable,6199 .ptr_sub => .{ ._, .sub },
6095 }, lhs_ty, dst_mcv, tmp_mcv);6200 else => unreachable,
6201 },
6202 lhs_ty,
6203 dst_mcv,
6204 tmp_mcv,
6205 );
6096 },6206 },
60976207
6098 .bool_or,6208 .bool_or,
6099 .bit_or,6209 .bit_or,
6100 => try self.genBinOpMir(.@"or", lhs_ty, dst_mcv, src_mcv),6210 => try self.genBinOpMir(.{ ._, .@"or" }, lhs_ty, dst_mcv, src_mcv),
61016211
6102 .bool_and,6212 .bool_and,
6103 .bit_and,6213 .bit_and,
6104 => try self.genBinOpMir(.@"and", lhs_ty, dst_mcv, src_mcv),6214 => try self.genBinOpMir(.{ ._, .@"and" }, lhs_ty, dst_mcv, src_mcv),
61056215
6106 .xor => try self.genBinOpMir(.xor, lhs_ty, dst_mcv, src_mcv),6216 .xor => try self.genBinOpMir(.{ ._, .xor }, lhs_ty, dst_mcv, src_mcv),
61076217
6108 .min,6218 .min,
6109 .max,6219 .max,
...@@ -6129,7 +6239,7 @@ fn genBinOp(...@@ -6129,7 +6239,7 @@ fn genBinOp(
6129 };6239 };
6130 defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock);6240 defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock);
61316241
6132 try self.genBinOpMir(.cmp, lhs_ty, dst_mcv, mat_src_mcv);6242 try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv);
61336243
6134 const int_info = lhs_ty.intInfo(self.target.*);6244 const int_info = lhs_ty.intInfo(self.target.*);
6135 const cc: Condition = switch (int_info.signedness) {6245 const cc: Condition = switch (int_info.signedness) {
...@@ -6206,7 +6316,7 @@ fn genBinOp(...@@ -6206,7 +6316,7 @@ fn genBinOp(
6206 }6316 }
62076317
6208 const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);6318 const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
6209 const mir_tag = if (@as(?Mir.Inst.Tag, switch (lhs_ty.zigTypeTag()) {6319 const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
6210 else => unreachable,6320 else => unreachable,
6211 .Float => switch (lhs_ty.floatBits(self.target.*)) {6321 .Float => switch (lhs_ty.floatBits(self.target.*)) {
6212 16 => if (self.hasFeature(.f16c)) {6322 16 => if (self.hasFeature(.f16c)) {
...@@ -6215,13 +6325,13 @@ fn genBinOp(...@@ -6215,13 +6325,13 @@ fn genBinOp(
6215 defer self.register_manager.unlockReg(tmp_lock);6325 defer self.register_manager.unlockReg(tmp_lock);
62166326
6217 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(6327 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
6218 .vpinsrw,6328 .{ ._, .vpinsrw },
6219 dst_reg,6329 dst_reg,
6220 dst_reg,6330 dst_reg,
6221 src_mcv.mem(.word),6331 src_mcv.mem(.word),
6222 Immediate.u(1),6332 Immediate.u(1),
6223 ) else try self.asmRegisterRegisterRegister(6333 ) else try self.asmRegisterRegisterRegister(
6224 .vpunpcklwd,6334 .{ ._, .vpunpcklwd },
6225 dst_reg,6335 dst_reg,
6226 dst_reg,6336 dst_reg,
6227 (if (src_mcv.isRegister())6337 (if (src_mcv.isRegister())
...@@ -6229,15 +6339,15 @@ fn genBinOp(...@@ -6229,15 +6339,15 @@ fn genBinOp(
6229 else6339 else
6230 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),6340 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6231 );6341 );
6232 try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);6342 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
6233 try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);6343 try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp_reg, dst_reg);
6234 try self.asmRegisterRegisterRegister(6344 try self.asmRegisterRegisterRegister(
6235 switch (air_tag) {6345 switch (air_tag) {
6236 .add => .vaddss,6346 .add => .{ ._, .vaddss },
6237 .sub => .vsubss,6347 .sub => .{ ._, .vsubss },
6238 .div_float, .div_trunc, .div_floor, .div_exact => .vdivss,6348 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivss },
6239 .max => .vmaxss,6349 .max => .{ ._, .vmaxss },
6240 .min => .vmaxss,6350 .min => .{ ._, .vmaxss },
6241 else => unreachable,6351 else => unreachable,
6242 },6352 },
6243 dst_reg,6353 dst_reg,
...@@ -6245,7 +6355,7 @@ fn genBinOp(...@@ -6245,7 +6355,7 @@ fn genBinOp(
6245 tmp_reg,6355 tmp_reg,
6246 );6356 );
6247 try self.asmRegisterRegisterImmediate(6357 try self.asmRegisterRegisterImmediate(
6248 .vcvtps2ph,6358 .{ ._, .vcvtps2ph },
6249 dst_reg,6359 dst_reg,
6250 dst_reg,6360 dst_reg,
6251 Immediate.u(0b1_00),6361 Immediate.u(0b1_00),
...@@ -6253,29 +6363,29 @@ fn genBinOp(...@@ -6253,29 +6363,29 @@ fn genBinOp(
6253 return dst_mcv;6363 return dst_mcv;
6254 } else null,6364 } else null,
6255 32 => switch (air_tag) {6365 32 => switch (air_tag) {
6256 .add => if (self.hasFeature(.avx)) .vaddss else .addss,6366 .add => if (self.hasFeature(.avx)) .{ ._, .vaddss } else .{ ._, .addss },
6257 .sub => if (self.hasFeature(.avx)) .vsubss else .subss,6367 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubss } else .{ ._, .subss },
6258 .mul => if (self.hasFeature(.avx)) .vmulss else .mulss,6368 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulss } else .{ ._, .mulss },
6259 .div_float,6369 .div_float,
6260 .div_trunc,6370 .div_trunc,
6261 .div_floor,6371 .div_floor,
6262 .div_exact,6372 .div_exact,
6263 => if (self.hasFeature(.avx)) .vdivss else .divss,6373 => if (self.hasFeature(.avx)) .{ ._, .vdivss } else .{ ._, .divss },
6264 .max => if (self.hasFeature(.avx)) .vmaxss else .maxss,6374 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxss } else .{ ._, .maxss },
6265 .min => if (self.hasFeature(.avx)) .vminss else .minss,6375 .min => if (self.hasFeature(.avx)) .{ ._, .vminss } else .{ ._, .minss },
6266 else => unreachable,6376 else => unreachable,
6267 },6377 },
6268 64 => switch (air_tag) {6378 64 => switch (air_tag) {
6269 .add => if (self.hasFeature(.avx)) .vaddsd else .addsd,6379 .add => if (self.hasFeature(.avx)) .{ ._, .vaddsd } else .{ ._, .addsd },
6270 .sub => if (self.hasFeature(.avx)) .vsubsd else .subsd,6380 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubsd } else .{ ._, .subsd },
6271 .mul => if (self.hasFeature(.avx)) .vmulsd else .mulsd,6381 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulsd } else .{ ._, .mulsd },
6272 .div_float,6382 .div_float,
6273 .div_trunc,6383 .div_trunc,
6274 .div_floor,6384 .div_floor,
6275 .div_exact,6385 .div_exact,
6276 => if (self.hasFeature(.avx)) .vdivsd else .divsd,6386 => if (self.hasFeature(.avx)) .{ ._, .vdivsd } else .{ ._, .divsd },
6277 .max => if (self.hasFeature(.avx)) .vmaxsd else .maxsd,6387 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxsd } else .{ ._, .maxsd },
6278 .min => if (self.hasFeature(.avx)) .vminsd else .minsd,6388 .min => if (self.hasFeature(.avx)) .{ ._, .vminsd } else .{ ._, .minsd },
6279 else => unreachable,6389 else => unreachable,
6280 },6390 },
6281 80, 128 => null,6391 80, 128 => null,
...@@ -6291,13 +6401,13 @@ fn genBinOp(...@@ -6291,13 +6401,13 @@ fn genBinOp(
6291 defer self.register_manager.unlockReg(tmp_lock);6401 defer self.register_manager.unlockReg(tmp_lock);
62926402
6293 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(6403 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
6294 .vpinsrw,6404 .{ ._, .vpinsrw },
6295 dst_reg,6405 dst_reg,
6296 dst_reg,6406 dst_reg,
6297 src_mcv.mem(.word),6407 src_mcv.mem(.word),
6298 Immediate.u(1),6408 Immediate.u(1),
6299 ) else try self.asmRegisterRegisterRegister(6409 ) else try self.asmRegisterRegisterRegister(
6300 .vpunpcklwd,6410 .{ ._, .vpunpcklwd },
6301 dst_reg,6411 dst_reg,
6302 dst_reg,6412 dst_reg,
6303 (if (src_mcv.isRegister())6413 (if (src_mcv.isRegister())
...@@ -6305,15 +6415,15 @@ fn genBinOp(...@@ -6305,15 +6415,15 @@ fn genBinOp(
6305 else6415 else
6306 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),6416 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6307 );6417 );
6308 try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);6418 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
6309 try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);6419 try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp_reg, dst_reg);
6310 try self.asmRegisterRegisterRegister(6420 try self.asmRegisterRegisterRegister(
6311 switch (air_tag) {6421 switch (air_tag) {
6312 .add => .vaddss,6422 .add => .{ ._, .vaddss },
6313 .sub => .vsubss,6423 .sub => .{ ._, .vsubss },
6314 .div_float, .div_trunc, .div_floor, .div_exact => .vdivss,6424 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivss },
6315 .max => .vmaxss,6425 .max => .{ ._, .vmaxss },
6316 .min => .vmaxss,6426 .min => .{ ._, .vmaxss },
6317 else => unreachable,6427 else => unreachable,
6318 },6428 },
6319 dst_reg,6429 dst_reg,
...@@ -6321,7 +6431,7 @@ fn genBinOp(...@@ -6321,7 +6431,7 @@ fn genBinOp(
6321 tmp_reg,6431 tmp_reg,
6322 );6432 );
6323 try self.asmRegisterRegisterImmediate(6433 try self.asmRegisterRegisterImmediate(
6324 .vcvtps2ph,6434 .{ ._, .vcvtps2ph },
6325 dst_reg,6435 dst_reg,
6326 dst_reg,6436 dst_reg,
6327 Immediate.u(0b1_00),6437 Immediate.u(0b1_00),
...@@ -6334,12 +6444,12 @@ fn genBinOp(...@@ -6334,12 +6444,12 @@ fn genBinOp(
6334 defer self.register_manager.unlockReg(tmp_lock);6444 defer self.register_manager.unlockReg(tmp_lock);
63356445
6336 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(6446 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
6337 .vpinsrd,6447 .{ ._, .vpinsrd },
6338 dst_reg,6448 dst_reg,
6339 src_mcv.mem(.dword),6449 src_mcv.mem(.dword),
6340 Immediate.u(1),6450 Immediate.u(1),
6341 ) else try self.asmRegisterRegisterRegister(6451 ) else try self.asmRegisterRegisterRegister(
6342 .vunpcklps,6452 .{ ._, .vunpcklps },
6343 dst_reg,6453 dst_reg,
6344 dst_reg,6454 dst_reg,
6345 (if (src_mcv.isRegister())6455 (if (src_mcv.isRegister())
...@@ -6347,15 +6457,20 @@ fn genBinOp(...@@ -6347,15 +6457,20 @@ fn genBinOp(
6347 else6457 else
6348 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),6458 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6349 );6459 );
6350 try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);6460 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
6351 try self.asmRegisterRegisterRegister(.vmovhlps, tmp_reg, dst_reg, dst_reg);6461 try self.asmRegisterRegisterRegister(
6462 .{ ._, .vmovhlps },
6463 tmp_reg,
6464 dst_reg,
6465 dst_reg,
6466 );
6352 try self.asmRegisterRegisterRegister(6467 try self.asmRegisterRegisterRegister(
6353 switch (air_tag) {6468 switch (air_tag) {
6354 .add => .vaddps,6469 .add => .{ ._, .vaddps },
6355 .sub => .vsubps,6470 .sub => .{ ._, .vsubps },
6356 .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,6471 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
6357 .max => .vmaxps,6472 .max => .{ ._, .vmaxps },
6358 .min => .vmaxps,6473 .min => .{ ._, .vmaxps },
6359 else => unreachable,6474 else => unreachable,
6360 },6475 },
6361 dst_reg,6476 dst_reg,
...@@ -6363,7 +6478,7 @@ fn genBinOp(...@@ -6363,7 +6478,7 @@ fn genBinOp(
6363 tmp_reg,6478 tmp_reg,
6364 );6479 );
6365 try self.asmRegisterRegisterImmediate(6480 try self.asmRegisterRegisterImmediate(
6366 .vcvtps2ph,6481 .{ ._, .vcvtps2ph },
6367 dst_reg,6482 dst_reg,
6368 dst_reg,6483 dst_reg,
6369 Immediate.u(0b1_00),6484 Immediate.u(0b1_00),
...@@ -6375,13 +6490,13 @@ fn genBinOp(...@@ -6375,13 +6490,13 @@ fn genBinOp(
6375 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);6490 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6376 defer self.register_manager.unlockReg(tmp_lock);6491 defer self.register_manager.unlockReg(tmp_lock);
63776492
6378 try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);6493 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
6379 if (src_mcv.isMemory()) try self.asmRegisterMemory(6494 if (src_mcv.isMemory()) try self.asmRegisterMemory(
6380 .vcvtph2ps,6495 .{ ._, .vcvtph2ps },
6381 tmp_reg,6496 tmp_reg,
6382 src_mcv.mem(.qword),6497 src_mcv.mem(.qword),
6383 ) else try self.asmRegisterRegister(6498 ) else try self.asmRegisterRegister(
6384 .vcvtph2ps,6499 .{ ._, .vcvtph2ps },
6385 tmp_reg,6500 tmp_reg,
6386 (if (src_mcv.isRegister())6501 (if (src_mcv.isRegister())
6387 src_mcv.getReg().?6502 src_mcv.getReg().?
...@@ -6390,11 +6505,11 @@ fn genBinOp(...@@ -6390,11 +6505,11 @@ fn genBinOp(
6390 );6505 );
6391 try self.asmRegisterRegisterRegister(6506 try self.asmRegisterRegisterRegister(
6392 switch (air_tag) {6507 switch (air_tag) {
6393 .add => .vaddps,6508 .add => .{ ._, .vaddps },
6394 .sub => .vsubps,6509 .sub => .{ ._, .vsubps },
6395 .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,6510 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
6396 .max => .vmaxps,6511 .max => .{ ._, .vmaxps },
6397 .min => .vmaxps,6512 .min => .{ ._, .vmaxps },
6398 else => unreachable,6513 else => unreachable,
6399 },6514 },
6400 dst_reg,6515 dst_reg,
...@@ -6402,7 +6517,7 @@ fn genBinOp(...@@ -6402,7 +6517,7 @@ fn genBinOp(
6402 tmp_reg,6517 tmp_reg,
6403 );6518 );
6404 try self.asmRegisterRegisterImmediate(6519 try self.asmRegisterRegisterImmediate(
6405 .vcvtps2ph,6520 .{ ._, .vcvtps2ph },
6406 dst_reg,6521 dst_reg,
6407 dst_reg,6522 dst_reg,
6408 Immediate.u(0b1_00),6523 Immediate.u(0b1_00),
...@@ -6414,13 +6529,13 @@ fn genBinOp(...@@ -6414,13 +6529,13 @@ fn genBinOp(
6414 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);6529 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6415 defer self.register_manager.unlockReg(tmp_lock);6530 defer self.register_manager.unlockReg(tmp_lock);
64166531
6417 try self.asmRegisterRegister(.vcvtph2ps, dst_reg.to256(), dst_reg);6532 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg.to256(), dst_reg);
6418 if (src_mcv.isMemory()) try self.asmRegisterMemory(6533 if (src_mcv.isMemory()) try self.asmRegisterMemory(
6419 .vcvtph2ps,6534 .{ ._, .vcvtph2ps },
6420 tmp_reg,6535 tmp_reg,
6421 src_mcv.mem(.xword),6536 src_mcv.mem(.xword),
6422 ) else try self.asmRegisterRegister(6537 ) else try self.asmRegisterRegister(
6423 .vcvtph2ps,6538 .{ ._, .vcvtph2ps },
6424 tmp_reg,6539 tmp_reg,
6425 (if (src_mcv.isRegister())6540 (if (src_mcv.isRegister())
6426 src_mcv.getReg().?6541 src_mcv.getReg().?
...@@ -6429,11 +6544,11 @@ fn genBinOp(...@@ -6429,11 +6544,11 @@ fn genBinOp(
6429 );6544 );
6430 try self.asmRegisterRegisterRegister(6545 try self.asmRegisterRegisterRegister(
6431 switch (air_tag) {6546 switch (air_tag) {
6432 .add => .vaddps,6547 .add => .{ ._, .vaddps },
6433 .sub => .vsubps,6548 .sub => .{ ._, .vsubps },
6434 .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,6549 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
6435 .max => .vmaxps,6550 .max => .{ ._, .vmaxps },
6436 .min => .vmaxps,6551 .min => .{ ._, .vmaxps },
6437 else => unreachable,6552 else => unreachable,
6438 },6553 },
6439 dst_reg.to256(),6554 dst_reg.to256(),
...@@ -6441,7 +6556,7 @@ fn genBinOp(...@@ -6441,7 +6556,7 @@ fn genBinOp(
6441 tmp_reg,6556 tmp_reg,
6442 );6557 );
6443 try self.asmRegisterRegisterImmediate(6558 try self.asmRegisterRegisterImmediate(
6444 .vcvtps2ph,6559 .{ ._, .vcvtps2ph },
6445 dst_reg,6560 dst_reg,
6446 dst_reg.to256(),6561 dst_reg.to256(),
6447 Immediate.u(0b1_00),6562 Immediate.u(0b1_00),
...@@ -6452,76 +6567,76 @@ fn genBinOp(...@@ -6452,76 +6567,76 @@ fn genBinOp(
6452 } else null,6567 } else null,
6453 32 => switch (lhs_ty.vectorLen()) {6568 32 => switch (lhs_ty.vectorLen()) {
6454 1 => switch (air_tag) {6569 1 => switch (air_tag) {
6455 .add => if (self.hasFeature(.avx)) .vaddss else .addss,6570 .add => if (self.hasFeature(.avx)) .{ ._, .vaddss } else .{ ._, .addss },
6456 .sub => if (self.hasFeature(.avx)) .vsubss else .subss,6571 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubss } else .{ ._, .subss },
6457 .mul => if (self.hasFeature(.avx)) .vmulss else .mulss,6572 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulss } else .{ ._, .mulss },
6458 .div_float,6573 .div_float,
6459 .div_trunc,6574 .div_trunc,
6460 .div_floor,6575 .div_floor,
6461 .div_exact,6576 .div_exact,
6462 => if (self.hasFeature(.avx)) .vdivss else .divss,6577 => if (self.hasFeature(.avx)) .{ ._, .vdivss } else .{ ._, .divss },
6463 .max => if (self.hasFeature(.avx)) .vmaxss else .maxss,6578 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxss } else .{ ._, .maxss },
6464 .min => if (self.hasFeature(.avx)) .vminss else .minss,6579 .min => if (self.hasFeature(.avx)) .{ ._, .vminss } else .{ ._, .minss },
6465 else => unreachable,6580 else => unreachable,
6466 },6581 },
6467 2...4 => switch (air_tag) {6582 2...4 => switch (air_tag) {
6468 .add => if (self.hasFeature(.avx)) .vaddps else .addps,6583 .add => if (self.hasFeature(.avx)) .{ ._, .vaddps } else .{ ._, .addps },
6469 .sub => if (self.hasFeature(.avx)) .vsubps else .subps,6584 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubps } else .{ ._, .subps },
6470 .mul => if (self.hasFeature(.avx)) .vmulps else .mulps,6585 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulps } else .{ ._, .mulps },
6471 .div_float,6586 .div_float,
6472 .div_trunc,6587 .div_trunc,
6473 .div_floor,6588 .div_floor,
6474 .div_exact,6589 .div_exact,
6475 => if (self.hasFeature(.avx)) .vdivps else .divps,6590 => if (self.hasFeature(.avx)) .{ ._, .vdivps } else .{ ._, .divps },
6476 .max => if (self.hasFeature(.avx)) .vmaxps else .maxps,6591 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxps } else .{ ._, .maxps },
6477 .min => if (self.hasFeature(.avx)) .vminps else .minps,6592 .min => if (self.hasFeature(.avx)) .{ ._, .vminps } else .{ ._, .minps },
6478 else => unreachable,6593 else => unreachable,
6479 },6594 },
6480 5...8 => if (self.hasFeature(.avx)) switch (air_tag) {6595 5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
6481 .add => .vaddps,6596 .add => .{ ._, .vaddps },
6482 .sub => .vsubps,6597 .sub => .{ ._, .vsubps },
6483 .mul => .vmulps,6598 .mul => .{ ._, .vmulps },
6484 .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,6599 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
6485 .max => .vmaxps,6600 .max => .{ ._, .vmaxps },
6486 .min => .vminps,6601 .min => .{ ._, .vminps },
6487 else => unreachable,6602 else => unreachable,
6488 } else null,6603 } else null,
6489 else => null,6604 else => null,
6490 },6605 },
6491 64 => switch (lhs_ty.vectorLen()) {6606 64 => switch (lhs_ty.vectorLen()) {
6492 1 => switch (air_tag) {6607 1 => switch (air_tag) {
6493 .add => if (self.hasFeature(.avx)) .vaddsd else .addsd,6608 .add => if (self.hasFeature(.avx)) .{ ._, .vaddsd } else .{ ._, .addsd },
6494 .sub => if (self.hasFeature(.avx)) .vsubsd else .subsd,6609 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubsd } else .{ ._, .subsd },
6495 .mul => if (self.hasFeature(.avx)) .vmulsd else .mulsd,6610 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulsd } else .{ ._, .mulsd },
6496 .div_float,6611 .div_float,
6497 .div_trunc,6612 .div_trunc,
6498 .div_floor,6613 .div_floor,
6499 .div_exact,6614 .div_exact,
6500 => if (self.hasFeature(.avx)) .vdivsd else .divsd,6615 => if (self.hasFeature(.avx)) .{ ._, .vdivsd } else .{ ._, .divsd },
6501 .max => if (self.hasFeature(.avx)) .vmaxsd else .maxsd,6616 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxsd } else .{ ._, .maxsd },
6502 .min => if (self.hasFeature(.avx)) .vminsd else .minsd,6617 .min => if (self.hasFeature(.avx)) .{ ._, .vminsd } else .{ ._, .minsd },
6503 else => unreachable,6618 else => unreachable,
6504 },6619 },
6505 2 => switch (air_tag) {6620 2 => switch (air_tag) {
6506 .add => if (self.hasFeature(.avx)) .vaddpd else .addpd,6621 .add => if (self.hasFeature(.avx)) .{ ._, .vaddpd } else .{ ._, .addpd },
6507 .sub => if (self.hasFeature(.avx)) .vsubpd else .subpd,6622 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubpd } else .{ ._, .subpd },
6508 .mul => if (self.hasFeature(.avx)) .vmulpd else .mulpd,6623 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulpd } else .{ ._, .mulpd },
6509 .div_float,6624 .div_float,
6510 .div_trunc,6625 .div_trunc,
6511 .div_floor,6626 .div_floor,
6512 .div_exact,6627 .div_exact,
6513 => if (self.hasFeature(.avx)) .vdivpd else .divpd,6628 => if (self.hasFeature(.avx)) .{ ._, .vdivpd } else .{ ._, .divpd },
6514 .max => if (self.hasFeature(.avx)) .vmaxpd else .maxpd,6629 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxpd } else .{ ._, .maxpd },
6515 .min => if (self.hasFeature(.avx)) .vminpd else .minpd,6630 .min => if (self.hasFeature(.avx)) .{ ._, .vminpd } else .{ ._, .minpd },
6516 else => unreachable,6631 else => unreachable,
6517 },6632 },
6518 3...4 => if (self.hasFeature(.avx)) switch (air_tag) {6633 3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
6519 .add => .vaddpd,6634 .add => .{ ._, .vaddpd },
6520 .sub => .vsubpd,6635 .sub => .{ ._, .vsubpd },
6521 .mul => .vmulpd,6636 .mul => .{ ._, .vmulpd },
6522 .div_float, .div_trunc, .div_floor, .div_exact => .vdivpd,6637 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivpd },
6523 .max => .vmaxpd,6638 .max => .{ ._, .vmaxpd },
6524 .min => .vminpd,6639 .min => .{ ._, .vminpd },
6525 else => unreachable,6640 else => unreachable,
6526 } else null,6641 } else null,
6527 else => null,6642 else => null,
...@@ -6583,7 +6698,13 @@ fn genBinOp(...@@ -6583,7 +6698,13 @@ fn genBinOp(
6583 return dst_mcv;6698 return dst_mcv;
6584}6699}
65856700
6586fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {6701fn genBinOpMir(
6702 self: *Self,
6703 mir_tag: Mir.Inst.FixedTag,
6704 ty: Type,
6705 dst_mcv: MCValue,
6706 src_mcv: MCValue,
6707) !void {
6587 const abi_size = @intCast(u32, ty.abiSize(self.target.*));6708 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
6588 switch (dst_mcv) {6709 switch (dst_mcv) {
6589 .none,6710 .none,
...@@ -6788,14 +6909,14 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -6788,14 +6909,14 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
6788 };6909 };
6789 var off: i32 = 0;6910 var off: i32 = 0;
6790 while (off < abi_size) : (off += 8) {6911 while (off < abi_size) : (off += 8) {
6791 const mir_limb_tag = switch (off) {6912 const mir_limb_tag: Mir.Inst.FixedTag = switch (off) {
6792 0 => mir_tag,6913 0 => mir_tag,
6793 else => switch (mir_tag) {6914 else => switch (mir_tag[1]) {
6794 .add => .adc,6915 .add => .{ ._, .adc },
6795 .sub, .cmp => .sbb,6916 .sub, .cmp => .{ ._, .sbb },
6796 .@"or", .@"and", .xor => mir_tag,6917 .@"or", .@"and", .xor => mir_tag,
6797 else => return self.fail("TODO genBinOpMir implement large ABI for {s}", .{6918 else => return self.fail("TODO genBinOpMir implement large ABI for {s}", .{
6798 @tagName(mir_tag),6919 @tagName(mir_tag[1]),
6799 }),6920 }),
6800 },6921 },
6801 };6922 };
...@@ -6967,14 +7088,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M...@@ -6967,14 +7088,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
6967 .reserved_frame,7088 .reserved_frame,
6968 => unreachable,7089 => unreachable,
6969 .register => |src_reg| try self.asmRegisterRegister(7090 .register => |src_reg| try self.asmRegisterRegister(
6970 .imul,7091 .{ .i_, .mul },
6971 dst_alias,7092 dst_alias,
6972 registerAlias(src_reg, abi_size),7093 registerAlias(src_reg, abi_size),
6973 ),7094 ),
6974 .immediate => |imm| {7095 .immediate => |imm| {
6975 if (math.cast(i32, imm)) |small| {7096 if (math.cast(i32, imm)) |small| {
6976 try self.asmRegisterRegisterImmediate(7097 try self.asmRegisterRegisterImmediate(
6977 .imul,7098 .{ .i_, .mul },
6978 dst_alias,7099 dst_alias,
6979 dst_alias,7100 dst_alias,
6980 Immediate.s(small),7101 Immediate.s(small),
...@@ -6994,19 +7115,19 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M...@@ -6994,19 +7115,19 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
6994 .lea_tlv,7115 .lea_tlv,
6995 .lea_frame,7116 .lea_frame,
6996 => try self.asmRegisterRegister(7117 => try self.asmRegisterRegister(
6997 .imul,7118 .{ .i_, .mul },
6998 dst_alias,7119 dst_alias,
6999 registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),7120 registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
7000 ),7121 ),
7001 .memory, .indirect, .load_frame => try self.asmRegisterMemory(7122 .memory, .indirect, .load_frame => try self.asmRegisterMemory(
7002 .imul,7123 .{ .i_, .mul },
7003 dst_alias,7124 dst_alias,
7004 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {7125 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
7005 .memory => |addr| .{7126 .memory => |addr| .{
7006 .base = .{ .reg = .ds },7127 .base = .{ .reg = .ds },
7007 .disp = math.cast(i32, @bitCast(i64, addr)) orelse7128 .disp = math.cast(i32, @bitCast(i64, addr)) orelse
7008 return self.asmRegisterRegister(7129 return self.asmRegisterRegister(
7009 .imul,7130 .{ .i_, .mul },
7010 dst_alias,7131 dst_alias,
7011 registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),7132 registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
7012 ),7133 ),
...@@ -7131,12 +7252,12 @@ fn genVarDbgInfo(...@@ -7131,12 +7252,12 @@ fn genVarDbgInfo(
7131}7252}
71327253
7133fn airTrap(self: *Self) !void {7254fn airTrap(self: *Self) !void {
7134 try self.asmOpOnly(.ud2);7255 try self.asmOpOnly(.{ ._, .ud2 });
7135 return self.finishAirBookkeeping();7256 return self.finishAirBookkeeping();
7136}7257}
71377258
7138fn airBreakpoint(self: *Self) !void {7259fn airBreakpoint(self: *Self) !void {
7139 try self.asmOpOnly(.int3);7260 try self.asmOpOnly(.{ ._, .int3 });
7140 return self.finishAirBookkeeping();7261 return self.finishAirBookkeeping();
7141}7262}
71427263
...@@ -7157,7 +7278,7 @@ fn airFence(self: *Self, inst: Air.Inst.Index) !void {...@@ -7157,7 +7278,7 @@ fn airFence(self: *Self, inst: Air.Inst.Index) !void {
7157 switch (order) {7278 switch (order) {
7158 .Unordered, .Monotonic => unreachable,7279 .Unordered, .Monotonic => unreachable,
7159 .Acquire, .Release, .AcqRel => {},7280 .Acquire, .Release, .AcqRel => {},
7160 .SeqCst => try self.asmOpOnly(.mfence),7281 .SeqCst => try self.asmOpOnly(.{ ._, .mfence }),
7161 }7282 }
7162 return self.finishAirBookkeeping();7283 return self.finishAirBookkeeping();
7163}7284}
...@@ -7251,7 +7372,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -7251,7 +7372,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7251 const atom = elf_file.getAtom(atom_index);7372 const atom = elf_file.getAtom(atom_index);
7252 _ = try atom.getOrCreateOffsetTableEntry(elf_file);7373 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
7253 const got_addr = atom.getOffsetTableAddress(elf_file);7374 const got_addr = atom.getOffsetTableAddress(elf_file);
7254 try self.asmMemory(.call, Memory.sib(.qword, .{7375 try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
7255 .base = .{ .reg = .ds },7376 .base = .{ .reg = .ds },
7256 .disp = @intCast(i32, got_addr),7377 .disp = @intCast(i32, got_addr),
7257 }));7378 }));
...@@ -7259,12 +7380,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -7259,12 +7380,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7259 const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);7380 const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
7260 const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;7381 const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
7261 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });7382 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
7262 try self.asmRegister(.call, .rax);7383 try self.asmRegister(.{ ._, .call }, .rax);
7263 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {7384 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
7264 const atom = try macho_file.getOrCreateAtomForDecl(func.owner_decl);7385 const atom = try macho_file.getOrCreateAtomForDecl(func.owner_decl);
7265 const sym_index = macho_file.getAtom(atom).getSymbolIndex().?;7386 const sym_index = macho_file.getAtom(atom).getSymbolIndex().?;
7266 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });7387 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
7267 try self.asmRegister(.call, .rax);7388 try self.asmRegister(.{ ._, .call }, .rax);
7268 } else if (self.bin_file.cast(link.File.Plan9)) |p9| {7389 } else if (self.bin_file.cast(link.File.Plan9)) |p9| {
7269 const decl_block_index = try p9.seeDecl(func.owner_decl);7390 const decl_block_index = try p9.seeDecl(func.owner_decl);
7270 const decl_block = p9.getDeclBlock(decl_block_index);7391 const decl_block = p9.getDeclBlock(decl_block_index);
...@@ -7273,7 +7394,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -7273,7 +7394,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7273 const got_addr = p9.bases.data;7394 const got_addr = p9.bases.data;
7274 const got_index = decl_block.got_index.?;7395 const got_index = decl_block.got_index.?;
7275 const fn_got_addr = got_addr + got_index * ptr_bytes;7396 const fn_got_addr = got_addr + got_index * ptr_bytes;
7276 try self.asmMemory(.call, Memory.sib(.qword, .{7397 try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
7277 .base = .{ .reg = .ds },7398 .base = .{ .reg = .ds },
7278 .disp = @intCast(i32, fn_got_addr),7399 .disp = @intCast(i32, fn_got_addr),
7279 }));7400 }));
...@@ -7296,7 +7417,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -7296,7 +7417,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7296 }),7417 }),
7297 } },7418 } },
7298 });7419 });
7299 try self.asmRegister(.call, .rax);7420 try self.asmRegister(.{ ._, .call }, .rax);
7300 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {7421 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
7301 const atom_index = try self.owner.getSymbolIndex(self);7422 const atom_index = try self.owner.getSymbolIndex(self);
7302 const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);7423 const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);
...@@ -7318,7 +7439,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -7318,7 +7439,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7318 assert(ty.zigTypeTag() == .Pointer);7439 assert(ty.zigTypeTag() == .Pointer);
7319 const mcv = try self.resolveInst(callee);7440 const mcv = try self.resolveInst(callee);
7320 try self.genSetReg(.rax, Type.usize, mcv);7441 try self.genSetReg(.rax, Type.usize, mcv);
7321 try self.asmRegister(.call, .rax);7442 try self.asmRegister(.{ ._, .call }, .rax);
7322 }7443 }
73237444
7324 var bt = self.liveness.iterateBigTomb(inst);7445 var bt = self.liveness.iterateBigTomb(inst);
...@@ -7408,7 +7529,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -7408,7 +7529,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
7408 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);7529 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
7409 const src_mcv = if (flipped) lhs_mcv else rhs_mcv;7530 const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
74107531
7411 try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv);7532 try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, src_mcv);
7412 break :result Condition.fromCompareOperator(7533 break :result Condition.fromCompareOperator(
7413 if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned,7534 if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned,
7414 if (flipped) op.reverse() else op,7535 if (flipped) op.reverse() else op,
...@@ -7442,13 +7563,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -7442,13 +7563,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
7442 defer self.register_manager.unlockReg(tmp2_lock);7563 defer self.register_manager.unlockReg(tmp2_lock);
74437564
7444 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(7565 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
7445 .vpinsrw,7566 .{ ._, .vpinsrw },
7446 tmp1_reg,7567 tmp1_reg,
7447 dst_reg.to128(),7568 dst_reg.to128(),
7448 src_mcv.mem(.word),7569 src_mcv.mem(.word),
7449 Immediate.u(1),7570 Immediate.u(1),
7450 ) else try self.asmRegisterRegisterRegister(7571 ) else try self.asmRegisterRegisterRegister(
7451 .vpunpcklwd,7572 .{ ._, .vpunpcklwd },
7452 tmp1_reg,7573 tmp1_reg,
7453 dst_reg.to128(),7574 dst_reg.to128(),
7454 (if (src_mcv.isRegister())7575 (if (src_mcv.isRegister())
...@@ -7456,14 +7577,24 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -7456,14 +7577,24 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
7456 else7577 else
7457 try self.copyToTmpRegister(ty, src_mcv)).to128(),7578 try self.copyToTmpRegister(ty, src_mcv)).to128(),
7458 );7579 );
7459 try self.asmRegisterRegister(.vcvtph2ps, tmp1_reg, tmp1_reg);7580 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, tmp1_reg, tmp1_reg);
7460 try self.asmRegisterRegister(.vmovshdup, tmp2_reg, tmp1_reg);7581 try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp2_reg, tmp1_reg);
7461 try self.genBinOpMir(.ucomiss, ty, tmp1_mcv, tmp2_mcv);7582 try self.genBinOpMir(.{ ._, .ucomiss }, ty, tmp1_mcv, tmp2_mcv);
7462 } else return self.fail("TODO implement airCmp for {}", .{7583 } else return self.fail("TODO implement airCmp for {}", .{
7463 ty.fmt(self.bin_file.options.module.?),7584 ty.fmt(self.bin_file.options.module.?),
7464 }),7585 }),
7465 32 => try self.genBinOpMir(.ucomiss, ty, .{ .register = dst_reg }, src_mcv),7586 32 => try self.genBinOpMir(
7466 64 => try self.genBinOpMir(.ucomisd, ty, .{ .register = dst_reg }, src_mcv),7587 .{ ._, .ucomiss },
7588 ty,
7589 .{ .register = dst_reg },
7590 src_mcv,
7591 ),
7592 64 => try self.genBinOpMir(
7593 .{ ._, .ucomisd },
7594 ty,
7595 .{ .register = dst_reg },
7596 src_mcv,
7597 ),
7467 else => return self.fail("TODO implement airCmp for {}", .{7598 else => return self.fail("TODO implement airCmp for {}", .{
7468 ty.fmt(self.bin_file.options.module.?),7599 ty.fmt(self.bin_file.options.module.?),
7469 }),7600 }),
...@@ -7507,7 +7638,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {...@@ -7507,7 +7638,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
7507 else => try self.copyToTmpRegister(op_ty, op_mcv),7638 else => try self.copyToTmpRegister(op_ty, op_mcv),
7508 };7639 };
7509 try self.asmRegisterMemory(7640 try self.asmRegisterMemory(
7510 .cmp,7641 .{ ._, .cmp },
7511 registerAlias(dst_reg, op_abi_size),7642 registerAlias(dst_reg, op_abi_size),
7512 Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),7643 Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),
7513 );7644 );
...@@ -7627,7 +7758,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {...@@ -7627,7 +7758,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
7627 },7758 },
7628 .register => |reg| {7759 .register => |reg| {
7629 try self.spillEflagsIfOccupied();7760 try self.spillEflagsIfOccupied();
7630 try self.asmRegisterImmediate(.@"test", reg, Immediate.u(1));7761 try self.asmRegisterImmediate(.{ ._, .@"test" }, reg, Immediate.u(1));
7631 return self.asmJccReloc(undefined, .e);7762 return self.asmJccReloc(undefined, .e);
7632 },7763 },
7633 .immediate,7764 .immediate,
...@@ -7730,13 +7861,13 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC...@@ -7730,13 +7861,13 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
7730 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));7861 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
7731 const alias_reg = registerAlias(opt_reg, some_abi_size);7862 const alias_reg = registerAlias(opt_reg, some_abi_size);
7732 assert(some_abi_size * 8 == alias_reg.bitSize());7863 assert(some_abi_size * 8 == alias_reg.bitSize());
7733 try self.asmRegisterRegister(.@"test", alias_reg, alias_reg);7864 try self.asmRegisterRegister(.{ ._, .@"test" }, alias_reg, alias_reg);
7734 return .{ .eflags = .z };7865 return .{ .eflags = .z };
7735 }7866 }
7736 assert(some_info.ty.tag() == .bool);7867 assert(some_info.ty.tag() == .bool);
7737 const opt_abi_size = @intCast(u32, opt_ty.abiSize(self.target.*));7868 const opt_abi_size = @intCast(u32, opt_ty.abiSize(self.target.*));
7738 try self.asmRegisterImmediate(7869 try self.asmRegisterImmediate(
7739 .bt,7870 .{ ._, .bt },
7740 registerAlias(opt_reg, opt_abi_size),7871 registerAlias(opt_reg, opt_abi_size),
7741 Immediate.u(@intCast(u6, some_info.off * 8)),7872 Immediate.u(@intCast(u6, some_info.off * 8)),
7742 );7873 );
...@@ -7755,7 +7886,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC...@@ -7755,7 +7886,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
7755 try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());7886 try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());
7756 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));7887 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
7757 try self.asmMemoryImmediate(7888 try self.asmMemoryImmediate(
7758 .cmp,7889 .{ ._, .cmp },
7759 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{7890 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
7760 .base = .{ .reg = addr_reg },7891 .base = .{ .reg = addr_reg },
7761 .disp = some_info.off,7892 .disp = some_info.off,
...@@ -7768,7 +7899,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC...@@ -7768,7 +7899,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
7768 .indirect, .load_frame => {7899 .indirect, .load_frame => {
7769 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));7900 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
7770 try self.asmMemoryImmediate(7901 try self.asmMemoryImmediate(
7771 .cmp,7902 .{ ._, .cmp },
7772 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {7903 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
7773 .indirect => |reg_off| .{7904 .indirect => |reg_off| .{
7774 .base = .{ .reg = reg_off.reg },7905 .base = .{ .reg = reg_off.reg },
...@@ -7810,7 +7941,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)...@@ -7810,7 +7941,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
78107941
7811 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));7942 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
7812 try self.asmMemoryImmediate(7943 try self.asmMemoryImmediate(
7813 .cmp,7944 .{ ._, .cmp },
7814 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{7945 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
7815 .base = .{ .reg = ptr_reg },7946 .base = .{ .reg = ptr_reg },
7816 .disp = some_info.off,7947 .disp = some_info.off,
...@@ -7841,14 +7972,24 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !...@@ -7841,14 +7972,24 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
7841 const tmp_reg = try self.copyToTmpRegister(ty, operand);7972 const tmp_reg = try self.copyToTmpRegister(ty, operand);
7842 if (err_off > 0) {7973 if (err_off > 0) {
7843 const shift = @intCast(u6, err_off * 8);7974 const shift = @intCast(u6, err_off * 8);
7844 try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{ .immediate = shift });7975 try self.genShiftBinOpMir(
7976 .{ ._r, .sh },
7977 ty,
7978 .{ .register = tmp_reg },
7979 .{ .immediate = shift },
7980 );
7845 } else {7981 } else {
7846 try self.truncateRegister(Type.anyerror, tmp_reg);7982 try self.truncateRegister(Type.anyerror, tmp_reg);
7847 }7983 }
7848 try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });7984 try self.genBinOpMir(
7985 .{ ._, .cmp },
7986 Type.anyerror,
7987 .{ .register = tmp_reg },
7988 .{ .immediate = 0 },
7989 );
7849 },7990 },
7850 .load_frame => |frame_addr| try self.genBinOpMir(7991 .load_frame => |frame_addr| try self.genBinOpMir(
7851 .cmp,7992 .{ ._, .cmp },
7852 Type.anyerror,7993 Type.anyerror,
7853 .{ .load_frame = .{7994 .{ .load_frame = .{
7854 .index = frame_addr.index,7995 .index = frame_addr.index,
...@@ -8073,7 +8214,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -8073,7 +8214,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
8073 try self.spillEflagsIfOccupied();8214 try self.spillEflagsIfOccupied();
8074 for (items, relocs, 0..) |item, *reloc, i| {8215 for (items, relocs, 0..) |item, *reloc, i| {
8075 const item_mcv = try self.resolveInst(item);8216 const item_mcv = try self.resolveInst(item);
8076 try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);8217 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);
8077 reloc.* = try self.asmJccReloc(undefined, if (i < relocs.len - 1) .e else .ne);8218 reloc.* = try self.asmJccReloc(undefined, if (i < relocs.len - 1) .e else .ne);
8078 }8219 }
80798220
...@@ -8284,7 +8425,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -8284,7 +8425,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
8284 .qword8425 .qword
8285 else8426 else
8286 null;8427 null;
8287 const mnem = mnem: {8428 const mnem_tag = Mir.Inst.FixedTag{ ._, mnem: {
8288 if (mnem_size) |_| {8429 if (mnem_size) |_| {
8289 if (std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])) |mnem| {8430 if (std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])) |mnem| {
8290 break :mnem mnem;8431 break :mnem mnem;
...@@ -8292,7 +8433,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -8292,7 +8433,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
8292 }8433 }
8293 break :mnem std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse8434 break :mnem std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse
8294 return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});8435 return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});
8295 };8436 } };
82968437
8297 var op_it = mem.tokenize(u8, mnem_it.rest(), ",");8438 var op_it = mem.tokenize(u8, mnem_it.rest(), ",");
8298 var ops = [1]encoder.Instruction.Operand{.none} ** 4;8439 var ops = [1]encoder.Instruction.Operand{.none} ** 4;
...@@ -8343,51 +8484,51 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -8343,51 +8484,51 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
8343 } else if (op_it.next()) |op_str| return self.fail("Extra operand: '{s}'", .{op_str});8484 } else if (op_it.next()) |op_str| return self.fail("Extra operand: '{s}'", .{op_str});
83448485
8345 (switch (ops[0]) {8486 (switch (ops[0]) {
8346 .none => self.asmOpOnly(mnem),8487 .none => self.asmOpOnly(mnem_tag),
8347 .reg => |reg0| switch (ops[1]) {8488 .reg => |reg0| switch (ops[1]) {
8348 .none => self.asmRegister(mnem, reg0),8489 .none => self.asmRegister(mnem_tag, reg0),
8349 .reg => |reg1| switch (ops[2]) {8490 .reg => |reg1| switch (ops[2]) {
8350 .none => self.asmRegisterRegister(mnem, reg1, reg0),8491 .none => self.asmRegisterRegister(mnem_tag, reg1, reg0),
8351 .reg => |reg2| switch (ops[3]) {8492 .reg => |reg2| switch (ops[3]) {
8352 .none => self.asmRegisterRegisterRegister(mnem, reg2, reg1, reg0),8493 .none => self.asmRegisterRegisterRegister(mnem_tag, reg2, reg1, reg0),
8353 else => error.InvalidInstruction,8494 else => error.InvalidInstruction,
8354 },8495 },
8355 .mem => |mem2| switch (ops[3]) {8496 .mem => |mem2| switch (ops[3]) {
8356 .none => self.asmMemoryRegisterRegister(mnem, mem2, reg1, reg0),8497 .none => self.asmMemoryRegisterRegister(mnem_tag, mem2, reg1, reg0),
8357 else => error.InvalidInstruction,8498 else => error.InvalidInstruction,
8358 },8499 },
8359 else => error.InvalidInstruction,8500 else => error.InvalidInstruction,
8360 },8501 },
8361 .mem => |mem1| switch (ops[2]) {8502 .mem => |mem1| switch (ops[2]) {
8362 .none => self.asmMemoryRegister(mnem, mem1, reg0),8503 .none => self.asmMemoryRegister(mnem_tag, mem1, reg0),
8363 else => error.InvalidInstruction,8504 else => error.InvalidInstruction,
8364 },8505 },
8365 else => error.InvalidInstruction,8506 else => error.InvalidInstruction,
8366 },8507 },
8367 .mem => |mem0| switch (ops[1]) {8508 .mem => |mem0| switch (ops[1]) {
8368 .none => self.asmMemory(mnem, mem0),8509 .none => self.asmMemory(mnem_tag, mem0),
8369 .reg => |reg1| switch (ops[2]) {8510 .reg => |reg1| switch (ops[2]) {
8370 .none => self.asmRegisterMemory(mnem, reg1, mem0),8511 .none => self.asmRegisterMemory(mnem_tag, reg1, mem0),
8371 else => error.InvalidInstruction,8512 else => error.InvalidInstruction,
8372 },8513 },
8373 else => error.InvalidInstruction,8514 else => error.InvalidInstruction,
8374 },8515 },
8375 .imm => |imm0| switch (ops[1]) {8516 .imm => |imm0| switch (ops[1]) {
8376 .none => self.asmImmediate(mnem, imm0),8517 .none => self.asmImmediate(mnem_tag, imm0),
8377 .reg => |reg1| switch (ops[2]) {8518 .reg => |reg1| switch (ops[2]) {
8378 .none => self.asmRegisterImmediate(mnem, reg1, imm0),8519 .none => self.asmRegisterImmediate(mnem_tag, reg1, imm0),
8379 .reg => |reg2| switch (ops[3]) {8520 .reg => |reg2| switch (ops[3]) {
8380 .none => self.asmRegisterRegisterImmediate(mnem, reg2, reg1, imm0),8521 .none => self.asmRegisterRegisterImmediate(mnem_tag, reg2, reg1, imm0),
8381 else => error.InvalidInstruction,8522 else => error.InvalidInstruction,
8382 },8523 },
8383 .mem => |mem2| switch (ops[3]) {8524 .mem => |mem2| switch (ops[3]) {
8384 .none => self.asmMemoryRegisterImmediate(mnem, mem2, reg1, imm0),8525 .none => self.asmMemoryRegisterImmediate(mnem_tag, mem2, reg1, imm0),
8385 else => error.InvalidInstruction,8526 else => error.InvalidInstruction,
8386 },8527 },
8387 else => error.InvalidInstruction,8528 else => error.InvalidInstruction,
8388 },8529 },
8389 .mem => |mem1| switch (ops[2]) {8530 .mem => |mem1| switch (ops[2]) {
8390 .none => self.asmMemoryImmediate(mnem, mem1, imm0),8531 .none => self.asmMemoryImmediate(mnem_tag, mem1, imm0),
8391 else => error.InvalidInstruction,8532 else => error.InvalidInstruction,
8392 },8533 },
8393 else => error.InvalidInstruction,8534 else => error.InvalidInstruction,
...@@ -8396,7 +8537,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -8396,7 +8537,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
8396 error.InvalidInstruction => return self.fail(8537 error.InvalidInstruction => return self.fail(
8397 "Invalid instruction: '{s} {s} {s} {s} {s}'",8538 "Invalid instruction: '{s} {s} {s} {s} {s}'",
8398 .{8539 .{
8399 @tagName(mnem),8540 @tagName(mnem_tag[1]),
8400 @tagName(ops[0]),8541 @tagName(ops[0]),
8401 @tagName(ops[1]),8542 @tagName(ops[1]),
8402 @tagName(ops[2]),8543 @tagName(ops[2]),
...@@ -8427,44 +8568,47 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -8427,44 +8568,47 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
8427 return self.finishAirResult(inst, result);8568 return self.finishAirResult(inst, result);
8428}8569}
84298570
8430fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.Tag {8571fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.FixedTag {
8431 switch (ty.zigTypeTag()) {8572 switch (ty.zigTypeTag()) {
8432 else => return .mov,8573 else => return .{ ._, .mov },
8433 .Float => switch (ty.floatBits(self.target.*)) {8574 .Float => switch (ty.floatBits(self.target.*)) {
8434 16 => unreachable, // needs special handling8575 16 => unreachable, // needs special handling
8435 32 => return if (self.hasFeature(.avx)) .vmovss else .movss,8576 32 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
8436 64 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,8577 64 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
8437 128 => return if (self.hasFeature(.avx))8578 128 => return if (self.hasFeature(.avx))
8438 if (aligned) .vmovaps else .vmovups8579 if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
8439 else if (aligned) .movaps else .movups,8580 else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
8440 else => {},8581 else => {},
8441 },8582 },
8442 .Vector => switch (ty.childType().zigTypeTag()) {8583 .Vector => switch (ty.childType().zigTypeTag()) {
8443 .Float => switch (ty.childType().floatBits(self.target.*)) {8584 .Float => switch (ty.childType().floatBits(self.target.*)) {
8444 16 => switch (ty.vectorLen()) {8585 16 => switch (ty.vectorLen()) {
8445 1 => unreachable, // needs special handling8586 1 => unreachable, // needs special handling
8446 2 => return if (self.hasFeature(.avx)) .vmovss else .movss,8587 2 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
8447 3...4 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,8588 3...4 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
8448 5...8 => return if (self.hasFeature(.avx))8589 5...8 => return if (self.hasFeature(.avx))
8449 if (aligned) .vmovaps else .vmovups8590 if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
8450 else if (aligned) .movaps else .movups,8591 else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
8451 9...16 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,8592 9...16 => if (self.hasFeature(.avx))
8593 return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
8452 else => {},8594 else => {},
8453 },8595 },
8454 32 => switch (ty.vectorLen()) {8596 32 => switch (ty.vectorLen()) {
8455 1 => return if (self.hasFeature(.avx)) .vmovss else .movss,8597 1 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
8456 2...4 => return if (self.hasFeature(.avx))8598 2...4 => return if (self.hasFeature(.avx))
8457 if (aligned) .vmovaps else .vmovups8599 if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
8458 else if (aligned) .movaps else .movups,8600 else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
8459 5...8 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,8601 5...8 => if (self.hasFeature(.avx))
8602 return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
8460 else => {},8603 else => {},
8461 },8604 },
8462 64 => switch (ty.vectorLen()) {8605 64 => switch (ty.vectorLen()) {
8463 1 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,8606 1 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
8464 2 => return if (self.hasFeature(.avx))8607 2 => return if (self.hasFeature(.avx))
8465 if (aligned) .vmovaps else .vmovups8608 if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
8466 else if (aligned) .movaps else .movups,8609 else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
8467 3...4 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,8610 3...4 => if (self.hasFeature(.avx))
8611 return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
8468 else => {},8612 else => {},
8469 },8613 },
8470 else => {},8614 else => {},
...@@ -8558,19 +8702,19 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -8558,19 +8702,19 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8558 if (imm == 0) {8702 if (imm == 0) {
8559 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit8703 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
8560 // register is the fastest way to zero a register.8704 // register is the fastest way to zero a register.
8561 try self.asmRegisterRegister(.xor, dst_reg.to32(), dst_reg.to32());8705 try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
8562 } else if (abi_size > 4 and math.cast(u32, imm) != null) {8706 } else if (abi_size > 4 and math.cast(u32, imm) != null) {
8563 // 32-bit moves zero-extend to 64-bit.8707 // 32-bit moves zero-extend to 64-bit.
8564 try self.asmRegisterImmediate(.mov, dst_reg.to32(), Immediate.u(imm));8708 try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(imm));
8565 } else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {8709 } else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
8566 try self.asmRegisterImmediate(8710 try self.asmRegisterImmediate(
8567 .mov,8711 .{ ._, .mov },
8568 registerAlias(dst_reg, abi_size),8712 registerAlias(dst_reg, abi_size),
8569 Immediate.s(@intCast(i32, @bitCast(i64, imm))),8713 Immediate.s(@intCast(i32, @bitCast(i64, imm))),
8570 );8714 );
8571 } else {8715 } else {
8572 try self.asmRegisterImmediate(8716 try self.asmRegisterImmediate(
8573 .mov,8717 .{ ._, .mov },
8574 registerAlias(dst_reg, abi_size),8718 registerAlias(dst_reg, abi_size),
8575 Immediate.u(imm),8719 Immediate.u(imm),
8576 );8720 );
...@@ -8579,18 +8723,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -8579,18 +8723,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8579 .register => |src_reg| if (dst_reg.id() != src_reg.id()) try self.asmRegisterRegister(8723 .register => |src_reg| if (dst_reg.id() != src_reg.id()) try self.asmRegisterRegister(
8580 if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))8724 if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))
8581 switch (ty.zigTypeTag()) {8725 switch (ty.zigTypeTag()) {
8582 else => .mov,8726 else => .{ ._, .mov },
8583 .Float, .Vector => .movaps,8727 .Float, .Vector => .{ ._, .movaps },
8584 }8728 }
8585 else switch (abi_size) {8729 else switch (abi_size) {
8586 2 => return try self.asmRegisterRegisterImmediate(8730 2 => return try self.asmRegisterRegisterImmediate(
8587 if (dst_reg.class() == .floating_point) .pinsrw else .pextrw,8731 if (dst_reg.class() == .floating_point) .{ ._, .pinsrw } else .{ ._, .pextrw },
8588 registerAlias(dst_reg, 4),8732 registerAlias(dst_reg, 4),
8589 registerAlias(src_reg, 4),8733 registerAlias(src_reg, 4),
8590 Immediate.u(0),8734 Immediate.u(0),
8591 ),8735 ),
8592 4 => .movd,8736 4 => .{ ._d, .mov },
8593 8 => .movq,8737 8 => .{ ._q, .mov },
8594 else => return self.fail(8738 else => return self.fail(
8595 "unsupported register copy from {s} to {s}",8739 "unsupported register copy from {s} to {s}",
8596 .{ @tagName(src_reg), @tagName(dst_reg) },8740 .{ @tagName(src_reg), @tagName(dst_reg) },
...@@ -8617,7 +8761,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -8617,7 +8761,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8617 });8761 });
8618 if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)8762 if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8619 try self.asmRegisterMemoryImmediate(8763 try self.asmRegisterMemoryImmediate(
8620 .pinsrw,8764 .{ ._, .pinsrw },
8621 registerAlias(dst_reg, abi_size),8765 registerAlias(dst_reg, abi_size),
8622 src_mem,8766 src_mem,
8623 Immediate.u(0),8767 Immediate.u(0),
...@@ -8627,14 +8771,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -8627,14 +8771,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8627 switch (src_mcv) {8771 switch (src_mcv) {
8628 .register_offset => |reg_off| switch (reg_off.off) {8772 .register_offset => |reg_off| switch (reg_off.off) {
8629 0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),8773 0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
8630 else => .lea,8774 else => .{ ._, .lea },
8631 },8775 },
8632 .indirect => try self.movMirTag(ty, false),8776 .indirect => try self.movMirTag(ty, false),
8633 .load_frame => |frame_addr| try self.movMirTag(8777 .load_frame => |frame_addr| try self.movMirTag(
8634 ty,8778 ty,
8635 self.getFrameAddrAlignment(frame_addr) >= ty.abiAlignment(self.target.*),8779 self.getFrameAddrAlignment(frame_addr) >= ty.abiAlignment(self.target.*),
8636 ),8780 ),
8637 .lea_frame => .lea,8781 .lea_frame => .{ ._, .lea },
8638 else => unreachable,8782 else => unreachable,
8639 },8783 },
8640 registerAlias(dst_reg, abi_size),8784 registerAlias(dst_reg, abi_size),
...@@ -8650,7 +8794,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -8650,7 +8794,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8650 });8794 });
8651 return if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)8795 return if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8652 self.asmRegisterMemoryImmediate(8796 self.asmRegisterMemoryImmediate(
8653 .pinsrw,8797 .{ ._, .pinsrw },
8654 registerAlias(dst_reg, abi_size),8798 registerAlias(dst_reg, abi_size),
8655 src_mem,8799 src_mem,
8656 Immediate.u(0),8800 Immediate.u(0),
...@@ -8694,7 +8838,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -8694,7 +8838,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8694 });8838 });
8695 if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)8839 if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8696 try self.asmRegisterMemoryImmediate(8840 try self.asmRegisterMemoryImmediate(
8697 .pinsrw,8841 .{ ._, .pinsrw },
8698 registerAlias(dst_reg, abi_size),8842 registerAlias(dst_reg, abi_size),
8699 src_mem,8843 src_mem,
8700 Immediate.u(0),8844 Immediate.u(0),
...@@ -8743,7 +8887,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -8743,7 +8887,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8743 } },8887 } },
8744 });8888 });
8745 // TODO: spill registers before calling8889 // TODO: spill registers before calling
8746 try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));8890 try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
8747 try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });8891 try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });
8748 } else return self.fail("TODO emit ptr to TLV sequence on {s}", .{8892 } else return self.fail("TODO emit ptr to TLV sequence on {s}", .{
8749 @tagName(self.bin_file.tag),8893 @tagName(self.bin_file.tag),
...@@ -8770,7 +8914,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal...@@ -8770,7 +8914,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
8770 else8914 else
8771 Immediate.u(@intCast(u32, imm));8915 Immediate.u(@intCast(u32, imm));
8772 try self.asmMemoryImmediate(8916 try self.asmMemoryImmediate(
8773 .mov,8917 .{ ._, .mov },
8774 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),8918 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
8775 immediate,8919 immediate,
8776 );8920 );
...@@ -8778,14 +8922,14 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal...@@ -8778,14 +8922,14 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
8778 3, 5...7 => unreachable,8922 3, 5...7 => unreachable,
8779 else => if (math.cast(i32, @bitCast(i64, imm))) |small| {8923 else => if (math.cast(i32, @bitCast(i64, imm))) |small| {
8780 try self.asmMemoryImmediate(8924 try self.asmMemoryImmediate(
8781 .mov,8925 .{ ._, .mov },
8782 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),8926 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
8783 Immediate.s(small),8927 Immediate.s(small),
8784 );8928 );
8785 } else {8929 } else {
8786 var offset: i32 = 0;8930 var offset: i32 = 0;
8787 while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(8931 while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
8788 .mov,8932 .{ ._, .mov },
8789 Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),8933 Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
8790 if (ty.isSignedInt())8934 if (ty.isSignedInt())
8791 Immediate.s(@truncate(8935 Immediate.s(@truncate(
...@@ -8808,7 +8952,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal...@@ -8808,7 +8952,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
8808 );8952 );
8809 if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)8953 if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8810 try self.asmMemoryRegisterImmediate(8954 try self.asmMemoryRegisterImmediate(
8811 .pextrw,8955 .{ ._, .pextrw },
8812 dst_mem,8956 dst_mem,
8813 src_reg.to128(),8957 src_reg.to128(),
8814 Immediate.u(0),8958 Immediate.u(0),
...@@ -8904,7 +9048,7 @@ fn genInlineMemcpyRegisterRegister(...@@ -8904,7 +9048,7 @@ fn genInlineMemcpyRegisterRegister(
8904 while (remainder > 0) {9048 while (remainder > 0) {
8905 const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));9049 const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
8906 try self.asmMemoryRegister(9050 try self.asmMemoryRegister(
8907 .mov,9051 .{ ._, .mov },
8908 Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{9052 Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
8909 .base = dst_reg,9053 .base = dst_reg,
8910 .disp = -next_offset,9054 .disp = -next_offset,
...@@ -8913,7 +9057,7 @@ fn genInlineMemcpyRegisterRegister(...@@ -8913,7 +9057,7 @@ fn genInlineMemcpyRegisterRegister(
8913 );9057 );
89149058
8915 if (nearest_power_of_two > 1) {9059 if (nearest_power_of_two > 1) {
8916 try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{9060 try self.genShiftBinOpMir(.{ ._r, .sh }, ty, .{ .register = tmp_reg }, .{
8917 .immediate = nearest_power_of_two * 8,9061 .immediate = nearest_power_of_two * 8,
8918 });9062 });
8919 }9063 }
...@@ -8924,8 +9068,8 @@ fn genInlineMemcpyRegisterRegister(...@@ -8924,8 +9068,8 @@ fn genInlineMemcpyRegisterRegister(
8924 } else {9068 } else {
8925 try self.asmMemoryRegister(9069 try self.asmMemoryRegister(
8926 switch (src_reg.class()) {9070 switch (src_reg.class()) {
8927 .general_purpose, .segment => .mov,9071 .general_purpose, .segment => .{ ._, .mov },
8928 .floating_point => .movss,9072 .floating_point => .{ ._, .movss },
8929 },9073 },
8930 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),9074 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
8931 registerAlias(src_reg, abi_size),9075 registerAlias(src_reg, abi_size),
...@@ -8938,11 +9082,7 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue...@@ -8938,11 +9082,7 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue
8938 try self.genSetReg(.rdi, Type.usize, dst_ptr);9082 try self.genSetReg(.rdi, Type.usize, dst_ptr);
8939 try self.genSetReg(.rsi, Type.usize, src_ptr);9083 try self.genSetReg(.rsi, Type.usize, src_ptr);
8940 try self.genSetReg(.rcx, Type.usize, len);9084 try self.genSetReg(.rcx, Type.usize, len);
8941 _ = try self.addInst(.{9085 try self.asmOpOnly(.{ .@"rep _sb", .mov });
8942 .tag = .mov,
8943 .ops = .none,
8944 .data = .{ .none = .{ .fixes = .@"rep _sb" } },
8945 });
8946}9086}
89479087
8948fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {9088fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {
...@@ -8950,11 +9090,7 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)...@@ -8950,11 +9090,7 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)
8950 try self.genSetReg(.rdi, Type.usize, dst_ptr);9090 try self.genSetReg(.rdi, Type.usize, dst_ptr);
8951 try self.genSetReg(.al, Type.u8, value);9091 try self.genSetReg(.al, Type.u8, value);
8952 try self.genSetReg(.rcx, Type.usize, len);9092 try self.genSetReg(.rcx, Type.usize, len);
8953 _ = try self.addInst(.{9093 try self.asmOpOnly(.{ .@"rep _sb", .sto });
8954 .tag = .sto,
8955 .ops = .none,
8956 .data = .{ .none = .{ .fixes = .@"rep _sb" } },
8957 });
8958}9094}
89599095
8960fn genLazySymbolRef(9096fn genLazySymbolRef(
...@@ -8972,14 +9108,14 @@ fn genLazySymbolRef(...@@ -8972,14 +9108,14 @@ fn genLazySymbolRef(
8972 const got_mem =9108 const got_mem =
8973 Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) });9109 Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) });
8974 switch (tag) {9110 switch (tag) {
8975 .lea, .mov => try self.asmRegisterMemory(.mov, reg.to64(), got_mem),9111 .lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), got_mem),
8976 .call => try self.asmMemory(.call, got_mem),9112 .call => try self.asmMemory(.{ ._, .call }, got_mem),
8977 else => unreachable,9113 else => unreachable,
8978 }9114 }
8979 switch (tag) {9115 switch (tag) {
8980 .lea, .call => {},9116 .lea, .call => {},
8981 .mov => try self.asmRegisterMemory(9117 .mov => try self.asmRegisterMemory(
8982 tag,9118 .{ ._, tag },
8983 reg.to64(),9119 reg.to64(),
8984 Memory.sib(.qword, .{ .base = .{ .reg = reg.to64() } }),9120 Memory.sib(.qword, .{ .base = .{ .reg = reg.to64() } }),
8985 ),9121 ),
...@@ -8996,7 +9132,7 @@ fn genLazySymbolRef(...@@ -8996,7 +9132,7 @@ fn genLazySymbolRef(
8996 }9132 }
8997 switch (tag) {9133 switch (tag) {
8998 .lea, .mov => {},9134 .lea, .mov => {},
8999 .call => try self.asmRegister(.call, reg),9135 .call => try self.asmRegister(.{ ._, .call }, reg),
9000 else => unreachable,9136 else => unreachable,
9001 }9137 }
9002 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {9138 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
...@@ -9010,7 +9146,7 @@ fn genLazySymbolRef(...@@ -9010,7 +9146,7 @@ fn genLazySymbolRef(
9010 }9146 }
9011 switch (tag) {9147 switch (tag) {
9012 .lea, .mov => {},9148 .lea, .mov => {},
9013 .call => try self.asmRegister(.call, reg),9149 .call => try self.asmRegister(.{ ._, .call }, reg),
9014 else => unreachable,9150 else => unreachable,
9015 }9151 }
9016 } else {9152 } else {
...@@ -9115,13 +9251,13 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {...@@ -9115,13 +9251,13 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
91159251
9116 try self.asmRegisterRegister(switch (dst_ty.floatBits(self.target.*)) {9252 try self.asmRegisterRegister(switch (dst_ty.floatBits(self.target.*)) {
9117 32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))9253 32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
9118 .cvtsi2ss9254 .{ ._, .cvtsi2ss }
9119 else9255 else
9120 return self.fail("TODO implement airIntToFloat from {} to {} without sse", .{9256 return self.fail("TODO implement airIntToFloat from {} to {} without sse", .{
9121 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),9257 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
9122 }),9258 }),
9123 64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))9259 64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
9124 .cvtsi2sd9260 .{ ._, .cvtsi2sd }
9125 else9261 else
9126 return self.fail("TODO implement airIntToFloat from {} to {} without sse2", .{9262 return self.fail("TODO implement airIntToFloat from {} to {} without sse2", .{
9127 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),9263 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
...@@ -9161,7 +9297,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {...@@ -9161,7 +9297,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
9161 },9297 },
9162 };9298 };
9163 try self.asmMemory(9299 try self.asmMemory(
9164 .fld,9300 .{ .f_, .ld },
9165 Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{9301 Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
9166 .base = .{ .frame = frame_addr.index },9302 .base = .{ .frame = frame_addr.index },
9167 .disp = frame_addr.off,9303 .disp = frame_addr.off,
...@@ -9171,7 +9307,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {...@@ -9171,7 +9307,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
9171 // convert9307 // convert
9172 const stack_dst = try self.allocRegOrMem(inst, false);9308 const stack_dst = try self.allocRegOrMem(inst, false);
9173 try self.asmMemory(9309 try self.asmMemory(
9174 .fisttp,9310 .{ .f_p, .istt },
9175 Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{9311 Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
9176 .base = .{ .frame = stack_dst.load_frame.index },9312 .base = .{ .frame = stack_dst.load_frame.index },
9177 .disp = stack_dst.load_frame.off,9313 .disp = stack_dst.load_frame.off,
...@@ -9227,22 +9363,11 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -9227,22 +9363,11 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
9227 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);9363 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
92289364
9229 try self.spillEflagsIfOccupied();9365 try self.spillEflagsIfOccupied();
9230 _ = try self.addInst(if (val_abi_size <= 8) .{9366 if (val_abi_size <= 8) try self.asmMemoryRegister(
9231 .tag = .cmpxchg,9367 .{ .@"lock _", .cmpxchg },
9232 .ops = .mr_sib,9368 ptr_mem,
9233 .data = .{ .rx = .{9369 registerAlias(new_reg.?, val_abi_size),
9234 .fixes = .@"lock _",9370 ) else try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
9235 .r1 = registerAlias(new_reg.?, val_abi_size),
9236 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9237 } },
9238 } else .{
9239 .tag = .cmpxchg,
9240 .ops = .m_sib,
9241 .data = .{ .x = .{
9242 .fixes = .@"lock _16b",
9243 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9244 } },
9245 });
92469371
9247 const result: MCValue = result: {9372 const result: MCValue = result: {
9248 if (self.liveness.isUnused(inst)) break :result .unreach;9373 if (self.liveness.isUnused(inst)) break :result .unreach;
...@@ -9340,21 +9465,17 @@ fn atomicOp(...@@ -9340,21 +9465,17 @@ fn atomicOp(
93409465
9341 try self.genSetReg(dst_reg, val_ty, val_mcv);9466 try self.genSetReg(dst_reg, val_ty, val_mcv);
9342 if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {9467 if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
9343 try self.genUnOpMir(.neg, val_ty, dst_mcv);9468 try self.genUnOpMir(.{ ._, .neg }, val_ty, dst_mcv);
9344 }9469 }
9345 _ = try self.addInst(.{9470 try self.asmMemoryRegister(
9346 .tag = tag,9471 switch (tag) {
9347 .ops = .mr_sib,9472 .mov, .xchg => .{ ._, tag },
9348 .data = .{ .rx = .{9473 .xadd, .add, .sub, .@"and", .@"or", .xor => .{ .@"lock _", tag },
9349 .fixes = switch (tag) {9474 else => unreachable,
9350 .mov, .xchg => ._,9475 },
9351 .xadd, .add, .sub, .@"and", .@"or", .xor => .@"lock _",9476 ptr_mem,
9352 else => unreachable,9477 registerAlias(dst_reg, val_abi_size),
9353 },9478 );
9354 .r1 = registerAlias(dst_reg, val_abi_size),
9355 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9356 } },
9357 });
93589479
9359 return if (unused) .unreach else dst_mcv;9480 return if (unused) .unreach else dst_mcv;
9360 },9481 },
...@@ -9364,22 +9485,22 @@ fn atomicOp(...@@ -9364,22 +9485,22 @@ fn atomicOp(
9364 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);9485 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
9365 defer self.register_manager.unlockReg(tmp_lock);9486 defer self.register_manager.unlockReg(tmp_lock);
93669487
9367 try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);9488 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);
9368 const loop = @intCast(u32, self.mir_instructions.len);9489 const loop = @intCast(u32, self.mir_instructions.len);
9369 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {9490 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
9370 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });9491 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
9371 }9492 }
9372 if (rmw_op) |op| switch (op) {9493 if (rmw_op) |op| switch (op) {
9373 .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),9494 .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
9374 .Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),9495 .Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv),
9375 .Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),9496 .Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv),
9376 .And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),9497 .And => try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv),
9377 .Nand => {9498 .Nand => {
9378 try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv);9499 try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv);
9379 try self.genUnOpMir(.not, val_ty, tmp_mcv);9500 try self.genUnOpMir(.{ ._, .not }, val_ty, tmp_mcv);
9380 },9501 },
9381 .Or => try self.genBinOpMir(.@"or", val_ty, tmp_mcv, val_mcv),9502 .Or => try self.genBinOpMir(.{ ._, .@"or" }, val_ty, tmp_mcv, val_mcv),
9382 .Xor => try self.genBinOpMir(.xor, val_ty, tmp_mcv, val_mcv),9503 .Xor => try self.genBinOpMir(.{ ._, .xor }, val_ty, tmp_mcv, val_mcv),
9383 .Min, .Max => {9504 .Min, .Max => {
9384 const cc: Condition = switch (if (val_ty.isAbiInt())9505 const cc: Condition = switch (if (val_ty.isAbiInt())
9385 val_ty.intInfo(self.target.*).signedness9506 val_ty.intInfo(self.target.*).signedness
...@@ -9397,7 +9518,7 @@ fn atomicOp(...@@ -9397,7 +9518,7 @@ fn atomicOp(
9397 },9518 },
9398 };9519 };
93999520
9400 try self.genBinOpMir(.cmp, val_ty, tmp_mcv, val_mcv);9521 try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
9401 const cmov_abi_size = @max(val_abi_size, 2);9522 const cmov_abi_size = @max(val_abi_size, 2);
9402 switch (val_mcv) {9523 switch (val_mcv) {
9403 .register => |val_reg| try self.asmCmovccRegisterRegister(9524 .register => |val_reg| try self.asmCmovccRegisterRegister(
...@@ -9421,24 +9542,20 @@ fn atomicOp(...@@ -9421,24 +9542,20 @@ fn atomicOp(
9421 }9542 }
9422 },9543 },
9423 };9544 };
9424 _ = try self.addInst(.{9545 try self.asmMemoryRegister(
9425 .tag = .cmpxchg,9546 .{ .@"lock _", .cmpxchg },
9426 .ops = .mr_sib,9547 ptr_mem,
9427 .data = .{ .rx = .{9548 registerAlias(tmp_reg, val_abi_size),
9428 .fixes = .@"lock _",9549 );
9429 .r1 = registerAlias(tmp_reg, val_abi_size),
9430 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9431 } },
9432 });
9433 _ = try self.asmJccReloc(loop, .ne);9550 _ = try self.asmJccReloc(loop, .ne);
9434 return if (unused) .unreach else .{ .register = .rax };9551 return if (unused) .unreach else .{ .register = .rax };
9435 } else {9552 } else {
9436 try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{9553 try self.asmRegisterMemory(.{ ._, .mov }, .rax, Memory.sib(.qword, .{
9437 .base = ptr_mem.sib.base,9554 .base = ptr_mem.sib.base,
9438 .scale_index = ptr_mem.scaleIndex(),9555 .scale_index = ptr_mem.scaleIndex(),
9439 .disp = ptr_mem.sib.disp + 0,9556 .disp = ptr_mem.sib.disp + 0,
9440 }));9557 }));
9441 try self.asmRegisterMemory(.mov, .rdx, Memory.sib(.qword, .{9558 try self.asmRegisterMemory(.{ ._, .mov }, .rdx, Memory.sib(.qword, .{
9442 .base = ptr_mem.sib.base,9559 .base = ptr_mem.sib.base,
9443 .scale_index = ptr_mem.scaleIndex(),9560 .scale_index = ptr_mem.scaleIndex(),
9444 .disp = ptr_mem.sib.disp + 8,9561 .disp = ptr_mem.sib.disp + 8,
...@@ -9453,58 +9570,51 @@ fn atomicOp(...@@ -9453,58 +9570,51 @@ fn atomicOp(
9453 const val_lo_mem = val_mem_mcv.mem(.qword);9570 const val_lo_mem = val_mem_mcv.mem(.qword);
9454 const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);9571 const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);
9455 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {9572 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
9456 try self.asmRegisterRegister(.mov, .rbx, .rax);9573 try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
9457 try self.asmRegisterRegister(.mov, .rcx, .rdx);9574 try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
9458 }9575 }
9459 if (rmw_op) |op| switch (op) {9576 if (rmw_op) |op| switch (op) {
9460 .Xchg => {9577 .Xchg => {
9461 try self.asmRegisterMemory(.mov, .rbx, val_lo_mem);9578 try self.asmRegisterMemory(.{ ._, .mov }, .rbx, val_lo_mem);
9462 try self.asmRegisterMemory(.mov, .rcx, val_hi_mem);9579 try self.asmRegisterMemory(.{ ._, .mov }, .rcx, val_hi_mem);
9463 },9580 },
9464 .Add => {9581 .Add => {
9465 try self.asmRegisterMemory(.add, .rbx, val_lo_mem);9582 try self.asmRegisterMemory(.{ ._, .add }, .rbx, val_lo_mem);
9466 try self.asmRegisterMemory(.adc, .rcx, val_hi_mem);9583 try self.asmRegisterMemory(.{ ._, .adc }, .rcx, val_hi_mem);
9467 },9584 },
9468 .Sub => {9585 .Sub => {
9469 try self.asmRegisterMemory(.sub, .rbx, val_lo_mem);9586 try self.asmRegisterMemory(.{ ._, .sub }, .rbx, val_lo_mem);
9470 try self.asmRegisterMemory(.sbb, .rcx, val_hi_mem);9587 try self.asmRegisterMemory(.{ ._, .sbb }, .rcx, val_hi_mem);
9471 },9588 },
9472 .And => {9589 .And => {
9473 try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);9590 try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem);
9474 try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);9591 try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem);
9475 },9592 },
9476 .Nand => {9593 .Nand => {
9477 try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);9594 try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem);
9478 try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);9595 try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem);
9479 try self.asmRegister(.not, .rbx);9596 try self.asmRegister(.{ ._, .not }, .rbx);
9480 try self.asmRegister(.not, .rcx);9597 try self.asmRegister(.{ ._, .not }, .rcx);
9481 },9598 },
9482 .Or => {9599 .Or => {
9483 try self.asmRegisterMemory(.@"or", .rbx, val_lo_mem);9600 try self.asmRegisterMemory(.{ ._, .@"or" }, .rbx, val_lo_mem);
9484 try self.asmRegisterMemory(.@"or", .rcx, val_hi_mem);9601 try self.asmRegisterMemory(.{ ._, .@"or" }, .rcx, val_hi_mem);
9485 },9602 },
9486 .Xor => {9603 .Xor => {
9487 try self.asmRegisterMemory(.xor, .rbx, val_lo_mem);9604 try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem);
9488 try self.asmRegisterMemory(.xor, .rcx, val_hi_mem);9605 try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem);
9489 },9606 },
9490 else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{9607 else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{
9491 val_ty.fmt(self.bin_file.options.module.?), @tagName(op),9608 val_ty.fmt(self.bin_file.options.module.?), @tagName(op),
9492 }),9609 }),
9493 };9610 };
9494 _ = try self.addInst(.{9611 try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
9495 .tag = .cmpxchg,
9496 .ops = .m_sib,
9497 .data = .{ .x = .{
9498 .fixes = .@"lock _16b",
9499 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9500 } },
9501 });
9502 _ = try self.asmJccReloc(loop, .ne);9612 _ = try self.asmJccReloc(loop, .ne);
95039613
9504 if (unused) return .unreach;9614 if (unused) return .unreach;
9505 const dst_mcv = try self.allocTempRegOrMem(val_ty, false);9615 const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
9506 try self.asmMemoryRegister(9616 try self.asmMemoryRegister(
9507 .mov,9617 .{ ._, .mov },
9508 Memory.sib(.qword, .{9618 Memory.sib(.qword, .{
9509 .base = .{ .frame = dst_mcv.load_frame.index },9619 .base = .{ .frame = dst_mcv.load_frame.index },
9510 .disp = dst_mcv.load_frame.off + 0,9620 .disp = dst_mcv.load_frame.off + 0,
...@@ -9512,7 +9622,7 @@ fn atomicOp(...@@ -9512,7 +9622,7 @@ fn atomicOp(
9512 .rax,9622 .rax,
9513 );9623 );
9514 try self.asmMemoryRegister(9624 try self.asmMemoryRegister(
9515 .mov,9625 .{ ._, .mov },
9516 Memory.sib(.qword, .{9626 Memory.sib(.qword, .{
9517 .base = .{ .frame = dst_mcv.load_frame.index },9627 .base = .{ .frame = dst_mcv.load_frame.index },
9518 .disp = dst_mcv.load_frame.off + 8,9628 .disp = dst_mcv.load_frame.off + 8,
...@@ -9664,8 +9774,13 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -9664,8 +9774,13 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
9664 .off = elem_abi_size,9774 .off = elem_abi_size,
9665 } });9775 } });
96669776
9667 try self.genBinOpMir(.sub, Type.usize, len_mcv, .{ .immediate = 1 });9777 try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 });
9668 try self.asmRegisterRegisterImmediate(.imul, len_reg, len_reg, Immediate.u(elem_abi_size));9778 try self.asmRegisterRegisterImmediate(
9779 .{ .i_, .mul },
9780 len_reg,
9781 len_reg,
9782 Immediate.u(elem_abi_size),
9783 );
9669 try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);9784 try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);
96709785
9671 try self.performReloc(skip_reloc);9786 try self.performReloc(skip_reloc);
...@@ -9803,7 +9918,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {...@@ -9803,7 +9918,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9803 try self.truncateRegister(err_ty, err_reg.to32());9918 try self.truncateRegister(err_ty, err_reg.to32());
98049919
9805 try self.asmRegisterMemory(9920 try self.asmRegisterMemory(
9806 .mov,9921 .{ ._, .mov },
9807 start_reg.to32(),9922 start_reg.to32(),
9808 Memory.sib(.dword, .{9923 Memory.sib(.dword, .{
9809 .base = .{ .reg = addr_reg.to64() },9924 .base = .{ .reg = addr_reg.to64() },
...@@ -9812,7 +9927,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {...@@ -9812,7 +9927,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9812 }),9927 }),
9813 );9928 );
9814 try self.asmRegisterMemory(9929 try self.asmRegisterMemory(
9815 .mov,9930 .{ ._, .mov },
9816 end_reg.to32(),9931 end_reg.to32(),
9817 Memory.sib(.dword, .{9932 Memory.sib(.dword, .{
9818 .base = .{ .reg = addr_reg.to64() },9933 .base = .{ .reg = addr_reg.to64() },
...@@ -9820,9 +9935,9 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {...@@ -9820,9 +9935,9 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9820 .disp = 8,9935 .disp = 8,
9821 }),9936 }),
9822 );9937 );
9823 try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());9938 try self.asmRegisterRegister(.{ ._, .sub }, end_reg.to32(), start_reg.to32());
9824 try self.asmRegisterMemory(9939 try self.asmRegisterMemory(
9825 .lea,9940 .{ ._, .lea },
9826 start_reg.to64(),9941 start_reg.to64(),
9827 Memory.sib(.byte, .{9942 Memory.sib(.byte, .{
9828 .base = .{ .reg = addr_reg.to64() },9943 .base = .{ .reg = addr_reg.to64() },
...@@ -9831,7 +9946,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {...@@ -9831,7 +9946,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9831 }),9946 }),
9832 );9947 );
9833 try self.asmRegisterMemory(9948 try self.asmRegisterMemory(
9834 .lea,9949 .{ ._, .lea },
9835 end_reg.to32(),9950 end_reg.to32(),
9836 Memory.sib(.byte, .{9951 Memory.sib(.byte, .{
9837 .base = .{ .reg = end_reg.to64() },9952 .base = .{ .reg = end_reg.to64() },
...@@ -9841,7 +9956,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {...@@ -9841,7 +9956,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
98419956
9842 const dst_mcv = try self.allocRegOrMem(inst, false);9957 const dst_mcv = try self.allocRegOrMem(inst, false);
9843 try self.asmMemoryRegister(9958 try self.asmMemoryRegister(
9844 .mov,9959 .{ ._, .mov },
9845 Memory.sib(.qword, .{9960 Memory.sib(.qword, .{
9846 .base = .{ .frame = dst_mcv.load_frame.index },9961 .base = .{ .frame = dst_mcv.load_frame.index },
9847 .disp = dst_mcv.load_frame.off,9962 .disp = dst_mcv.load_frame.off,
...@@ -9849,7 +9964,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {...@@ -9849,7 +9964,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9849 start_reg.to64(),9964 start_reg.to64(),
9850 );9965 );
9851 try self.asmMemoryRegister(9966 try self.asmMemoryRegister(
9852 .mov,9967 .{ ._, .mov },
9853 Memory.sib(.qword, .{9968 Memory.sib(.qword, .{
9854 .base = .{ .frame = dst_mcv.load_frame.index },9969 .base = .{ .frame = dst_mcv.load_frame.index },
9855 .disp = dst_mcv.load_frame.off + 8,9970 .disp = dst_mcv.load_frame.off + 8,
...@@ -9945,13 +10060,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -9945,13 +10060,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
9945 try self.truncateRegister(elem_ty, elem_reg);10060 try self.truncateRegister(elem_ty, elem_reg);
9946 }10061 }
9947 if (elem_bit_off > 0) try self.genShiftBinOpMir(10062 if (elem_bit_off > 0) try self.genShiftBinOpMir(
9948 .shl,10063 .{ ._l, .sh },
9949 elem_ty,10064 elem_ty,
9950 .{ .register = elem_reg },10065 .{ .register = elem_reg },
9951 .{ .immediate = elem_bit_off },10066 .{ .immediate = elem_bit_off },
9952 );10067 );
9953 try self.genBinOpMir(10068 try self.genBinOpMir(
9954 .@"or",10069 .{ ._, .@"or" },
9955 elem_ty,10070 elem_ty,
9956 .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },10071 .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
9957 .{ .register = elem_reg },10072 .{ .register = elem_reg },
...@@ -9962,13 +10077,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -9962,13 +10077,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
9962 try self.truncateRegister(elem_ty, registerAlias(reg, elem_abi_size));10077 try self.truncateRegister(elem_ty, registerAlias(reg, elem_abi_size));
9963 }10078 }
9964 try self.genShiftBinOpMir(10079 try self.genShiftBinOpMir(
9965 .shr,10080 .{ ._r, .sh },
9966 elem_ty,10081 elem_ty,
9967 .{ .register = reg },10082 .{ .register = reg },
9968 .{ .immediate = elem_abi_bits - elem_bit_off },10083 .{ .immediate = elem_abi_bits - elem_bit_off },
9969 );10084 );
9970 try self.genBinOpMir(10085 try self.genBinOpMir(
9971 .@"or",10086 .{ ._, .@"or" },
9972 elem_ty,10087 elem_ty,
9973 .{ .load_frame = .{10088 .{ .load_frame = .{
9974 .index = frame_index,10089 .index = frame_index,
...@@ -10078,25 +10193,25 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {...@@ -10078,25 +10193,25 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
10078 }10193 }
1007910194
10080 const mir_tag = if (@as(10195 const mir_tag = if (@as(
10081 ?Mir.Inst.Tag,10196 ?Mir.Inst.FixedTag,
10082 if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))10197 if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))
10083 switch (ty.zigTypeTag()) {10198 switch (ty.zigTypeTag()) {
10084 .Float => switch (ty.floatBits(self.target.*)) {10199 .Float => switch (ty.floatBits(self.target.*)) {
10085 32 => .vfmadd132ss,10200 32 => .{ ._, .vfmadd132ss },
10086 64 => .vfmadd132sd,10201 64 => .{ ._, .vfmadd132sd },
10087 16, 80, 128 => null,10202 16, 80, 128 => null,
10088 else => unreachable,10203 else => unreachable,
10089 },10204 },
10090 .Vector => switch (ty.childType().zigTypeTag()) {10205 .Vector => switch (ty.childType().zigTypeTag()) {
10091 .Float => switch (ty.childType().floatBits(self.target.*)) {10206 .Float => switch (ty.childType().floatBits(self.target.*)) {
10092 32 => switch (ty.vectorLen()) {10207 32 => switch (ty.vectorLen()) {
10093 1 => .vfmadd132ss,10208 1 => .{ ._, .vfmadd132ss },
10094 2...8 => .vfmadd132ps,10209 2...8 => .{ ._, .vfmadd132ps },
10095 else => null,10210 else => null,
10096 },10211 },
10097 64 => switch (ty.vectorLen()) {10212 64 => switch (ty.vectorLen()) {
10098 1 => .vfmadd132sd,10213 1 => .{ ._, .vfmadd132sd },
10099 2...4 => .vfmadd132pd,10214 2...4 => .{ ._, .vfmadd132pd },
10100 else => null,10215 else => null,
10101 },10216 },
10102 16, 80, 128 => null,10217 16, 80, 128 => null,
...@@ -10109,21 +10224,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {...@@ -10109,21 +10224,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
10109 else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))10224 else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
10110 switch (ty.zigTypeTag()) {10225 switch (ty.zigTypeTag()) {
10111 .Float => switch (ty.floatBits(self.target.*)) {10226 .Float => switch (ty.floatBits(self.target.*)) {
10112 32 => .vfmadd213ss,10227 32 => .{ ._, .vfmadd213ss },
10113 64 => .vfmadd213sd,10228 64 => .{ ._, .vfmadd213sd },
10114 16, 80, 128 => null,10229 16, 80, 128 => null,
10115 else => unreachable,10230 else => unreachable,
10116 },10231 },
10117 .Vector => switch (ty.childType().zigTypeTag()) {10232 .Vector => switch (ty.childType().zigTypeTag()) {
10118 .Float => switch (ty.childType().floatBits(self.target.*)) {10233 .Float => switch (ty.childType().floatBits(self.target.*)) {
10119 32 => switch (ty.vectorLen()) {10234 32 => switch (ty.vectorLen()) {
10120 1 => .vfmadd213ss,10235 1 => .{ ._, .vfmadd213ss },
10121 2...8 => .vfmadd213ps,10236 2...8 => .{ ._, .vfmadd213ps },
10122 else => null,10237 else => null,
10123 },10238 },
10124 64 => switch (ty.vectorLen()) {10239 64 => switch (ty.vectorLen()) {
10125 1 => .vfmadd213sd,10240 1 => .{ ._, .vfmadd213sd },
10126 2...4 => .vfmadd213pd,10241 2...4 => .{ ._, .vfmadd213pd },
10127 else => null,10242 else => null,
10128 },10243 },
10129 16, 80, 128 => null,10244 16, 80, 128 => null,
...@@ -10136,21 +10251,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {...@@ -10136,21 +10251,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
10136 else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))10251 else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
10137 switch (ty.zigTypeTag()) {10252 switch (ty.zigTypeTag()) {
10138 .Float => switch (ty.floatBits(self.target.*)) {10253 .Float => switch (ty.floatBits(self.target.*)) {
10139 32 => .vfmadd231ss,10254 32 => .{ ._, .vfmadd231ss },
10140 64 => .vfmadd231sd,10255 64 => .{ ._, .vfmadd231sd },
10141 16, 80, 128 => null,10256 16, 80, 128 => null,
10142 else => unreachable,10257 else => unreachable,
10143 },10258 },
10144 .Vector => switch (ty.childType().zigTypeTag()) {10259 .Vector => switch (ty.childType().zigTypeTag()) {
10145 .Float => switch (ty.childType().floatBits(self.target.*)) {10260 .Float => switch (ty.childType().floatBits(self.target.*)) {
10146 32 => switch (ty.vectorLen()) {10261 32 => switch (ty.vectorLen()) {
10147 1 => .vfmadd231ss,10262 1 => .{ ._, .vfmadd231ss },
10148 2...8 => .vfmadd231ps,10263 2...8 => .{ ._, .vfmadd231ps },
10149 else => null,10264 else => null,
10150 },10265 },
10151 64 => switch (ty.vectorLen()) {10266 64 => switch (ty.vectorLen()) {
10152 1 => .vfmadd231sd,10267 1 => .{ ._, .vfmadd231sd },
10153 2...4 => .vfmadd231pd,10268 2...4 => .{ ._, .vfmadd231pd },
10154 else => null,10269 else => null,
10155 },10270 },
10156 16, 80, 128 => null,10271 16, 80, 128 => null,
...@@ -10522,17 +10637,37 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {...@@ -10522,17 +10637,37 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
10522 switch (int_info.signedness) {10637 switch (int_info.signedness) {
10523 .signed => {10638 .signed => {
10524 const shift = @intCast(u6, max_reg_bit_width - int_info.bits);10639 const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
10525 try self.genShiftBinOpMir(.sal, Type.isize, .{ .register = reg }, .{ .immediate = shift });10640 try self.genShiftBinOpMir(
10526 try self.genShiftBinOpMir(.sar, Type.isize, .{ .register = reg }, .{ .immediate = shift });10641 .{ ._l, .sa },
10642 Type.isize,
10643 .{ .register = reg },
10644 .{ .immediate = shift },
10645 );
10646 try self.genShiftBinOpMir(
10647 .{ ._r, .sa },
10648 Type.isize,
10649 .{ .register = reg },
10650 .{ .immediate = shift },
10651 );
10527 },10652 },
10528 .unsigned => {10653 .unsigned => {
10529 const shift = @intCast(u6, max_reg_bit_width - int_info.bits);10654 const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
10530 const mask = (~@as(u64, 0)) >> shift;10655 const mask = (~@as(u64, 0)) >> shift;
10531 if (int_info.bits <= 32) {10656 if (int_info.bits <= 32) {
10532 try self.genBinOpMir(.@"and", Type.u32, .{ .register = reg }, .{ .immediate = mask });10657 try self.genBinOpMir(
10658 .{ ._, .@"and" },
10659 Type.u32,
10660 .{ .register = reg },
10661 .{ .immediate = mask },
10662 );
10533 } else {10663 } else {
10534 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });10664 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
10535 try self.genBinOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .register = tmp_reg });10665 try self.genBinOpMir(
10666 .{ ._, .@"and" },
10667 Type.usize,
10668 .{ .register = reg },
10669 .{ .register = tmp_reg },
10670 );
10536 }10671 }
10537 },10672 },
10538 }10673 }
src/arch/x86_64/Lower.zig+6-3
...@@ -286,10 +286,10 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -286,10 +286,10 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
286 .rri_s, .rri_u => inst.data.rri.fixes,286 .rri_s, .rri_u => inst.data.rri.fixes,
287 .ri_s, .ri_u => inst.data.ri.fixes,287 .ri_s, .ri_u => inst.data.ri.fixes,
288 .ri64, .rm_sib, .rm_rip, .mr_sib, .mr_rip => inst.data.rx.fixes,288 .ri64, .rm_sib, .rm_rip, .mr_sib, .mr_rip => inst.data.rx.fixes,
289 .mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => ._,
290 .mrr_sib, .mrr_rip, .rrm_sib, .rrm_rip => inst.data.rrx.fixes,289 .mrr_sib, .mrr_rip, .rrm_sib, .rrm_rip => inst.data.rrx.fixes,
291 .rmi_sib, .rmi_rip, .mri_sib, .mri_rip => inst.data.rix.fixes,290 .rmi_sib, .rmi_rip, .mri_sib, .mri_rip => inst.data.rix.fixes,
292 .rrmi_sib, .rrmi_rip => inst.data.rrix.fixes,291 .rrmi_sib, .rrmi_rip => inst.data.rrix.fixes,
292 .mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => inst.data.x.fixes,
293 .m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes,293 .m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes,
294 .extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._,294 .extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._,
295 else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),295 else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),
...@@ -356,8 +356,11 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -356,8 +356,11 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
356 .{ .mem = lower.mem(inst.ops, inst.data.x.payload) },356 .{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
357 },357 },
358 .mi_sib_s, .mi_sib_u, .mi_rip_u, .mi_rip_s => &.{358 .mi_sib_s, .mi_sib_u, .mi_rip_u, .mi_rip_s => &.{
359 .{ .mem = lower.mem(inst.ops, inst.data.ix.payload) },359 .{ .mem = lower.mem(inst.ops, inst.data.x.payload + 1) },
360 .{ .imm = lower.imm(inst.ops, inst.data.ix.i) },360 .{ .imm = lower.imm(
361 inst.ops,
362 lower.mir.extraData(Mir.Imm32, inst.data.x.payload).data.imm,
363 ) },
361 },364 },
362 .rm_sib, .rm_rip => &.{365 .rm_sib, .rm_rip => &.{
363 .{ .reg = inst.data.rx.r1 },366 .{ .reg = inst.data.rx.r1 },
src/arch/x86_64/Mir.zig+62-38
...@@ -36,6 +36,18 @@ pub const Inst = struct {...@@ -36,6 +36,18 @@ pub const Inst = struct {
36 /// ___36 /// ___
37 @"_",37 @"_",
3838
39 /// Integer __
40 i_,
41
42 /// ___ Left
43 _l,
44 /// ___ Left Double
45 _ld,
46 /// ___ Right
47 _r,
48 /// ___ Right Double
49 _rd,
50
39 /// ___ Above51 /// ___ Above
40 _a,52 _a,
41 /// ___ Above Or Equal53 /// ___ Above Or Equal
...@@ -53,7 +65,7 @@ pub const Inst = struct {...@@ -53,7 +65,7 @@ pub const Inst = struct {
53 /// ___ Greater Or Equal65 /// ___ Greater Or Equal
54 _ge,66 _ge,
55 /// ___ Less67 /// ___ Less
56 _l,68 //_l,
57 /// ___ Less Or Equal69 /// ___ Less Or Equal
58 _le,70 _le,
59 /// ___ Not Above71 /// ___ Not Above
...@@ -97,6 +109,15 @@ pub const Inst = struct {...@@ -97,6 +109,15 @@ pub const Inst = struct {
97 /// ___ Zero109 /// ___ Zero
98 _z,110 _z,
99111
112 /// ___ Byte
113 //_b,
114 /// ___ Word
115 _w,
116 /// ___ Doubleword
117 _d,
118 /// ___ QuadWord
119 _q,
120
100 /// ___ String121 /// ___ String
101 //_s,122 //_s,
102 /// ___ String Byte123 /// ___ String Byte
...@@ -165,6 +186,18 @@ pub const Inst = struct {...@@ -165,6 +186,18 @@ pub const Inst = struct {
165186
166 /// Locked ___187 /// Locked ___
167 @"lock _",188 @"lock _",
189 /// ___ And Complement
190 //_c,
191 /// Locked ___ And Complement
192 @"lock _c",
193 /// ___ And Reset
194 //_r,
195 /// Locked ___ And Reset
196 @"lock _r",
197 /// ___ And Set
198 //_s,
199 /// Locked ___ And Set
200 @"lock _s",
168 /// ___ 8 Bytes201 /// ___ 8 Bytes
169 _8b,202 _8b,
170 /// Locked ___ 8 Bytes203 /// Locked ___ 8 Bytes
...@@ -174,6 +207,11 @@ pub const Inst = struct {...@@ -174,6 +207,11 @@ pub const Inst = struct {
174 /// Locked ___ 16 Bytes207 /// Locked ___ 16 Bytes
175 @"lock _16b",208 @"lock _16b",
176209
210 /// Float ___
211 f_,
212 /// Float ___ Pop
213 f_p,
214
177 /// Packed ___215 /// Packed ___
178 p_,216 p_,
179 /// Packed ___ Byte217 /// Packed ___ Byte
...@@ -250,13 +288,10 @@ pub const Inst = struct {...@@ -250,13 +288,10 @@ pub const Inst = struct {
250 /// Byte swap288 /// Byte swap
251 bswap,289 bswap,
252 /// Bit test290 /// Bit test
253 bt,
254 /// Bit test and complement291 /// Bit test and complement
255 btc,
256 /// Bit test and reset292 /// Bit test and reset
257 btr,
258 /// Bit test and set293 /// Bit test and set
259 bts,294 bt,
260 /// Call295 /// Call
261 call,296 call,
262 /// Convert byte to word297 /// Convert byte to word
...@@ -280,21 +315,18 @@ pub const Inst = struct {...@@ -280,21 +315,18 @@ pub const Inst = struct {
280 /// Convert word to doubleword315 /// Convert word to doubleword
281 cwde,316 cwde,
282 /// Unsigned division317 /// Unsigned division
283 div,
284 /// Store integer with truncation
285 fisttp,
286 /// Load floating-point value
287 fld,
288 /// Signed division318 /// Signed division
289 idiv,319 div,
290 /// Signed multiplication
291 imul,
292 ///320 ///
293 int3,321 int3,
322 /// Store integer with truncation
323 istt,
294 /// Conditional jump324 /// Conditional jump
295 j,325 j,
296 /// Jump326 /// Jump
297 jmp,327 jmp,
328 /// Load floating-point value
329 ld,
298 /// Load effective address330 /// Load effective address
299 lea,331 lea,
300 /// Load string332 /// Load string
...@@ -307,20 +339,17 @@ pub const Inst = struct {...@@ -307,20 +339,17 @@ pub const Inst = struct {
307 mfence,339 mfence,
308 /// Move340 /// Move
309 /// Move data from string to string341 /// Move data from string to string
342 /// Move doubleword
343 /// Move quadword
310 mov,344 mov,
311 /// Move data after swapping bytes345 /// Move data after swapping bytes
312 movbe,346 movbe,
313 /// Move doubleword
314 movd,
315 /// Move quadword
316 movq,
317 /// Move with sign extension347 /// Move with sign extension
318 movsx,348 movsx,
319 /// Move with sign extension
320 movsxd,
321 /// Move with zero extension349 /// Move with zero extension
322 movzx,350 movzx,
323 /// Multiply351 /// Multiply
352 /// Signed multiplication
324 mul,353 mul,
325 /// Two's complement negation354 /// Two's complement negation
326 neg,355 neg,
...@@ -337,19 +366,16 @@ pub const Inst = struct {...@@ -337,19 +366,16 @@ pub const Inst = struct {
337 /// Push366 /// Push
338 push,367 push,
339 /// Rotate left through carry368 /// Rotate left through carry
340 rcl,
341 /// Rotate right through carry369 /// Rotate right through carry
342 rcr,370 rc,
343 /// Return371 /// Return
344 ret,372 ret,
345 /// Rotate left373 /// Rotate left
346 rol,
347 /// Rotate right374 /// Rotate right
348 ror,375 ro,
349 /// Arithmetic shift left376 /// Arithmetic shift left
350 sal,
351 /// Arithmetic shift right377 /// Arithmetic shift right
352 sar,378 sa,
353 /// Integer subtraction with borrow379 /// Integer subtraction with borrow
354 sbb,380 sbb,
355 /// Scan string381 /// Scan string
...@@ -359,13 +385,10 @@ pub const Inst = struct {...@@ -359,13 +385,10 @@ pub const Inst = struct {
359 /// Store fence385 /// Store fence
360 sfence,386 sfence,
361 /// Logical shift left387 /// Logical shift left
362 shl,
363 /// Double precision shift left388 /// Double precision shift left
364 shld,
365 /// Logical shift right389 /// Logical shift right
366 shr,
367 /// Double precision shift right390 /// Double precision shift right
368 shrd,391 sh,
369 /// Subtract392 /// Subtract
370 sub,393 sub,
371 /// Store string394 /// Store string
...@@ -730,6 +753,8 @@ pub const Inst = struct {...@@ -730,6 +753,8 @@ pub const Inst = struct {
730 pseudo,753 pseudo,
731 };754 };
732755
756 pub const FixedTag = struct { Fixes, Tag };
757
733 pub const Ops = enum(u8) {758 pub const Ops = enum(u8) {
734 /// No data associated with this instruction (only mnemonic is used).759 /// No data associated with this instruction (only mnemonic is used).
735 none,760 none,
...@@ -800,16 +825,16 @@ pub const Inst = struct {...@@ -800,16 +825,16 @@ pub const Inst = struct {
800 /// Uses `x` with extra data of type `MemoryRip`.825 /// Uses `x` with extra data of type `MemoryRip`.
801 m_rip,826 m_rip,
802 /// Memory (SIB), immediate (unsigned) operands.827 /// Memory (SIB), immediate (unsigned) operands.
803 /// Uses `ix` payload with extra data of type `MemorySib`.828 /// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
804 mi_sib_u,829 mi_sib_u,
805 /// Memory (RIP), immediate (unsigned) operands.830 /// Memory (RIP), immediate (unsigned) operands.
806 /// Uses `ix` payload with extra data of type `MemoryRip`.831 /// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
807 mi_rip_u,832 mi_rip_u,
808 /// Memory (SIB), immediate (sign-extend) operands.833 /// Memory (SIB), immediate (sign-extend) operands.
809 /// Uses `ix` payload with extra data of type `MemorySib`.834 /// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
810 mi_sib_s,835 mi_sib_s,
811 /// Memory (RIP), immediate (sign-extend) operands.836 /// Memory (RIP), immediate (sign-extend) operands.
812 /// Uses `ix` payload with extra data of type `MemoryRip`.837 /// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
813 mi_rip_s,838 mi_rip_s,
814 /// Memory (SIB), register operands.839 /// Memory (SIB), register operands.
815 /// Uses `rx` payload with extra data of type `MemorySib`.840 /// Uses `rx` payload with extra data of type `MemorySib`.
...@@ -974,11 +999,6 @@ pub const Inst = struct {...@@ -974,11 +999,6 @@ pub const Inst = struct {
974 r1: Register,999 r1: Register,
975 payload: u32,1000 payload: u32,
976 },1001 },
977 /// Immediate, followed by Custom payload found in extra.
978 ix: struct {
979 i: u32,
980 payload: u32,
981 },
982 /// Register, register, followed by Custom payload found in extra.1002 /// Register, register, followed by Custom payload found in extra.
983 rrx: struct {1003 rrx: struct {
984 fixes: Fixes = ._,1004 fixes: Fixes = ._,
...@@ -1081,6 +1101,10 @@ pub const RegisterList = struct {...@@ -1081,6 +1101,10 @@ pub const RegisterList = struct {
1081 }1101 }
1082};1102};
10831103
1104pub const Imm32 = struct {
1105 imm: u32,
1106};
1107
1084pub const Imm64 = struct {1108pub const Imm64 = struct {
1085 msb: u32,1109 msb: u32,
1086 lsb: u32,1110 lsb: u32,