authorgravatar for joachim.schmidt557@outlook.comJoachim Schmidt <joachim.schmidt557@outlook.com> 2022-09-23 19:45:15+02:00
committergravatar for joachim.schmidt557@outlook.comJoachim Schmidt <joachim.schmidt557@outlook.com> 2022-10-20 16:14:52+02:00
logd8fddb535ca425be8db42e3d86e3715cc6ebad56
tree8de74ee981e180e2f7b81f11071d02e0afa07ff4
parent5838fe89c1c17ed2cda76fed36f310d9d557b42f
signaturelock-open Commit is signed but in an unrecognized format.

stage2 AArch64: move cmp to new allocRegs mechanism

Remove cmp from binOp in the process

1 files changed, 580 insertions(+), 98 deletions(-)

src/arch/aarch64/CodeGen.zig+580-98
...@@ -1265,6 +1265,376 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {...@@ -1265,6 +1265,376 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1265 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1265 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1266}1266}
12671267
1268/// An argument to a Mir instruction which is read (and possibly also
1269/// written to) by the respective instruction
1270const ReadArg = struct {
1271 ty: Type,
1272 bind: Bind,
1273 class: RegisterManager.RegisterBitSet,
1274 reg: *Register,
1275
1276 const Bind = union(enum) {
1277 inst: Air.Inst.Ref,
1278 mcv: MCValue,
1279
1280 fn resolveToMcv(bind: Bind, function: *Self) InnerError!MCValue {
1281 return switch (bind) {
1282 .inst => |inst| try function.resolveInst(inst),
1283 .mcv => |mcv| mcv,
1284 };
1285 }
1286
1287 fn resolveToImmediate(bind: Bind, function: *Self) InnerError!?u64 {
1288 switch (bind) {
1289 .inst => |inst| {
1290 // TODO resolve independently of inst_table
1291 const mcv = try function.resolveInst(inst);
1292 switch (mcv) {
1293 .immediate => |imm| return imm,
1294 else => return null,
1295 }
1296 },
1297 .mcv => |mcv| {
1298 switch (mcv) {
1299 .immediate => |imm| return imm,
1300 else => return null,
1301 }
1302 },
1303 }
1304 }
1305 };
1306};
1307
1308/// An argument to a Mir instruction which is written to (but not read
1309/// from) by the respective instruction
1310const WriteArg = struct {
1311 ty: Type,
1312 bind: Bind,
1313 class: RegisterManager.RegisterBitSet,
1314 reg: *Register,
1315
1316 const Bind = union(enum) {
1317 reg: Register,
1318 none: void,
1319 };
1320};
1321
1322/// Holds all data necessary for enabling the potential reuse of
1323/// operand registers as destinations
1324const ReuseMetadata = struct {
1325 corresponding_inst: Air.Inst.Index,
1326
1327 /// Maps every element index of read_args to the corresponding
1328 /// index in the Air instruction
1329 ///
1330 /// When the order of read_args corresponds exactly to the order
1331 /// of the inputs of the Air instruction, this would be e.g.
1332 /// &.{ 0, 1 }. However, when the order is not the same or some
1333 /// inputs to the Air instruction are omitted (e.g. when they can
1334 /// be represented as immediates to the Mir instruction),
1335 /// operand_mapping should reflect that fact.
1336 operand_mapping: []const Liveness.OperandInt,
1337};
1338
1339/// Allocate a set of registers for use as arguments for a Mir
1340/// instruction
1341///
1342/// If the Mir instruction these registers are allocated for
1343/// corresponds exactly to a single Air instruction, populate
1344/// reuse_metadata in order to enable potential reuse of an operand as
1345/// the destination (provided that that operand dies in this
1346/// instruction).
1347///
1348/// Reusing an operand register as destination is the only time two
1349/// arguments may share the same register. In all other cases,
1350/// allocRegs guarantees that a register will never be allocated to
1351/// more than one argument.
1352///
1353/// Furthermore, allocReg guarantees that all arguments which are
1354/// already bound to registers before calling allocRegs will not
1355/// change their register binding. This is done by locking these
1356/// registers.
1357fn allocRegs(
1358 self: *Self,
1359 read_args: []const ReadArg,
1360 write_args: []const WriteArg,
1361 reuse_metadata: ?ReuseMetadata,
1362) InnerError!void {
1363 // Air instructions have exactly one output
1364 assert(!(reuse_metadata != null and write_args.len != 1)); // see note above
1365
1366 // The operand mapping is a 1:1 mapping of read args to their
1367 // corresponding operand index in the Air instruction
1368 assert(!(reuse_metadata != null and reuse_metadata.?.operand_mapping.len != read_args.len)); // see note above
1369
1370 const locks = try self.gpa.alloc(?RegisterLock, read_args.len + write_args.len);
1371 defer self.gpa.free(locks);
1372 const read_locks = locks[0..read_args.len];
1373 const write_locks = locks[read_args.len..];
1374
1375 std.mem.set(?RegisterLock, locks, null);
1376 defer for (locks) |lock| {
1377 if (lock) |locked_reg| self.register_manager.unlockReg(locked_reg);
1378 };
1379
1380 // When we reuse a read_arg as a destination, the corresponding
1381 // MCValue of the read_arg will be set to .dead. In that case, we
1382 // skip allocating this read_arg.
1383 var reused_read_arg: ?usize = null;
1384
1385 // Lock all args which are already allocated to registers
1386 for (read_args) |arg, i| {
1387 const mcv = try arg.bind.resolveToMcv(self);
1388 if (mcv == .register) {
1389 read_locks[i] = self.register_manager.lockReg(mcv.register);
1390 }
1391 }
1392
1393 for (write_args) |arg, i| {
1394 if (arg.bind == .reg) {
1395 write_locks[i] = self.register_manager.lockReg(arg.bind.reg);
1396 }
1397 }
1398
1399 // Allocate registers for all args which aren't allocated to
1400 // registers yet
1401 for (read_args) |arg, i| {
1402 const mcv = try arg.bind.resolveToMcv(self);
1403 if (mcv == .register) {
1404 arg.reg.* = mcv.register;
1405 } else {
1406 const track_inst: ?Air.Inst.Index = switch (arg.bind) {
1407 .inst => |inst| Air.refToIndex(inst).?,
1408 else => null,
1409 };
1410 const raw_reg = try self.register_manager.allocReg(track_inst, gp);
1411 arg.reg.* = self.registerAlias(raw_reg, arg.ty);
1412 read_locks[i] = self.register_manager.lockReg(arg.reg.*);
1413 }
1414 }
1415
1416 if (reuse_metadata != null) {
1417 const inst = reuse_metadata.?.corresponding_inst;
1418 const operand_mapping = reuse_metadata.?.operand_mapping;
1419 const arg = write_args[0];
1420 if (arg.bind == .reg) {
1421 arg.reg.* = arg.bind.reg;
1422 } else {
1423 reuse_operand: for (read_args) |read_arg, i| {
1424 if (read_arg.bind == .inst) {
1425 const operand = read_arg.bind.inst;
1426 const mcv = try self.resolveInst(operand);
1427 if (mcv == .register and
1428 std.meta.eql(arg.class, read_arg.class) and
1429 self.reuseOperand(inst, operand, operand_mapping[i], mcv))
1430 {
1431 arg.reg.* = mcv.register;
1432 write_locks[0] = null;
1433 reused_read_arg = i;
1434 break :reuse_operand;
1435 }
1436 }
1437 } else {
1438 const raw_reg = try self.register_manager.allocReg(inst, arg.class);
1439 arg.reg.* = self.registerAlias(raw_reg, arg.ty);
1440 write_locks[0] = self.register_manager.lockReg(arg.reg.*);
1441 }
1442 }
1443 } else {
1444 for (write_args) |arg, i| {
1445 if (arg.bind == .reg) {
1446 arg.reg.* = arg.bind.reg;
1447 } else {
1448 const raw_reg = try self.register_manager.allocReg(null, arg.class);
1449 arg.reg.* = self.registerAlias(raw_reg, arg.ty);
1450 write_locks[i] = self.register_manager.lockReg(arg.reg.*);
1451 }
1452 }
1453 }
1454
1455 // For all read_args which need to be moved from non-register to
1456 // register, perform the move
1457 for (read_args) |arg, i| {
1458 if (reused_read_arg) |j| {
1459 // Check whether this read_arg was reused
1460 if (i == j) continue;
1461 }
1462
1463 const mcv = try arg.bind.resolveToMcv(self);
1464 if (mcv != .register) {
1465 if (arg.bind == .inst) {
1466 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1467 const inst = Air.refToIndex(arg.bind.inst).?;
1468
1469 // Overwrite the MCValue associated with this inst
1470 branch.inst_table.putAssumeCapacity(inst, .{ .register = arg.reg.* });
1471
1472 // If the previous MCValue occupied some space we track, we
1473 // need to make sure it is marked as free now.
1474 switch (mcv) {
1475 .condition_flags => {
1476 assert(self.condition_flags_inst.? == inst);
1477 self.condition_flags_inst = null;
1478 },
1479 .register => |prev_reg| {
1480 assert(!self.register_manager.isRegFree(prev_reg));
1481 self.register_manager.freeReg(prev_reg);
1482 },
1483 else => {},
1484 }
1485 }
1486
1487 try self.genSetReg(arg.ty, arg.reg.*, mcv);
1488 }
1489 }
1490}
1491
1492/// Wrapper around allocRegs and addInst tailored for specific Mir
1493/// instructions which are binary operations acting on two registers
1494///
1495/// Returns the destination register
1496fn binOpRegisterNew(
1497 self: *Self,
1498 mir_tag: Mir.Inst.Tag,
1499 lhs_bind: ReadArg.Bind,
1500 rhs_bind: ReadArg.Bind,
1501 lhs_ty: Type,
1502 rhs_ty: Type,
1503 maybe_inst: ?Air.Inst.Index,
1504) !MCValue {
1505 var lhs_reg: Register = undefined;
1506 var rhs_reg: Register = undefined;
1507 var dest_reg: Register = undefined;
1508
1509 const read_args = [_]ReadArg{
1510 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
1511 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
1512 };
1513 const write_args = [_]WriteArg{
1514 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1515 };
1516 try self.allocRegs(
1517 &read_args,
1518 &write_args,
1519 if (maybe_inst) |inst| .{
1520 .corresponding_inst = inst,
1521 .operand_mapping = &.{ 0, 1 },
1522 } else null,
1523 );
1524
1525 const mir_data: Mir.Inst.Data = switch (mir_tag) {
1526 .add_shifted_register,
1527 .adds_shifted_register,
1528 .sub_shifted_register,
1529 .subs_shifted_register,
1530 => .{ .rrr_imm6_shift = .{
1531 .rd = dest_reg,
1532 .rn = lhs_reg,
1533 .rm = rhs_reg,
1534 .imm6 = 0,
1535 .shift = .lsl,
1536 } },
1537 .mul,
1538 .lsl_register,
1539 .asr_register,
1540 .lsr_register,
1541 .sdiv,
1542 .udiv,
1543 => .{ .rrr = .{
1544 .rd = dest_reg,
1545 .rn = lhs_reg,
1546 .rm = rhs_reg,
1547 } },
1548 .smull,
1549 .umull,
1550 => .{ .rrr = .{
1551 .rd = dest_reg.toX(),
1552 .rn = lhs_reg,
1553 .rm = rhs_reg,
1554 } },
1555 .and_shifted_register,
1556 .orr_shifted_register,
1557 .eor_shifted_register,
1558 => .{ .rrr_imm6_logical_shift = .{
1559 .rd = dest_reg,
1560 .rn = lhs_reg,
1561 .rm = rhs_reg,
1562 .imm6 = 0,
1563 .shift = .lsl,
1564 } },
1565 else => unreachable,
1566 };
1567
1568 _ = try self.addInst(.{
1569 .tag = mir_tag,
1570 .data = mir_data,
1571 });
1572
1573 return MCValue{ .register = dest_reg };
1574}
1575
1576/// Wrapper around allocRegs and addInst tailored for specific Mir
1577/// instructions which are binary operations acting on a register and
1578/// an immediate
1579///
1580/// Returns the destination register
1581fn binOpImmediateNew(
1582 self: *Self,
1583 mir_tag: Mir.Inst.Tag,
1584 lhs_bind: ReadArg.Bind,
1585 rhs_immediate: u32,
1586 lhs_ty: Type,
1587 lhs_and_rhs_swapped: bool,
1588 maybe_inst: ?Air.Inst.Index,
1589) !MCValue {
1590 var lhs_reg: Register = undefined;
1591 var dest_reg: Register = undefined;
1592
1593 const read_args = [_]ReadArg{
1594 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
1595 };
1596 const write_args = [_]WriteArg{
1597 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1598 };
1599 const operand_mapping: []const Liveness.OperandInt = if (lhs_and_rhs_swapped) &.{1} else &.{0};
1600 try self.allocRegs(
1601 &read_args,
1602 &write_args,
1603 if (maybe_inst) |inst| .{
1604 .corresponding_inst = inst,
1605 .operand_mapping = operand_mapping,
1606 } else null,
1607 );
1608
1609 const mir_data: Mir.Inst.Data = switch (mir_tag) {
1610 .add_immediate,
1611 .adds_immediate,
1612 .sub_immediate,
1613 .subs_immediate,
1614 => .{ .rr_imm12_sh = .{
1615 .rd = dest_reg,
1616 .rn = lhs_reg,
1617 .imm12 = @intCast(u12, rhs_immediate),
1618 } },
1619 .lsl_immediate,
1620 .asr_immediate,
1621 .lsr_immediate,
1622 => .{ .rr_shift = .{
1623 .rd = dest_reg,
1624 .rn = lhs_reg,
1625 .shift = @intCast(u6, rhs_immediate),
1626 } },
1627 else => unreachable,
1628 };
1629
1630 _ = try self.addInst(.{
1631 .tag = mir_tag,
1632 .data = mir_data,
1633 });
1634
1635 return MCValue{ .register = dest_reg };
1636}
1637
1268/// Don't call this function directly. Use binOp instead.1638/// Don't call this function directly. Use binOp instead.
1269///1639///
1270/// Calling this function signals an intention to generate a Mir1640/// Calling this function signals an intention to generate a Mir
...@@ -1342,7 +1712,6 @@ fn binOpRegister(...@@ -1342,7 +1712,6 @@ fn binOpRegister(
1342 defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);1712 defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
13431713
1344 const dest_reg = switch (mir_tag) {1714 const dest_reg = switch (mir_tag) {
1345 .cmp_shifted_register => undefined, // cmp has no destination register
1346 else => if (metadata) |md| blk: {1715 else => if (metadata) |md| blk: {
1347 if (lhs_is_register and self.reuseOperand(md.inst, md.lhs, 0, lhs)) {1716 if (lhs_is_register and self.reuseOperand(md.inst, md.lhs, 0, lhs)) {
1348 break :blk lhs_reg;1717 break :blk lhs_reg;
...@@ -1373,12 +1742,6 @@ fn binOpRegister(...@@ -1373,12 +1742,6 @@ fn binOpRegister(
1373 .imm6 = 0,1742 .imm6 = 0,
1374 .shift = .lsl,1743 .shift = .lsl,
1375 } },1744 } },
1376 .cmp_shifted_register => .{ .rr_imm6_shift = .{
1377 .rn = lhs_reg,
1378 .rm = rhs_reg,
1379 .imm6 = 0,
1380 .shift = .lsl,
1381 } },
1382 .mul,1745 .mul,
1383 .lsl_register,1746 .lsl_register,
1384 .asr_register,1747 .asr_register,
...@@ -1469,7 +1832,6 @@ fn binOpImmediate(...@@ -1469,7 +1832,6 @@ fn binOpImmediate(
1469 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);1832 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
14701833
1471 const dest_reg = switch (mir_tag) {1834 const dest_reg = switch (mir_tag) {
1472 .cmp_immediate => undefined, // cmp has no destination register
1473 else => if (metadata) |md| blk: {1835 else => if (metadata) |md| blk: {
1474 if (lhs_is_register and self.reuseOperand(1836 if (lhs_is_register and self.reuseOperand(
1475 md.inst,1837 md.inst,
...@@ -1508,10 +1870,6 @@ fn binOpImmediate(...@@ -1508,10 +1870,6 @@ fn binOpImmediate(
1508 .rn = lhs_reg,1870 .rn = lhs_reg,
1509 .shift = @intCast(u6, rhs.immediate),1871 .shift = @intCast(u6, rhs.immediate),
1510 } },1872 } },
1511 .cmp_immediate => .{ .r_imm12_sh = .{
1512 .rn = lhs_reg,
1513 .imm12 = @intCast(u12, rhs.immediate),
1514 } },
1515 else => unreachable,1873 else => unreachable,
1516 };1874 };
15171875
...@@ -1554,7 +1912,6 @@ fn binOp(...@@ -1554,7 +1912,6 @@ fn binOp(
1554 switch (tag) {1912 switch (tag) {
1555 .add,1913 .add,
1556 .sub,1914 .sub,
1557 .cmp_eq,
1558 => {1915 => {
1559 switch (lhs_ty.zigTypeTag()) {1916 switch (lhs_ty.zigTypeTag()) {
1560 .Float => return self.fail("TODO binary operations on floats", .{}),1917 .Float => return self.fail("TODO binary operations on floats", .{}),
...@@ -1568,13 +1925,12 @@ fn binOp(...@@ -1568,13 +1925,12 @@ fn binOp(
1568 // operands1925 // operands
1569 const lhs_immediate_ok = switch (tag) {1926 const lhs_immediate_ok = switch (tag) {
1570 .add => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12),1927 .add => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12),
1571 .sub, .cmp_eq => false,1928 .sub => false,
1572 else => unreachable,1929 else => unreachable,
1573 };1930 };
1574 const rhs_immediate_ok = switch (tag) {1931 const rhs_immediate_ok = switch (tag) {
1575 .add,1932 .add,
1576 .sub,1933 .sub,
1577 .cmp_eq,
1578 => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12),1934 => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12),
1579 else => unreachable,1935 else => unreachable,
1580 };1936 };
...@@ -1582,13 +1938,11 @@ fn binOp(...@@ -1582,13 +1938,11 @@ fn binOp(
1582 const mir_tag_register: Mir.Inst.Tag = switch (tag) {1938 const mir_tag_register: Mir.Inst.Tag = switch (tag) {
1583 .add => .add_shifted_register,1939 .add => .add_shifted_register,
1584 .sub => .sub_shifted_register,1940 .sub => .sub_shifted_register,
1585 .cmp_eq => .cmp_shifted_register,
1586 else => unreachable,1941 else => unreachable,
1587 };1942 };
1588 const mir_tag_immediate: Mir.Inst.Tag = switch (tag) {1943 const mir_tag_immediate: Mir.Inst.Tag = switch (tag) {
1589 .add => .add_immediate,1944 .add => .add_immediate,
1590 .sub => .sub_immediate,1945 .sub => .sub_immediate,
1591 .cmp_eq => .cmp_immediate,
1592 else => unreachable,1946 else => unreachable,
1593 };1947 };
15941948
...@@ -2052,7 +2406,15 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2052,7 +2406,15 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
2052 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);2406 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
20532407
2054 // cmp dest, truncated2408 // cmp dest, truncated
2055 _ = try self.binOp(.cmp_eq, dest, .{ .register = truncated_reg }, lhs_ty, lhs_ty, null);2409 _ = try self.addInst(.{
2410 .tag = .cmp_shifted_register,
2411 .data = .{ .rr_imm6_shift = .{
2412 .rn = dest_reg,
2413 .rm = truncated_reg,
2414 .imm6 = 0,
2415 .shift = .lsl,
2416 } },
2417 });
20562418
2057 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });2419 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
2058 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .condition_flags = .ne });2420 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .condition_flags = .ne });
...@@ -2333,14 +2695,13 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2333,14 +2695,13 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2333 } },2695 } },
2334 });2696 });
23352697
2336 _ = try self.binOp(2698 _ = try self.addInst(.{
2337 .cmp_eq,2699 .tag = .cmp_immediate,
2338 .{ .register = dest_high_reg },2700 .data = .{ .r_imm12_sh = .{
2339 .{ .immediate = 0 },2701 .rn = dest_high_reg,
2340 Type.usize,2702 .imm12 = 0,
2341 Type.usize,2703 } },
2342 null,2704 });
2343 );
23442705
2345 if (int_info.bits < 64) {2706 if (int_info.bits < 64) {
2346 // lsr dest_high, dest, #shift2707 // lsr dest_high, dest, #shift
...@@ -2353,14 +2714,13 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2353,14 +2714,13 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2353 } },2714 } },
2354 });2715 });
23552716
2356 _ = try self.binOp(2717 _ = try self.addInst(.{
2357 .cmp_eq,2718 .tag = .cmp_immediate,
2358 .{ .register = dest_high_reg },2719 .data = .{ .r_imm12_sh = .{
2359 .{ .immediate = 0 },2720 .rn = dest_high_reg,
2360 Type.usize,2721 .imm12 = 0,
2361 Type.usize,2722 } },
2362 null,2723 });
2363 );
2364 }2724 }
2365 },2725 },
2366 }2726 }
...@@ -2388,8 +2748,6 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2388,8 +2748,6 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2388 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;2748 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2389 if (self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ extra.lhs, extra.rhs, .none });2749 if (self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ extra.lhs, extra.rhs, .none });
2390 const result: MCValue = result: {2750 const result: MCValue = result: {
2391 const lhs = try self.resolveInst(extra.lhs);
2392 const rhs = try self.resolveInst(extra.rhs);
2393 const lhs_ty = self.air.typeOf(extra.lhs);2751 const lhs_ty = self.air.typeOf(extra.lhs);
2394 const rhs_ty = self.air.typeOf(extra.rhs);2752 const rhs_ty = self.air.typeOf(extra.rhs);
23952753
...@@ -2405,33 +2763,113 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2405,33 +2763,113 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2405 if (int_info.bits <= 64) {2763 if (int_info.bits <= 64) {
2406 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);2764 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);
24072765
2408 const lhs_lock: ?RegisterLock = if (lhs == .register)
2409 self.register_manager.lockRegAssumeUnused(lhs.register)
2410 else
2411 null;
2412 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
2413
2414 try self.spillCompareFlagsIfOccupied();2766 try self.spillCompareFlagsIfOccupied();
2415 self.condition_flags_inst = null;
24162767
2417 // lsl dest, lhs, rhs2768 const lhs_bind: ReadArg.Bind = .{ .inst = extra.lhs };
2418 const dest = try self.binOp(.shl, lhs, rhs, lhs_ty, rhs_ty, null);2769 const rhs_bind: ReadArg.Bind = .{ .inst = extra.rhs };
2419 const dest_reg = dest.register;2770
2420 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);2771 var lhs_reg: Register = undefined;
2421 defer self.register_manager.unlockReg(dest_reg_lock);2772 var rhs_reg: Register = undefined;
2773 var dest_reg: Register = undefined;
2774 var reconstructed_reg: Register = undefined;
2775
2776 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
2777 if (rhs_immediate) |imm| {
2778 const read_args = [_]ReadArg{
2779 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
2780 };
2781 const write_args = [_]WriteArg{
2782 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
2783 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &reconstructed_reg },
2784 };
2785 try self.allocRegs(
2786 &read_args,
2787 &write_args,
2788 null,
2789 );
2790
2791 // lsl dest, lhs, rhs
2792 _ = try self.addInst(.{
2793 .tag = .lsl_immediate,
2794 .data = .{ .rr_shift = .{
2795 .rd = dest_reg,
2796 .rn = lhs_reg,
2797 .shift = @intCast(u6, imm),
2798 } },
2799 });
2800
2801 try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits);
2802
2803 // asr/lsr reconstructed, dest, rhs
2804 _ = try self.addInst(.{
2805 .tag = switch (int_info.signedness) {
2806 .signed => Mir.Inst.Tag.asr_immediate,
2807 .unsigned => Mir.Inst.Tag.lsr_immediate,
2808 },
2809 .data = .{ .rr_shift = .{
2810 .rd = reconstructed_reg,
2811 .rn = dest_reg,
2812 .shift = @intCast(u6, imm),
2813 } },
2814 });
2815 } else {
2816 const read_args = [_]ReadArg{
2817 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
2818 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
2819 };
2820 const write_args = [_]WriteArg{
2821 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
2822 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &reconstructed_reg },
2823 };
2824 try self.allocRegs(
2825 &read_args,
2826 &write_args,
2827 null,
2828 );
2829
2830 // lsl dest, lhs, rhs
2831 _ = try self.addInst(.{
2832 .tag = .lsl_register,
2833 .data = .{ .rrr = .{
2834 .rd = dest_reg,
2835 .rn = lhs_reg,
2836 .rm = rhs_reg,
2837 } },
2838 });
24222839
2423 // asr/lsr reconstructed, dest, rhs2840 try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits);
2424 const reconstructed = try self.binOp(.shr, dest, rhs, lhs_ty, rhs_ty, null);2841
2842 // asr/lsr reconstructed, dest, rhs
2843 _ = try self.addInst(.{
2844 .tag = switch (int_info.signedness) {
2845 .signed => Mir.Inst.Tag.asr_register,
2846 .unsigned => Mir.Inst.Tag.lsr_register,
2847 },
2848 .data = .{ .rrr = .{
2849 .rd = reconstructed_reg,
2850 .rn = dest_reg,
2851 .rm = rhs_reg,
2852 } },
2853 });
2854 }
24252855
2426 // cmp lhs, reconstructed2856 // cmp lhs, reconstructed
2427 _ = try self.binOp(.cmp_eq, lhs, reconstructed, lhs_ty, lhs_ty, null);2857 _ = try self.addInst(.{
2858 .tag = .cmp_shifted_register,
2859 .data = .{ .rr_imm6_shift = .{
2860 .rn = lhs_reg,
2861 .rm = reconstructed_reg,
2862 .imm6 = 0,
2863 .shift = .lsl,
2864 } },
2865 });
24282866
2429 try self.genSetStack(lhs_ty, stack_offset, dest);2867 try self.genSetStack(lhs_ty, stack_offset, .{ .register = dest_reg });
2430 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .condition_flags = .ne });2868 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .condition_flags = .ne });
24312869
2432 break :result MCValue{ .stack_offset = stack_offset };2870 break :result MCValue{ .stack_offset = stack_offset };
2433 } else {2871 } else {
2434 return self.fail("TODO overflow operations on integers > u64/i64", .{});2872 return self.fail("TODO ARM overflow operations on integers > u32/i32", .{});
2435 }2873 }
2436 },2874 },
2437 else => unreachable,2875 else => unreachable,
...@@ -3634,54 +4072,100 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -3634,54 +4072,100 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
36344072
3635fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {4073fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
3636 const bin_op = self.air.instructions.items(.data)[inst].bin_op;4074 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
3637 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {4075 const lhs_ty = self.air.typeOf(bin_op.lhs);
3638 const lhs = try self.resolveInst(bin_op.lhs);
3639 const rhs = try self.resolveInst(bin_op.rhs);
3640 const lhs_ty = self.air.typeOf(bin_op.lhs);
3641
3642 var int_buffer: Type.Payload.Bits = undefined;
3643 const int_ty = switch (lhs_ty.zigTypeTag()) {
3644 .Vector => return self.fail("TODO AArch64 cmp vectors", .{}),
3645 .Enum => lhs_ty.intTagType(&int_buffer),
3646 .Int => lhs_ty,
3647 .Bool => Type.initTag(.u1),
3648 .Pointer => Type.usize,
3649 .ErrorSet => Type.initTag(.u16),
3650 .Optional => blk: {
3651 var opt_buffer: Type.Payload.ElemType = undefined;
3652 const payload_ty = lhs_ty.optionalChild(&opt_buffer);
3653 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
3654 break :blk Type.initTag(.u1);
3655 } else if (lhs_ty.isPtrLikeOptional()) {
3656 break :blk Type.usize;
3657 } else {
3658 return self.fail("TODO AArch64 cmp non-pointer optionals", .{});
3659 }
3660 },
3661 .Float => return self.fail("TODO AArch64 cmp floats", .{}),
3662 else => unreachable,
3663 };
36644076
3665 const int_info = int_ty.intInfo(self.target.*);4077 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else blk: {
3666 if (int_info.bits <= 64) {4078 break :blk try self.cmp(.{ .inst = bin_op.lhs }, .{ .inst = bin_op.rhs }, lhs_ty, op);
3667 _ = try self.binOp(.cmp_eq, lhs, rhs, int_ty, int_ty, BinOpMetadata{4079 };
3668 .inst = inst,4080
3669 .lhs = bin_op.lhs,4081 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3670 .rhs = bin_op.rhs,4082}
3671 });4083
4084fn cmp(
4085 self: *Self,
4086 lhs: ReadArg.Bind,
4087 rhs: ReadArg.Bind,
4088 lhs_ty: Type,
4089 op: math.CompareOperator,
4090) !MCValue {
4091 var int_buffer: Type.Payload.Bits = undefined;
4092 const int_ty = switch (lhs_ty.zigTypeTag()) {
4093 .Optional => blk: {
4094 var opt_buffer: Type.Payload.ElemType = undefined;
4095 const payload_ty = lhs_ty.optionalChild(&opt_buffer);
4096 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
4097 break :blk Type.initTag(.u1);
4098 } else if (lhs_ty.isPtrLikeOptional()) {
4099 break :blk Type.usize;
4100 } else {
4101 return self.fail("TODO ARM cmp non-pointer optionals", .{});
4102 }
4103 },
4104 .Float => return self.fail("TODO ARM cmp floats", .{}),
4105 .Enum => lhs_ty.intTagType(&int_buffer),
4106 .Int => lhs_ty,
4107 .Bool => Type.initTag(.u1),
4108 .Pointer => Type.usize,
4109 .ErrorSet => Type.initTag(.u16),
4110 else => unreachable,
4111 };
4112
4113 const int_info = int_ty.intInfo(self.target.*);
4114 if (int_info.bits <= 64) {
4115 try self.spillCompareFlagsIfOccupied();
36724116
3673 try self.spillCompareFlagsIfOccupied();4117 var lhs_reg: Register = undefined;
3674 self.condition_flags_inst = inst;4118 var rhs_reg: Register = undefined;
36754119
3676 break :result switch (int_info.signedness) {4120 const rhs_immediate = try rhs.resolveToImmediate(self);
3677 .signed => MCValue{ .condition_flags = Condition.fromCompareOperatorSigned(op) },4121 const rhs_immediate_ok = if (rhs_immediate) |imm| imm <= std.math.maxInt(u12) else false;
3678 .unsigned => MCValue{ .condition_flags = Condition.fromCompareOperatorUnsigned(op) },4122
4123 if (rhs_immediate_ok) {
4124 const read_args = [_]ReadArg{
4125 .{ .ty = int_ty, .bind = lhs, .class = gp, .reg = &lhs_reg },
3679 };4126 };
4127 try self.allocRegs(
4128 &read_args,
4129 &.{},
4130 null, // we won't be able to reuse a register as there are no write_regs
4131 );
4132
4133 _ = try self.addInst(.{
4134 .tag = .cmp_immediate,
4135 .data = .{ .r_imm12_sh = .{
4136 .rn = lhs_reg,
4137 .imm12 = @intCast(u12, rhs_immediate.?),
4138 } },
4139 });
3680 } else {4140 } else {
3681 return self.fail("TODO AArch64 cmp for ints > 64 bits", .{});4141 const read_args = [_]ReadArg{
4142 .{ .ty = int_ty, .bind = lhs, .class = gp, .reg = &lhs_reg },
4143 .{ .ty = int_ty, .bind = rhs, .class = gp, .reg = &rhs_reg },
4144 };
4145 try self.allocRegs(
4146 &read_args,
4147 &.{},
4148 null, // we won't be able to reuse a register as there are no write_regs
4149 );
4150
4151 _ = try self.addInst(.{
4152 .tag = .cmp_shifted_register,
4153 .data = .{ .rr_imm6_shift = .{
4154 .rn = lhs_reg,
4155 .rm = rhs_reg,
4156 .imm6 = 0,
4157 .shift = .lsl,
4158 } },
4159 });
3682 }4160 }
3683 };4161
3684 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });4162 return switch (int_info.signedness) {
4163 .signed => MCValue{ .condition_flags = Condition.fromCompareOperatorSigned(op) },
4164 .unsigned => MCValue{ .condition_flags = Condition.fromCompareOperatorUnsigned(op) },
4165 };
4166 } else {
4167 return self.fail("TODO AArch64 cmp for ints > 64 bits", .{});
4168 }
3685}4169}
36864170
3687fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {4171fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
...@@ -3926,15 +4410,13 @@ fn isNonNull(self: *Self, operand: MCValue) !MCValue {...@@ -3926,15 +4410,13 @@ fn isNonNull(self: *Self, operand: MCValue) !MCValue {
39264410
3927fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {4411fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
3928 const error_type = ty.errorUnionSet();4412 const error_type = ty.errorUnionSet();
3929 const error_int_type = Type.initTag(.u16);
39304413
3931 if (error_type.errorSetIsEmpty()) {4414 if (error_type.errorSetIsEmpty()) {
3932 return MCValue{ .immediate = 0 }; // always false4415 return MCValue{ .immediate = 0 }; // always false
3933 }4416 }
39344417
3935 const error_mcv = try self.errUnionErr(operand, ty);4418 const error_mcv = try self.errUnionErr(operand, ty);
3936 _ = try self.binOp(.cmp_eq, error_mcv, .{ .immediate = 0 }, error_int_type, error_int_type, null);4419 return try self.cmp(.{ .mcv = error_mcv }, .{ .mcv = .{ .immediate = 0 } }, error_type, .gt);
3937 return MCValue{ .condition_flags = .hi };
3938}4420}
39394421
3940fn isNonErr(self: *Self, ty: Type, operand: MCValue) !MCValue {4422fn isNonErr(self: *Self, ty: Type, operand: MCValue) !MCValue {