authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-08-05 16:56:27-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-08-05 16:56:27-07:00
log40babaa537329c8425579a47ba4767f298f5d869
treefdd4b5bd0a464867d3ae934d65421cf9ae4b870c
parentab6b53a259180aab9c9fc4c57d981ba9ecd6e2fa
parentf46c80b267396d02b5008bf8c426e0eb886a05d2
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #12338 from joachimschmidt557/stage2-aarch64

stage2 AArch64: fixes for removing builtin.stage2_arch

4 files changed, 844 insertions(+), 171 deletions(-)

src/arch/aarch64/CodeGen.zig+700-161
......@@ -166,10 +166,12 @@ const MCValue = union(enum) {
166166 /// the type is u1) or true (if the type in bool) iff the
167167 /// specified condition is true.
168168 condition_flags: Condition,
169 /// The value is a function argument passed via the stack.
170 stack_argument_offset: u32,
169171
170172 fn isMemory(mcv: MCValue) bool {
171173 return switch (mcv) {
172 .memory, .stack_offset => true,
174 .memory, .stack_offset, .stack_argument_offset => true,
173175 else => false,
174176 };
175177 }
......@@ -192,6 +194,7 @@ const MCValue = union(enum) {
192194 .condition_flags,
193195 .ptr_stack_offset,
194196 .undef,
197 .stack_argument_offset,
195198 => false,
196199
197200 .register,
......@@ -337,6 +340,7 @@ pub fn generate(
337340 .prev_di_line = module_fn.lbrace_line,
338341 .prev_di_column = module_fn.lbrace_column,
339342 .stack_size = mem.alignForwardGeneric(u32, function.max_end_stack, function.stack_align),
343 .saved_regs_stack_space = function.saved_regs_stack_space,
340344 };
341345 defer emit.deinit();
342346
......@@ -414,6 +418,23 @@ fn gen(self: *Self) !void {
414418 // sub sp, sp, #reloc
415419 const backpatch_reloc = try self.addNop();
416420
421 if (self.ret_mcv == .stack_offset) {
422 // The address of where to store the return value is in x0
423 // (or w0 when pointer size is 32 bits). As this register
424 // might get overwritten along the way, save the address
425 // to the stack.
426 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
427 const ptr_bytes = @divExact(ptr_bits, 8);
428 const ret_ptr_reg = registerAlias(.x0, ptr_bytes);
429
430 const stack_offset = mem.alignForwardGeneric(u32, self.next_stack_offset, ptr_bytes) + ptr_bytes;
431 self.next_stack_offset = stack_offset;
432 self.max_end_stack = @maximum(self.max_end_stack, self.next_stack_offset);
433
434 try self.genSetStack(Type.usize, stack_offset, MCValue{ .register = ret_ptr_reg });
435 self.ret_mcv = MCValue{ .stack_offset = stack_offset };
436 }
437
417438 _ = try self.addInst(.{
418439 .tag = .dbg_prologue_end,
419440 .data = .{ .nop = {} },
......@@ -540,33 +561,38 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
540561
541562 switch (air_tags[inst]) {
542563 // zig fmt: off
543 .add => try self.airBinOp(inst, .add),
544 .addwrap => try self.airBinOp(inst, .addwrap),
545 .sub => try self.airBinOp(inst, .sub),
546 .subwrap => try self.airBinOp(inst, .subwrap),
547 .mul => try self.airBinOp(inst, .mul),
548 .mulwrap => try self.airBinOp(inst, .mulwrap),
549 .shl => try self.airBinOp(inst, .shl),
550 .shl_exact => try self.airBinOp(inst, .shl_exact),
551 .bool_and => try self.airBinOp(inst, .bool_and),
552 .bool_or => try self.airBinOp(inst, .bool_or),
553 .bit_and => try self.airBinOp(inst, .bit_and),
554 .bit_or => try self.airBinOp(inst, .bit_or),
555 .xor => try self.airBinOp(inst, .xor),
556 .shr => try self.airBinOp(inst, .shr),
557 .shr_exact => try self.airBinOp(inst, .shr_exact),
558
559 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
560 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
564 .add => try self.airBinOp(inst, .add),
565 .addwrap => try self.airBinOp(inst, .addwrap),
566 .sub => try self.airBinOp(inst, .sub),
567 .subwrap => try self.airBinOp(inst, .subwrap),
568 .mul => try self.airBinOp(inst, .mul),
569 .mulwrap => try self.airBinOp(inst, .mulwrap),
570 .shl => try self.airBinOp(inst, .shl),
571 .shl_exact => try self.airBinOp(inst, .shl_exact),
572 .bool_and => try self.airBinOp(inst, .bool_and),
573 .bool_or => try self.airBinOp(inst, .bool_or),
574 .bit_and => try self.airBinOp(inst, .bit_and),
575 .bit_or => try self.airBinOp(inst, .bit_or),
576 .xor => try self.airBinOp(inst, .xor),
577 .shr => try self.airBinOp(inst, .shr),
578 .shr_exact => try self.airBinOp(inst, .shr_exact),
579 .div_float => try self.airBinOp(inst, .div_float),
580 .div_trunc => try self.airBinOp(inst, .div_trunc),
581 .div_floor => try self.airBinOp(inst, .div_floor),
582 .div_exact => try self.airBinOp(inst, .div_exact),
583 .rem => try self.airBinOp(inst, .rem),
584 .mod => try self.airBinOp(inst, .mod),
585
586 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
587 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
588
589 .min => try self.airMin(inst),
590 .max => try self.airMax(inst),
561591
562592 .add_sat => try self.airAddSat(inst),
563593 .sub_sat => try self.airSubSat(inst),
564594 .mul_sat => try self.airMulSat(inst),
565 .rem => try self.airRem(inst),
566 .mod => try self.airMod(inst),
567595 .shl_sat => try self.airShlSat(inst),
568 .min => try self.airMin(inst),
569 .max => try self.airMax(inst),
570596 .slice => try self.airSlice(inst),
571597
572598 .sqrt,
......@@ -591,8 +617,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
591617 .mul_with_overflow => try self.airMulWithOverflow(inst),
592618 .shl_with_overflow => try self.airShlWithOverflow(inst),
593619
594 .div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst),
595
596620 .cmp_lt => try self.airCmp(inst, .lt),
597621 .cmp_lte => try self.airCmp(inst, .lte),
598622 .cmp_eq => try self.airCmp(inst, .eq),
......@@ -1008,17 +1032,43 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
10081032 if (self.liveness.isUnused(inst))
10091033 return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
10101034
1011 const operand_ty = self.air.typeOf(ty_op.operand);
1012 const operand = try self.resolveInst(ty_op.operand);
1013 const info_a = operand_ty.intInfo(self.target.*);
1014 const info_b = self.air.typeOfIndex(inst).intInfo(self.target.*);
1015 if (info_a.signedness != info_b.signedness)
1016 return self.fail("TODO gen intcast sign safety in semantic analysis", .{});
1035 const operand = ty_op.operand;
1036 const operand_mcv = try self.resolveInst(operand);
1037 const operand_ty = self.air.typeOf(operand);
1038 const operand_info = operand_ty.intInfo(self.target.*);
1039
1040 const dest_ty = self.air.typeOfIndex(inst);
1041 const dest_abi_size = dest_ty.abiSize(self.target.*);
1042 const dest_info = dest_ty.intInfo(self.target.*);
1043
1044 const result: MCValue = result: {
1045 const operand_lock: ?RegisterLock = switch (operand_mcv) {
1046 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1047 else => null,
1048 };
1049 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
1050
1051 const truncated: MCValue = switch (operand_mcv) {
1052 .register => |r| MCValue{ .register = registerAlias(r, dest_abi_size) },
1053 else => operand_mcv,
1054 };
10171055
1018 if (info_a.bits == info_b.bits)
1019 return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
1056 if (dest_info.bits > operand_info.bits) {
1057 const dest_mcv = try self.allocRegOrMem(inst, true);
1058 try self.setRegOrMem(self.air.typeOfIndex(inst), dest_mcv, truncated);
1059 break :result dest_mcv;
1060 } else {
1061 if (self.reuseOperand(inst, operand, 0, truncated)) {
1062 break :result truncated;
1063 } else {
1064 const dest_mcv = try self.allocRegOrMem(inst, true);
1065 try self.setRegOrMem(self.air.typeOfIndex(inst), dest_mcv, truncated);
1066 break :result dest_mcv;
1067 }
1068 }
1069 };
10201070
1021 return self.fail("TODO implement intCast for {}", .{self.target.cpu.arch});
1071 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
10221072}
10231073
10241074fn truncRegister(
......@@ -1044,6 +1094,8 @@ fn truncRegister(
10441094 });
10451095 },
10461096 32, 64 => {
1097 assert(dest_reg.size() == operand_reg.size());
1098
10471099 _ = try self.addInst(.{
10481100 .tag = .mov_register,
10491101 .data = .{ .rr = .{
......@@ -1099,7 +1151,7 @@ fn trunc(
10991151
11001152 return MCValue{ .register = dest_reg };
11011153 } else {
1102 return self.fail("TODO: truncate to ints > 32 bits", .{});
1154 return self.fail("TODO: truncate to ints > 64 bits", .{});
11031155 }
11041156}
11051157
......@@ -1262,6 +1314,9 @@ fn binOpRegister(
12621314 const lhs_is_register = lhs == .register;
12631315 const rhs_is_register = rhs == .register;
12641316
1317 if (lhs_is_register) assert(lhs.register == registerAlias(lhs.register, lhs_ty.abiSize(self.target.*)));
1318 if (rhs_is_register) assert(rhs.register == registerAlias(rhs.register, rhs_ty.abiSize(self.target.*)));
1319
12651320 const lhs_lock: ?RegisterLock = if (lhs_is_register)
12661321 self.register_manager.lockReg(lhs.register)
12671322 else
......@@ -1291,13 +1346,22 @@ fn binOpRegister(
12911346 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
12921347 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
12931348
1294 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
1349 const rhs_reg = if (rhs_is_register)
1350 // lhs is almost always equal to rhs, except in shifts. In
1351 // order to guarantee that registers will have equal sizes, we
1352 // use the register alias of rhs corresponding to the size of
1353 // lhs.
1354 registerAlias(rhs.register, lhs_ty.abiSize(self.target.*))
1355 else blk: {
12951356 const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: {
12961357 break :inst Air.refToIndex(md.rhs).?;
12971358 } else null;
12981359
12991360 const raw_reg = try self.register_manager.allocReg(track_inst, gp);
1300 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
1361
1362 // Here, we deliberately use lhs as lhs and rhs may differ in
1363 // the case of shifts. See comment above.
1364 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
13011365
13021366 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
13031367
......@@ -1348,6 +1412,8 @@ fn binOpRegister(
13481412 .lsl_register,
13491413 .asr_register,
13501414 .lsr_register,
1415 .sdiv,
1416 .udiv,
13511417 => .{ .rrr = .{
13521418 .rd = dest_reg,
13531419 .rn = lhs_reg,
......@@ -1404,6 +1470,8 @@ fn binOpImmediate(
14041470) !MCValue {
14051471 const lhs_is_register = lhs == .register;
14061472
1473 if (lhs_is_register) assert(lhs.register == registerAlias(lhs.register, lhs_ty.abiSize(self.target.*)));
1474
14071475 const lhs_lock: ?RegisterLock = if (lhs_is_register)
14081476 self.register_manager.lockReg(lhs.register)
14091477 else
......@@ -1586,6 +1654,151 @@ fn binOp(
15861654 else => unreachable,
15871655 }
15881656 },
1657 .div_float => {
1658 switch (lhs_ty.zigTypeTag()) {
1659 .Float => return self.fail("TODO div_float", .{}),
1660 .Vector => return self.fail("TODO div_float on vectors", .{}),
1661 else => unreachable,
1662 }
1663 },
1664 .div_trunc, .div_floor, .div_exact => {
1665 switch (lhs_ty.zigTypeTag()) {
1666 .Float => return self.fail("TODO div on floats", .{}),
1667 .Vector => return self.fail("TODO div on vectors", .{}),
1668 .Int => {
1669 assert(lhs_ty.eql(rhs_ty, mod));
1670 const int_info = lhs_ty.intInfo(self.target.*);
1671 if (int_info.bits <= 64) {
1672 switch (int_info.signedness) {
1673 .signed => {
1674 switch (tag) {
1675 .div_trunc, .div_exact => {
1676 // TODO optimize integer division by constants
1677 return try self.binOpRegister(.sdiv, lhs, rhs, lhs_ty, rhs_ty, metadata);
1678 },
1679 .div_floor => return self.fail("TODO div_floor on signed integers", .{}),
1680 else => unreachable,
1681 }
1682 },
1683 .unsigned => {
1684 // TODO optimize integer division by constants
1685 return try self.binOpRegister(.udiv, lhs, rhs, lhs_ty, rhs_ty, metadata);
1686 },
1687 }
1688 } else {
1689 return self.fail("TODO integer division for ints with bits > 64", .{});
1690 }
1691 },
1692 else => unreachable,
1693 }
1694 },
1695 .rem, .mod => {
1696 switch (lhs_ty.zigTypeTag()) {
1697 .Float => return self.fail("TODO rem/mod on floats", .{}),
1698 .Vector => return self.fail("TODO rem/mod on vectors", .{}),
1699 .Int => {
1700 assert(lhs_ty.eql(rhs_ty, mod));
1701 const int_info = lhs_ty.intInfo(self.target.*);
1702 if (int_info.bits <= 64) {
1703 if (int_info.signedness == .signed and tag == .mod) {
1704 return self.fail("TODO mod on signed integers", .{});
1705 } else {
1706 const lhs_is_register = lhs == .register;
1707 const rhs_is_register = rhs == .register;
1708
1709 const lhs_lock: ?RegisterLock = if (lhs_is_register)
1710 self.register_manager.lockReg(lhs.register)
1711 else
1712 null;
1713 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
1714
1715 const rhs_lock: ?RegisterLock = if (rhs_is_register)
1716 self.register_manager.lockReg(rhs.register)
1717 else
1718 null;
1719 defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);
1720
1721 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1722
1723 const lhs_reg = if (lhs_is_register) lhs.register else blk: {
1724 const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: {
1725 break :inst Air.refToIndex(md.lhs).?;
1726 } else null;
1727
1728 const raw_reg = try self.register_manager.allocReg(track_inst, gp);
1729 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1730
1731 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
1732
1733 break :blk reg;
1734 };
1735 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
1736 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
1737
1738 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
1739 const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: {
1740 break :inst Air.refToIndex(md.rhs).?;
1741 } else null;
1742
1743 const raw_reg = try self.register_manager.allocReg(track_inst, gp);
1744 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
1745
1746 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
1747
1748 break :blk reg;
1749 };
1750 const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
1751 defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
1752
1753 const dest_regs: [2]Register = blk: {
1754 const raw_regs = try self.register_manager.allocRegs(2, .{ null, null }, gp);
1755 const abi_size = lhs_ty.abiSize(self.target.*);
1756 break :blk .{
1757 registerAlias(raw_regs[0], abi_size),
1758 registerAlias(raw_regs[1], abi_size),
1759 };
1760 };
1761 const dest_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dest_regs);
1762 defer for (dest_regs_locks) |reg| {
1763 self.register_manager.unlockReg(reg);
1764 };
1765 const quotient_reg = dest_regs[0];
1766 const remainder_reg = dest_regs[1];
1767
1768 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
1769 if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
1770
1771 _ = try self.addInst(.{
1772 .tag = switch (int_info.signedness) {
1773 .signed => .sdiv,
1774 .unsigned => .udiv,
1775 },
1776 .data = .{ .rrr = .{
1777 .rd = quotient_reg,
1778 .rn = lhs_reg,
1779 .rm = rhs_reg,
1780 } },
1781 });
1782
1783 _ = try self.addInst(.{
1784 .tag = .msub,
1785 .data = .{ .rrrr = .{
1786 .rd = remainder_reg,
1787 .rn = quotient_reg,
1788 .rm = rhs_reg,
1789 .ra = lhs_reg,
1790 } },
1791 });
1792
1793 return MCValue{ .register = remainder_reg };
1794 }
1795 } else {
1796 return self.fail("TODO rem/mod for integers with bits > 64", .{});
1797 }
1798 },
1799 else => unreachable,
1800 }
1801 },
15891802 .addwrap,
15901803 .subwrap,
15911804 .mulwrap,
......@@ -1869,7 +2082,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
18692082 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
18702083
18712084 // cmp dest, truncated
1872 _ = try self.binOp(.cmp_eq, dest, .{ .register = truncated_reg }, Type.usize, Type.usize, null);
2085 _ = try self.binOp(.cmp_eq, dest, .{ .register = truncated_reg }, lhs_ty, lhs_ty, null);
18732086
18742087 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
18752088 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .condition_flags = .ne });
......@@ -2257,24 +2470,6 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
22572470 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
22582471}
22592472
2260fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
2261 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2262 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement div for {}", .{self.target.cpu.arch});
2263 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2264}
2265
2266fn airRem(self: *Self, inst: Air.Inst.Index) !void {
2267 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2268 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement rem for {}", .{self.target.cpu.arch});
2269 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2270}
2271
2272fn airMod(self: *Self, inst: Air.Inst.Index) !void {
2273 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2274 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mod for {}", .{self.target.cpu.arch});
2275 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2276}
2277
22782473fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
22792474 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
22802475 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
......@@ -2313,6 +2508,9 @@ fn errUnionErr(self: *Self, error_union_mcv: MCValue, error_union_ty: Type) !MCV
23132508 const err_offset = @intCast(u32, errUnionErrorOffset(payload_ty, self.target.*));
23142509 switch (error_union_mcv) {
23152510 .register => return self.fail("TODO errUnionErr for registers", .{}),
2511 .stack_argument_offset => |off| {
2512 return MCValue{ .stack_argument_offset = off + err_offset };
2513 },
23162514 .stack_offset => |off| {
23172515 return MCValue{ .stack_offset = off - err_offset };
23182516 },
......@@ -2347,6 +2545,9 @@ fn errUnionPayload(self: *Self, error_union_mcv: MCValue, error_union_ty: Type)
23472545 const payload_offset = @intCast(u32, errUnionPayloadOffset(payload_ty, self.target.*));
23482546 switch (error_union_mcv) {
23492547 .register => return self.fail("TODO errUnionPayload for registers", .{}),
2548 .stack_argument_offset => |off| {
2549 return MCValue{ .stack_argument_offset = off + payload_offset };
2550 },
23502551 .stack_offset => |off| {
23512552 return MCValue{ .stack_offset = off - payload_offset };
23522553 },
......@@ -2436,21 +2637,28 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
24362637 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
24372638}
24382639
2640fn slicePtr(mcv: MCValue) MCValue {
2641 switch (mcv) {
2642 .dead, .unreach, .none => unreachable,
2643 .register => unreachable, // a slice doesn't fit in one register
2644 .stack_argument_offset => |off| {
2645 return MCValue{ .stack_argument_offset = off };
2646 },
2647 .stack_offset => |off| {
2648 return MCValue{ .stack_offset = off };
2649 },
2650 .memory => |addr| {
2651 return MCValue{ .memory = addr };
2652 },
2653 else => unreachable, // invalid MCValue for a slice
2654 }
2655}
2656
24392657fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
24402658 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
24412659 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
24422660 const mcv = try self.resolveInst(ty_op.operand);
2443 switch (mcv) {
2444 .dead, .unreach, .none => unreachable,
2445 .register => unreachable, // a slice doesn't fit in one register
2446 .stack_offset => |off| {
2447 break :result MCValue{ .stack_offset = off };
2448 },
2449 .memory => |addr| {
2450 break :result MCValue{ .memory = addr };
2451 },
2452 else => return self.fail("TODO implement slice_len for {}", .{mcv}),
2453 }
2661 break :result slicePtr(mcv);
24542662 };
24552663 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
24562664}
......@@ -2464,6 +2672,9 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
24642672 switch (mcv) {
24652673 .dead, .unreach, .none => unreachable,
24662674 .register => unreachable, // a slice doesn't fit in one register
2675 .stack_argument_offset => |off| {
2676 break :result MCValue{ .stack_argument_offset = off + ptr_bytes };
2677 },
24672678 .stack_offset => |off| {
24682679 break :result MCValue{ .stack_offset = off - ptr_bytes };
24692680 },
......@@ -2514,6 +2725,9 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
25142725
25152726 if (!is_volatile and self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
25162727 const result: MCValue = result: {
2728 const slice_ty = self.air.typeOf(bin_op.lhs);
2729 const elem_ty = slice_ty.childType();
2730 const elem_size = elem_ty.abiSize(self.target.*);
25172731 const slice_mcv = try self.resolveInst(bin_op.lhs);
25182732
25192733 // TODO optimize for the case where the index is a constant,
......@@ -2521,10 +2735,6 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
25212735 const index_mcv = try self.resolveInst(bin_op.rhs);
25222736 const index_is_register = index_mcv == .register;
25232737
2524 const slice_ty = self.air.typeOf(bin_op.lhs);
2525 const elem_ty = slice_ty.childType();
2526 const elem_size = elem_ty.abiSize(self.target.*);
2527
25282738 var buf: Type.SlicePtrFieldTypeBuffer = undefined;
25292739 const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
25302740
......@@ -2534,15 +2744,17 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
25342744 null;
25352745 defer if (index_lock) |reg| self.register_manager.unlockReg(reg);
25362746
2537 const base_mcv: MCValue = switch (slice_mcv) {
2538 .stack_offset => |off| .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, .{ .stack_offset = off }) },
2539 else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}),
2540 };
2541 const base_lock = self.register_manager.lockRegAssumeUnused(base_mcv.register);
2542 defer self.register_manager.unlockReg(base_lock);
2747 const base_mcv = slicePtr(slice_mcv);
25432748
25442749 switch (elem_size) {
25452750 else => {
2751 const base_reg = switch (base_mcv) {
2752 .register => |r| r,
2753 else => try self.copyToTmpRegister(slice_ptr_field_type, base_mcv),
2754 };
2755 const base_reg_lock = self.register_manager.lockRegAssumeUnused(base_reg);
2756 defer self.register_manager.unlockReg(base_reg_lock);
2757
25462758 const dest = try self.allocRegOrMem(inst, true);
25472759 const addr = try self.binOp(.ptr_add, base_mcv, index_mcv, slice_ptr_field_type, Type.usize, null);
25482760 try self.load(dest, addr, slice_ptr_field_type);
......@@ -2557,7 +2769,16 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
25572769fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
25582770 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
25592771 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2560 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_elem_ptr for {}", .{self.target.cpu.arch});
2772 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2773 const slice_mcv = try self.resolveInst(extra.lhs);
2774 const index_mcv = try self.resolveInst(extra.rhs);
2775 const base_mcv = slicePtr(slice_mcv);
2776
2777 const slice_ty = self.air.typeOf(extra.lhs);
2778
2779 const addr = try self.binOp(.ptr_add, base_mcv, index_mcv, slice_ty, Type.usize, null);
2780 break :result addr;
2781 };
25612782 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
25622783}
25632784
......@@ -2577,7 +2798,15 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
25772798fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
25782799 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
25792800 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2580 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch});
2801 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2802 const ptr_mcv = try self.resolveInst(extra.lhs);
2803 const index_mcv = try self.resolveInst(extra.rhs);
2804
2805 const ptr_ty = self.air.typeOf(extra.lhs);
2806
2807 const addr = try self.binOp(.ptr_add, ptr_mcv, index_mcv, ptr_ty, Type.usize, null);
2808 break :result addr;
2809 };
25812810 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
25822811}
25832812
......@@ -2726,6 +2955,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
27262955 },
27272956 .memory,
27282957 .stack_offset,
2958 .stack_argument_offset,
27292959 .got_load,
27302960 .direct_load,
27312961 => {
......@@ -2907,6 +3137,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
29073137 defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
29083138
29093139 switch (value) {
3140 .dead => unreachable,
3141 .undef => unreachable,
29103142 .register => |value_reg| {
29113143 try self.genStrRegister(value_reg, addr_reg, value_ty);
29123144 },
......@@ -2920,13 +3152,48 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
29203152 try self.genSetReg(value_ty, tmp_reg, value);
29213153 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
29223154 } else {
2923 return self.fail("TODO implement memcpy", .{});
3155 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
3156 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
3157 defer for (regs_locks) |reg| {
3158 self.register_manager.unlockReg(reg);
3159 };
3160
3161 const src_reg = addr_reg;
3162 const dst_reg = regs[0];
3163 const len_reg = regs[1];
3164 const count_reg = regs[2];
3165 const tmp_reg = regs[3];
3166
3167 switch (value) {
3168 .stack_offset => |off| {
3169 // sub src_reg, fp, #off
3170 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
3171 },
3172 .memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = @intCast(u32, addr) }),
3173 .stack_argument_offset => |off| {
3174 _ = try self.addInst(.{
3175 .tag = .ldr_ptr_stack_argument,
3176 .data = .{ .load_store_stack = .{
3177 .rt = src_reg,
3178 .offset = off,
3179 } },
3180 });
3181 },
3182 else => return self.fail("TODO store {} to register", .{value}),
3183 }
3184
3185 // mov len, #abi_size
3186 try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
3187
3188 // memcpy(src, dst, len)
3189 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
29243190 }
29253191 },
29263192 }
29273193 },
29283194 .memory,
29293195 .stack_offset,
3196 .stack_argument_offset,
29303197 .got_load,
29313198 .direct_load,
29323199 => {
......@@ -3005,10 +3272,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
30053272 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
30063273 const mcv = try self.resolveInst(operand);
30073274 const struct_ty = self.air.typeOf(operand);
3275 const struct_field_ty = struct_ty.structFieldType(index);
30083276 const struct_field_offset = @intCast(u32, struct_ty.structFieldOffset(index, self.target.*));
30093277
30103278 switch (mcv) {
30113279 .dead, .unreach => unreachable,
3280 .stack_argument_offset => |off| {
3281 break :result MCValue{ .stack_argument_offset = off + struct_field_offset };
3282 },
30123283 .stack_offset => |off| {
30133284 break :result MCValue{ .stack_offset = off - struct_field_offset };
30143285 },
......@@ -3016,29 +3287,28 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
30163287 break :result MCValue{ .memory = addr + struct_field_offset };
30173288 },
30183289 .register_with_overflow => |rwo| {
3019 switch (index) {
3020 0 => {
3021 // get wrapped value: return register
3022 break :result MCValue{ .register = rwo.reg };
3023 },
3024 1 => {
3025 // TODO return special MCValue condition flags
3026 // get overflow bit: set register to C flag
3027 // resp. V flag
3028 const raw_dest_reg = try self.register_manager.allocReg(null, gp);
3029 const dest_reg = raw_dest_reg.to32();
3290 const reg_lock = self.register_manager.lockRegAssumeUnused(rwo.reg);
3291 defer self.register_manager.unlockReg(reg_lock);
30303292
3031 _ = try self.addInst(.{
3032 .tag = .cset,
3033 .data = .{ .r_cond = .{
3034 .rd = dest_reg,
3035 .cond = rwo.flag,
3036 } },
3037 });
3293 const field: MCValue = switch (index) {
3294 // get wrapped value: return register
3295 0 => MCValue{ .register = rwo.reg },
3296
3297 // get overflow bit: return C or V flag
3298 1 => MCValue{ .condition_flags = rwo.flag },
30383299
3039 break :result MCValue{ .register = dest_reg };
3040 },
30413300 else => unreachable,
3301 };
3302
3303 if (self.reuseOperand(inst, operand, 0, field)) {
3304 break :result field;
3305 } else {
3306 // Copy to new register
3307 const raw_dest_reg = try self.register_manager.allocReg(null, gp);
3308 const dest_reg = registerAlias(raw_dest_reg, struct_field_ty.abiSize(self.target.*));
3309 try self.genSetReg(struct_field_ty, dest_reg, field);
3310
3311 break :result MCValue{ .register = dest_reg };
30423312 }
30433313 },
30443314 else => return self.fail("TODO implement codegen struct_field_val for {}", .{mcv}),
......@@ -3143,6 +3413,31 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
31433413 // saving compare flags may require a new caller-saved register
31443414 try self.spillCompareFlagsIfOccupied();
31453415
3416 if (info.return_value == .stack_offset) {
3417 log.debug("airCall: return by reference", .{});
3418 const ret_ty = fn_ty.fnReturnType();
3419 const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
3420 const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));
3421 const stack_offset = try self.allocMem(inst, ret_abi_size, ret_abi_align);
3422
3423 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
3424 const ptr_bytes = @divExact(ptr_bits, 8);
3425 const ret_ptr_reg = registerAlias(.x0, ptr_bytes);
3426
3427 var ptr_ty_payload: Type.Payload.ElemType = .{
3428 .base = .{ .tag = .single_mut_pointer },
3429 .data = ret_ty,
3430 };
3431 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
3432 try self.register_manager.getReg(ret_ptr_reg, null);
3433 try self.genSetReg(ptr_ty, ret_ptr_reg, .{ .ptr_stack_offset = stack_offset });
3434
3435 info.return_value = .{ .stack_offset = stack_offset };
3436 }
3437
3438 // Make space for the arguments passed via the stack
3439 self.max_end_stack += info.stack_byte_count;
3440
31463441 for (info.args) |mc_arg, arg_i| {
31473442 const arg = args[arg_i];
31483443 const arg_ty = self.air.typeOf(arg);
......@@ -3154,12 +3449,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
31543449 try self.register_manager.getReg(reg, null);
31553450 try self.genSetReg(arg_ty, reg, arg_mcv);
31563451 },
3157 .stack_offset => {
3158 return self.fail("TODO implement calling with parameters in memory", .{});
3159 },
3160 .ptr_stack_offset => {
3161 return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
3162 },
3452 .stack_offset => unreachable,
3453 .stack_argument_offset => |offset| try self.genSetStackArgument(
3454 arg_ty,
3455 offset,
3456 arg_mcv,
3457 ),
31633458 else => unreachable,
31643459 }
31653460 }
......@@ -3303,8 +3598,15 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
33033598 },
33043599 .stack_offset => {
33053600 // Return result by reference
3306 // TODO
3307 return self.fail("TODO implement airRet for {}", .{self.ret_mcv});
3601 //
3602 // self.ret_mcv is an address to where this function
3603 // should store its result into
3604 var ptr_ty_payload: Type.Payload.ElemType = .{
3605 .base = .{ .tag = .single_mut_pointer },
3606 .data = ret_ty,
3607 };
3608 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
3609 try self.store(self.ret_mcv, operand, ptr_ty, ret_ty);
33083610 },
33093611 else => unreachable,
33103612 }
......@@ -3330,10 +3632,34 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
33303632 },
33313633 .stack_offset => {
33323634 // Return result by reference
3333 // TODO
3334 return self.fail("TODO implement airRetLoad for {}", .{self.ret_mcv});
3635 //
3636 // self.ret_mcv is an address to where this function
3637 // should store its result into
3638 //
3639 // If the operand is a ret_ptr instruction, we are done
3640 // here. Else we need to load the result from the location
3641 // pointed to by the operand and store it to the result
3642 // location.
3643 const op_inst = Air.refToIndex(un_op).?;
3644 if (self.air.instructions.items(.tag)[op_inst] != .ret_ptr) {
3645 const abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
3646 const abi_align = ret_ty.abiAlignment(self.target.*);
3647
3648 // This is essentially allocMem without the
3649 // instruction tracking
3650 if (abi_align > self.stack_align)
3651 self.stack_align = abi_align;
3652 // TODO find a free slot instead of always appending
3653 const offset = mem.alignForwardGeneric(u32, self.next_stack_offset, abi_align) + abi_size;
3654 self.next_stack_offset = offset;
3655 self.max_end_stack = @maximum(self.max_end_stack, self.next_stack_offset);
3656
3657 const tmp_mcv = MCValue{ .stack_offset = offset };
3658 try self.load(tmp_mcv, ptr, ptr_ty);
3659 try self.store(self.ret_mcv, tmp_mcv, ptr_ty, ret_ty);
3660 }
33353661 },
3336 else => unreachable,
3662 else => unreachable, // invalid return result
33373663 }
33383664
33393665 try self.exitlude_jump_relocs.append(self.gpa, try self.addNop());
......@@ -3635,40 +3961,14 @@ fn isNonNull(self: *Self, operand: MCValue) !MCValue {
36353961
36363962fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
36373963 const error_type = ty.errorUnionSet();
3638 const payload_type = ty.errorUnionPayload();
3964 const error_int_type = Type.initTag(.u16);
36393965
36403966 if (error_type.errorSetIsEmpty()) {
36413967 return MCValue{ .immediate = 0 }; // always false
36423968 }
36433969
3644 const err_off = errUnionErrorOffset(payload_type, self.target.*);
3645 switch (operand) {
3646 .stack_offset => |off| {
3647 const offset = off - @intCast(u32, err_off);
3648 const tmp_reg = try self.copyToTmpRegister(Type.anyerror, .{ .stack_offset = offset });
3649 _ = try self.addInst(.{
3650 .tag = .cmp_immediate,
3651 .data = .{ .r_imm12_sh = .{
3652 .rn = tmp_reg,
3653 .imm12 = 0,
3654 } },
3655 });
3656 },
3657 .register => |reg| {
3658 if (err_off > 0 or payload_type.hasRuntimeBitsIgnoreComptime()) {
3659 return self.fail("TODO implement isErr for register operand with payload bits", .{});
3660 }
3661 _ = try self.addInst(.{
3662 .tag = .cmp_immediate,
3663 .data = .{ .r_imm12_sh = .{
3664 .rn = reg,
3665 .imm12 = 0,
3666 } },
3667 });
3668 },
3669 else => return self.fail("TODO implement isErr for {}", .{operand}),
3670 }
3671
3970 const error_mcv = try self.errUnionErr(operand, ty);
3971 _ = try self.binOp(.cmp_eq, error_mcv, .{ .immediate = 0 }, error_int_type, error_int_type, null);
36723972 return MCValue{ .condition_flags = .hi };
36733973}
36743974
......@@ -3886,7 +4186,7 @@ fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
38864186 block_data.mcv = switch (operand_mcv) {
38874187 .none, .dead, .unreach => unreachable,
38884188 .register, .stack_offset, .memory => operand_mcv,
3889 .immediate, .condition_flags => blk: {
4189 .immediate, .stack_argument_offset, .condition_flags => blk: {
38904190 const new_mcv = try self.allocRegOrMem(block, true);
38914191 try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv);
38924192 break :blk new_mcv;
......@@ -4128,6 +4428,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
41284428 .got_load,
41294429 .direct_load,
41304430 .memory,
4431 .stack_argument_offset,
41314432 .stack_offset,
41324433 => {
41334434 switch (mcv) {
......@@ -4166,6 +4467,15 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
41664467 // sub src_reg, fp, #off
41674468 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
41684469 },
4470 .stack_argument_offset => |off| {
4471 _ = try self.addInst(.{
4472 .tag = .ldr_ptr_stack_argument,
4473 .data = .{ .load_store_stack = .{
4474 .rt = src_reg,
4475 .offset = off,
4476 } },
4477 });
4478 },
41694479 .memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = addr }),
41704480 .got_load,
41714481 .direct_load,
......@@ -4269,6 +4579,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
42694579 }
42704580 },
42714581 .register => |src_reg| {
4582 assert(src_reg.size() == reg.size());
4583
42724584 // If the registers are the same, nothing to do.
42734585 if (src_reg.id() == reg.id())
42744586 return;
......@@ -4330,6 +4642,196 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
43304642 else => unreachable,
43314643 }
43324644 },
4645 .stack_argument_offset => |off| {
4646 const abi_size = ty.abiSize(self.target.*);
4647
4648 switch (abi_size) {
4649 1, 2, 4, 8 => {
4650 const tag: Mir.Inst.Tag = switch (abi_size) {
4651 1 => if (ty.isSignedInt()) Mir.Inst.Tag.ldrsb_stack_argument else .ldrb_stack_argument,
4652 2 => if (ty.isSignedInt()) Mir.Inst.Tag.ldrsh_stack_argument else .ldrh_stack_argument,
4653 4, 8 => .ldr_stack_argument,
4654 else => unreachable, // unexpected abi size
4655 };
4656
4657 _ = try self.addInst(.{
4658 .tag = tag,
4659 .data = .{ .load_store_stack = .{
4660 .rt = reg,
4661 .offset = @intCast(u32, off),
4662 } },
4663 });
4664 },
4665 3, 5, 6, 7 => return self.fail("TODO implement genSetReg types size {}", .{abi_size}),
4666 else => unreachable,
4667 }
4668 },
4669 }
4670}
4671
4672fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
4673 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4674 switch (mcv) {
4675 .dead => unreachable,
4676 .none, .unreach => return,
4677 .undef => {
4678 if (!self.wantSafety())
4679 return; // The already existing value will do just fine.
4680 // TODO Upgrade this to a memset call when we have that available.
4681 switch (ty.abiSize(self.target.*)) {
4682 1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
4683 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
4684 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
4685 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
4686 else => return self.fail("TODO implement memset", .{}),
4687 }
4688 },
4689 .register => |reg| {
4690 switch (abi_size) {
4691 1, 2, 4, 8 => {
4692 const tag: Mir.Inst.Tag = switch (abi_size) {
4693 1 => .strb_immediate,
4694 2 => .strh_immediate,
4695 4, 8 => .str_immediate,
4696 else => unreachable, // unexpected abi size
4697 };
4698 const rt = registerAlias(reg, abi_size);
4699 const offset = switch (abi_size) {
4700 1 => blk: {
4701 if (math.cast(u12, stack_offset)) |imm| {
4702 break :blk Instruction.LoadStoreOffset.imm(imm);
4703 } else {
4704 return self.fail("TODO genSetStackArgument byte with larger offset", .{});
4705 }
4706 },
4707 2 => blk: {
4708 assert(std.mem.isAlignedGeneric(u32, stack_offset, 2)); // misaligned stack entry
4709 if (math.cast(u12, @divExact(stack_offset, 2))) |imm| {
4710 break :blk Instruction.LoadStoreOffset.imm(imm);
4711 } else {
4712 return self.fail("TODO getSetStackArgument halfword with larger offset", .{});
4713 }
4714 },
4715 4, 8 => blk: {
4716 const alignment = abi_size;
4717 assert(std.mem.isAlignedGeneric(u32, stack_offset, alignment)); // misaligned stack entry
4718 if (math.cast(u12, @divExact(stack_offset, alignment))) |imm| {
4719 break :blk Instruction.LoadStoreOffset.imm(imm);
4720 } else {
4721 return self.fail("TODO genSetStackArgument with larger offset", .{});
4722 }
4723 },
4724 else => unreachable,
4725 };
4726
4727 _ = try self.addInst(.{
4728 .tag = tag,
4729 .data = .{ .load_store_register_immediate = .{
4730 .rt = rt,
4731 .rn = .sp,
4732 .offset = offset.immediate,
4733 } },
4734 });
4735 },
4736 else => return self.fail("TODO genSetStackArgument other types abi_size={}", .{abi_size}),
4737 }
4738 },
4739 .register_with_overflow => {
4740 return self.fail("TODO implement genSetStackArgument {}", .{mcv});
4741 },
4742 .got_load,
4743 .direct_load,
4744 .memory,
4745 .stack_argument_offset,
4746 .stack_offset,
4747 => {
4748 if (abi_size <= 4) {
4749 const reg = try self.copyToTmpRegister(ty, mcv);
4750 return self.genSetStackArgument(ty, stack_offset, MCValue{ .register = reg });
4751 } else {
4752 var ptr_ty_payload: Type.Payload.ElemType = .{
4753 .base = .{ .tag = .single_mut_pointer },
4754 .data = ty,
4755 };
4756 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
4757
4758 // TODO call extern memcpy
4759 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
4760 const regs_locks = self.register_manager.lockRegsAssumeUnused(5, regs);
4761 defer for (regs_locks) |reg| {
4762 self.register_manager.unlockReg(reg);
4763 };
4764
4765 const src_reg = regs[0];
4766 const dst_reg = regs[1];
4767 const len_reg = regs[2];
4768 const count_reg = regs[3];
4769 const tmp_reg = regs[4];
4770
4771 switch (mcv) {
4772 .stack_offset => |off| {
4773 // sub src_reg, fp, #off
4774 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
4775 },
4776 .stack_argument_offset => |off| {
4777 _ = try self.addInst(.{
4778 .tag = .ldr_ptr_stack_argument,
4779 .data = .{ .load_store_stack = .{
4780 .rt = src_reg,
4781 .offset = off,
4782 } },
4783 });
4784 },
4785 .memory => |addr| try self.genSetReg(ptr_ty, src_reg, .{ .immediate = @intCast(u32, addr) }),
4786 .got_load,
4787 .direct_load,
4788 => |sym_index| {
4789 const tag: Mir.Inst.Tag = switch (mcv) {
4790 .got_load => .load_memory_ptr_got,
4791 .direct_load => .load_memory_ptr_direct,
4792 else => unreachable,
4793 };
4794 const mod = self.bin_file.options.module.?;
4795 _ = try self.addInst(.{
4796 .tag = tag,
4797 .data = .{
4798 .payload = try self.addExtra(Mir.LoadMemoryPie{
4799 .register = @enumToInt(src_reg),
4800 .atom_index = mod.declPtr(self.mod_fn.owner_decl).link.macho.sym_index,
4801 .sym_index = sym_index,
4802 }),
4803 },
4804 });
4805 },
4806 else => unreachable,
4807 }
4808
4809 // add dst_reg, sp, #stack_offset
4810 _ = try self.addInst(.{
4811 .tag = .add_immediate,
4812 .data = .{ .rr_imm12_sh = .{
4813 .rd = dst_reg,
4814 .rn = .sp,
4815 .imm12 = math.cast(u12, stack_offset) orelse {
4816 return self.fail("TODO load: set reg to stack offset with all possible offsets", .{});
4817 },
4818 } },
4819 });
4820
4821 // mov len, #abi_size
4822 try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
4823
4824 // memcpy(src, dst, len)
4825 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
4826 }
4827 },
4828 .condition_flags,
4829 .immediate,
4830 .ptr_stack_offset,
4831 => {
4832 const reg = try self.copyToTmpRegister(ty, mcv);
4833 return self.genSetStackArgument(ty, stack_offset, MCValue{ .register = reg });
4834 },
43334835 }
43344836}
43354837
......@@ -4799,11 +5301,27 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
47995301 result.stack_align = 1;
48005302 return result;
48015303 },
4802 .Unspecified, .C => {
5304 .C => {
48035305 // ARM64 Procedure Call Standard
48045306 var ncrn: usize = 0; // Next Core Register Number
48055307 var nsaa: u32 = 0; // Next stacked argument address
48065308
5309 if (ret_ty.zigTypeTag() == .NoReturn) {
5310 result.return_value = .{ .unreach = {} };
5311 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {
5312 result.return_value = .{ .none = {} };
5313 } else {
5314 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
5315 if (ret_ty_size == 0) {
5316 assert(ret_ty.isError());
5317 result.return_value = .{ .immediate = 0 };
5318 } else if (ret_ty_size <= 8) {
5319 result.return_value = .{ .register = registerAlias(c_abi_int_return_regs[0], ret_ty_size) };
5320 } else {
5321 return self.fail("TODO support more return types for ARM backend", .{});
5322 }
5323 }
5324
48075325 for (param_types) |ty, i| {
48085326 const param_size = @intCast(u32, ty.abiSize(self.target.*));
48095327 if (param_size == 0) {
......@@ -4837,7 +5355,7 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
48375355 }
48385356 }
48395357
4840 result.args[i] = .{ .stack_offset = nsaa };
5358 result.args[i] = .{ .stack_argument_offset = nsaa };
48415359 nsaa += param_size;
48425360 }
48435361 }
......@@ -4845,28 +5363,49 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
48455363 result.stack_byte_count = nsaa;
48465364 result.stack_align = 16;
48475365 },
4848 else => return self.fail("TODO implement function parameters for {} on aarch64", .{cc}),
4849 }
4850
4851 if (ret_ty.zigTypeTag() == .NoReturn) {
4852 result.return_value = .{ .unreach = {} };
4853 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {
4854 result.return_value = .{ .none = {} };
4855 } else switch (cc) {
4856 .Naked => unreachable,
4857 .Unspecified, .C => {
4858 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
4859 if (ret_ty_size == 0) {
4860 assert(ret_ty.isError());
4861 result.return_value = .{ .immediate = 0 };
4862 } else if (ret_ty_size <= 8) {
4863 result.return_value = .{ .register = registerAlias(c_abi_int_return_regs[0], ret_ty_size) };
5366 .Unspecified => {
5367 if (ret_ty.zigTypeTag() == .NoReturn) {
5368 result.return_value = .{ .unreach = {} };
5369 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {
5370 result.return_value = .{ .none = {} };
48645371 } else {
4865 return self.fail("TODO support more return types for ARM backend", .{});
5372 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
5373 if (ret_ty_size == 0) {
5374 assert(ret_ty.isError());
5375 result.return_value = .{ .immediate = 0 };
5376 } else if (ret_ty_size <= 8) {
5377 result.return_value = .{ .register = registerAlias(.x0, ret_ty_size) };
5378 } else {
5379 // The result is returned by reference, not by
5380 // value. This means that x0 (or w0 when pointer
5381 // size is 32 bits) will contain the address of
5382 // where this function should write the result
5383 // into.
5384 result.return_value = .{ .stack_offset = 0 };
5385 }
48665386 }
5387
5388 var stack_offset: u32 = 0;
5389
5390 for (param_types) |ty, i| {
5391 if (ty.abiSize(self.target.*) > 0) {
5392 const param_size = @intCast(u32, ty.abiSize(self.target.*));
5393 const param_alignment = ty.abiAlignment(self.target.*);
5394
5395 stack_offset = std.mem.alignForwardGeneric(u32, stack_offset, param_alignment);
5396 result.args[i] = .{ .stack_argument_offset = stack_offset };
5397 stack_offset += param_size;
5398 } else {
5399 result.args[i] = .{ .none = {} };
5400 }
5401 }
5402
5403 result.stack_byte_count = stack_offset;
5404 result.stack_align = 16;
48675405 },
4868 else => return self.fail("TODO implement function return values for {}", .{cc}),
5406 else => return self.fail("TODO implement function parameters for {} on aarch64", .{cc}),
48695407 }
5408
48705409 return result;
48715410}
48725411
src/arch/aarch64/Emit.zig+109-10
......@@ -27,14 +27,21 @@ code: *std.ArrayList(u8),
2727
2828prev_di_line: u32,
2929prev_di_column: u32,
30
3031/// Relative to the beginning of `code`.
3132prev_di_pc: usize,
3233
34/// The amount of stack space consumed by the saved callee-saved
35/// registers in bytes
36saved_regs_stack_space: u32,
37
3338/// The branch type of every branch
3439branch_types: std.AutoHashMapUnmanaged(Mir.Inst.Index, BranchType) = .{},
40
3541/// For every forward branch, maps the target instruction to a list of
3642/// branches which branch to this target instruction
3743branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUnmanaged(Mir.Inst.Index)) = .{},
44
3845/// For backward branches: stores the code offset of the target
3946/// instruction
4047///
......@@ -42,6 +49,8 @@ branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUn
4249/// instruction
4350code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
4451
52/// The final stack frame size of the function (already aligned to the
53/// respective stack alignment). Does not include prologue stack space.
4554stack_size: u32,
4655
4756const InnerError = error{
......@@ -82,9 +91,11 @@ pub fn emitMir(
8291 .sub_immediate => try emit.mirAddSubtractImmediate(inst),
8392 .subs_immediate => try emit.mirAddSubtractImmediate(inst),
8493
85 .asr_register => try emit.mirShiftRegister(inst),
86 .lsl_register => try emit.mirShiftRegister(inst),
87 .lsr_register => try emit.mirShiftRegister(inst),
94 .asr_register => try emit.mirDataProcessing2Source(inst),
95 .lsl_register => try emit.mirDataProcessing2Source(inst),
96 .lsr_register => try emit.mirDataProcessing2Source(inst),
97 .sdiv => try emit.mirDataProcessing2Source(inst),
98 .udiv => try emit.mirDataProcessing2Source(inst),
8899
89100 .asr_immediate => try emit.mirShiftImmediate(inst),
90101 .lsl_immediate => try emit.mirShiftImmediate(inst),
......@@ -148,6 +159,13 @@ pub fn emitMir(
148159 .strb_stack => try emit.mirLoadStoreStack(inst),
149160 .strh_stack => try emit.mirLoadStoreStack(inst),
150161
162 .ldr_stack_argument => try emit.mirLoadStackArgument(inst),
163 .ldr_ptr_stack_argument => try emit.mirLoadStackArgument(inst),
164 .ldrb_stack_argument => try emit.mirLoadStackArgument(inst),
165 .ldrh_stack_argument => try emit.mirLoadStackArgument(inst),
166 .ldrsb_stack_argument => try emit.mirLoadStackArgument(inst),
167 .ldrsh_stack_argument => try emit.mirLoadStackArgument(inst),
168
151169 .ldr_register => try emit.mirLoadStoreRegisterRegister(inst),
152170 .ldrb_register => try emit.mirLoadStoreRegisterRegister(inst),
153171 .ldrh_register => try emit.mirLoadStoreRegisterRegister(inst),
......@@ -172,6 +190,7 @@ pub fn emitMir(
172190 .movk => try emit.mirMoveWideImmediate(inst),
173191 .movz => try emit.mirMoveWideImmediate(inst),
174192
193 .msub => try emit.mirDataProcessing3Source(inst),
175194 .mul => try emit.mirDataProcessing3Source(inst),
176195 .smulh => try emit.mirDataProcessing3Source(inst),
177196 .smull => try emit.mirDataProcessing3Source(inst),
......@@ -504,7 +523,7 @@ fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
504523 }
505524}
506525
507fn mirShiftRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
526fn mirDataProcessing2Source(emit: *Emit, inst: Mir.Inst.Index) !void {
508527 const tag = emit.mir.instructions.items(.tag)[inst];
509528 const rrr = emit.mir.instructions.items(.data)[inst].rrr;
510529 const rd = rrr.rd;
......@@ -515,6 +534,8 @@ fn mirShiftRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
515534 .asr_register => try emit.writeInstruction(Instruction.asrRegister(rd, rn, rm)),
516535 .lsl_register => try emit.writeInstruction(Instruction.lslRegister(rd, rn, rm)),
517536 .lsr_register => try emit.writeInstruction(Instruction.lsrRegister(rd, rn, rm)),
537 .sdiv => try emit.writeInstruction(Instruction.sdiv(rd, rn, rm)),
538 .udiv => try emit.writeInstruction(Instruction.udiv(rd, rn, rm)),
518539 else => unreachable,
519540 }
520541}
......@@ -920,6 +941,67 @@ fn mirLoadStoreRegisterPair(emit: *Emit, inst: Mir.Inst.Index) !void {
920941 }
921942}
922943
944fn mirLoadStackArgument(emit: *Emit, inst: Mir.Inst.Index) !void {
945 const tag = emit.mir.instructions.items(.tag)[inst];
946 const load_store_stack = emit.mir.instructions.items(.data)[inst].load_store_stack;
947 const rt = load_store_stack.rt;
948
949 const raw_offset = emit.stack_size + emit.saved_regs_stack_space + load_store_stack.offset;
950 switch (tag) {
951 .ldr_ptr_stack_argument => {
952 const offset = if (math.cast(u12, raw_offset)) |imm| imm else {
953 return emit.fail("TODO load stack argument ptr with larger offset", .{});
954 };
955
956 switch (tag) {
957 .ldr_ptr_stack_argument => try emit.writeInstruction(Instruction.add(rt, .sp, offset, false)),
958 else => unreachable,
959 }
960 },
961 .ldrb_stack_argument, .ldrsb_stack_argument => {
962 const offset = if (math.cast(u12, raw_offset)) |imm| Instruction.LoadStoreOffset.imm(imm) else {
963 return emit.fail("TODO load stack argument byte with larger offset", .{});
964 };
965
966 switch (tag) {
967 .ldrb_stack_argument => try emit.writeInstruction(Instruction.ldrb(rt, .sp, offset)),
968 .ldrsb_stack_argument => try emit.writeInstruction(Instruction.ldrsb(rt, .sp, offset)),
969 else => unreachable,
970 }
971 },
972 .ldrh_stack_argument, .ldrsh_stack_argument => {
973 assert(std.mem.isAlignedGeneric(u32, raw_offset, 2)); // misaligned stack entry
974 const offset = if (math.cast(u12, @divExact(raw_offset, 2))) |imm| Instruction.LoadStoreOffset.imm(imm) else {
975 return emit.fail("TODO load stack argument halfword with larger offset", .{});
976 };
977
978 switch (tag) {
979 .ldrh_stack_argument => try emit.writeInstruction(Instruction.ldrh(rt, .sp, offset)),
980 .ldrsh_stack_argument => try emit.writeInstruction(Instruction.ldrsh(rt, .sp, offset)),
981 else => unreachable,
982 }
983 },
984 .ldr_stack_argument => {
985 const alignment: u32 = switch (rt.size()) {
986 32 => 4,
987 64 => 8,
988 else => unreachable,
989 };
990
991 assert(std.mem.isAlignedGeneric(u32, raw_offset, alignment)); // misaligned stack entry
992 const offset = if (math.cast(u12, @divExact(raw_offset, alignment))) |imm| Instruction.LoadStoreOffset.imm(imm) else {
993 return emit.fail("TODO load stack argument with larger offset", .{});
994 };
995
996 switch (tag) {
997 .ldr_stack_argument => try emit.writeInstruction(Instruction.ldr(rt, .sp, offset)),
998 else => unreachable,
999 }
1000 },
1001 else => unreachable,
1002 }
1003}
1004
9231005fn mirLoadStoreStack(emit: *Emit, inst: Mir.Inst.Index) !void {
9241006 const tag = emit.mir.instructions.items(.tag)[inst];
9251007 const load_store_stack = emit.mir.instructions.items(.data)[inst].load_store_stack;
......@@ -1059,14 +1141,31 @@ fn mirMoveWideImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
10591141
10601142fn mirDataProcessing3Source(emit: *Emit, inst: Mir.Inst.Index) !void {
10611143 const tag = emit.mir.instructions.items(.tag)[inst];
1062 const rrr = emit.mir.instructions.items(.data)[inst].rrr;
10631144
10641145 switch (tag) {
1065 .mul => try emit.writeInstruction(Instruction.mul(rrr.rd, rrr.rn, rrr.rm)),
1066 .smulh => try emit.writeInstruction(Instruction.smulh(rrr.rd, rrr.rn, rrr.rm)),
1067 .smull => try emit.writeInstruction(Instruction.smull(rrr.rd, rrr.rn, rrr.rm)),
1068 .umulh => try emit.writeInstruction(Instruction.umulh(rrr.rd, rrr.rn, rrr.rm)),
1069 .umull => try emit.writeInstruction(Instruction.umull(rrr.rd, rrr.rn, rrr.rm)),
1146 .mul,
1147 .smulh,
1148 .smull,
1149 .umulh,
1150 .umull,
1151 => {
1152 const rrr = emit.mir.instructions.items(.data)[inst].rrr;
1153 switch (tag) {
1154 .mul => try emit.writeInstruction(Instruction.mul(rrr.rd, rrr.rn, rrr.rm)),
1155 .smulh => try emit.writeInstruction(Instruction.smulh(rrr.rd, rrr.rn, rrr.rm)),
1156 .smull => try emit.writeInstruction(Instruction.smull(rrr.rd, rrr.rn, rrr.rm)),
1157 .umulh => try emit.writeInstruction(Instruction.umulh(rrr.rd, rrr.rn, rrr.rm)),
1158 .umull => try emit.writeInstruction(Instruction.umull(rrr.rd, rrr.rn, rrr.rm)),
1159 else => unreachable,
1160 }
1161 },
1162 .msub => {
1163 const rrrr = emit.mir.instructions.items(.data)[inst].rrrr;
1164 switch (tag) {
1165 .msub => try emit.writeInstruction(Instruction.msub(rrrr.rd, rrrr.rn, rrrr.rm, rrrr.ra)),
1166 else => unreachable,
1167 }
1168 },
10701169 else => unreachable,
10711170 }
10721171}
src/arch/aarch64/Mir.zig+27
......@@ -92,20 +92,28 @@ pub const Inst = struct {
9292 load_memory_ptr_direct,
9393 /// Load Pair of Registers
9494 ldp,
95 /// Pseudo-instruction: Load pointer to stack argument
96 ldr_ptr_stack_argument,
9597 /// Pseudo-instruction: Load from stack
9698 ldr_stack,
99 /// Pseudo-instruction: Load from stack argument
100 ldr_stack_argument,
97101 /// Load Register (immediate)
98102 ldr_immediate,
99103 /// Load Register (register)
100104 ldr_register,
101105 /// Pseudo-instruction: Load byte from stack
102106 ldrb_stack,
107 /// Pseudo-instruction: Load byte from stack argument
108 ldrb_stack_argument,
103109 /// Load Register Byte (immediate)
104110 ldrb_immediate,
105111 /// Load Register Byte (register)
106112 ldrb_register,
107113 /// Pseudo-instruction: Load halfword from stack
108114 ldrh_stack,
115 /// Pseudo-instruction: Load halfword from stack argument
116 ldrh_stack_argument,
109117 /// Load Register Halfword (immediate)
110118 ldrh_immediate,
111119 /// Load Register Halfword (register)
......@@ -114,10 +122,14 @@ pub const Inst = struct {
114122 ldrsb_immediate,
115123 /// Pseudo-instruction: Load signed byte from stack
116124 ldrsb_stack,
125 /// Pseudo-instruction: Load signed byte from stack argument
126 ldrsb_stack_argument,
117127 /// Load Register Signed Halfword (immediate)
118128 ldrsh_immediate,
119129 /// Pseudo-instruction: Load signed halfword from stack
120130 ldrsh_stack,
131 /// Pseudo-instruction: Load signed halfword from stack argument
132 ldrsh_stack_argument,
121133 /// Load Register Signed Word (immediate)
122134 ldrsw_immediate,
123135 /// Logical Shift Left (immediate)
......@@ -136,6 +148,8 @@ pub const Inst = struct {
136148 movk,
137149 /// Move wide with zero
138150 movz,
151 /// Multiply-subtract
152 msub,
139153 /// Multiply
140154 mul,
141155 /// Bitwise NOT
......@@ -152,6 +166,8 @@ pub const Inst = struct {
152166 ret,
153167 /// Signed bitfield extract
154168 sbfx,
169 /// Signed divide
170 sdiv,
155171 /// Signed multiply high
156172 smulh,
157173 /// Signed multiply long
......@@ -200,6 +216,8 @@ pub const Inst = struct {
200216 tst_immediate,
201217 /// Unsigned bitfield extract
202218 ubfx,
219 /// Unsigned divide
220 udiv,
203221 /// Unsigned multiply high
204222 umulh,
205223 /// Unsigned multiply long
......@@ -430,6 +448,15 @@ pub const Inst = struct {
430448 rn: Register,
431449 offset: bits.Instruction.LoadStorePairOffset,
432450 },
451 /// Four registers
452 ///
453 /// Used by e.g. msub
454 rrrr: struct {
455 rd: Register,
456 rn: Register,
457 rm: Register,
458 ra: Register,
459 },
433460 /// Debug info: line and column
434461 ///
435462 /// Used by e.g. dbg_line
src/arch/aarch64/bits.zig+8
......@@ -1698,6 +1698,14 @@ pub const Instruction = union(enum) {
16981698
16991699 // Data processing (2 source)
17001700
1701 pub fn udiv(rd: Register, rn: Register, rm: Register) Instruction {
1702 return dataProcessing2Source(0b0, 0b000010, rd, rn, rm);
1703 }
1704
1705 pub fn sdiv(rd: Register, rn: Register, rm: Register) Instruction {
1706 return dataProcessing2Source(0b0, 0b000011, rd, rn, rm);
1707 }
1708
17011709 pub fn lslv(rd: Register, rn: Register, rm: Register) Instruction {
17021710 return dataProcessing2Source(0b0, 0b001000, rd, rn, rm);
17031711 }