authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2021-11-10 22:19:22-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2021-11-10 22:19:22-08:00
log53523ef5d0413459bd2eb9d84d2338f2bc49d417
tree7ca061b0fbd524aed732e7393449c41220536dfb
parentf3ba72cf5a7e2c40c6a10fd987e7f4486c2b9300
parent4168b01e7a6fcae6f541d098491a4e1e00041f1c
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #10129 from joachimschmidt557/stage2-aarch64

stage2 AArch64: Implement conditional branches

5 files changed, 702 insertions(+), 96 deletions(-)

src/arch/aarch64/CodeGen.zig+257-34
......@@ -177,7 +177,7 @@ const StackAllocation = struct {
177177};
178178
179179const BlockData = struct {
180 relocs: std.ArrayListUnmanaged(Reloc),
180 relocs: std.ArrayListUnmanaged(Mir.Inst.Index),
181181 /// The first break instruction encounters `null` here and chooses a
182182 /// machine code value for the block result, populating this field.
183183 /// Following break instructions encounter that value and use it for
......@@ -185,18 +185,6 @@ const BlockData = struct {
185185 mcv: MCValue,
186186};
187187
188const Reloc = union(enum) {
189 /// The value is an offset into the `Function` `code` from the beginning.
190 /// To perform the reloc, write 32-bit signed little-endian integer
191 /// which is a relative jump, based on the address following the reloc.
192 rel32: usize,
193 /// A branch in the ARM instruction set
194 arm_branch: struct {
195 pos: usize,
196 cond: @import("../arm/bits.zig").Condition,
197 },
198};
199
200188const BigTomb = struct {
201189 function: *Self,
202190 inst: Air.Inst.Index,
......@@ -426,6 +414,12 @@ fn gen(self: *Self) !void {
426414 });
427415 }
428416
417 // add sp, sp, #stack_size
418 _ = try self.addInst(.{
419 .tag = .add_immediate,
420 .data = .{ .rr_imm12_sh = .{ .rd = .xzr, .rn = .xzr, .imm12 = @intCast(u12, aligned_stack_end) } },
421 });
422
429423 // ldp fp, lr, [sp], #16
430424 _ = try self.addInst(.{
431425 .tag = .ldp,
......@@ -437,12 +431,6 @@ fn gen(self: *Self) !void {
437431 } },
438432 });
439433
440 // add sp, sp, #stack_size
441 _ = try self.addInst(.{
442 .tag = .add_immediate,
443 .data = .{ .rr_imm12_sh = .{ .rd = .xzr, .rn = .xzr, .imm12 = @intCast(u12, aligned_stack_end) } },
444 });
445
446434 // ret lr
447435 _ = try self.addInst(.{
448436 .tag = .ret,
......@@ -1358,7 +1346,9 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
13581346 const stack_offset = try self.allocMem(inst, abi_size, abi_align);
13591347 try self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
13601348
1361 break :blk MCValue{ .stack_offset = stack_offset };
1349 // TODO correct loading and storing from memory
1350 // break :blk MCValue{ .stack_offset = stack_offset };
1351 break :blk result;
13621352 },
13631353 else => result,
13641354 };
......@@ -1634,8 +1624,6 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
16341624}
16351625
16361626fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1637 _ = op;
1638
16391627 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
16401628 if (self.liveness.isUnused(inst))
16411629 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
......@@ -1646,10 +1634,79 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
16461634
16471635 const lhs = try self.resolveInst(bin_op.lhs);
16481636 const rhs = try self.resolveInst(bin_op.rhs);
1649 _ = lhs;
1650 _ = rhs;
1637 const result: MCValue = result: {
1638 const lhs_is_register = lhs == .register;
1639 const rhs_is_register = rhs == .register;
1640 // lhs should always be a register
1641 const rhs_should_be_register = switch (rhs) {
1642 .immediate => |imm| imm < 0 or imm > std.math.maxInt(u12),
1643 else => true,
1644 };
1645
1646 var lhs_mcv = lhs;
1647 var rhs_mcv = rhs;
1648
1649 // Allocate registers
1650 if (rhs_should_be_register) {
1651 if (!lhs_is_register and !rhs_is_register) {
1652 const regs = try self.register_manager.allocRegs(2, .{
1653 Air.refToIndex(bin_op.rhs).?, Air.refToIndex(bin_op.lhs).?,
1654 }, &.{});
1655 lhs_mcv = MCValue{ .register = regs[0] };
1656 rhs_mcv = MCValue{ .register = regs[1] };
1657 } else if (!rhs_is_register) {
1658 rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(bin_op.rhs).?, &.{}) };
1659 }
1660 }
1661 if (!lhs_is_register) {
1662 lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(bin_op.lhs).?, &.{}) };
1663 }
1664
1665 // Move the operands to the newly allocated registers
1666 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1667 if (lhs_mcv == .register and !lhs_is_register) {
1668 try self.genSetReg(ty, lhs_mcv.register, lhs);
1669 branch.inst_table.putAssumeCapacity(Air.refToIndex(bin_op.lhs).?, lhs);
1670 }
1671 if (rhs_mcv == .register and !rhs_is_register) {
1672 try self.genSetReg(ty, rhs_mcv.register, rhs);
1673 branch.inst_table.putAssumeCapacity(Air.refToIndex(bin_op.rhs).?, rhs);
1674 }
1675
1676 // The destination register is not present in the cmp instruction
1677 // The signedness of the integer does not matter for the cmp instruction
1678 switch (rhs_mcv) {
1679 .register => |reg| {
1680 _ = try self.addInst(.{
1681 .tag = .cmp_shifted_register,
1682 .data = .{ .rrr_imm6_shift = .{
1683 .rd = .xzr,
1684 .rn = lhs_mcv.register,
1685 .rm = reg,
1686 .imm6 = 0,
1687 .shift = .lsl,
1688 } },
1689 });
1690 },
1691 .immediate => |imm| {
1692 _ = try self.addInst(.{
1693 .tag = .cmp_immediate,
1694 .data = .{ .rr_imm12_sh = .{
1695 .rd = .xzr,
1696 .rn = lhs_mcv.register,
1697 .imm12 = @intCast(u12, imm),
1698 } },
1699 });
1700 },
1701 else => unreachable,
1702 }
16511703
1652 return self.fail("TODO implement cmp for {}", .{self.target.cpu.arch});
1704 break :result switch (ty.isSignedInt()) {
1705 true => MCValue{ .compare_flags_signed = op },
1706 false => MCValue{ .compare_flags_unsigned = op },
1707 };
1708 };
1709 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
16531710}
16541711
16551712fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
......@@ -1667,9 +1724,153 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
16671724}
16681725
16691726fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
1670 _ = inst;
1727 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
1728 const cond = try self.resolveInst(pl_op.operand);
1729 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
1730 const then_body = self.air.extra[extra.end..][0..extra.data.then_body_len];
1731 const else_body = self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len];
1732 const liveness_condbr = self.liveness.getCondBr(inst);
1733
1734 const reloc: Mir.Inst.Index = switch (cond) {
1735 .compare_flags_signed,
1736 .compare_flags_unsigned,
1737 => try self.addInst(.{
1738 .tag = .b_cond,
1739 .data = .{
1740 .inst_cond = .{
1741 .inst = undefined, // populated later through performReloc
1742 .cond = switch (cond) {
1743 .compare_flags_signed => |cmp_op| blk: {
1744 // Here we map to the opposite condition because the jump is to the false branch.
1745 const condition = Instruction.Condition.fromCompareOperatorSigned(cmp_op);
1746 break :blk condition.negate();
1747 },
1748 .compare_flags_unsigned => |cmp_op| blk: {
1749 // Here we map to the opposite condition because the jump is to the false branch.
1750 const condition = Instruction.Condition.fromCompareOperatorUnsigned(cmp_op);
1751 break :blk condition.negate();
1752 },
1753 else => unreachable,
1754 },
1755 },
1756 },
1757 }),
1758 else => return self.fail("TODO implement condr when condition is {s}", .{@tagName(cond)}),
1759 };
1760
1761 // Capture the state of register and stack allocation state so that we can revert to it.
1762 const parent_next_stack_offset = self.next_stack_offset;
1763 const parent_free_registers = self.register_manager.free_registers;
1764 var parent_stack = try self.stack.clone(self.gpa);
1765 defer parent_stack.deinit(self.gpa);
1766 const parent_registers = self.register_manager.registers;
1767
1768 try self.branch_stack.append(.{});
1769
1770 try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
1771 for (liveness_condbr.then_deaths) |operand| {
1772 self.processDeath(operand);
1773 }
1774 try self.genBody(then_body);
1775
1776 // Revert to the previous register and stack allocation state.
1777
1778 var saved_then_branch = self.branch_stack.pop();
1779 defer saved_then_branch.deinit(self.gpa);
1780
1781 self.register_manager.registers = parent_registers;
1782
1783 self.stack.deinit(self.gpa);
1784 self.stack = parent_stack;
1785 parent_stack = .{};
1786
1787 self.next_stack_offset = parent_next_stack_offset;
1788 self.register_manager.free_registers = parent_free_registers;
1789
1790 try self.performReloc(reloc);
1791 const else_branch = self.branch_stack.addOneAssumeCapacity();
1792 else_branch.* = .{};
1793
1794 try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
1795 for (liveness_condbr.else_deaths) |operand| {
1796 self.processDeath(operand);
1797 }
1798 try self.genBody(else_body);
1799
1800 // At this point, each branch will possibly have conflicting values for where
1801 // each instruction is stored. They agree, however, on which instructions are alive/dead.
1802 // We use the first ("then") branch as canonical, and here emit
1803 // instructions into the second ("else") branch to make it conform.
1804 // We continue respect the data structure semantic guarantees of the else_branch so
1805 // that we can use all the code emitting abstractions. This is why at the bottom we
1806 // assert that parent_branch.free_registers equals the saved_then_branch.free_registers
1807 // rather than assigning it.
1808 const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2];
1809 try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count());
1810
1811 const else_slice = else_branch.inst_table.entries.slice();
1812 const else_keys = else_slice.items(.key);
1813 const else_values = else_slice.items(.value);
1814 for (else_keys) |else_key, else_idx| {
1815 const else_value = else_values[else_idx];
1816 const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) |then_entry| blk: {
1817 // The instruction's MCValue is overridden in both branches.
1818 parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value);
1819 if (else_value == .dead) {
1820 assert(then_entry.value == .dead);
1821 continue;
1822 }
1823 break :blk then_entry.value;
1824 } else blk: {
1825 if (else_value == .dead)
1826 continue;
1827 // The instruction is only overridden in the else branch.
1828 var i: usize = self.branch_stack.items.len - 2;
1829 while (true) {
1830 i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
1831 if (self.branch_stack.items[i].inst_table.get(else_key)) |mcv| {
1832 assert(mcv != .dead);
1833 break :blk mcv;
1834 }
1835 }
1836 };
1837 log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv });
1838 // TODO make sure the destination stack offset / register does not already have something
1839 // going on there.
1840 try self.setRegOrMem(self.air.typeOfIndex(else_key), canon_mcv, else_value);
1841 // TODO track the new register / stack allocation
1842 }
1843 try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count());
1844 const then_slice = saved_then_branch.inst_table.entries.slice();
1845 const then_keys = then_slice.items(.key);
1846 const then_values = then_slice.items(.value);
1847 for (then_keys) |then_key, then_idx| {
1848 const then_value = then_values[then_idx];
1849 // We already deleted the items from this table that matched the else_branch.
1850 // So these are all instructions that are only overridden in the then branch.
1851 parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
1852 if (then_value == .dead)
1853 continue;
1854 const parent_mcv = blk: {
1855 var i: usize = self.branch_stack.items.len - 2;
1856 while (true) {
1857 i -= 1;
1858 if (self.branch_stack.items[i].inst_table.get(then_key)) |mcv| {
1859 assert(mcv != .dead);
1860 break :blk mcv;
1861 }
1862 }
1863 };
1864 log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value });
1865 // TODO make sure the destination stack offset / register does not already have something
1866 // going on there.
1867 try self.setRegOrMem(self.air.typeOfIndex(then_key), parent_mcv, then_value);
1868 // TODO track the new register / stack allocation
1869 }
16711870
1672 return self.fail("TODO implement condbr {}", .{self.target.cpu.arch});
1871 self.branch_stack.pop().deinit(self.gpa);
1872
1873 return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
16731874}
16741875
16751876fn isNull(self: *Self, operand: MCValue) !MCValue {
......@@ -1860,10 +2061,12 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
18602061 return self.fail("TODO airSwitch for {}", .{self.target.cpu.arch});
18612062}
18622063
1863fn performReloc(self: *Self, reloc: Reloc) !void {
1864 switch (reloc) {
1865 .rel32 => return self.fail("TODO reloc.rel32 for {}", .{self.target.cpu.arch}),
1866 .arm_branch => return self.fail("TODO reloc.arm_branch for {}", .{self.target.cpu.arch}),
2064fn performReloc(self: *Self, inst: Mir.Inst.Index) !void {
2065 const tag = self.mir_instructions.items(.tag)[inst];
2066 switch (tag) {
2067 .b_cond => self.mir_instructions.items(.data)[inst].inst_cond.inst = @intCast(Air.Inst.Index, self.mir_instructions.len),
2068 .b => self.mir_instructions.items(.data)[inst].inst = @intCast(Air.Inst.Index, self.mir_instructions.len),
2069 else => unreachable,
18672070 }
18682071}
18692072
......@@ -1903,7 +2106,10 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {
19032106 // Emit a jump with a relocation. It will be patched up after the block ends.
19042107 try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
19052108
1906 return self.fail("TODO implement brvoid for {}", .{self.target.cpu.arch});
2109 block_data.relocs.appendAssumeCapacity(try self.addInst(.{
2110 .tag = .b,
2111 .data = .{ .inst = undefined }, // populated later through performReloc
2112 }));
19072113}
19082114
19092115fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
......@@ -2050,8 +2256,6 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
20502256 return self.fail("TODO implement set stack variable from embedded_in_code", .{});
20512257 },
20522258 .register => |reg| {
2053 _ = reg;
2054
20552259 const abi_size = ty.abiSize(self.target.*);
20562260 const adj_off = stack_offset + abi_size;
20572261
......@@ -2115,6 +2319,25 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
21152319 else => unreachable, // unexpected register size
21162320 }
21172321 },
2322 .compare_flags_unsigned,
2323 .compare_flags_signed,
2324 => |op| {
2325 const condition = switch (mcv) {
2326 .compare_flags_unsigned => Instruction.Condition.fromCompareOperatorUnsigned(op),
2327 .compare_flags_signed => Instruction.Condition.fromCompareOperatorSigned(op),
2328 else => unreachable,
2329 };
2330
2331 _ = try self.addInst(.{
2332 .tag = .cset,
2333 .data = .{ .rrr_cond = .{
2334 .rd = reg,
2335 .rn = .xzr,
2336 .rm = .xzr,
2337 .cond = condition,
2338 } },
2339 });
2340 },
21182341 .immediate => |x| {
21192342 _ = try self.addInst(.{
21202343 .tag = .movz,
src/arch/aarch64/Emit.zig+180-60
......@@ -14,6 +14,7 @@ const DW = std.dwarf;
1414const leb128 = std.leb;
1515const Instruction = bits.Instruction;
1616const Register = bits.Register;
17const log = std.log.scoped(.aarch64_emit);
1718const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
1819
1920mir: Mir,
......@@ -47,9 +48,16 @@ const InnerError = error{
4748};
4849
4950const BranchType = enum {
51 b_cond,
5052 unconditional_branch_immediate,
5153
52 const default = BranchType.unconditional_branch_immediate;
54 fn default(tag: Mir.Inst.Tag) BranchType {
55 return switch (tag) {
56 .b, .bl => .unconditional_branch_immediate,
57 .b_cond => .b_cond,
58 else => unreachable,
59 };
60 }
5361};
5462
5563pub fn emitMir(
......@@ -65,8 +73,11 @@ pub fn emitMir(
6573 const inst = @intCast(u32, index);
6674 switch (tag) {
6775 .add_immediate => try emit.mirAddSubtractImmediate(inst),
76 .cmp_immediate => try emit.mirAddSubtractImmediate(inst),
6877 .sub_immediate => try emit.mirAddSubtractImmediate(inst),
6978
79 .b_cond => try emit.mirConditionalBranchImmediate(inst),
80
7081 .b => try emit.mirBranch(inst),
7182 .bl => try emit.mirBranch(inst),
7283
......@@ -78,6 +89,10 @@ pub fn emitMir(
7889
7990 .call_extern => try emit.mirCallExtern(inst),
8091
92 .cmp_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
93
94 .cset => try emit.mirConditionalSelect(inst),
95
8196 .dbg_line => try emit.mirDbgLine(inst),
8297
8398 .dbg_prologue_end => try emit.mirDebugPrologueEnd(),
......@@ -107,29 +122,50 @@ pub fn emitMir(
107122}
108123
109124pub fn deinit(emit: *Emit) void {
125 var iter = emit.branch_forward_origins.valueIterator();
126 while (iter.next()) |origin_list| {
127 origin_list.deinit(emit.bin_file.allocator);
128 }
129
110130 emit.branch_types.deinit(emit.bin_file.allocator);
111131 emit.branch_forward_origins.deinit(emit.bin_file.allocator);
112132 emit.code_offset_mapping.deinit(emit.bin_file.allocator);
113133 emit.* = undefined;
114134}
115135
116fn optimalBranchType(emit: *Emit, offset: i64) !BranchType {
136fn optimalBranchType(emit: *Emit, tag: Mir.Inst.Tag, offset: i64) !BranchType {
117137 assert(offset & 0b11 == 0);
118138
119 // TODO handle conditional branches
120 if (std.math.cast(i26, offset >> 2)) |_| {
121 return BranchType.unconditional_branch_immediate;
122 } else |_| {
123 return emit.fail("TODO support branches larger than +-128 MiB", .{});
139 switch (tag) {
140 .b, .bl => {
141 if (std.math.cast(i26, offset >> 2)) |_| {
142 return BranchType.unconditional_branch_immediate;
143 } else |_| {
144 return emit.fail("TODO support branches larger than +-128 MiB", .{});
145 }
146 },
147 .b_cond => {
148 if (std.math.cast(i19, offset >> 2)) |_| {
149 return BranchType.b_cond;
150 } else |_| {
151 return emit.fail("TODO support conditional branches larger than +-1 MiB", .{});
152 }
153 },
154 else => unreachable,
124155 }
125156}
126157
127158fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize {
128159 const tag = emit.mir.instructions.items(.tag)[inst];
129 switch (tag) {
130 .b, .bl => switch (emit.branch_types.get(inst).?) {
160
161 if (isBranch(tag)) {
162 switch (emit.branch_types.get(inst).?) {
131163 .unconditional_branch_immediate => return 4,
132 },
164 .b_cond => return 4,
165 }
166 }
167
168 switch (tag) {
133169 .load_memory => {
134170 if (emit.bin_file.options.pie) {
135171 // adrp, ldr
......@@ -146,10 +182,32 @@ fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize {
146182 return 5 * 4;
147183 }
148184 },
185 .call_extern => return 4,
186 .dbg_line,
187 .dbg_epilogue_begin,
188 .dbg_prologue_end,
189 => return 0,
149190 else => return 4,
150191 }
151192}
152193
194fn isBranch(tag: Mir.Inst.Tag) bool {
195 return switch (tag) {
196 .b, .bl, .b_cond => true,
197 else => false,
198 };
199}
200
201fn branchTarget(emit: *Emit, inst: Mir.Inst.Index) Mir.Inst.Index {
202 const tag = emit.mir.instructions.items(.tag)[inst];
203
204 switch (tag) {
205 .b, .bl => return emit.mir.instructions.items(.data)[inst].inst,
206 .b_cond => return emit.mir.instructions.items(.data)[inst].inst_cond.inst,
207 else => unreachable,
208 }
209}
210
153211fn lowerBranches(emit: *Emit) !void {
154212 const mir_tags = emit.mir.instructions.items(.tag);
155213 const allocator = emit.bin_file.allocator;
......@@ -162,41 +220,38 @@ fn lowerBranches(emit: *Emit) !void {
162220 // generating MIR
163221 for (mir_tags) |tag, index| {
164222 const inst = @intCast(u32, index);
165 switch (tag) {
166 .b, .bl => {
167 const target_inst = emit.mir.instructions.items(.data)[inst].inst;
168
169 // Remember this branch instruction
170 try emit.branch_types.put(allocator, inst, BranchType.default);
171
172 // Forward branches require some extra stuff: We only
173 // know their offset once we arrive at the target
174 // instruction. Therefore, we need to be able to
175 // access the branch instruction when we visit the
176 // target instruction in order to manipulate its type
177 // etc.
178 if (target_inst > inst) {
179 // Remember the branch instruction index
180 try emit.code_offset_mapping.put(allocator, inst, 0);
181
182 if (emit.branch_forward_origins.getPtr(target_inst)) |origin_list| {
183 try origin_list.append(allocator, inst);
184 } else {
185 var origin_list: std.ArrayListUnmanaged(Mir.Inst.Index) = .{};
186 try origin_list.append(allocator, inst);
187 try emit.branch_forward_origins.put(allocator, target_inst, origin_list);
188 }
223 if (isBranch(tag)) {
224 const target_inst = emit.branchTarget(inst);
225
226 // Remember this branch instruction
227 try emit.branch_types.put(allocator, inst, BranchType.default(tag));
228
229 // Forward branches require some extra stuff: We only
230 // know their offset once we arrive at the target
231 // instruction. Therefore, we need to be able to
232 // access the branch instruction when we visit the
233 // target instruction in order to manipulate its type
234 // etc.
235 if (target_inst > inst) {
236 // Remember the branch instruction index
237 try emit.code_offset_mapping.put(allocator, inst, 0);
238
239 if (emit.branch_forward_origins.getPtr(target_inst)) |origin_list| {
240 try origin_list.append(allocator, inst);
241 } else {
242 var origin_list: std.ArrayListUnmanaged(Mir.Inst.Index) = .{};
243 try origin_list.append(allocator, inst);
244 try emit.branch_forward_origins.put(allocator, target_inst, origin_list);
189245 }
246 }
190247
191 // Remember the target instruction index so that we
192 // update the real code offset in all future passes
193 //
194 // putNoClobber may not be used as the put operation
195 // may clobber the entry when multiple branches branch
196 // to the same target instruction
197 try emit.code_offset_mapping.put(allocator, target_inst, 0);
198 },
199 else => {}, // not a branch
248 // Remember the target instruction index so that we
249 // update the real code offset in all future passes
250 //
251 // putNoClobber may not be used as the put operation
252 // may clobber the entry when multiple branches branch
253 // to the same target instruction
254 try emit.code_offset_mapping.put(allocator, target_inst, 0);
200255 }
201256 }
202257
......@@ -220,21 +275,20 @@ fn lowerBranches(emit: *Emit) !void {
220275
221276 // If this instruction is a backward branch, calculate the
222277 // offset, which may potentially update the branch type
223 switch (tag) {
224 .b, .bl => {
225 const target_inst = emit.mir.instructions.items(.data)[inst].inst;
226 if (target_inst < inst) {
227 const target_offset = emit.code_offset_mapping.get(target_inst).?;
228 const offset = @intCast(i64, target_offset) - @intCast(i64, current_code_offset + 8);
229 const branch_type = emit.branch_types.getPtr(inst).?;
230 const optimal_branch_type = try emit.optimalBranchType(offset);
231 if (branch_type.* != optimal_branch_type) {
232 branch_type.* = optimal_branch_type;
233 all_branches_lowered = false;
234 }
278 if (isBranch(tag)) {
279 const target_inst = emit.branchTarget(inst);
280 if (target_inst < inst) {
281 const target_offset = emit.code_offset_mapping.get(target_inst).?;
282 const offset = @intCast(i64, target_offset) - @intCast(i64, current_code_offset);
283 const branch_type = emit.branch_types.getPtr(inst).?;
284 const optimal_branch_type = try emit.optimalBranchType(tag, offset);
285 if (branch_type.* != optimal_branch_type) {
286 branch_type.* = optimal_branch_type;
287 all_branches_lowered = false;
235288 }
236 },
237 else => {},
289
290 log.debug("lowerBranches: branch {} has offset {}", .{ inst, offset });
291 }
238292 }
239293
240294 // If this instruction is the target of one or more
......@@ -242,14 +296,17 @@ fn lowerBranches(emit: *Emit) !void {
242296 // potentially update the branch type
243297 if (emit.branch_forward_origins.get(inst)) |origin_list| {
244298 for (origin_list.items) |forward_branch_inst| {
299 const branch_tag = emit.mir.instructions.items(.tag)[forward_branch_inst];
245300 const forward_branch_inst_offset = emit.code_offset_mapping.get(forward_branch_inst).?;
246 const offset = @intCast(i64, forward_branch_inst_offset) - @intCast(i64, current_code_offset + 8);
301 const offset = @intCast(i64, current_code_offset) - @intCast(i64, forward_branch_inst_offset);
247302 const branch_type = emit.branch_types.getPtr(forward_branch_inst).?;
248 const optimal_branch_type = try emit.optimalBranchType(offset);
303 const optimal_branch_type = try emit.optimalBranchType(branch_tag, offset);
249304 if (branch_type.* != optimal_branch_type) {
250305 branch_type.* = optimal_branch_type;
251306 all_branches_lowered = false;
252307 }
308
309 log.debug("lowerBranches: branch {} has offset {}", .{ forward_branch_inst, offset });
253310 }
254311 }
255312
......@@ -347,6 +404,12 @@ fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
347404 rr_imm12_sh.imm12,
348405 rr_imm12_sh.sh == 1,
349406 )),
407 .cmp_immediate => try emit.writeInstruction(Instruction.subs(
408 rr_imm12_sh.rd,
409 rr_imm12_sh.rn,
410 rr_imm12_sh.imm12,
411 rr_imm12_sh.sh == 1,
412 )),
350413 .sub_immediate => try emit.writeInstruction(Instruction.sub(
351414 rr_imm12_sh.rd,
352415 rr_imm12_sh.rn,
......@@ -357,12 +420,37 @@ fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
357420 }
358421}
359422
423fn mirConditionalBranchImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
424 const tag = emit.mir.instructions.items(.tag)[inst];
425 const inst_cond = emit.mir.instructions.items(.data)[inst].inst_cond;
426
427 const offset = @intCast(i64, emit.code_offset_mapping.get(inst_cond.inst).?) - @intCast(i64, emit.code.items.len);
428 const branch_type = emit.branch_types.get(inst).?;
429 log.debug("mirConditionalBranchImmediate: {} offset={}", .{ inst, offset });
430
431 switch (branch_type) {
432 .b_cond => switch (tag) {
433 .b_cond => try emit.writeInstruction(Instruction.bCond(inst_cond.cond, @intCast(i21, offset))),
434 else => unreachable,
435 },
436 else => unreachable,
437 }
438}
439
360440fn mirBranch(emit: *Emit, inst: Mir.Inst.Index) !void {
361441 const tag = emit.mir.instructions.items(.tag)[inst];
362442 const target_inst = emit.mir.instructions.items(.data)[inst].inst;
363443
364 const offset = @intCast(i64, emit.code_offset_mapping.get(target_inst).?) - @intCast(i64, emit.code.items.len + 8);
444 log.debug("branch {}(tag: {}) -> {}(tag: {})", .{
445 inst,
446 tag,
447 target_inst,
448 emit.mir.instructions.items(.tag)[target_inst],
449 });
450
451 const offset = @intCast(i64, emit.code_offset_mapping.get(target_inst).?) - @intCast(i64, emit.code.items.len);
365452 const branch_type = emit.branch_types.get(inst).?;
453 log.debug("mirBranch: {} offset={}", .{ inst, offset });
366454
367455 switch (branch_type) {
368456 .unconditional_branch_immediate => switch (tag) {
......@@ -370,6 +458,7 @@ fn mirBranch(emit: *Emit, inst: Mir.Inst.Index) !void {
370458 .bl => try emit.writeInstruction(Instruction.bl(@intCast(i28, offset))),
371459 else => unreachable,
372460 },
461 else => unreachable,
373462 }
374463}
375464
......@@ -453,6 +542,37 @@ fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) !void {
453542 }
454543}
455544
545fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
546 const tag = emit.mir.instructions.items(.tag)[inst];
547 const rrr_imm6_shift = emit.mir.instructions.items(.data)[inst].rrr_imm6_shift;
548
549 switch (tag) {
550 .cmp_shifted_register => try emit.writeInstruction(Instruction.subsShiftedRegister(
551 rrr_imm6_shift.rd,
552 rrr_imm6_shift.rn,
553 rrr_imm6_shift.rm,
554 rrr_imm6_shift.shift,
555 rrr_imm6_shift.imm6,
556 )),
557 else => unreachable,
558 }
559}
560
561fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void {
562 const tag = emit.mir.instructions.items(.tag)[inst];
563 const rrr_cond = emit.mir.instructions.items(.data)[inst].rrr_cond;
564
565 switch (tag) {
566 .cset => try emit.writeInstruction(Instruction.csinc(
567 rrr_cond.rd,
568 rrr_cond.rn,
569 rrr_cond.rm,
570 rrr_cond.cond,
571 )),
572 else => unreachable,
573 }
574}
575
456576fn mirLoadMemory(emit: *Emit, inst: Mir.Inst.Index) !void {
457577 assert(emit.mir.instructions.items(.tag)[inst] == .load_memory);
458578 const payload = emit.mir.instructions.items(.data)[inst].payload;
src/arch/aarch64/Mir.zig+36-2
......@@ -26,6 +26,8 @@ pub const Inst = struct {
2626 pub const Tag = enum(u16) {
2727 /// Add (immediate)
2828 add_immediate,
29 /// Branch conditionally
30 b_cond,
2931 /// Branch
3032 b,
3133 /// Branch with Link
......@@ -36,13 +38,19 @@ pub const Inst = struct {
3638 brk,
3739 /// Pseudo-instruction: Call extern
3840 call_extern,
41 /// Compare (immediate)
42 cmp_immediate,
43 /// Compare (shifted register)
44 cmp_shifted_register,
45 /// Conditional set
46 cset,
3947 /// Pseudo-instruction: End of prologue
4048 dbg_prologue_end,
4149 /// Pseudo-instruction: Beginning of epilogue
4250 dbg_epilogue_begin,
4351 /// Pseudo-instruction: Update debug line
4452 dbg_line,
45 /// Psuedo-instruction: Load memory
53 /// Pseudo-instruction: Load memory
4654 ///
4755 /// Payload is `LoadMemory`
4856 load_memory,
......@@ -97,7 +105,7 @@ pub const Inst = struct {
97105 ///
98106 /// Used by e.g. nop
99107 nop: void,
100 /// Another instruction.
108 /// Another instruction
101109 ///
102110 /// Used by e.g. b
103111 inst: Index,
......@@ -117,6 +125,13 @@ pub const Inst = struct {
117125 ///
118126 /// Used by e.g. blr
119127 reg: Register,
128 /// Another instruction and a condition
129 ///
130 /// Used by e.g. b_cond
131 inst_cond: struct {
132 inst: Index,
133 cond: bits.Instruction.Condition,
134 },
120135 /// A register, an unsigned 16-bit immediate, and an optional shift
121136 ///
122137 /// Used by e.g. movz
......@@ -141,6 +156,25 @@ pub const Inst = struct {
141156 imm12: u12,
142157 sh: u1 = 0,
143158 },
159 /// Three registers and a shift (shift type and 6-bit amount)
160 ///
161 /// Used by e.g. cmp_shifted_register
162 rrr_imm6_shift: struct {
163 rd: Register,
164 rn: Register,
165 rm: Register,
166 imm6: u6,
167 shift: bits.Instruction.AddSubtractShiftedRegisterShift,
168 },
169 /// Three registers and a condition
170 ///
171 /// Used by e.g. cset
172 rrr_cond: struct {
173 rd: Register,
174 rn: Register,
175 rm: Register,
176 cond: bits.Instruction.Condition,
177 },
144178 /// Three registers and a LoadStoreOffset
145179 ///
146180 /// Used by e.g. str_register
src/arch/aarch64/bits.zig+200
......@@ -295,6 +295,18 @@ pub const Instruction = union(enum) {
295295 op: u1,
296296 sf: u1,
297297 },
298 add_subtract_shifted_register: packed struct {
299 rd: u5,
300 rn: u5,
301 imm6: u6,
302 rm: u5,
303 fixed_1: u1 = 0b0,
304 shift: u2,
305 fixed_2: u5 = 0b01011,
306 s: u1,
307 op: u1,
308 sf: u1,
309 },
298310 conditional_branch: struct {
299311 cond: u4,
300312 o0: u1,
......@@ -309,6 +321,17 @@ pub const Instruction = union(enum) {
309321 fixed: u6 = 0b011010,
310322 sf: u1,
311323 },
324 conditional_select: struct {
325 rd: u5,
326 rn: u5,
327 op2: u2,
328 cond: u4,
329 rm: u5,
330 fixed: u8 = 0b11010100,
331 s: u1,
332 op: u1,
333 sf: u1,
334 },
312335
313336 pub const Shift = struct {
314337 shift: Type = .lsl,
......@@ -376,6 +399,57 @@ pub const Instruction = union(enum) {
376399 /// Integer: Always
377400 /// Floating point: Always
378401 nv,
402
403 /// Converts a std.math.CompareOperator into a condition flag,
404 /// i.e. returns the condition that is true iff the result of the
405 /// comparison is true. Assumes signed comparison
406 pub fn fromCompareOperatorSigned(op: std.math.CompareOperator) Condition {
407 return switch (op) {
408 .gte => .ge,
409 .gt => .gt,
410 .neq => .ne,
411 .lt => .lt,
412 .lte => .le,
413 .eq => .eq,
414 };
415 }
416
417 /// Converts a std.math.CompareOperator into a condition flag,
418 /// i.e. returns the condition that is true iff the result of the
419 /// comparison is true. Assumes unsigned comparison
420 pub fn fromCompareOperatorUnsigned(op: std.math.CompareOperator) Condition {
421 return switch (op) {
422 .gte => .cs,
423 .gt => .hi,
424 .neq => .ne,
425 .lt => .cc,
426 .lte => .ls,
427 .eq => .eq,
428 };
429 }
430
431 /// Returns the condition which is true iff the given condition is
432 /// false (if such a condition exists)
433 pub fn negate(cond: Condition) Condition {
434 return switch (cond) {
435 .eq => .ne,
436 .ne => .eq,
437 .cs => .cc,
438 .cc => .cs,
439 .mi => .pl,
440 .pl => .mi,
441 .vs => .vc,
442 .vc => .vs,
443 .hi => .ls,
444 .ls => .hi,
445 .ge => .lt,
446 .lt => .ge,
447 .gt => .le,
448 .le => .gt,
449 .al => unreachable,
450 .nv => unreachable,
451 };
452 }
379453 };
380454
381455 pub fn toU32(self: Instruction) u32 {
......@@ -391,9 +465,11 @@ pub const Instruction = union(enum) {
391465 .no_operation => |v| @bitCast(u32, v),
392466 .logical_shifted_register => |v| @bitCast(u32, v),
393467 .add_subtract_immediate => |v| @bitCast(u32, v),
468 .add_subtract_shifted_register => |v| @bitCast(u32, v),
394469 // TODO once packed structs work, this can be refactored
395470 .conditional_branch => |v| @as(u32, v.cond) | (@as(u32, v.o0) << 4) | (@as(u32, v.imm19) << 5) | (@as(u32, v.o1) << 24) | (@as(u32, v.fixed) << 25),
396471 .compare_and_branch => |v| @as(u32, v.rt) | (@as(u32, v.imm19) << 5) | (@as(u32, v.op) << 24) | (@as(u32, v.fixed) << 25) | (@as(u32, v.sf) << 31),
472 .conditional_select => |v| @as(u32, v.rd) | @as(u32, v.rn) << 5 | @as(u32, v.op2) << 10 | @as(u32, v.cond) << 12 | @as(u32, v.rm) << 16 | @as(u32, v.fixed) << 21 | @as(u32, v.s) << 29 | @as(u32, v.op) << 30 | @as(u32, v.sf) << 31,
397473 };
398474 }
399475
......@@ -804,6 +880,35 @@ pub const Instruction = union(enum) {
804880 };
805881 }
806882
883 pub const AddSubtractShiftedRegisterShift = enum(u2) { lsl, lsr, asr, _ };
884
885 fn addSubtractShiftedRegister(
886 op: u1,
887 s: u1,
888 shift: AddSubtractShiftedRegisterShift,
889 rd: Register,
890 rn: Register,
891 rm: Register,
892 imm6: u6,
893 ) Instruction {
894 return Instruction{
895 .add_subtract_shifted_register = .{
896 .rd = rd.id(),
897 .rn = rn.id(),
898 .imm6 = imm6,
899 .rm = rm.id(),
900 .shift = @enumToInt(shift),
901 .s = s,
902 .op = op,
903 .sf = switch (rd.size()) {
904 32 => 0b0,
905 64 => 0b1,
906 else => unreachable, // unexpected register size
907 },
908 },
909 };
910 }
911
807912 fn conditionalBranch(
808913 o0: u1,
809914 o1: u1,
......@@ -841,6 +946,33 @@ pub const Instruction = union(enum) {
841946 };
842947 }
843948
949 fn conditionalSelect(
950 op2: u2,
951 op: u1,
952 s: u1,
953 rd: Register,
954 rn: Register,
955 rm: Register,
956 cond: Condition,
957 ) Instruction {
958 return Instruction{
959 .conditional_select = .{
960 .rd = rd.id(),
961 .rn = rn.id(),
962 .op2 = op2,
963 .cond = @enumToInt(cond),
964 .rm = rm.id(),
965 .s = s,
966 .op = op,
967 .sf = switch (rd.size()) {
968 32 => 0b0,
969 64 => 0b1,
970 else => unreachable, // unexpected register size
971 },
972 },
973 };
974 }
975
844976 // Helper functions for assembly syntax functions
845977
846978 // Move wide (immediate)
......@@ -1055,6 +1187,48 @@ pub const Instruction = union(enum) {
10551187 return addSubtractImmediate(0b1, 0b1, rd, rn, imm, shift);
10561188 }
10571189
1190 // Add/subtract (shifted register)
1191
1192 pub fn addShiftedRegister(
1193 rd: Register,
1194 rn: Register,
1195 rm: Register,
1196 shift: AddSubtractShiftedRegisterShift,
1197 imm6: u6,
1198 ) Instruction {
1199 return addSubtractShiftedRegister(0b0, 0b0, shift, rd, rn, rm, imm6);
1200 }
1201
1202 pub fn addsShiftedRegister(
1203 rd: Register,
1204 rn: Register,
1205 rm: Register,
1206 shift: AddSubtractShiftedRegisterShift,
1207 imm6: u6,
1208 ) Instruction {
1209 return addSubtractShiftedRegister(0b0, 0b1, shift, rd, rn, rm, imm6);
1210 }
1211
1212 pub fn subShiftedRegister(
1213 rd: Register,
1214 rn: Register,
1215 rm: Register,
1216 shift: AddSubtractShiftedRegisterShift,
1217 imm6: u6,
1218 ) Instruction {
1219 return addSubtractShiftedRegister(0b1, 0b0, shift, rd, rn, rm, imm6);
1220 }
1221
1222 pub fn subsShiftedRegister(
1223 rd: Register,
1224 rn: Register,
1225 rm: Register,
1226 shift: AddSubtractShiftedRegisterShift,
1227 imm6: u6,
1228 ) Instruction {
1229 return addSubtractShiftedRegister(0b1, 0b1, shift, rd, rn, rm, imm6);
1230 }
1231
10581232 // Conditional branch
10591233
10601234 pub fn bCond(cond: Condition, offset: i21) Instruction {
......@@ -1070,6 +1244,24 @@ pub const Instruction = union(enum) {
10701244 pub fn cbnz(rt: Register, offset: i21) Instruction {
10711245 return compareAndBranch(0b1, rt, offset);
10721246 }
1247
1248 // Conditional select
1249
1250 pub fn csel(rd: Register, rn: Register, rm: Register, cond: Condition) Instruction {
1251 return conditionalSelect(0b00, 0b0, 0b0, rd, rn, rm, cond);
1252 }
1253
1254 pub fn csinc(rd: Register, rn: Register, rm: Register, cond: Condition) Instruction {
1255 return conditionalSelect(0b01, 0b0, 0b0, rd, rn, rm, cond);
1256 }
1257
1258 pub fn csinv(rd: Register, rn: Register, rm: Register, cond: Condition) Instruction {
1259 return conditionalSelect(0b00, 0b1, 0b0, rd, rn, rm, cond);
1260 }
1261
1262 pub fn csneg(rd: Register, rn: Register, rm: Register, cond: Condition) Instruction {
1263 return conditionalSelect(0b01, 0b1, 0b0, rd, rn, rm, cond);
1264 }
10731265};
10741266
10751267test {
......@@ -1231,6 +1423,14 @@ test "serialize instructions" {
12311423 .inst = Instruction.cbz(.x10, 40),
12321424 .expected = 0b1_011010_0_0000000000000001010_01010,
12331425 },
1426 .{ // add x0, x1, x2, lsl #5
1427 .inst = Instruction.addShiftedRegister(.x0, .x1, .x2, .lsl, 5),
1428 .expected = 0b1_0_0_01011_00_0_00010_000101_00001_00000,
1429 },
1430 .{ // csinc x1, x2, x4, eq
1431 .inst = Instruction.csinc(.x1, .x2, .x4, .eq),
1432 .expected = 0b1_0_0_11010100_00100_0000_0_1_00010_00001,
1433 },
12341434 };
12351435
12361436 for (testcases) |case| {
test/stage2/aarch64.zig+29
......@@ -68,4 +68,33 @@ pub fn addCases(ctx: *TestContext) !void {
6868 "",
6969 );
7070 }
71
72 {
73 var case = ctx.exe("conditional branches", linux_aarch64);
74
75 case.addCompareOutput(
76 \\pub fn main() void {
77 \\ foo(123);
78 \\}
79 \\
80 \\fn foo(x: u64) void {
81 \\ if (x > 42) {
82 \\ print();
83 \\ }
84 \\}
85 \\
86 \\fn print() void {
87 \\ asm volatile ("svc #0"
88 \\ :
89 \\ : [number] "{x8}" (64),
90 \\ [arg1] "{x0}" (1),
91 \\ [arg2] "{x1}" (@ptrToInt("Hello, World!\n")),
92 \\ [arg3] "{x2}" ("Hello, World!\n".len),
93 \\ : "memory", "cc"
94 \\ );
95 \\}
96 ,
97 "Hello, World!\n",
98 );
99 }
71100}