| ... | @@ -213,7 +213,15 @@ const StackAllocation = struct { | ... | @@ -213,7 +213,15 @@ const StackAllocation = struct { |
| 213 | }; | 213 | }; |
| 214 | | 214 | |
| 215 | const BlockData = struct { | 215 | const BlockData = struct { |
| 216 | relocs: std.ArrayListUnmanaged(Mir.Inst.Index), | 216 | relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{}, |
| | 217 | branch: ?Branch = null, |
| | 218 | branch_depth: u32, |
| | 219 | |
| | 220 | fn deinit(self: *BlockData, gpa: Allocator) void { |
| | 221 | if (self.branch) |*branch| branch.deinit(gpa); |
| | 222 | self.relocs.deinit(gpa); |
| | 223 | self.* = undefined; |
| | 224 | } |
| 217 | }; | 225 | }; |
| 218 | | 226 | |
| 219 | const BigTomb = struct { | 227 | const BigTomb = struct { |
| ... | @@ -5233,11 +5241,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5233,11 +5241,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { |
| 5233 | // that death now instead of later as this has an effect on | 5241 | // that death now instead of later as this has an effect on |
| 5234 | // whether it needs to be spilled in the branches | 5242 | // whether it needs to be spilled in the branches |
| 5235 | if (self.liveness.operandDies(inst, 0)) { | 5243 | if (self.liveness.operandDies(inst, 0)) { |
| 5236 | const op_int = @enumToInt(pl_op.operand); | 5244 | if (Air.refToIndex(pl_op.operand)) |op_inst| self.processDeath(op_inst); |
| 5237 | if (op_int >= Air.Inst.Ref.typed_value_map.len) { | | |
| 5238 | const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len); | | |
| 5239 | self.processDeath(op_index); | | |
| 5240 | } | | |
| 5241 | } | 5245 | } |
| 5242 | | 5246 | |
| 5243 | // Capture the state of register and stack allocation state so that we can revert to it. | 5247 | // Capture the state of register and stack allocation state so that we can revert to it. |
| ... | @@ -5290,10 +5294,10 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5290,10 +5294,10 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { |
| 5290 | for (self.branch_stack.items) |bs| { | 5294 | for (self.branch_stack.items) |bs| { |
| 5291 | log.debug("{}", .{bs.fmtDebug()}); | 5295 | log.debug("{}", .{bs.fmtDebug()}); |
| 5292 | } | 5296 | } |
| 5293 | | | |
| 5294 | log.debug("Then branch: {}", .{then_branch.fmtDebug()}); | 5297 | log.debug("Then branch: {}", .{then_branch.fmtDebug()}); |
| 5295 | log.debug("Else branch: {}", .{else_branch.fmtDebug()}); | 5298 | log.debug("Else branch: {}", .{else_branch.fmtDebug()}); |
| 5296 | try self.canonicaliseBranches(true, &then_branch, &else_branch); | 5299 | |
| | 5300 | try self.canonicaliseBranches(true, &then_branch, &else_branch, true); |
| 5297 | | 5301 | |
| 5298 | // We already took care of pl_op.operand earlier, so we're going | 5302 | // We already took care of pl_op.operand earlier, so we're going |
| 5299 | // to pass .none here | 5303 | // to pass .none here |
| ... | @@ -5599,11 +5603,16 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5599,11 +5603,16 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void { |
| 5599 | } | 5603 | } |
| 5600 | | 5604 | |
| 5601 | fn airBlock(self: *Self, inst: Air.Inst.Index) !void { | 5605 | fn airBlock(self: *Self, inst: Air.Inst.Index) !void { |
| 5602 | try self.blocks.putNoClobber(self.gpa, inst, .{ | 5606 | // A block is a setup to be able to jump to the end. |
| 5603 | // A block is a setup to be able to jump to the end. | 5607 | const branch_depth = @intCast(u32, self.branch_stack.items.len); |
| 5604 | .relocs = .{}, | 5608 | try self.blocks.putNoClobber(self.gpa, inst, .{ .branch_depth = branch_depth }); |
| 5605 | }); | 5609 | defer { |
| 5606 | defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa); | 5610 | var block_data = self.blocks.fetchRemove(inst).?.value; |
| | 5611 | block_data.deinit(self.gpa); |
| | 5612 | } |
| | 5613 | |
| | 5614 | try self.branch_stack.append(.{}); |
| | 5615 | defer _ = self.branch_stack.pop(); |
| 5607 | | 5616 | |
| 5608 | const ty = self.air.typeOfIndex(inst); | 5617 | const ty = self.air.typeOfIndex(inst); |
| 5609 | const unused = !ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(inst); | 5618 | const unused = !ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(inst); |
| ... | @@ -5613,8 +5622,8 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5613,8 +5622,8 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void { |
| 5613 | // this field. Following break instructions will use that MCValue to put | 5622 | // this field. Following break instructions will use that MCValue to put |
| 5614 | // their block results. | 5623 | // their block results. |
| 5615 | const result: MCValue = if (unused) .dead else .none; | 5624 | const result: MCValue = if (unused) .dead else .none; |
| 5616 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | 5625 | const branch = &self.branch_stack.items[branch_depth]; |
| 5617 | branch.inst_table.putAssumeCapacityNoClobber(inst, result); | 5626 | try branch.inst_table.putNoClobber(self.gpa, inst, result); |
| 5618 | } | 5627 | } |
| 5619 | | 5628 | |
| 5620 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; | 5629 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| ... | @@ -5622,7 +5631,23 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5622,7 +5631,23 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void { |
| 5622 | const body = self.air.extra[extra.end..][0..extra.data.body_len]; | 5631 | const body = self.air.extra[extra.end..][0..extra.data.body_len]; |
| 5623 | try self.genBody(body); | 5632 | try self.genBody(body); |
| 5624 | | 5633 | |
| 5625 | for (self.blocks.getPtr(inst).?.relocs.items) |reloc| try self.performReloc(reloc); | 5634 | const block_data = self.blocks.getPtr(inst).?; |
| | 5635 | { |
| | 5636 | const src_branch = block_data.branch orelse self.branch_stack.items[branch_depth]; |
| | 5637 | const dst_branch = &self.branch_stack.items[branch_depth - 1]; |
| | 5638 | try dst_branch.inst_table.ensureUnusedCapacity(self.gpa, src_branch.inst_table.count()); |
| | 5639 | var it = src_branch.inst_table.iterator(); |
| | 5640 | while (it.next()) |entry| { |
| | 5641 | const tracked_inst = entry.key_ptr.*; |
| | 5642 | const tracked_value = entry.value_ptr.*; |
| | 5643 | if (dst_branch.inst_table.fetchPutAssumeCapacity(tracked_inst, tracked_value)) |old_entry| { |
| | 5644 | self.freeValue(old_entry.value); |
| | 5645 | } |
| | 5646 | self.getValue(tracked_value, tracked_inst); |
| | 5647 | } |
| | 5648 | } |
| | 5649 | |
| | 5650 | for (block_data.relocs.items) |reloc| try self.performReloc(reloc); |
| 5626 | | 5651 | |
| 5627 | const result = if (unused) .dead else self.getResolvedInstValue(inst).?.*; | 5652 | const result = if (unused) .dead else self.getResolvedInstValue(inst).?.*; |
| 5628 | self.getValue(result, inst); | 5653 | self.getValue(result, inst); |
| ... | @@ -5647,11 +5672,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5647,11 +5672,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 5647 | // that death now instead of later as this has an effect on | 5672 | // that death now instead of later as this has an effect on |
| 5648 | // whether it needs to be spilled in the branches | 5673 | // whether it needs to be spilled in the branches |
| 5649 | if (self.liveness.operandDies(inst, 0)) { | 5674 | if (self.liveness.operandDies(inst, 0)) { |
| 5650 | const op_int = @enumToInt(pl_op.operand); | 5675 | if (Air.refToIndex(pl_op.operand)) |op_inst| self.processDeath(op_inst); |
| 5651 | if (op_int >= Air.Inst.Ref.typed_value_map.len) { | | |
| 5652 | const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len); | | |
| 5653 | self.processDeath(op_index); | | |
| 5654 | } | | |
| 5655 | } | 5676 | } |
| 5656 | | 5677 | |
| 5657 | log.debug("airSwitch: %{d}", .{inst}); | 5678 | log.debug("airSwitch: %{d}", .{inst}); |
| ... | @@ -5704,8 +5725,9 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5704,8 +5725,9 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 5704 | errdefer case_branch.deinit(self.gpa); | 5725 | errdefer case_branch.deinit(self.gpa); |
| 5705 | | 5726 | |
| 5706 | log.debug("Case-{d} branch: {}", .{ case_i, case_branch.fmtDebug() }); | 5727 | log.debug("Case-{d} branch: {}", .{ case_i, case_branch.fmtDebug() }); |
| | 5728 | const final = case_i == cases_len - 1; |
| 5707 | if (prev_branch) |*canon_branch| { | 5729 | if (prev_branch) |*canon_branch| { |
| 5708 | try self.canonicaliseBranches(case_i == cases_len - 1, canon_branch, &case_branch); | 5730 | try self.canonicaliseBranches(final, canon_branch, &case_branch, true); |
| 5709 | canon_branch.deinit(self.gpa); | 5731 | canon_branch.deinit(self.gpa); |
| 5710 | } | 5732 | } |
| 5711 | prev_branch = case_branch; | 5733 | prev_branch = case_branch; |
| ... | @@ -5740,7 +5762,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5740,7 +5762,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 5740 | | 5762 | |
| 5741 | log.debug("Else branch: {}", .{else_branch.fmtDebug()}); | 5763 | log.debug("Else branch: {}", .{else_branch.fmtDebug()}); |
| 5742 | if (prev_branch) |*canon_branch| { | 5764 | if (prev_branch) |*canon_branch| { |
| 5743 | try self.canonicaliseBranches(true, canon_branch, &else_branch); | 5765 | try self.canonicaliseBranches(true, canon_branch, &else_branch, true); |
| 5744 | canon_branch.deinit(self.gpa); | 5766 | canon_branch.deinit(self.gpa); |
| 5745 | } | 5767 | } |
| 5746 | prev_branch = else_branch; | 5768 | prev_branch = else_branch; |
| ... | @@ -5756,27 +5778,30 @@ fn canonicaliseBranches( | ... | @@ -5756,27 +5778,30 @@ fn canonicaliseBranches( |
| 5756 | update_parent: bool, | 5778 | update_parent: bool, |
| 5757 | canon_branch: *Branch, | 5779 | canon_branch: *Branch, |
| 5758 | target_branch: *const Branch, | 5780 | target_branch: *const Branch, |
| | 5781 | comptime assert_same_deaths: bool, |
| 5759 | ) !void { | 5782 | ) !void { |
| 5760 | const parent_branch = | 5783 | const parent_branch = |
| 5761 | if (update_parent) &self.branch_stack.items[self.branch_stack.items.len - 1] else undefined; | 5784 | if (update_parent) &self.branch_stack.items[self.branch_stack.items.len - 1] else undefined; |
| 5762 | if (update_parent) try self.ensureProcessDeathCapacity(target_branch.inst_table.count()); | | |
| 5763 | | 5785 | |
| 5764 | const target_slice = target_branch.inst_table.entries.slice(); | 5786 | if (update_parent) try self.ensureProcessDeathCapacity(target_branch.inst_table.count()); |
| 5765 | for (target_slice.items(.key), target_slice.items(.value)) |target_key, target_value| { | 5787 | var target_it = target_branch.inst_table.iterator(); |
| | 5788 | while (target_it.next()) |target_entry| { |
| | 5789 | const target_key = target_entry.key_ptr.*; |
| | 5790 | const target_value = target_entry.value_ptr.*; |
| 5766 | const canon_mcv = if (canon_branch.inst_table.fetchSwapRemove(target_key)) |canon_entry| blk: { | 5791 | const canon_mcv = if (canon_branch.inst_table.fetchSwapRemove(target_key)) |canon_entry| blk: { |
| 5767 | // The instruction's MCValue is overridden in both branches. | 5792 | // The instruction's MCValue is overridden in both branches. |
| 5768 | if (update_parent) { | 5793 | if (update_parent) { |
| 5769 | parent_branch.inst_table.putAssumeCapacity(target_key, canon_entry.value); | 5794 | parent_branch.inst_table.putAssumeCapacity(target_key, canon_entry.value); |
| 5770 | } | 5795 | } |
| 5771 | if (target_value == .dead) { | 5796 | if (target_value == .dead) { |
| 5772 | assert(canon_entry.value == .dead); | 5797 | if (assert_same_deaths) assert(canon_entry.value == .dead); |
| 5773 | continue; | 5798 | continue; |
| 5774 | } | 5799 | } |
| 5775 | break :blk canon_entry.value; | 5800 | break :blk canon_entry.value; |
| 5776 | } else blk: { | 5801 | } else blk: { |
| 5777 | if (target_value == .dead) continue; | 5802 | if (target_value == .dead) continue; |
| 5778 | // The instruction is only overridden in the else branch. | 5803 | // The instruction is only overridden in the else branch. |
| 5779 | // If integer overflows occurs, the question is: why wasn't the instruction marked dead? | 5804 | // If integer overflow occurs, the question is: why wasn't the instruction marked dead? |
| 5780 | break :blk self.getResolvedInstValue(target_key).?.*; | 5805 | break :blk self.getResolvedInstValue(target_key).?.*; |
| 5781 | }; | 5806 | }; |
| 5782 | log.debug("consolidating target_entry {d} {}=>{}", .{ target_key, target_value, canon_mcv }); | 5807 | log.debug("consolidating target_entry {d} {}=>{}", .{ target_key, target_value, canon_mcv }); |
| ... | @@ -5786,9 +5811,12 @@ fn canonicaliseBranches( | ... | @@ -5786,9 +5811,12 @@ fn canonicaliseBranches( |
| 5786 | self.freeValue(target_value); | 5811 | self.freeValue(target_value); |
| 5787 | // TODO track the new register / stack allocation | 5812 | // TODO track the new register / stack allocation |
| 5788 | } | 5813 | } |
| | 5814 | |
| 5789 | if (update_parent) try self.ensureProcessDeathCapacity(canon_branch.inst_table.count()); | 5815 | if (update_parent) try self.ensureProcessDeathCapacity(canon_branch.inst_table.count()); |
| 5790 | const canon_slice = canon_branch.inst_table.entries.slice(); | 5816 | var canon_it = canon_branch.inst_table.iterator(); |
| 5791 | for (canon_slice.items(.key), canon_slice.items(.value)) |canon_key, canon_value| { | 5817 | while (canon_it.next()) |canon_entry| { |
| | 5818 | const canon_key = canon_entry.key_ptr.*; |
| | 5819 | const canon_value = canon_entry.value_ptr.*; |
| 5792 | // We already deleted the items from this table that matched the target_branch. | 5820 | // We already deleted the items from this table that matched the target_branch. |
| 5793 | // So these are all instructions that are only overridden in the canon branch. | 5821 | // So these are all instructions that are only overridden in the canon branch. |
| 5794 | const parent_mcv = | 5822 | const parent_mcv = |
| ... | @@ -5821,22 +5849,19 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void { | ... | @@ -5821,22 +5849,19 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void { |
| 5821 | } | 5849 | } |
| 5822 | | 5850 | |
| 5823 | fn airBr(self: *Self, inst: Air.Inst.Index) !void { | 5851 | fn airBr(self: *Self, inst: Air.Inst.Index) !void { |
| 5824 | const branch = self.air.instructions.items(.data)[inst].br; | 5852 | const br = self.air.instructions.items(.data)[inst].br; |
| 5825 | try self.br(inst, branch.block_inst, branch.operand); | 5853 | const block = br.block_inst; |
| 5826 | return self.finishAir(inst, .dead, .{ branch.operand, .none, .none }); | | |
| 5827 | } | | |
| 5828 | | 5854 | |
| 5829 | fn br(self: *Self, inst: Air.Inst.Index, block: Air.Inst.Index, operand: Air.Inst.Ref) !void { | | |
| 5830 | // The first break instruction encounters `.none` here and chooses a | 5855 | // The first break instruction encounters `.none` here and chooses a |
| 5831 | // machine code value for the block result, populating this field. | 5856 | // machine code value for the block result, populating this field. |
| 5832 | // Following break instructions encounter that value and use it for | 5857 | // Following break instructions encounter that value and use it for |
| 5833 | // the location to store their block results. | 5858 | // the location to store their block results. |
| 5834 | if (self.getResolvedInstValue(block)) |dst_mcv| { | 5859 | if (self.getResolvedInstValue(block)) |dst_mcv| { |
| 5835 | const src_mcv = try self.resolveInst(operand); | 5860 | const src_mcv = try self.resolveInst(br.operand); |
| 5836 | switch (dst_mcv.*) { | 5861 | switch (dst_mcv.*) { |
| 5837 | .none => { | 5862 | .none => { |
| 5838 | const result = result: { | 5863 | const result = result: { |
| 5839 | if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result src_mcv; | 5864 | if (self.reuseOperand(inst, br.operand, 0, src_mcv)) break :result src_mcv; |
| 5840 | | 5865 | |
| 5841 | const new_mcv = try self.allocRegOrMem(block, true); | 5866 | const new_mcv = try self.allocRegOrMem(block, true); |
| 5842 | try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, src_mcv); | 5867 | try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, src_mcv); |
| ... | @@ -5848,16 +5873,50 @@ fn br(self: *Self, inst: Air.Inst.Index, block: Air.Inst.Index, operand: Air.Ins | ... | @@ -5848,16 +5873,50 @@ fn br(self: *Self, inst: Air.Inst.Index, block: Air.Inst.Index, operand: Air.Ins |
| 5848 | else => try self.setRegOrMem(self.air.typeOfIndex(block), dst_mcv.*, src_mcv), | 5873 | else => try self.setRegOrMem(self.air.typeOfIndex(block), dst_mcv.*, src_mcv), |
| 5849 | } | 5874 | } |
| 5850 | } | 5875 | } |
| 5851 | return self.brVoid(block); | | |
| 5852 | } | | |
| 5853 | | 5876 | |
| 5854 | fn brVoid(self: *Self, block: Air.Inst.Index) !void { | 5877 | // Process operand death early so that it is properly accounted for in the Branch below. |
| | 5878 | if (self.liveness.operandDies(inst, 0)) { |
| | 5879 | if (Air.refToIndex(br.operand)) |op_inst| self.processDeath(op_inst); |
| | 5880 | } |
| | 5881 | |
| 5855 | const block_data = self.blocks.getPtr(block).?; | 5882 | const block_data = self.blocks.getPtr(block).?; |
| | 5883 | { |
| | 5884 | var branch = Branch{}; |
| | 5885 | errdefer branch.deinit(self.gpa); |
| | 5886 | |
| | 5887 | var branch_i = self.branch_stack.items.len - 1; |
| | 5888 | while (branch_i >= block_data.branch_depth) : (branch_i -= 1) { |
| | 5889 | const table = &self.branch_stack.items[branch_i].inst_table; |
| | 5890 | try branch.inst_table.ensureUnusedCapacity(self.gpa, table.count()); |
| | 5891 | var it = table.iterator(); |
| | 5892 | while (it.next()) |entry| { |
| | 5893 | const gop = branch.inst_table.getOrPutAssumeCapacity(entry.key_ptr.*); |
| | 5894 | if (!gop.found_existing) gop.value_ptr.* = entry.value_ptr.*; |
| | 5895 | } |
| | 5896 | } |
| | 5897 | |
| | 5898 | if (block_data.branch) |*prev_branch| { |
| | 5899 | log.debug("brVoid: %{d}", .{inst}); |
| | 5900 | log.debug("Upper branches:", .{}); |
| | 5901 | for (self.branch_stack.items) |bs| { |
| | 5902 | log.debug("{}", .{bs.fmtDebug()}); |
| | 5903 | } |
| | 5904 | log.debug("Prev branch: {}", .{prev_branch.fmtDebug()}); |
| | 5905 | log.debug("Cur branch: {}", .{branch.fmtDebug()}); |
| | 5906 | |
| | 5907 | try self.canonicaliseBranches(false, prev_branch, &branch, false); |
| | 5908 | prev_branch.deinit(self.gpa); |
| | 5909 | } |
| | 5910 | block_data.branch = branch; |
| | 5911 | } |
| | 5912 | |
| 5856 | // Emit a jump with a relocation. It will be patched up after the block ends. | 5913 | // Emit a jump with a relocation. It will be patched up after the block ends. |
| 5857 | try block_data.relocs.ensureUnusedCapacity(self.gpa, 1); | 5914 | try block_data.relocs.ensureUnusedCapacity(self.gpa, 1); |
| 5858 | // Leave the jump offset undefined | 5915 | // Leave the jump offset undefined |
| 5859 | const jmp_reloc = try self.asmJmpReloc(undefined); | 5916 | const jmp_reloc = try self.asmJmpReloc(undefined); |
| 5860 | block_data.relocs.appendAssumeCapacity(jmp_reloc); | 5917 | block_data.relocs.appendAssumeCapacity(jmp_reloc); |
| | 5918 | |
| | 5919 | self.finishAirBookkeeping(); |
| 5861 | } | 5920 | } |
| 5862 | | 5921 | |
| 5863 | fn airAsm(self: *Self, inst: Air.Inst.Index) !void { | 5922 | fn airAsm(self: *Self, inst: Air.Inst.Index) !void { |
Jacob Young <jacobly@ziglang.org> 2023-03-24 04:40:45-04:00
src