authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-08 23:20:13-04:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2023-04-20 20:49:35+01:00
logf18ee1e2a2d4fb7ece2741efbfe4bda8b056068e
tree4e32ae7b8011a2ea37f695d240f376209d8a2770
parent488d804a1c205583e22aee4076e41f497b3ef1b0
signaturelock-open Commit is signed but in an unrecognized format.

x86_64: add block death workaround

1 files changed, 52 insertions(+), 37 deletions(-)

src/arch/x86_64/CodeGen.zig+52-37
......@@ -89,8 +89,8 @@ register_manager: RegisterManager = .{},
8989/// Maps offset to what is stored there.
9090stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
9191
92/// Index of the current scope.
93scope_index: u32 = 0,
92/// Generation of the current scope, increments by 1 for every entered scope.
93scope_generation: u32 = 0,
9494
9595/// Offset from the stack base, representing the end of the stack frame.
9696max_end_stack: u32 = 0,
......@@ -116,7 +116,7 @@ pub const MCValue = union(enum) {
116116 /// Control flow will not allow this value to be observed.
117117 unreach,
118118 /// No more references to this value remain.
119 /// The payload is the value of scope_index at the point where the death occurred
119 /// The payload is the value of scope_generation at the point where the death occurred
120120 dead: u32,
121121 /// The value is undefined.
122122 undef,
......@@ -253,9 +253,9 @@ const InstTracking = struct {
253253 self.short = .{ .register = reg };
254254 }
255255
256 fn resurrect(self: *InstTracking, scope_index: u32) void {
256 fn resurrect(self: *InstTracking, scope_generation: u32) void {
257257 switch (self.short) {
258 .dead => |die_index| if (die_index >= scope_index) {
258 .dead => |die_generation| if (die_generation >= scope_generation) {
259259 self.short = self.long;
260260 },
261261 else => {},
......@@ -268,7 +268,7 @@ const InstTracking = struct {
268268 }
269269
270270 fn reuse(self: *InstTracking, function: *Self) void {
271 self.short = .{ .dead = function.scope_index };
271 self.short = .{ .dead = function.scope_generation };
272272 }
273273};
274274
......@@ -280,10 +280,12 @@ const StackAllocation = struct {
280280
281281const BlockData = struct {
282282 relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{},
283 deaths: std.ArrayListUnmanaged(u32) = .{}, // inst_tracking indices
283284 state: State,
284285
285286 fn deinit(self: *BlockData, gpa: Allocator) void {
286287 self.relocs.deinit(gpa);
288 self.deaths.deinit(gpa);
287289 self.* = undefined;
288290 }
289291};
......@@ -1354,30 +1356,25 @@ const State = struct {
13541356 registers: RegisterManager.TrackedRegisters,
13551357 free_registers: RegisterManager.RegisterBitSet,
13561358 inst_tracking_len: u32,
1357 scope_index: u32,
1359 scope_generation: u32,
13581360};
13591361
13601362fn initRetroactiveState(self: *Self) State {
13611363 var state: State = undefined;
13621364 state.inst_tracking_len = @intCast(u32, self.inst_tracking.count());
1363 state.scope_index = self.scope_index;
1365 state.scope_generation = self.scope_generation;
13641366 return state;
13651367}
13661368
1367fn saveRetroactiveState(self: *Self, state: *State, comptime hack_around_liveness_bug: bool) !void {
1369fn saveRetroactiveState(self: *Self, state: *State) !void {
13681370 try self.spillEflagsIfOccupied();
13691371 state.registers = self.register_manager.registers;
13701372 state.free_registers = self.register_manager.free_registers;
1371 if (hack_around_liveness_bug) for (0..state.registers.len) |index| {
1372 if (state.free_registers.isSet(index)) continue;
1373 if (self.inst_tracking.getIndex(state.registers[index]).? < state.inst_tracking_len) continue;
1374 state.free_registers.set(index);
1375 };
13761373}
13771374
13781375fn saveState(self: *Self) !State {
13791376 var state = self.initRetroactiveState();
1380 try self.saveRetroactiveState(&state, false);
1377 try self.saveRetroactiveState(&state);
13811378 return state;
13821379}
13831380
......@@ -1388,19 +1385,12 @@ fn restoreState(self: *Self, state: State, comptime opts: struct {
13881385 close_scope: bool,
13891386}) !void {
13901387 if (opts.close_scope) {
1391 if (std.debug.runtime_safety) {
1392 for (self.inst_tracking.values()[state.inst_tracking_len..]) |tracking| {
1393 switch (tracking.short) {
1394 .dead, .unreach => {},
1395 else => unreachable,
1396 }
1397 }
1398 }
1388 for (self.inst_tracking.values()[state.inst_tracking_len..]) |*tracking| tracking.die(self);
13991389 self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
14001390 }
14011391
1402 if (opts.resurrect)
1403 for (self.inst_tracking.values()) |*tracking| tracking.resurrect(state.scope_index);
1392 if (opts.resurrect) for (self.inst_tracking.values()) |*tracking|
1393 tracking.resurrect(state.scope_generation);
14041394
14051395 for (0..state.registers.len) |index| {
14061396 const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
......@@ -6146,7 +6136,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
61466136
61476137 const outer_state = try self.saveState();
61486138 {
6149 self.scope_index += 1;
6139 self.scope_generation += 1;
61506140 const inner_state = try self.saveState();
61516141
61526142 for (liveness_condbr.then_deaths) |operand| self.processDeath(operand);
......@@ -6162,6 +6152,12 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
61626152
61636153 for (liveness_condbr.else_deaths) |operand| self.processDeath(operand);
61646154 try self.genBody(else_body);
6155 try self.restoreState(inner_state, .{
6156 .emit_instructions = false,
6157 .update_tracking = true,
6158 .resurrect = true,
6159 .close_scope = true,
6160 });
61656161 }
61666162 try self.restoreState(outer_state, .{
61676163 .emit_instructions = false,
......@@ -6473,7 +6469,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
64736469 const body = self.air.extra[loop.end..][0..loop.data.body_len];
64746470 const jmp_target = @intCast(u32, self.mir_instructions.len);
64756471
6476 self.scope_index += 1;
6472 self.scope_generation += 1;
64776473 const state = try self.saveState();
64786474
64796475 try self.genBody(body);
......@@ -6501,7 +6497,7 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
65016497 .short = if (ty.isNoReturn()) .unreach else .none,
65026498 });
65036499
6504 self.scope_index += 1;
6500 self.scope_generation += 1;
65056501 try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() });
65066502 defer {
65076503 var block_data = self.blocks.fetchRemove(inst).?.value;
......@@ -6515,6 +6511,7 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
65156511
65166512 const tracking = self.inst_tracking.getPtr(inst).?;
65176513 const block_data = self.blocks.getPtr(inst).?;
6514 for (block_data.deaths.items) |tracking_index| self.inst_tracking.values()[tracking_index].die(self);
65186515 if (tracking.short != .unreach) try self.restoreState(block_data.state, .{
65196516 .emit_instructions = false,
65206517 .update_tracking = true,
......@@ -6547,7 +6544,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
65476544
65486545 const outer_state = try self.saveState();
65496546 {
6550 self.scope_index += 1;
6547 self.scope_generation += 1;
65516548 const inner_state = try self.saveState();
65526549
65536550 while (case_i < switch_br.data.cases_len) : (case_i += 1) {
......@@ -6572,13 +6569,12 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
65726569 for (liveness.deaths[case_i]) |operand| self.processDeath(operand);
65736570
65746571 try self.genBody(case_body);
6575 if (case_i < switch_br.data.cases_len - 1 or switch_br.data.else_body_len > 0)
6576 try self.restoreState(inner_state, .{
6577 .emit_instructions = false,
6578 .update_tracking = true,
6579 .resurrect = true,
6580 .close_scope = true,
6581 });
6572 try self.restoreState(inner_state, .{
6573 .emit_instructions = false,
6574 .update_tracking = true,
6575 .resurrect = true,
6576 .close_scope = true,
6577 });
65826578
65836579 for (relocs) |reloc| try self.performReloc(reloc);
65846580 }
......@@ -6590,6 +6586,12 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
65906586 for (liveness.deaths[else_deaths]) |operand| self.processDeath(operand);
65916587
65926588 try self.genBody(else_body);
6589 try self.restoreState(inner_state, .{
6590 .emit_instructions = false,
6591 .update_tracking = true,
6592 .resurrect = true,
6593 .close_scope = true,
6594 });
65936595 }
65946596 }
65956597 try self.restoreState(outer_state, .{
......@@ -6631,6 +6633,19 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
66316633 const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
66326634 const block_data = self.blocks.getPtr(br.block_inst).?;
66336635 if (block_tracking.long == .unreach) {
6636 // .unreach is used to mean that we are the first branch
6637
6638 // We need to compute a list of deaths for later. This list needs to include
6639 // instructions that was born before, and has died since, the target block.
6640 for (self.inst_tracking.values()[0..block_data.state.inst_tracking_len], 0..) |
6641 *tracking,
6642 tracked_index,
6643 | switch (tracking.short) {
6644 .dead => |die_generation| if (die_generation >= block_data.state.scope_generation)
6645 try block_data.deaths.append(self.gpa, @intCast(u32, tracked_index)),
6646 else => {},
6647 };
6648
66346649 const result = result: {
66356650 if (block_unused) break :result .none;
66366651 if (self.reuseOperand(inst, br.operand, 0, src_mcv)) break :result src_mcv;
......@@ -6640,7 +6655,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
66406655 break :result new_mcv;
66416656 };
66426657 block_tracking.* = InstTracking.init(result);
6643 try self.saveRetroactiveState(&block_data.state, true);
6658 try self.saveRetroactiveState(&block_data.state);
66446659 self.freeValue(result);
66456660 } else {
66466661 if (!block_unused) try self.setRegOrMem(block_ty, block_tracking.short, src_mcv);