| ... | ... | @@ -265,12 +265,15 @@ pub fn generate( |
| 265 | 265 | const fn_type = fn_owner_decl.ty; |
| 266 | 266 | |
| 267 | 267 | var branch_stack = std.ArrayList(Branch).init(bin_file.allocator); |
| 268 | try branch_stack.ensureUnusedCapacity(2); |
| 269 | // The outermost branch is used for constants only. |
| 270 | branch_stack.appendAssumeCapacity(.{}); |
| 271 | branch_stack.appendAssumeCapacity(.{}); |
| 268 | 272 | defer { |
| 269 | | assert(branch_stack.items.len == 1); |
| 270 | | branch_stack.items[0].deinit(bin_file.allocator); |
| 273 | assert(branch_stack.items.len == 2); |
| 274 | for (branch_stack.items) |*branch| branch.deinit(bin_file.allocator); |
| 271 | 275 | branch_stack.deinit(); |
| 272 | 276 | } |
| 273 | | try branch_stack.append(.{}); |
| 274 | 277 | |
| 275 | 278 | var function = Self{ |
| 276 | 279 | .gpa = bin_file.allocator, |
| ... | ... | @@ -1070,20 +1073,29 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 1070 | 1073 | if (self.air_bookkeeping < old_air_bookkeeping + 1) { |
| 1071 | 1074 | std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[inst] }); |
| 1072 | 1075 | } |
| 1076 | |
| 1077 | { // check consistency of tracked registers |
| 1078 | var it = self.register_manager.free_registers.iterator(.{ .kind = .unset }); |
| 1079 | while (it.next()) |index| { |
| 1080 | const tracked_inst = self.register_manager.registers[index]; |
| 1081 | switch (air_tags[tracked_inst]) { |
| 1082 | .block => {}, |
| 1083 | else => assert(RegisterManager.indexOfRegIntoTracked( |
| 1084 | switch (self.getResolvedInstValue(tracked_inst).?) { |
| 1085 | .register => |reg| reg, |
| 1086 | .register_overflow => |ro| ro.reg, |
| 1087 | else => unreachable, |
| 1088 | }, |
| 1089 | ).? == index), |
| 1090 | } |
| 1091 | } |
| 1092 | } |
| 1073 | 1093 | } |
| 1074 | 1094 | } |
| 1075 | 1095 | } |
| 1076 | 1096 | |
| 1077 | | /// Asserts there is already capacity to insert into top branch inst_table. |
| 1078 | | fn processDeath(self: *Self, inst: Air.Inst.Index) void { |
| 1079 | | const air_tags = self.air.instructions.items(.tag); |
| 1080 | | if (air_tags[inst] == .constant) return; // Constants are immortal. |
| 1081 | | const prev_value = self.getResolvedInstValue(inst) orelse return; |
| 1082 | | log.debug("%{d} => {}", .{ inst, MCValue.dead }); |
| 1083 | | // When editing this function, note that the logic must synchronize with `reuseOperand`. |
| 1084 | | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| 1085 | | branch.inst_table.putAssumeCapacity(inst, .dead); |
| 1086 | | switch (prev_value) { |
| 1097 | fn freeValue(self: *Self, value: MCValue) void { |
| 1098 | switch (value) { |
| 1087 | 1099 | .register => |reg| { |
| 1088 | 1100 | self.register_manager.freeReg(reg); |
| 1089 | 1101 | }, |
| ... | ... | @@ -1098,6 +1110,18 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void { |
| 1098 | 1110 | } |
| 1099 | 1111 | } |
| 1100 | 1112 | |
| 1113 | /// Asserts there is already capacity to insert into top branch inst_table. |
| 1114 | fn processDeath(self: *Self, inst: Air.Inst.Index) void { |
| 1115 | const air_tags = self.air.instructions.items(.tag); |
| 1116 | if (air_tags[inst] == .constant) return; // Constants are immortal. |
| 1117 | const prev_value = self.getResolvedInstValue(inst) orelse return; |
| 1118 | log.debug("%{d} => {}", .{ inst, MCValue.dead }); |
| 1119 | // When editing this function, note that the logic must synchronize with `reuseOperand`. |
| 1120 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| 1121 | branch.inst_table.putAssumeCapacity(inst, .dead); |
| 1122 | self.freeValue(prev_value); |
| 1123 | } |
| 1124 | |
| 1101 | 1125 | /// Called when there are no operands, and the instruction is always unreferenced. |
| 1102 | 1126 | fn finishAirBookkeeping(self: *Self) void { |
| 1103 | 1127 | if (std.debug.runtime_safety) { |
| ... | ... | @@ -1140,13 +1164,17 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live |
| 1140 | 1164 | }, |
| 1141 | 1165 | else => {}, |
| 1142 | 1166 | } |
| 1167 | } else switch (result) { |
| 1168 | .none, .dead, .unreach => {}, |
| 1169 | else => unreachable, // Why didn't the result die? |
| 1143 | 1170 | } |
| 1144 | 1171 | self.finishAirBookkeeping(); |
| 1145 | 1172 | } |
| 1146 | 1173 | |
| 1147 | 1174 | fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void { |
| 1175 | // In addition to the caller's needs, we need enough space to spill every register and eflags. |
| 1148 | 1176 | const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table; |
| 1149 | | try table.ensureUnusedCapacity(self.gpa, additional_count); |
| 1177 | try table.ensureUnusedCapacity(self.gpa, additional_count + self.register_manager.registers.len + 1); |
| 1150 | 1178 | } |
| 1151 | 1179 | |
| 1152 | 1180 | fn allocMem(self: *Self, inst: ?Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 { |
| ... | ... | @@ -1252,12 +1280,15 @@ fn captureState(self: *Self) !State { |
| 1252 | 1280 | }; |
| 1253 | 1281 | } |
| 1254 | 1282 | |
| 1255 | | fn revertState(self: *Self, state: State) void { |
| 1283 | fn revertState(self: *Self, state: State) !void { |
| 1284 | var stack = try state.stack.clone(self.gpa); |
| 1285 | errdefer stack.deinit(self.gpa); |
| 1286 | |
| 1256 | 1287 | self.register_manager.registers = state.registers; |
| 1257 | 1288 | self.eflags_inst = state.eflags_inst; |
| 1258 | 1289 | |
| 1259 | 1290 | self.stack.deinit(self.gpa); |
| 1260 | | self.stack = state.stack; |
| 1291 | self.stack = stack; |
| 1261 | 1292 | |
| 1262 | 1293 | self.next_stack_offset = state.next_stack_offset; |
| 1263 | 1294 | self.register_manager.free_registers = state.free_registers; |
| ... | ... | @@ -1277,7 +1308,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void |
| 1277 | 1308 | else => {}, |
| 1278 | 1309 | } |
| 1279 | 1310 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| 1280 | | try branch.inst_table.put(self.gpa, inst, stack_mcv); |
| 1311 | branch.inst_table.putAssumeCapacity(inst, stack_mcv); |
| 1281 | 1312 | try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{}); |
| 1282 | 1313 | } |
| 1283 | 1314 | |
| ... | ... | @@ -1294,7 +1325,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void { |
| 1294 | 1325 | log.debug("spilling %{d} to mcv {any}", .{ inst_to_save, new_mcv }); |
| 1295 | 1326 | |
| 1296 | 1327 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| 1297 | | try branch.inst_table.put(self.gpa, inst_to_save, new_mcv); |
| 1328 | branch.inst_table.putAssumeCapacity(inst_to_save, new_mcv); |
| 1298 | 1329 | |
| 1299 | 1330 | self.eflags_inst = null; |
| 1300 | 1331 | |
| ... | ... | @@ -1347,13 +1378,23 @@ fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Ty |
| 1347 | 1378 | } |
| 1348 | 1379 | |
| 1349 | 1380 | fn airAlloc(self: *Self, inst: Air.Inst.Index) !void { |
| 1350 | | const stack_offset = try self.allocMemPtr(inst); |
| 1351 | | return self.finishAir(inst, .{ .ptr_stack_offset = @intCast(i32, stack_offset) }, .{ .none, .none, .none }); |
| 1381 | const result: MCValue = result: { |
| 1382 | if (self.liveness.isUnused(inst)) break :result .dead; |
| 1383 | |
| 1384 | const stack_offset = try self.allocMemPtr(inst); |
| 1385 | break :result .{ .ptr_stack_offset = @intCast(i32, stack_offset) }; |
| 1386 | }; |
| 1387 | return self.finishAir(inst, result, .{ .none, .none, .none }); |
| 1352 | 1388 | } |
| 1353 | 1389 | |
| 1354 | 1390 | fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void { |
| 1355 | | const stack_offset = try self.allocMemPtr(inst); |
| 1356 | | return self.finishAir(inst, .{ .ptr_stack_offset = @intCast(i32, stack_offset) }, .{ .none, .none, .none }); |
| 1391 | const result: MCValue = result: { |
| 1392 | if (self.liveness.isUnused(inst)) break :result .dead; |
| 1393 | |
| 1394 | const stack_offset = try self.allocMemPtr(inst); |
| 1395 | break :result .{ .ptr_stack_offset = @intCast(i32, stack_offset) }; |
| 1396 | }; |
| 1397 | return self.finishAir(inst, result, .{ .none, .none, .none }); |
| 1357 | 1398 | } |
| 1358 | 1399 | |
| 1359 | 1400 | fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void { |
| ... | ... | @@ -1992,11 +2033,6 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void { |
| 1992 | 2033 | }, |
| 1993 | 2034 | .register => |reg| { |
| 1994 | 2035 | // TODO reuse operand |
| 1995 | | self.register_manager.getRegAssumeFree(.rcx, null); |
| 1996 | | const rcx_lock = |
| 1997 | | if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null; |
| 1998 | | defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); |
| 1999 | | |
| 2000 | 2036 | const eu_lock = self.register_manager.lockReg(reg); |
| 2001 | 2037 | defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); |
| 2002 | 2038 | |
| ... | ... | @@ -2047,11 +2083,6 @@ fn genUnwrapErrorUnionPayloadMir( |
| 2047 | 2083 | }, |
| 2048 | 2084 | .register => |reg| { |
| 2049 | 2085 | // TODO reuse operand |
| 2050 | | self.register_manager.getRegAssumeFree(.rcx, null); |
| 2051 | | const rcx_lock = |
| 2052 | | if (payload_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null; |
| 2053 | | defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); |
| 2054 | | |
| 2055 | 2086 | const eu_lock = self.register_manager.lockReg(reg); |
| 2056 | 2087 | defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); |
| 2057 | 2088 | |
| ... | ... | @@ -2877,17 +2908,18 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { |
| 2877 | 2908 | const imm_0000_1111 = Immediate.u(mask / 0b0001_0001); |
| 2878 | 2909 | const imm_0000_0001 = Immediate.u(mask / 0b1111_1111); |
| 2879 | 2910 | |
| 2880 | | const tmp_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) |
| 2881 | | src_mcv.register |
| 2911 | const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) |
| 2912 | src_mcv |
| 2882 | 2913 | else |
| 2883 | | try self.copyToTmpRegister(src_ty, src_mcv); |
| 2884 | | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 2885 | | defer self.register_manager.unlockReg(tmp_lock); |
| 2886 | | |
| 2887 | | const dst_reg = try self.register_manager.allocReg(inst, gp); |
| 2914 | try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv); |
| 2915 | const dst_reg = dst_mcv.register; |
| 2888 | 2916 | const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); |
| 2889 | 2917 | defer self.register_manager.unlockReg(dst_lock); |
| 2890 | 2918 | |
| 2919 | const tmp_reg = try self.register_manager.allocReg(null, gp); |
| 2920 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 2921 | defer self.register_manager.unlockReg(tmp_lock); |
| 2922 | |
| 2891 | 2923 | { |
| 2892 | 2924 | const dst = registerAlias(dst_reg, src_abi_size); |
| 2893 | 2925 | const tmp = registerAlias(tmp_reg, src_abi_size); |
| ... | ... | @@ -2896,9 +2928,9 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { |
| 2896 | 2928 | else |
| 2897 | 2929 | undefined; |
| 2898 | 2930 | |
| 2899 | | // tmp = operand |
| 2900 | | try self.asmRegisterRegister(.mov, dst, tmp); |
| 2901 | 2931 | // dst = operand |
| 2932 | try self.asmRegisterRegister(.mov, tmp, dst); |
| 2933 | // tmp = operand |
| 2902 | 2934 | try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1)); |
| 2903 | 2935 | // tmp = operand >> 1 |
| 2904 | 2936 | if (src_abi_size > 4) { |
| ... | ... | @@ -2948,7 +2980,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { |
| 2948 | 2980 | } |
| 2949 | 2981 | // dst = (temp3 * 0x01...01) >> (bits - 8) |
| 2950 | 2982 | } |
| 2951 | | break :result .{ .register = dst_reg }; |
| 2983 | break :result dst_mcv; |
| 2952 | 2984 | }; |
| 2953 | 2985 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 2954 | 2986 | } |
| ... | ... | @@ -3796,8 +3828,6 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue |
| 3796 | 3828 | |
| 3797 | 3829 | /// Clobbers .rcx for non-immediate shift value. |
| 3798 | 3830 | fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shift: MCValue) !void { |
| 3799 | | assert(reg.to64() != .rcx); |
| 3800 | | |
| 3801 | 3831 | switch (tag) { |
| 3802 | 3832 | .sal, .sar, .shl, .shr => {}, |
| 3803 | 3833 | else => unreachable, |
| ... | ... | @@ -4612,23 +4642,24 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void { |
| 4612 | 4642 | const src_index = self.air.instructions.items(.data)[inst].arg.src_index; |
| 4613 | 4643 | const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index); |
| 4614 | 4644 | |
| 4615 | | if (self.liveness.isUnused(inst)) |
| 4616 | | return self.finishAirBookkeeping(); |
| 4645 | const result: MCValue = result: { |
| 4646 | if (self.liveness.isUnused(inst)) break :result .dead; |
| 4617 | 4647 | |
| 4618 | | const dst_mcv: MCValue = switch (mcv) { |
| 4619 | | .register => |reg| blk: { |
| 4620 | | self.register_manager.getRegAssumeFree(reg.to64(), inst); |
| 4621 | | break :blk MCValue{ .register = reg }; |
| 4622 | | }, |
| 4623 | | .stack_offset => |off| blk: { |
| 4624 | | const offset = @intCast(i32, self.max_end_stack) - off + 16; |
| 4625 | | break :blk MCValue{ .stack_offset = -offset }; |
| 4626 | | }, |
| 4627 | | else => return self.fail("TODO implement arg for {}", .{mcv}), |
| 4648 | const dst_mcv: MCValue = switch (mcv) { |
| 4649 | .register => |reg| blk: { |
| 4650 | self.register_manager.getRegAssumeFree(reg.to64(), inst); |
| 4651 | break :blk MCValue{ .register = reg }; |
| 4652 | }, |
| 4653 | .stack_offset => |off| blk: { |
| 4654 | const offset = @intCast(i32, self.max_end_stack) - off + 16; |
| 4655 | break :blk MCValue{ .stack_offset = -offset }; |
| 4656 | }, |
| 4657 | else => return self.fail("TODO implement arg for {}", .{mcv}), |
| 4658 | }; |
| 4659 | try self.genArgDbgInfo(ty, name, dst_mcv); |
| 4660 | break :result dst_mcv; |
| 4628 | 4661 | }; |
| 4629 | | try self.genArgDbgInfo(ty, name, dst_mcv); |
| 4630 | | |
| 4631 | | return self.finishAir(inst, dst_mcv, .{ .none, .none, .none }); |
| 4662 | return self.finishAir(inst, result, .{ .none, .none, .none }); |
| 4632 | 4663 | } |
| 4633 | 4664 | |
| 4634 | 4665 | fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void { |
| ... | ... | @@ -4924,6 +4955,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier |
| 4924 | 4955 | } |
| 4925 | 4956 | |
| 4926 | 4957 | const result: MCValue = result: { |
| 4958 | if (self.liveness.isUnused(inst)) break :result .dead; |
| 4959 | |
| 4927 | 4960 | switch (info.return_value) { |
| 4928 | 4961 | .register => { |
| 4929 | 4962 | // Save function return value in a new register |
| ... | ... | @@ -5137,7 +5170,10 @@ fn genTry( |
| 5137 | 5170 | const reloc = try self.genCondBrMir(Type.anyerror, is_err_mcv); |
| 5138 | 5171 | try self.genBody(body); |
| 5139 | 5172 | try self.performReloc(reloc); |
| 5140 | | const result = try self.genUnwrapErrorUnionPayloadMir(inst, err_union_ty, err_union); |
| 5173 | const result = if (self.liveness.isUnused(inst)) |
| 5174 | .dead |
| 5175 | else |
| 5176 | try self.genUnwrapErrorUnionPayloadMir(inst, err_union_ty, err_union); |
| 5141 | 5177 | return result; |
| 5142 | 5178 | } |
| 5143 | 5179 | |
| ... | ... | @@ -5234,7 +5270,8 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { |
| 5234 | 5270 | } |
| 5235 | 5271 | |
| 5236 | 5272 | // Capture the state of register and stack allocation state so that we can revert to it. |
| 5237 | | const saved_state = try self.captureState(); |
| 5273 | var saved_state = try self.captureState(); |
| 5274 | defer saved_state.deinit(self.gpa); |
| 5238 | 5275 | |
| 5239 | 5276 | { |
| 5240 | 5277 | try self.branch_stack.append(.{}); |
| ... | ... | @@ -5252,7 +5289,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { |
| 5252 | 5289 | var then_branch = self.branch_stack.pop(); |
| 5253 | 5290 | defer then_branch.deinit(self.gpa); |
| 5254 | 5291 | |
| 5255 | | self.revertState(saved_state); |
| 5292 | try self.revertState(saved_state); |
| 5256 | 5293 | |
| 5257 | 5294 | try self.performReloc(reloc); |
| 5258 | 5295 | |
| ... | ... | @@ -5286,9 +5323,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { |
| 5286 | 5323 | |
| 5287 | 5324 | log.debug("Then branch: {}", .{then_branch.fmtDebug()}); |
| 5288 | 5325 | log.debug("Else branch: {}", .{else_branch.fmtDebug()}); |
| 5289 | | |
| 5290 | | const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| 5291 | | try self.canonicaliseBranches(parent_branch, &then_branch, &else_branch); |
| 5326 | try self.canonicaliseBranches(true, &then_branch, &else_branch); |
| 5292 | 5327 | |
| 5293 | 5328 | // We already took care of pl_op.operand earlier, so we're going |
| 5294 | 5329 | // to pass .none here |
| ... | ... | @@ -5423,10 +5458,6 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) ! |
| 5423 | 5458 | try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 }); |
| 5424 | 5459 | }, |
| 5425 | 5460 | .register => |reg| { |
| 5426 | | self.register_manager.getRegAssumeFree(.rcx, null); |
| 5427 | | const rcx_lock = if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null; |
| 5428 | | defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); |
| 5429 | | |
| 5430 | 5461 | const eu_lock = self.register_manager.lockReg(reg); |
| 5431 | 5462 | defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); |
| 5432 | 5463 | |
| ... | ... | @@ -5606,7 +5637,7 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void { |
| 5606 | 5637 | // break instruction will choose a MCValue for the block result and overwrite |
| 5607 | 5638 | // this field. Following break instructions will use that MCValue to put their |
| 5608 | 5639 | // block results. |
| 5609 | | .mcv = .none, |
| 5640 | .mcv = if (self.liveness.isUnused(inst)) .dead else .none, |
| 5610 | 5641 | }); |
| 5611 | 5642 | defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa); |
| 5612 | 5643 | |
| ... | ... | @@ -5646,21 +5677,29 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 5646 | 5677 | } |
| 5647 | 5678 | } |
| 5648 | 5679 | |
| 5649 | | var branch_stack = std.ArrayList(Branch).init(self.gpa); |
| 5650 | | defer { |
| 5651 | | for (branch_stack.items) |*bs| { |
| 5652 | | bs.deinit(self.gpa); |
| 5653 | | } |
| 5654 | | branch_stack.deinit(); |
| 5680 | log.debug("airSwitch: %{d}", .{inst}); |
| 5681 | log.debug("Upper branches:", .{}); |
| 5682 | for (self.branch_stack.items) |bs| { |
| 5683 | log.debug("{}", .{bs.fmtDebug()}); |
| 5655 | 5684 | } |
| 5656 | | try branch_stack.ensureTotalCapacityPrecise(switch_br.data.cases_len + 1); |
| 5657 | 5685 | |
| 5686 | var prev_branch: ?Branch = null; |
| 5687 | defer if (prev_branch) |*branch| branch.deinit(self.gpa); |
| 5688 | |
| 5689 | // Capture the state of register and stack allocation state so that we can revert to it. |
| 5690 | var saved_state = try self.captureState(); |
| 5691 | defer saved_state.deinit(self.gpa); |
| 5692 | |
| 5693 | const cases_len = switch_br.data.cases_len + @boolToInt(switch_br.data.else_body_len > 0); |
| 5658 | 5694 | while (case_i < switch_br.data.cases_len) : (case_i += 1) { |
| 5659 | 5695 | const case = self.air.extraData(Air.SwitchBr.Case, extra_index); |
| 5660 | 5696 | const items = @ptrCast([]const Air.Inst.Ref, self.air.extra[case.end..][0..case.data.items_len]); |
| 5661 | 5697 | const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len]; |
| 5662 | 5698 | extra_index = case.end + items.len + case_body.len; |
| 5663 | 5699 | |
| 5700 | // Revert to the previous register and stack allocation state. |
| 5701 | if (prev_branch) |_| try self.revertState(saved_state); |
| 5702 | |
| 5664 | 5703 | var relocs = try self.gpa.alloc(u32, items.len); |
| 5665 | 5704 | defer self.gpa.free(relocs); |
| 5666 | 5705 | |
| ... | ... | @@ -5671,12 +5710,9 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 5671 | 5710 | reloc.* = try self.asmJccReloc(undefined, .ne); |
| 5672 | 5711 | } |
| 5673 | 5712 | |
| 5674 | | // Capture the state of register and stack allocation state so that we can revert to it. |
| 5675 | | const saved_state = try self.captureState(); |
| 5676 | | |
| 5677 | 5713 | { |
| 5678 | | try self.branch_stack.append(.{}); |
| 5679 | | errdefer _ = self.branch_stack.pop(); |
| 5714 | if (cases_len > 1) try self.branch_stack.append(.{}); |
| 5715 | errdefer _ = if (cases_len > 1) self.branch_stack.pop(); |
| 5680 | 5716 | |
| 5681 | 5717 | try self.ensureProcessDeathCapacity(liveness.deaths[case_i].len); |
| 5682 | 5718 | for (liveness.deaths[case_i]) |operand| { |
| ... | ... | @@ -5686,25 +5722,31 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 5686 | 5722 | try self.genBody(case_body); |
| 5687 | 5723 | } |
| 5688 | 5724 | |
| 5689 | | branch_stack.appendAssumeCapacity(self.branch_stack.pop()); |
| 5690 | | |
| 5691 | | // Revert to the previous register and stack allocation state. |
| 5692 | | self.revertState(saved_state); |
| 5725 | // Consolidate returned MCValues between prongs like we do in airCondBr. |
| 5726 | if (cases_len > 1) { |
| 5727 | var case_branch = self.branch_stack.pop(); |
| 5728 | errdefer case_branch.deinit(self.gpa); |
| 5693 | 5729 | |
| 5694 | | for (relocs) |reloc| { |
| 5695 | | try self.performReloc(reloc); |
| 5730 | log.debug("Case-{d} branch: {}", .{ case_i, case_branch.fmtDebug() }); |
| 5731 | if (prev_branch) |*canon_branch| { |
| 5732 | try self.canonicaliseBranches(case_i == cases_len - 1, canon_branch, &case_branch); |
| 5733 | canon_branch.deinit(self.gpa); |
| 5734 | } |
| 5735 | prev_branch = case_branch; |
| 5696 | 5736 | } |
| 5737 | |
| 5738 | for (relocs) |reloc| try self.performReloc(reloc); |
| 5697 | 5739 | } |
| 5698 | 5740 | |
| 5699 | 5741 | if (switch_br.data.else_body_len > 0) { |
| 5700 | 5742 | const else_body = self.air.extra[extra_index..][0..switch_br.data.else_body_len]; |
| 5701 | 5743 | |
| 5702 | | // Capture the state of register and stack allocation state so that we can revert to it. |
| 5703 | | const saved_state = try self.captureState(); |
| 5744 | // Revert to the previous register and stack allocation state. |
| 5745 | if (prev_branch) |_| try self.revertState(saved_state); |
| 5704 | 5746 | |
| 5705 | 5747 | { |
| 5706 | | try self.branch_stack.append(.{}); |
| 5707 | | errdefer _ = self.branch_stack.pop(); |
| 5748 | if (cases_len > 1) try self.branch_stack.append(.{}); |
| 5749 | errdefer _ = if (cases_len > 1) self.branch_stack.pop(); |
| 5708 | 5750 | |
| 5709 | 5751 | const else_deaths = liveness.deaths.len - 1; |
| 5710 | 5752 | try self.ensureProcessDeathCapacity(liveness.deaths[else_deaths].len); |
| ... | ... | @@ -5715,53 +5757,48 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 5715 | 5757 | try self.genBody(else_body); |
| 5716 | 5758 | } |
| 5717 | 5759 | |
| 5718 | | branch_stack.appendAssumeCapacity(self.branch_stack.pop()); |
| 5760 | // Consolidate returned MCValues between a prong and the else branch like we do in airCondBr. |
| 5761 | if (cases_len > 1) { |
| 5762 | var else_branch = self.branch_stack.pop(); |
| 5763 | errdefer else_branch.deinit(self.gpa); |
| 5719 | 5764 | |
| 5720 | | // Revert to the previous register and stack allocation state. |
| 5721 | | self.revertState(saved_state); |
| 5722 | | } |
| 5723 | | |
| 5724 | | // Consolidate returned MCValues between prongs and else branch like we do |
| 5725 | | // in airCondBr. |
| 5726 | | log.debug("airSwitch: %{d}", .{inst}); |
| 5727 | | log.debug("Upper branches:", .{}); |
| 5728 | | for (self.branch_stack.items) |bs| { |
| 5729 | | log.debug("{}", .{bs.fmtDebug()}); |
| 5730 | | } |
| 5731 | | for (branch_stack.items, 0..) |bs, i| { |
| 5732 | | log.debug("Case-{d} branch: {}", .{ i, bs.fmtDebug() }); |
| 5733 | | } |
| 5734 | | |
| 5735 | | // TODO: can we reduce the complexity of this algorithm? |
| 5736 | | const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| 5737 | | var i: usize = branch_stack.items.len; |
| 5738 | | while (i > 1) : (i -= 1) { |
| 5739 | | const canon_branch = &branch_stack.items[i - 2]; |
| 5740 | | const target_branch = &branch_stack.items[i - 1]; |
| 5741 | | try self.canonicaliseBranches(parent_branch, canon_branch, target_branch); |
| 5765 | log.debug("Else branch: {}", .{else_branch.fmtDebug()}); |
| 5766 | if (prev_branch) |*canon_branch| { |
| 5767 | try self.canonicaliseBranches(true, canon_branch, &else_branch); |
| 5768 | canon_branch.deinit(self.gpa); |
| 5769 | } |
| 5770 | prev_branch = else_branch; |
| 5771 | } |
| 5742 | 5772 | } |
| 5743 | 5773 | |
| 5744 | | // We already took care of pl_op.operand earlier, so we're going |
| 5745 | | // to pass .none here |
| 5774 | // We already took care of pl_op.operand earlier, so we're going to pass .none here |
| 5746 | 5775 | return self.finishAir(inst, .unreach, .{ .none, .none, .none }); |
| 5747 | 5776 | } |
| 5748 | 5777 | |
| 5749 | | fn canonicaliseBranches(self: *Self, parent_branch: *Branch, canon_branch: *Branch, target_branch: *Branch) !void { |
| 5750 | | try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, target_branch.inst_table.count()); |
| 5778 | fn canonicaliseBranches( |
| 5779 | self: *Self, |
| 5780 | update_parent: bool, |
| 5781 | canon_branch: *Branch, |
| 5782 | target_branch: *const Branch, |
| 5783 | ) !void { |
| 5784 | const parent_branch = |
| 5785 | if (update_parent) &self.branch_stack.items[self.branch_stack.items.len - 1] else undefined; |
| 5786 | if (update_parent) try self.ensureProcessDeathCapacity(target_branch.inst_table.count()); |
| 5751 | 5787 | |
| 5752 | 5788 | const target_slice = target_branch.inst_table.entries.slice(); |
| 5753 | 5789 | for (target_slice.items(.key), target_slice.items(.value)) |target_key, target_value| { |
| 5754 | 5790 | const canon_mcv = if (canon_branch.inst_table.fetchSwapRemove(target_key)) |canon_entry| blk: { |
| 5755 | 5791 | // The instruction's MCValue is overridden in both branches. |
| 5756 | | parent_branch.inst_table.putAssumeCapacity(target_key, canon_entry.value); |
| 5792 | if (update_parent) { |
| 5793 | parent_branch.inst_table.putAssumeCapacity(target_key, canon_entry.value); |
| 5794 | } |
| 5757 | 5795 | if (target_value == .dead) { |
| 5758 | 5796 | assert(canon_entry.value == .dead); |
| 5759 | 5797 | continue; |
| 5760 | 5798 | } |
| 5761 | 5799 | break :blk canon_entry.value; |
| 5762 | 5800 | } else blk: { |
| 5763 | | if (target_value == .dead) |
| 5764 | | continue; |
| 5801 | if (target_value == .dead) continue; |
| 5765 | 5802 | // The instruction is only overridden in the else branch. |
| 5766 | 5803 | // If integer overflows occurs, the question is: why wasn't the instruction marked dead? |
| 5767 | 5804 | break :blk self.getResolvedInstValue(target_key).?; |
| ... | ... | @@ -5770,22 +5807,25 @@ fn canonicaliseBranches(self: *Self, parent_branch: *Branch, canon_branch: *Bran |
| 5770 | 5807 | // TODO make sure the destination stack offset / register does not already have something |
| 5771 | 5808 | // going on there. |
| 5772 | 5809 | try self.setRegOrMem(self.air.typeOfIndex(target_key), canon_mcv, target_value); |
| 5810 | self.freeValue(target_value); |
| 5773 | 5811 | // TODO track the new register / stack allocation |
| 5774 | 5812 | } |
| 5775 | | try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, canon_branch.inst_table.count()); |
| 5813 | if (update_parent) try self.ensureProcessDeathCapacity(canon_branch.inst_table.count()); |
| 5776 | 5814 | const canon_slice = canon_branch.inst_table.entries.slice(); |
| 5777 | 5815 | for (canon_slice.items(.key), canon_slice.items(.value)) |canon_key, canon_value| { |
| 5778 | 5816 | // We already deleted the items from this table that matched the target_branch. |
| 5779 | 5817 | // So these are all instructions that are only overridden in the canon branch. |
| 5780 | | parent_branch.inst_table.putAssumeCapacity(canon_key, canon_value); |
| 5781 | | log.debug("canon_value = {}", .{canon_value}); |
| 5782 | | if (canon_value == .dead) |
| 5783 | | continue; |
| 5784 | | const parent_mcv = self.getResolvedInstValue(canon_key).?; |
| 5818 | const parent_mcv = |
| 5819 | if (canon_value != .dead) self.getResolvedInstValue(canon_key).? else undefined; |
| 5820 | if (update_parent) { |
| 5821 | parent_branch.inst_table.putAssumeCapacity(canon_key, canon_value); |
| 5822 | } |
| 5823 | if (canon_value == .dead) continue; |
| 5785 | 5824 | log.debug("consolidating canon_entry {d} {}=>{}", .{ canon_key, parent_mcv, canon_value }); |
| 5786 | 5825 | // TODO make sure the destination stack offset / register does not already have something |
| 5787 | 5826 | // going on there. |
| 5788 | | try self.setRegOrMem(self.air.typeOfIndex(canon_key), parent_mcv, canon_value); |
| 5827 | try self.setRegOrMem(self.air.typeOfIndex(canon_key), canon_value, parent_mcv); |
| 5828 | self.freeValue(parent_mcv); |
| 5789 | 5829 | // TODO track the new register / stack allocation |
| 5790 | 5830 | } |
| 5791 | 5831 | } |
| ... | ... | @@ -5811,11 +5851,9 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void { |
| 5811 | 5851 | |
| 5812 | 5852 | fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void { |
| 5813 | 5853 | const block_data = self.blocks.getPtr(block).?; |
| 5814 | | |
| 5815 | | if (self.air.typeOf(operand).hasRuntimeBits()) { |
| 5854 | if (block_data.mcv != .dead and self.air.typeOf(operand).hasRuntimeBits()) { |
| 5816 | 5855 | const operand_mcv = try self.resolveInst(operand); |
| 5817 | | const block_mcv = block_data.mcv; |
| 5818 | | if (block_mcv == .none) { |
| 5856 | if (block_data.mcv == .none) { |
| 5819 | 5857 | block_data.mcv = switch (operand_mcv) { |
| 5820 | 5858 | .none, .dead, .unreach => unreachable, |
| 5821 | 5859 | .register, .stack_offset, .memory => operand_mcv, |
| ... | ... | @@ -5827,7 +5865,7 @@ fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void { |
| 5827 | 5865 | else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}), |
| 5828 | 5866 | }; |
| 5829 | 5867 | } else { |
| 5830 | | try self.setRegOrMem(self.air.typeOfIndex(block), block_mcv, operand_mcv); |
| 5868 | try self.setRegOrMem(self.air.typeOfIndex(block), block_data.mcv, operand_mcv); |
| 5831 | 5869 | } |
| 5832 | 5870 | } |
| 5833 | 5871 | return self.brVoid(block); |
| ... | ... | @@ -6916,7 +6954,8 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void { |
| 6916 | 6954 | const pl_op = self.air.instructions.items(.data)[inst].pl_op; |
| 6917 | 6955 | const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; |
| 6918 | 6956 | |
| 6919 | | const dst_reg = try self.register_manager.allocReg(inst, gp); |
| 6957 | const unused = self.liveness.isUnused(inst); |
| 6958 | const dst_reg = try self.register_manager.allocReg(if (unused) null else inst, gp); |
| 6920 | 6959 | |
| 6921 | 6960 | const ptr_ty = self.air.typeOf(pl_op.operand); |
| 6922 | 6961 | const ptr_mcv = try self.resolveInst(pl_op.operand); |
| ... | ... | @@ -6924,7 +6963,6 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void { |
| 6924 | 6963 | const val_ty = self.air.typeOf(extra.operand); |
| 6925 | 6964 | const val_mcv = try self.resolveInst(extra.operand); |
| 6926 | 6965 | |
| 6927 | | const unused = self.liveness.isUnused(inst); |
| 6928 | 6966 | try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering()); |
| 6929 | 6967 | const result: MCValue = if (unused) .dead else .{ .register = dst_reg }; |
| 6930 | 6968 | return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none }); |
Jacob Young <jacobly@ziglang.org> 2023-03-23 00:27:25-04:00
src