| ... | @@ -213,12 +213,15 @@ const StackAllocation = struct { | ... | @@ -213,12 +213,15 @@ const StackAllocation = struct { |
| 213 | }; | 213 | }; |
| 214 | | 214 | |
| 215 | const BlockData = struct { | 215 | const BlockData = struct { |
| 216 | relocs: std.ArrayListUnmanaged(Mir.Inst.Index), | 216 | relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{}, |
| 217 | /// The first break instruction encounters `null` here and chooses a | 217 | branch: Branch = .{}, |
| 218 | /// machine code value for the block result, populating this field. | 218 | branch_depth: u32, |
| 219 | /// Following break instructions encounter that value and use it for | 219 | |
| 220 | /// the location to store their block results. | 220 | fn deinit(self: *BlockData, gpa: Allocator) void { |
| 221 | mcv: MCValue, | 221 | self.branch.deinit(gpa); |
| | 222 | self.relocs.deinit(gpa); |
| | 223 | self.* = undefined; |
| | 224 | } |
| 222 | }; | 225 | }; |
| 223 | | 226 | |
| 224 | const BigTomb = struct { | 227 | const BigTomb = struct { |
| ... | @@ -265,12 +268,15 @@ pub fn generate( | ... | @@ -265,12 +268,15 @@ pub fn generate( |
| 265 | const fn_type = fn_owner_decl.ty; | 268 | const fn_type = fn_owner_decl.ty; |
| 266 | | 269 | |
| 267 | var branch_stack = std.ArrayList(Branch).init(bin_file.allocator); | 270 | var branch_stack = std.ArrayList(Branch).init(bin_file.allocator); |
| | 271 | try branch_stack.ensureUnusedCapacity(2); |
| | 272 | // The outermost branch is used for constants only. |
| | 273 | branch_stack.appendAssumeCapacity(.{}); |
| | 274 | branch_stack.appendAssumeCapacity(.{}); |
| 268 | defer { | 275 | defer { |
| 269 | assert(branch_stack.items.len == 1); | 276 | assert(branch_stack.items.len == 2); |
| 270 | branch_stack.items[0].deinit(bin_file.allocator); | 277 | for (branch_stack.items) |*branch| branch.deinit(bin_file.allocator); |
| 271 | branch_stack.deinit(); | 278 | branch_stack.deinit(); |
| 272 | } | 279 | } |
| 273 | try branch_stack.append(.{}); | | |
| 274 | | 280 | |
| 275 | var function = Self{ | 281 | var function = Self{ |
| 276 | .gpa = bin_file.allocator, | 282 | .gpa = bin_file.allocator, |
| ... | @@ -1070,20 +1076,36 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -1070,20 +1076,36 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 1070 | if (self.air_bookkeeping < old_air_bookkeeping + 1) { | 1076 | if (self.air_bookkeeping < old_air_bookkeeping + 1) { |
| 1071 | std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[inst] }); | 1077 | std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[inst] }); |
| 1072 | } | 1078 | } |
| | 1079 | |
| | 1080 | { // check consistency of tracked registers |
| | 1081 | var it = self.register_manager.free_registers.iterator(.{ .kind = .unset }); |
| | 1082 | while (it.next()) |index| { |
| | 1083 | const tracked_inst = self.register_manager.registers[index]; |
| | 1084 | const tracked_mcv = self.getResolvedInstValue(tracked_inst).?.*; |
| | 1085 | assert(RegisterManager.indexOfRegIntoTracked(switch (tracked_mcv) { |
| | 1086 | .register => |reg| reg, |
| | 1087 | .register_overflow => |ro| ro.reg, |
| | 1088 | else => unreachable, |
| | 1089 | }).? == index); |
| | 1090 | } |
| | 1091 | } |
| 1073 | } | 1092 | } |
| 1074 | } | 1093 | } |
| 1075 | } | 1094 | } |
| 1076 | | 1095 | |
| 1077 | /// Asserts there is already capacity to insert into top branch inst_table. | 1096 | fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void { |
| 1078 | fn processDeath(self: *Self, inst: Air.Inst.Index) void { | 1097 | const reg = switch (value) { |
| 1079 | const air_tags = self.air.instructions.items(.tag); | 1098 | .register => |reg| reg, |
| 1080 | if (air_tags[inst] == .constant) return; // Constants are immortal. | 1099 | .register_overflow => |ro| ro.reg, |
| 1081 | const prev_value = self.getResolvedInstValue(inst) orelse return; | 1100 | else => return, |
| 1082 | log.debug("%{d} => {}", .{ inst, MCValue.dead }); | 1101 | }; |
| 1083 | // When editing this function, note that the logic must synchronize with `reuseOperand`. | 1102 | if (self.register_manager.isRegFree(reg)) { |
| 1084 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | 1103 | self.register_manager.getRegAssumeFree(reg, inst); |
| 1085 | branch.inst_table.putAssumeCapacity(inst, .dead); | 1104 | } |
| 1086 | switch (prev_value) { | 1105 | } |
| | 1106 | |
| | 1107 | fn freeValue(self: *Self, value: MCValue) void { |
| | 1108 | switch (value) { |
| 1087 | .register => |reg| { | 1109 | .register => |reg| { |
| 1088 | self.register_manager.freeReg(reg); | 1110 | self.register_manager.freeReg(reg); |
| 1089 | }, | 1111 | }, |
| ... | @@ -1098,6 +1120,18 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void { | ... | @@ -1098,6 +1120,18 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void { |
| 1098 | } | 1120 | } |
| 1099 | } | 1121 | } |
| 1100 | | 1122 | |
| | 1123 | /// Asserts there is already capacity to insert into top branch inst_table. |
| | 1124 | fn processDeath(self: *Self, inst: Air.Inst.Index) void { |
| | 1125 | const air_tags = self.air.instructions.items(.tag); |
| | 1126 | if (air_tags[inst] == .constant) return; // Constants are immortal. |
| | 1127 | const prev_value = (self.getResolvedInstValue(inst) orelse return).*; |
| | 1128 | log.debug("%{d} => {}", .{ inst, MCValue.dead }); |
| | 1129 | // When editing this function, note that the logic must synchronize with `reuseOperand`. |
| | 1130 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| | 1131 | branch.inst_table.putAssumeCapacity(inst, .dead); |
| | 1132 | self.freeValue(prev_value); |
| | 1133 | } |
| | 1134 | |
| 1101 | /// Called when there are no operands, and the instruction is always unreferenced. | 1135 | /// Called when there are no operands, and the instruction is always unreferenced. |
| 1102 | fn finishAirBookkeeping(self: *Self) void { | 1136 | fn finishAirBookkeeping(self: *Self) void { |
| 1103 | if (std.debug.runtime_safety) { | 1137 | if (std.debug.runtime_safety) { |
| ... | @@ -1121,32 +1155,21 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live | ... | @@ -1121,32 +1155,21 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live |
| 1121 | log.debug("%{d} => {}", .{ inst, result }); | 1155 | log.debug("%{d} => {}", .{ inst, result }); |
| 1122 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | 1156 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| 1123 | branch.inst_table.putAssumeCapacityNoClobber(inst, result); | 1157 | branch.inst_table.putAssumeCapacityNoClobber(inst, result); |
| 1124 | | 1158 | // In some cases, an operand may be reused as the result. |
| 1125 | // In some cases (such as bitcast), an operand | 1159 | // If that operand died and was a register, it was freed by |
| 1126 | // may be the same MCValue as the result. If | 1160 | // processDeath, so we have to "re-allocate" the register. |
| 1127 | // that operand died and was a register, it | 1161 | self.getValue(result, inst); |
| 1128 | // was freed by processDeath. We have to | 1162 | } else switch (result) { |
| 1129 | // "re-allocate" the register. | 1163 | .none, .dead, .unreach => {}, |
| 1130 | switch (result) { | 1164 | else => unreachable, // Why didn't the result die? |
| 1131 | .register => |reg| { | | |
| 1132 | if (self.register_manager.isRegFree(reg)) { | | |
| 1133 | self.register_manager.getRegAssumeFree(reg, inst); | | |
| 1134 | } | | |
| 1135 | }, | | |
| 1136 | .register_overflow => |ro| { | | |
| 1137 | if (self.register_manager.isRegFree(ro.reg)) { | | |
| 1138 | self.register_manager.getRegAssumeFree(ro.reg, inst); | | |
| 1139 | } | | |
| 1140 | }, | | |
| 1141 | else => {}, | | |
| 1142 | } | | |
| 1143 | } | 1165 | } |
| 1144 | self.finishAirBookkeeping(); | 1166 | self.finishAirBookkeeping(); |
| 1145 | } | 1167 | } |
| 1146 | | 1168 | |
| 1147 | fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void { | 1169 | fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void { |
| | 1170 | // In addition to the caller's needs, we need enough space to spill every register and eflags. |
| 1148 | const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table; | 1171 | const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table; |
| 1149 | try table.ensureUnusedCapacity(self.gpa, additional_count); | 1172 | try table.ensureUnusedCapacity(self.gpa, additional_count + self.register_manager.registers.len + 1); |
| 1150 | } | 1173 | } |
| 1151 | | 1174 | |
| 1152 | fn allocMem(self: *Self, inst: ?Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 { | 1175 | fn allocMem(self: *Self, inst: ?Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 { |
| ... | @@ -1231,42 +1254,29 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_ | ... | @@ -1231,42 +1254,29 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_ |
| 1231 | } | 1254 | } |
| 1232 | | 1255 | |
| 1233 | const State = struct { | 1256 | const State = struct { |
| 1234 | next_stack_offset: u32, | | |
| 1235 | registers: abi.RegisterManager.TrackedRegisters, | 1257 | registers: abi.RegisterManager.TrackedRegisters, |
| 1236 | free_registers: abi.RegisterManager.RegisterBitSet, | 1258 | free_registers: abi.RegisterManager.RegisterBitSet, |
| 1237 | eflags_inst: ?Air.Inst.Index, | 1259 | eflags_inst: ?Air.Inst.Index, |
| 1238 | stack: std.AutoHashMapUnmanaged(u32, StackAllocation), | | |
| 1239 | | | |
| 1240 | fn deinit(state: *State, gpa: Allocator) void { | | |
| 1241 | state.stack.deinit(gpa); | | |
| 1242 | } | | |
| 1243 | }; | 1260 | }; |
| 1244 | | 1261 | |
| 1245 | fn captureState(self: *Self) !State { | 1262 | fn captureState(self: *Self) State { |
| 1246 | return State{ | 1263 | return State{ |
| 1247 | .next_stack_offset = self.next_stack_offset, | | |
| 1248 | .registers = self.register_manager.registers, | 1264 | .registers = self.register_manager.registers, |
| 1249 | .free_registers = self.register_manager.free_registers, | 1265 | .free_registers = self.register_manager.free_registers, |
| 1250 | .eflags_inst = self.eflags_inst, | 1266 | .eflags_inst = self.eflags_inst, |
| 1251 | .stack = try self.stack.clone(self.gpa), | | |
| 1252 | }; | 1267 | }; |
| 1253 | } | 1268 | } |
| 1254 | | 1269 | |
| 1255 | fn revertState(self: *Self, state: State) void { | 1270 | fn revertState(self: *Self, state: State) void { |
| 1256 | self.register_manager.registers = state.registers; | | |
| 1257 | self.eflags_inst = state.eflags_inst; | 1271 | self.eflags_inst = state.eflags_inst; |
| 1258 | | | |
| 1259 | self.stack.deinit(self.gpa); | | |
| 1260 | self.stack = state.stack; | | |
| 1261 | | | |
| 1262 | self.next_stack_offset = state.next_stack_offset; | | |
| 1263 | self.register_manager.free_registers = state.free_registers; | 1272 | self.register_manager.free_registers = state.free_registers; |
| | 1273 | self.register_manager.registers = state.registers; |
| 1264 | } | 1274 | } |
| 1265 | | 1275 | |
| 1266 | pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void { | 1276 | pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void { |
| 1267 | const stack_mcv = try self.allocRegOrMem(inst, false); | 1277 | const stack_mcv = try self.allocRegOrMem(inst, false); |
| 1268 | log.debug("spilling %{d} to stack mcv {any}", .{ inst, stack_mcv }); | 1278 | log.debug("spilling %{d} to stack mcv {any}", .{ inst, stack_mcv }); |
| 1269 | const reg_mcv = self.getResolvedInstValue(inst).?; | 1279 | const reg_mcv = self.getResolvedInstValue(inst).?.*; |
| 1270 | switch (reg_mcv) { | 1280 | switch (reg_mcv) { |
| 1271 | .register => |other| { | 1281 | .register => |other| { |
| 1272 | assert(reg.to64() == other.to64()); | 1282 | assert(reg.to64() == other.to64()); |
| ... | @@ -1277,13 +1287,13 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void | ... | @@ -1277,13 +1287,13 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void |
| 1277 | else => {}, | 1287 | else => {}, |
| 1278 | } | 1288 | } |
| 1279 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | 1289 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| 1280 | try branch.inst_table.put(self.gpa, inst, stack_mcv); | 1290 | branch.inst_table.putAssumeCapacity(inst, stack_mcv); |
| 1281 | try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{}); | 1291 | try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{}); |
| 1282 | } | 1292 | } |
| 1283 | | 1293 | |
| 1284 | pub fn spillEflagsIfOccupied(self: *Self) !void { | 1294 | pub fn spillEflagsIfOccupied(self: *Self) !void { |
| 1285 | if (self.eflags_inst) |inst_to_save| { | 1295 | if (self.eflags_inst) |inst_to_save| { |
| 1286 | const mcv = self.getResolvedInstValue(inst_to_save).?; | 1296 | const mcv = self.getResolvedInstValue(inst_to_save).?.*; |
| 1287 | const new_mcv = switch (mcv) { | 1297 | const new_mcv = switch (mcv) { |
| 1288 | .register_overflow => try self.allocRegOrMem(inst_to_save, false), | 1298 | .register_overflow => try self.allocRegOrMem(inst_to_save, false), |
| 1289 | .eflags => try self.allocRegOrMem(inst_to_save, true), | 1299 | .eflags => try self.allocRegOrMem(inst_to_save, true), |
| ... | @@ -1294,7 +1304,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void { | ... | @@ -1294,7 +1304,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void { |
| 1294 | log.debug("spilling %{d} to mcv {any}", .{ inst_to_save, new_mcv }); | 1304 | log.debug("spilling %{d} to mcv {any}", .{ inst_to_save, new_mcv }); |
| 1295 | | 1305 | |
| 1296 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | 1306 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| 1297 | try branch.inst_table.put(self.gpa, inst_to_save, new_mcv); | 1307 | branch.inst_table.putAssumeCapacity(inst_to_save, new_mcv); |
| 1298 | | 1308 | |
| 1299 | self.eflags_inst = null; | 1309 | self.eflags_inst = null; |
| 1300 | | 1310 | |
| ... | @@ -1347,13 +1357,23 @@ fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Ty | ... | @@ -1347,13 +1357,23 @@ fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Ty |
| 1347 | } | 1357 | } |
| 1348 | | 1358 | |
| 1349 | fn airAlloc(self: *Self, inst: Air.Inst.Index) !void { | 1359 | fn airAlloc(self: *Self, inst: Air.Inst.Index) !void { |
| 1350 | const stack_offset = try self.allocMemPtr(inst); | 1360 | const result: MCValue = result: { |
| 1351 | return self.finishAir(inst, .{ .ptr_stack_offset = @intCast(i32, stack_offset) }, .{ .none, .none, .none }); | 1361 | if (self.liveness.isUnused(inst)) break :result .dead; |
| | 1362 | |
| | 1363 | const stack_offset = try self.allocMemPtr(inst); |
| | 1364 | break :result .{ .ptr_stack_offset = @intCast(i32, stack_offset) }; |
| | 1365 | }; |
| | 1366 | return self.finishAir(inst, result, .{ .none, .none, .none }); |
| 1352 | } | 1367 | } |
| 1353 | | 1368 | |
| 1354 | fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void { | 1369 | fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void { |
| 1355 | const stack_offset = try self.allocMemPtr(inst); | 1370 | const result: MCValue = result: { |
| 1356 | return self.finishAir(inst, .{ .ptr_stack_offset = @intCast(i32, stack_offset) }, .{ .none, .none, .none }); | 1371 | if (self.liveness.isUnused(inst)) break :result .dead; |
| | 1372 | |
| | 1373 | const stack_offset = try self.allocMemPtr(inst); |
| | 1374 | break :result .{ .ptr_stack_offset = @intCast(i32, stack_offset) }; |
| | 1375 | }; |
| | 1376 | return self.finishAir(inst, result, .{ .none, .none, .none }); |
| 1357 | } | 1377 | } |
| 1358 | | 1378 | |
| 1359 | fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void { | 1379 | fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void { |
| ... | @@ -1992,11 +2012,6 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1992,11 +2012,6 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void { |
| 1992 | }, | 2012 | }, |
| 1993 | .register => |reg| { | 2013 | .register => |reg| { |
| 1994 | // TODO reuse operand | 2014 | // TODO reuse operand |
| 1995 | self.register_manager.getRegAssumeFree(.rcx, null); | | |
| 1996 | const rcx_lock = | | |
| 1997 | if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null; | | |
| 1998 | defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 1999 | | | |
| 2000 | const eu_lock = self.register_manager.lockReg(reg); | 2015 | const eu_lock = self.register_manager.lockReg(reg); |
| 2001 | defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); | 2016 | defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); |
| 2002 | | 2017 | |
| ... | @@ -2047,11 +2062,6 @@ fn genUnwrapErrorUnionPayloadMir( | ... | @@ -2047,11 +2062,6 @@ fn genUnwrapErrorUnionPayloadMir( |
| 2047 | }, | 2062 | }, |
| 2048 | .register => |reg| { | 2063 | .register => |reg| { |
| 2049 | // TODO reuse operand | 2064 | // TODO reuse operand |
| 2050 | self.register_manager.getRegAssumeFree(.rcx, null); | | |
| 2051 | const rcx_lock = | | |
| 2052 | if (payload_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null; | | |
| 2053 | defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 2054 | | | |
| 2055 | const eu_lock = self.register_manager.lockReg(reg); | 2065 | const eu_lock = self.register_manager.lockReg(reg); |
| 2056 | defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); | 2066 | defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); |
| 2057 | | 2067 | |
| ... | @@ -2749,7 +2759,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2749,7 +2759,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { |
| 2749 | }; | 2759 | }; |
| 2750 | defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); | 2760 | defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); |
| 2751 | | 2761 | |
| 2752 | const dst_reg = try self.register_manager.allocReg(inst, gp); | 2762 | const dst_reg = try self.register_manager.allocReg(null, gp); |
| 2753 | const dst_mcv = MCValue{ .register = dst_reg }; | 2763 | const dst_mcv = MCValue{ .register = dst_reg }; |
| 2754 | const dst_lock = self.register_manager.lockReg(dst_reg); | 2764 | const dst_lock = self.register_manager.lockReg(dst_reg); |
| 2755 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); | 2765 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| ... | @@ -2764,14 +2774,14 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2764,14 +2774,14 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { |
| 2764 | } | 2774 | } |
| 2765 | | 2775 | |
| 2766 | const src_bits = src_ty.bitSize(self.target.*); | 2776 | const src_bits = src_ty.bitSize(self.target.*); |
| 2767 | const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits }); | 2777 | const width_mcv = |
| 2768 | const width_mcv = MCValue{ .register = width_reg }; | 2778 | try self.copyToRegisterWithInstTracking(inst, dst_ty, .{ .immediate = src_bits }); |
| 2769 | try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv); | 2779 | try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv); |
| 2770 | | 2780 | |
| 2771 | const dst_abi_size = @intCast(u32, @max(dst_ty.abiSize(self.target.*), 2)); | 2781 | const dst_abi_size = @intCast(u32, @max(dst_ty.abiSize(self.target.*), 2)); |
| 2772 | try self.asmCmovccRegisterRegister( | 2782 | try self.asmCmovccRegisterRegister( |
| 2773 | registerAlias(dst_reg, dst_abi_size), | 2783 | registerAlias(dst_reg, dst_abi_size), |
| 2774 | registerAlias(width_reg, dst_abi_size), | 2784 | registerAlias(width_mcv.register, dst_abi_size), |
| 2775 | .z, | 2785 | .z, |
| 2776 | ); | 2786 | ); |
| 2777 | | 2787 | |
| ... | @@ -2835,7 +2845,6 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2835,7 +2845,6 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { |
| 2835 | registerAlias(width_reg, abi_size), | 2845 | registerAlias(width_reg, abi_size), |
| 2836 | .z, | 2846 | .z, |
| 2837 | ); | 2847 | ); |
| 2838 | | | |
| 2839 | break :result dst_mcv; | 2848 | break :result dst_mcv; |
| 2840 | }; | 2849 | }; |
| 2841 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); | 2850 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| ... | @@ -2877,17 +2886,18 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2877,17 +2886,18 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { |
| 2877 | const imm_0000_1111 = Immediate.u(mask / 0b0001_0001); | 2886 | const imm_0000_1111 = Immediate.u(mask / 0b0001_0001); |
| 2878 | const imm_0000_0001 = Immediate.u(mask / 0b1111_1111); | 2887 | const imm_0000_0001 = Immediate.u(mask / 0b1111_1111); |
| 2879 | | 2888 | |
| 2880 | const tmp_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) | 2889 | const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) |
| 2881 | src_mcv.register | 2890 | src_mcv |
| 2882 | else | 2891 | else |
| 2883 | try self.copyToTmpRegister(src_ty, src_mcv); | 2892 | try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv); |
| 2884 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | 2893 | const dst_reg = dst_mcv.register; |
| 2885 | defer self.register_manager.unlockReg(tmp_lock); | | |
| 2886 | | | |
| 2887 | const dst_reg = try self.register_manager.allocReg(inst, gp); | | |
| 2888 | const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); | 2894 | const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); |
| 2889 | defer self.register_manager.unlockReg(dst_lock); | 2895 | defer self.register_manager.unlockReg(dst_lock); |
| 2890 | | 2896 | |
| | 2897 | const tmp_reg = try self.register_manager.allocReg(null, gp); |
| | 2898 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| | 2899 | defer self.register_manager.unlockReg(tmp_lock); |
| | 2900 | |
| 2891 | { | 2901 | { |
| 2892 | const dst = registerAlias(dst_reg, src_abi_size); | 2902 | const dst = registerAlias(dst_reg, src_abi_size); |
| 2893 | const tmp = registerAlias(tmp_reg, src_abi_size); | 2903 | const tmp = registerAlias(tmp_reg, src_abi_size); |
| ... | @@ -2896,9 +2906,9 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2896,9 +2906,9 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { |
| 2896 | else | 2906 | else |
| 2897 | undefined; | 2907 | undefined; |
| 2898 | | 2908 | |
| 2899 | // tmp = operand | | |
| 2900 | try self.asmRegisterRegister(.mov, dst, tmp); | | |
| 2901 | // dst = operand | 2909 | // dst = operand |
| | 2910 | try self.asmRegisterRegister(.mov, tmp, dst); |
| | 2911 | // tmp = operand |
| 2902 | try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1)); | 2912 | try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1)); |
| 2903 | // tmp = operand >> 1 | 2913 | // tmp = operand >> 1 |
| 2904 | if (src_abi_size > 4) { | 2914 | if (src_abi_size > 4) { |
| ... | @@ -2948,7 +2958,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2948,7 +2958,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { |
| 2948 | } | 2958 | } |
| 2949 | // dst = (temp3 * 0x01...01) >> (bits - 8) | 2959 | // dst = (temp3 * 0x01...01) >> (bits - 8) |
| 2950 | } | 2960 | } |
| 2951 | break :result .{ .register = dst_reg }; | 2961 | break :result dst_mcv; |
| 2952 | }; | 2962 | }; |
| 2953 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); | 2963 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 2954 | } | 2964 | } |
| ... | @@ -3170,8 +3180,8 @@ fn reuseOperand( | ... | @@ -3170,8 +3180,8 @@ fn reuseOperand( |
| 3170 | .register => |reg| { | 3180 | .register => |reg| { |
| 3171 | // If it's in the registers table, need to associate the register with the | 3181 | // If it's in the registers table, need to associate the register with the |
| 3172 | // new instruction. | 3182 | // new instruction. |
| 3173 | if (RegisterManager.indexOfRegIntoTracked(reg)) |index| { | 3183 | if (!self.register_manager.isRegFree(reg)) { |
| 3174 | if (!self.register_manager.isRegFree(reg)) { | 3184 | if (RegisterManager.indexOfRegIntoTracked(reg)) |index| { |
| 3175 | self.register_manager.registers[index] = inst; | 3185 | self.register_manager.registers[index] = inst; |
| 3176 | } | 3186 | } |
| 3177 | } | 3187 | } |
| ... | @@ -3510,7 +3520,7 @@ fn airStore(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -3510,7 +3520,7 @@ fn airStore(self: *Self, inst: Air.Inst.Index) !void { |
| 3510 | const value_ty = self.air.typeOf(bin_op.rhs); | 3520 | const value_ty = self.air.typeOf(bin_op.rhs); |
| 3511 | log.debug("airStore(%{d}): {} <- {}", .{ inst, ptr, value }); | 3521 | log.debug("airStore(%{d}): {} <- {}", .{ inst, ptr, value }); |
| 3512 | try self.store(ptr, value, ptr_ty, value_ty); | 3522 | try self.store(ptr, value, ptr_ty, value_ty); |
| 3513 | return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); | 3523 | return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 3514 | } | 3524 | } |
| 3515 | | 3525 | |
| 3516 | fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void { | 3526 | fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void { |
| ... | @@ -3796,8 +3806,6 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue | ... | @@ -3796,8 +3806,6 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue |
| 3796 | | 3806 | |
| 3797 | /// Clobbers .rcx for non-immediate shift value. | 3807 | /// Clobbers .rcx for non-immediate shift value. |
| 3798 | fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shift: MCValue) !void { | 3808 | fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shift: MCValue) !void { |
| 3799 | assert(reg.to64() != .rcx); | | |
| 3800 | | | |
| 3801 | switch (tag) { | 3809 | switch (tag) { |
| 3802 | .sal, .sar, .shl, .shr => {}, | 3810 | .sal, .sar, .shl, .shr => {}, |
| 3803 | else => unreachable, | 3811 | else => unreachable, |
| ... | @@ -4612,23 +4620,24 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -4612,23 +4620,24 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void { |
| 4612 | const src_index = self.air.instructions.items(.data)[inst].arg.src_index; | 4620 | const src_index = self.air.instructions.items(.data)[inst].arg.src_index; |
| 4613 | const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index); | 4621 | const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index); |
| 4614 | | 4622 | |
| 4615 | if (self.liveness.isUnused(inst)) | 4623 | const result: MCValue = result: { |
| 4616 | return self.finishAirBookkeeping(); | 4624 | if (self.liveness.isUnused(inst)) break :result .dead; |
| 4617 | | 4625 | |
| 4618 | const dst_mcv: MCValue = switch (mcv) { | 4626 | const dst_mcv: MCValue = switch (mcv) { |
| 4619 | .register => |reg| blk: { | 4627 | .register => |reg| blk: { |
| 4620 | self.register_manager.getRegAssumeFree(reg.to64(), inst); | 4628 | self.register_manager.getRegAssumeFree(reg.to64(), inst); |
| 4621 | break :blk MCValue{ .register = reg }; | 4629 | break :blk MCValue{ .register = reg }; |
| 4622 | }, | 4630 | }, |
| 4623 | .stack_offset => |off| blk: { | 4631 | .stack_offset => |off| blk: { |
| 4624 | const offset = @intCast(i32, self.max_end_stack) - off + 16; | 4632 | const offset = @intCast(i32, self.max_end_stack) - off + 16; |
| 4625 | break :blk MCValue{ .stack_offset = -offset }; | 4633 | break :blk MCValue{ .stack_offset = -offset }; |
| 4626 | }, | 4634 | }, |
| 4627 | else => return self.fail("TODO implement arg for {}", .{mcv}), | 4635 | else => return self.fail("TODO implement arg for {}", .{mcv}), |
| | 4636 | }; |
| | 4637 | try self.genArgDbgInfo(ty, name, dst_mcv); |
| | 4638 | break :result dst_mcv; |
| 4628 | }; | 4639 | }; |
| 4629 | try self.genArgDbgInfo(ty, name, dst_mcv); | 4640 | return self.finishAir(inst, result, .{ .none, .none, .none }); |
| 4630 | | | |
| 4631 | return self.finishAir(inst, dst_mcv, .{ .none, .none, .none }); | | |
| 4632 | } | 4641 | } |
| 4633 | | 4642 | |
| 4634 | fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void { | 4643 | fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void { |
| ... | @@ -4924,6 +4933,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier | ... | @@ -4924,6 +4933,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier |
| 4924 | } | 4933 | } |
| 4925 | | 4934 | |
| 4926 | const result: MCValue = result: { | 4935 | const result: MCValue = result: { |
| | 4936 | if (self.liveness.isUnused(inst)) break :result .dead; |
| | 4937 | |
| 4927 | switch (info.return_value) { | 4938 | switch (info.return_value) { |
| 4928 | .register => { | 4939 | .register => { |
| 4929 | // Save function return value in a new register | 4940 | // Save function return value in a new register |
| ... | @@ -5137,7 +5148,10 @@ fn genTry( | ... | @@ -5137,7 +5148,10 @@ fn genTry( |
| 5137 | const reloc = try self.genCondBrMir(Type.anyerror, is_err_mcv); | 5148 | const reloc = try self.genCondBrMir(Type.anyerror, is_err_mcv); |
| 5138 | try self.genBody(body); | 5149 | try self.genBody(body); |
| 5139 | try self.performReloc(reloc); | 5150 | try self.performReloc(reloc); |
| 5140 | const result = try self.genUnwrapErrorUnionPayloadMir(inst, err_union_ty, err_union); | 5151 | const result = if (self.liveness.isUnused(inst)) |
| | 5152 | .dead |
| | 5153 | else |
| | 5154 | try self.genUnwrapErrorUnionPayloadMir(inst, err_union_ty, err_union); |
| 5141 | return result; | 5155 | return result; |
| 5142 | } | 5156 | } |
| 5143 | | 5157 | |
| ... | @@ -5226,15 +5240,11 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5226,15 +5240,11 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { |
| 5226 | // that death now instead of later as this has an effect on | 5240 | // that death now instead of later as this has an effect on |
| 5227 | // whether it needs to be spilled in the branches | 5241 | // whether it needs to be spilled in the branches |
| 5228 | if (self.liveness.operandDies(inst, 0)) { | 5242 | if (self.liveness.operandDies(inst, 0)) { |
| 5229 | const op_int = @enumToInt(pl_op.operand); | 5243 | if (Air.refToIndex(pl_op.operand)) |op_inst| self.processDeath(op_inst); |
| 5230 | if (op_int >= Air.Inst.Ref.typed_value_map.len) { | | |
| 5231 | const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len); | | |
| 5232 | self.processDeath(op_index); | | |
| 5233 | } | | |
| 5234 | } | 5244 | } |
| 5235 | | 5245 | |
| 5236 | // Capture the state of register and stack allocation state so that we can revert to it. | 5246 | // Capture the state of register and stack allocation state so that we can revert to it. |
| 5237 | const saved_state = try self.captureState(); | 5247 | const saved_state = self.captureState(); |
| 5238 | | 5248 | |
| 5239 | { | 5249 | { |
| 5240 | try self.branch_stack.append(.{}); | 5250 | try self.branch_stack.append(.{}); |
| ... | @@ -5283,12 +5293,10 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5283,12 +5293,10 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { |
| 5283 | for (self.branch_stack.items) |bs| { | 5293 | for (self.branch_stack.items) |bs| { |
| 5284 | log.debug("{}", .{bs.fmtDebug()}); | 5294 | log.debug("{}", .{bs.fmtDebug()}); |
| 5285 | } | 5295 | } |
| 5286 | | | |
| 5287 | log.debug("Then branch: {}", .{then_branch.fmtDebug()}); | 5296 | log.debug("Then branch: {}", .{then_branch.fmtDebug()}); |
| 5288 | log.debug("Else branch: {}", .{else_branch.fmtDebug()}); | 5297 | log.debug("Else branch: {}", .{else_branch.fmtDebug()}); |
| 5289 | | 5298 | |
| 5290 | const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | 5299 | try self.canonicaliseBranches(true, &then_branch, &else_branch, true, true); |
| 5291 | try self.canonicaliseBranches(parent_branch, &then_branch, &else_branch); | | |
| 5292 | | 5300 | |
| 5293 | // We already took care of pl_op.operand earlier, so we're going | 5301 | // We already took care of pl_op.operand earlier, so we're going |
| 5294 | // to pass .none here | 5302 | // to pass .none here |
| ... | @@ -5423,10 +5431,6 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) ! | ... | @@ -5423,10 +5431,6 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) ! |
| 5423 | try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 }); | 5431 | try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 }); |
| 5424 | }, | 5432 | }, |
| 5425 | .register => |reg| { | 5433 | .register => |reg| { |
| 5426 | self.register_manager.getRegAssumeFree(.rcx, null); | | |
| 5427 | const rcx_lock = if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null; | | |
| 5428 | defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 5429 | | | |
| 5430 | const eu_lock = self.register_manager.lockReg(reg); | 5434 | const eu_lock = self.register_manager.lockReg(reg); |
| 5431 | defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); | 5435 | defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); |
| 5432 | | 5436 | |
| ... | @@ -5598,27 +5602,46 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5598,27 +5602,46 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void { |
| 5598 | } | 5602 | } |
| 5599 | | 5603 | |
| 5600 | fn airBlock(self: *Self, inst: Air.Inst.Index) !void { | 5604 | fn airBlock(self: *Self, inst: Air.Inst.Index) !void { |
| 5601 | try self.blocks.putNoClobber(self.gpa, inst, .{ | 5605 | // A block is a setup to be able to jump to the end. |
| 5602 | // A block is a setup to be able to jump to the end. | 5606 | const branch_depth = @intCast(u32, self.branch_stack.items.len); |
| 5603 | .relocs = .{}, | 5607 | try self.blocks.putNoClobber(self.gpa, inst, .{ .branch_depth = branch_depth }); |
| 5604 | // It also acts as a receptacle for break operands. | 5608 | defer { |
| 5605 | // Here we use `MCValue.none` to represent a null value so that the first | 5609 | var block_data = self.blocks.fetchRemove(inst).?.value; |
| 5606 | // break instruction will choose a MCValue for the block result and overwrite | 5610 | block_data.deinit(self.gpa); |
| 5607 | // this field. Following break instructions will use that MCValue to put their | 5611 | } |
| 5608 | // block results. | | |
| 5609 | .mcv = .none, | | |
| 5610 | }); | | |
| 5611 | defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa); | | |
| 5612 | | 5612 | |
| 5613 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; | 5613 | const ty = self.air.typeOfIndex(inst); |
| 5614 | const extra = self.air.extraData(Air.Block, ty_pl.payload); | 5614 | const unused = !ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(inst); |
| 5615 | const body = self.air.extra[extra.end..][0..extra.data.body_len]; | | |
| 5616 | try self.genBody(body); | | |
| 5617 | | 5615 | |
| 5618 | for (self.blocks.getPtr(inst).?.relocs.items) |reloc| try self.performReloc(reloc); | 5616 | { |
| | 5617 | // Here we use `.none` to represent a null value so that the first break |
| | 5618 | // instruction will choose a MCValue for the block result and overwrite |
| | 5619 | // this field. Following break instructions will use that MCValue to put |
| | 5620 | // their block results. |
| | 5621 | const result: MCValue = if (unused) .dead else .none; |
| | 5622 | const branch = &self.branch_stack.items[branch_depth - 1]; |
| | 5623 | try branch.inst_table.putNoClobber(self.gpa, inst, result); |
| | 5624 | } |
| 5619 | | 5625 | |
| 5620 | const result = self.blocks.getPtr(inst).?.mcv; | 5626 | { |
| 5621 | return self.finishAir(inst, result, .{ .none, .none, .none }); | 5627 | try self.branch_stack.append(.{}); |
| | 5628 | errdefer _ = self.branch_stack.pop(); |
| | 5629 | |
| | 5630 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| | 5631 | const extra = self.air.extraData(Air.Block, ty_pl.payload); |
| | 5632 | const body = self.air.extra[extra.end..][0..extra.data.body_len]; |
| | 5633 | try self.genBody(body); |
| | 5634 | } |
| | 5635 | |
| | 5636 | const block_data = self.blocks.getPtr(inst).?; |
| | 5637 | const target_branch = self.branch_stack.pop(); |
| | 5638 | try self.canonicaliseBranches(true, &block_data.branch, &target_branch, false, false); |
| | 5639 | |
| | 5640 | for (block_data.relocs.items) |reloc| try self.performReloc(reloc); |
| | 5641 | |
| | 5642 | const result = if (unused) .dead else self.getResolvedInstValue(inst).?.*; |
| | 5643 | self.getValue(result, inst); |
| | 5644 | self.finishAirBookkeeping(); |
| 5622 | } | 5645 | } |
| 5623 | | 5646 | |
| 5624 | fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { | 5647 | fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| ... | @@ -5639,28 +5662,31 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5639,28 +5662,31 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 5639 | // that death now instead of later as this has an effect on | 5662 | // that death now instead of later as this has an effect on |
| 5640 | // whether it needs to be spilled in the branches | 5663 | // whether it needs to be spilled in the branches |
| 5641 | if (self.liveness.operandDies(inst, 0)) { | 5664 | if (self.liveness.operandDies(inst, 0)) { |
| 5642 | const op_int = @enumToInt(pl_op.operand); | 5665 | if (Air.refToIndex(pl_op.operand)) |op_inst| self.processDeath(op_inst); |
| 5643 | if (op_int >= Air.Inst.Ref.typed_value_map.len) { | | |
| 5644 | const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len); | | |
| 5645 | self.processDeath(op_index); | | |
| 5646 | } | | |
| 5647 | } | 5666 | } |
| 5648 | | 5667 | |
| 5649 | var branch_stack = std.ArrayList(Branch).init(self.gpa); | 5668 | log.debug("airSwitch: %{d}", .{inst}); |
| 5650 | defer { | 5669 | log.debug("Upper branches:", .{}); |
| 5651 | for (branch_stack.items) |*bs| { | 5670 | for (self.branch_stack.items) |bs| { |
| 5652 | bs.deinit(self.gpa); | 5671 | log.debug("{}", .{bs.fmtDebug()}); |
| 5653 | } | | |
| 5654 | branch_stack.deinit(); | | |
| 5655 | } | 5672 | } |
| 5656 | try branch_stack.ensureTotalCapacityPrecise(switch_br.data.cases_len + 1); | | |
| 5657 | | 5673 | |
| | 5674 | var prev_branch: ?Branch = null; |
| | 5675 | defer if (prev_branch) |*branch| branch.deinit(self.gpa); |
| | 5676 | |
| | 5677 | // Capture the state of register and stack allocation state so that we can revert to it. |
| | 5678 | const saved_state = self.captureState(); |
| | 5679 | |
| | 5680 | const cases_len = switch_br.data.cases_len + @boolToInt(switch_br.data.else_body_len > 0); |
| 5658 | while (case_i < switch_br.data.cases_len) : (case_i += 1) { | 5681 | while (case_i < switch_br.data.cases_len) : (case_i += 1) { |
| 5659 | const case = self.air.extraData(Air.SwitchBr.Case, extra_index); | 5682 | const case = self.air.extraData(Air.SwitchBr.Case, extra_index); |
| 5660 | const items = @ptrCast([]const Air.Inst.Ref, self.air.extra[case.end..][0..case.data.items_len]); | 5683 | const items = @ptrCast([]const Air.Inst.Ref, self.air.extra[case.end..][0..case.data.items_len]); |
| 5661 | const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len]; | 5684 | const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len]; |
| 5662 | extra_index = case.end + items.len + case_body.len; | 5685 | extra_index = case.end + items.len + case_body.len; |
| 5663 | | 5686 | |
| | 5687 | // Revert to the previous register and stack allocation state. |
| | 5688 | if (prev_branch) |_| self.revertState(saved_state); |
| | 5689 | |
| 5664 | var relocs = try self.gpa.alloc(u32, items.len); | 5690 | var relocs = try self.gpa.alloc(u32, items.len); |
| 5665 | defer self.gpa.free(relocs); | 5691 | defer self.gpa.free(relocs); |
| 5666 | | 5692 | |
| ... | @@ -5671,12 +5697,9 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5671,12 +5697,9 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 5671 | reloc.* = try self.asmJccReloc(undefined, .ne); | 5697 | reloc.* = try self.asmJccReloc(undefined, .ne); |
| 5672 | } | 5698 | } |
| 5673 | | 5699 | |
| 5674 | // Capture the state of register and stack allocation state so that we can revert to it. | | |
| 5675 | const saved_state = try self.captureState(); | | |
| 5676 | | | |
| 5677 | { | 5700 | { |
| 5678 | try self.branch_stack.append(.{}); | 5701 | if (cases_len > 1) try self.branch_stack.append(.{}); |
| 5679 | errdefer _ = self.branch_stack.pop(); | 5702 | errdefer _ = if (cases_len > 1) self.branch_stack.pop(); |
| 5680 | | 5703 | |
| 5681 | try self.ensureProcessDeathCapacity(liveness.deaths[case_i].len); | 5704 | try self.ensureProcessDeathCapacity(liveness.deaths[case_i].len); |
| 5682 | for (liveness.deaths[case_i]) |operand| { | 5705 | for (liveness.deaths[case_i]) |operand| { |
| ... | @@ -5686,25 +5709,32 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5686,25 +5709,32 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 5686 | try self.genBody(case_body); | 5709 | try self.genBody(case_body); |
| 5687 | } | 5710 | } |
| 5688 | | 5711 | |
| 5689 | branch_stack.appendAssumeCapacity(self.branch_stack.pop()); | 5712 | // Consolidate returned MCValues between prongs like we do in airCondBr. |
| 5690 | | 5713 | if (cases_len > 1) { |
| 5691 | // Revert to the previous register and stack allocation state. | 5714 | var case_branch = self.branch_stack.pop(); |
| 5692 | self.revertState(saved_state); | 5715 | errdefer case_branch.deinit(self.gpa); |
| 5693 | | 5716 | |
| 5694 | for (relocs) |reloc| { | 5717 | log.debug("Case-{d} branch: {}", .{ case_i, case_branch.fmtDebug() }); |
| 5695 | try self.performReloc(reloc); | 5718 | const final = case_i == cases_len - 1; |
| | 5719 | if (prev_branch) |*canon_branch| { |
| | 5720 | try self.canonicaliseBranches(final, canon_branch, &case_branch, true, true); |
| | 5721 | canon_branch.deinit(self.gpa); |
| | 5722 | } |
| | 5723 | prev_branch = case_branch; |
| 5696 | } | 5724 | } |
| | 5725 | |
| | 5726 | for (relocs) |reloc| try self.performReloc(reloc); |
| 5697 | } | 5727 | } |
| 5698 | | 5728 | |
| 5699 | if (switch_br.data.else_body_len > 0) { | 5729 | if (switch_br.data.else_body_len > 0) { |
| 5700 | const else_body = self.air.extra[extra_index..][0..switch_br.data.else_body_len]; | 5730 | const else_body = self.air.extra[extra_index..][0..switch_br.data.else_body_len]; |
| 5701 | | 5731 | |
| 5702 | // Capture the state of register and stack allocation state so that we can revert to it. | 5732 | // Revert to the previous register and stack allocation state. |
| 5703 | const saved_state = try self.captureState(); | 5733 | if (prev_branch) |_| self.revertState(saved_state); |
| 5704 | | 5734 | |
| 5705 | { | 5735 | { |
| 5706 | try self.branch_stack.append(.{}); | 5736 | if (cases_len > 1) try self.branch_stack.append(.{}); |
| 5707 | errdefer _ = self.branch_stack.pop(); | 5737 | errdefer _ = if (cases_len > 1) self.branch_stack.pop(); |
| 5708 | | 5738 | |
| 5709 | const else_deaths = liveness.deaths.len - 1; | 5739 | const else_deaths = liveness.deaths.len - 1; |
| 5710 | try self.ensureProcessDeathCapacity(liveness.deaths[else_deaths].len); | 5740 | try self.ensureProcessDeathCapacity(liveness.deaths[else_deaths].len); |
| ... | @@ -5715,78 +5745,103 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5715,78 +5745,103 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 5715 | try self.genBody(else_body); | 5745 | try self.genBody(else_body); |
| 5716 | } | 5746 | } |
| 5717 | | 5747 | |
| 5718 | branch_stack.appendAssumeCapacity(self.branch_stack.pop()); | 5748 | // Consolidate returned MCValues between a prong and the else branch like we do in airCondBr. |
| 5719 | | 5749 | if (cases_len > 1) { |
| 5720 | // Revert to the previous register and stack allocation state. | 5750 | var else_branch = self.branch_stack.pop(); |
| 5721 | self.revertState(saved_state); | 5751 | errdefer else_branch.deinit(self.gpa); |
| 5722 | } | | |
| 5723 | | 5752 | |
| 5724 | // Consolidate returned MCValues between prongs and else branch like we do | 5753 | log.debug("Else branch: {}", .{else_branch.fmtDebug()}); |
| 5725 | // in airCondBr. | 5754 | if (prev_branch) |*canon_branch| { |
| 5726 | log.debug("airSwitch: %{d}", .{inst}); | 5755 | try self.canonicaliseBranches(true, canon_branch, &else_branch, true, true); |
| 5727 | log.debug("Upper branches:", .{}); | 5756 | canon_branch.deinit(self.gpa); |
| 5728 | for (self.branch_stack.items) |bs| { | 5757 | } |
| 5729 | log.debug("{}", .{bs.fmtDebug()}); | 5758 | prev_branch = else_branch; |
| 5730 | } | 5759 | } |
| 5731 | for (branch_stack.items, 0..) |bs, i| { | | |
| 5732 | log.debug("Case-{d} branch: {}", .{ i, bs.fmtDebug() }); | | |
| 5733 | } | | |
| 5734 | | | |
| 5735 | // TODO: can we reduce the complexity of this algorithm? | | |
| 5736 | const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | | |
| 5737 | var i: usize = branch_stack.items.len; | | |
| 5738 | while (i > 1) : (i -= 1) { | | |
| 5739 | const canon_branch = &branch_stack.items[i - 2]; | | |
| 5740 | const target_branch = &branch_stack.items[i - 1]; | | |
| 5741 | try self.canonicaliseBranches(parent_branch, canon_branch, target_branch); | | |
| 5742 | } | 5760 | } |
| 5743 | | 5761 | |
| 5744 | // We already took care of pl_op.operand earlier, so we're going | 5762 | // We already took care of pl_op.operand earlier, so we're going to pass .none here |
| 5745 | // to pass .none here | | |
| 5746 | return self.finishAir(inst, .unreach, .{ .none, .none, .none }); | 5763 | return self.finishAir(inst, .unreach, .{ .none, .none, .none }); |
| 5747 | } | 5764 | } |
| 5748 | | 5765 | |
| 5749 | fn canonicaliseBranches(self: *Self, parent_branch: *Branch, canon_branch: *Branch, target_branch: *Branch) !void { | 5766 | fn canonicaliseBranches( |
| 5750 | try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, target_branch.inst_table.count()); | 5767 | self: *Self, |
| | 5768 | update_parent: bool, |
| | 5769 | canon_branch: *Branch, |
| | 5770 | target_branch: *const Branch, |
| | 5771 | comptime set_values: bool, |
| | 5772 | comptime assert_same_deaths: bool, |
| | 5773 | ) !void { |
| | 5774 | var hazard_map = std.AutoHashMap(MCValue, void).init(self.gpa); |
| | 5775 | defer hazard_map.deinit(); |
| | 5776 | |
| | 5777 | const parent_branch = |
| | 5778 | if (update_parent) &self.branch_stack.items[self.branch_stack.items.len - 1] else undefined; |
| 5751 | | 5779 | |
| 5752 | const target_slice = target_branch.inst_table.entries.slice(); | 5780 | if (update_parent) try self.ensureProcessDeathCapacity(target_branch.inst_table.count()); |
| 5753 | for (target_slice.items(.key), target_slice.items(.value)) |target_key, target_value| { | 5781 | var target_it = target_branch.inst_table.iterator(); |
| | 5782 | while (target_it.next()) |target_entry| { |
| | 5783 | const target_key = target_entry.key_ptr.*; |
| | 5784 | const target_value = target_entry.value_ptr.*; |
| 5754 | const canon_mcv = if (canon_branch.inst_table.fetchSwapRemove(target_key)) |canon_entry| blk: { | 5785 | const canon_mcv = if (canon_branch.inst_table.fetchSwapRemove(target_key)) |canon_entry| blk: { |
| 5755 | // The instruction's MCValue is overridden in both branches. | 5786 | // The instruction's MCValue is overridden in both branches. |
| 5756 | parent_branch.inst_table.putAssumeCapacity(target_key, canon_entry.value); | | |
| 5757 | if (target_value == .dead) { | 5787 | if (target_value == .dead) { |
| 5758 | assert(canon_entry.value == .dead); | 5788 | if (update_parent) { |
| | 5789 | parent_branch.inst_table.putAssumeCapacity(target_key, .dead); |
| | 5790 | } |
| | 5791 | if (assert_same_deaths) assert(canon_entry.value == .dead); |
| 5759 | continue; | 5792 | continue; |
| 5760 | } | 5793 | } |
| | 5794 | if (update_parent) { |
| | 5795 | parent_branch.inst_table.putAssumeCapacity(target_key, canon_entry.value); |
| | 5796 | } |
| 5761 | break :blk canon_entry.value; | 5797 | break :blk canon_entry.value; |
| 5762 | } else blk: { | 5798 | } else blk: { |
| 5763 | if (target_value == .dead) | 5799 | if (target_value == .dead) { |
| | 5800 | if (update_parent) { |
| | 5801 | parent_branch.inst_table.putAssumeCapacity(target_key, .dead); |
| | 5802 | } |
| 5764 | continue; | 5803 | continue; |
| | 5804 | } |
| 5765 | // The instruction is only overridden in the else branch. | 5805 | // The instruction is only overridden in the else branch. |
| 5766 | // If integer overflows occurs, the question is: why wasn't the instruction marked dead? | 5806 | // If integer overflow occurs, the question is: why wasn't the instruction marked dead? |
| 5767 | break :blk self.getResolvedInstValue(target_key).?; | 5807 | break :blk self.getResolvedInstValue(target_key).?.*; |
| 5768 | }; | 5808 | }; |
| 5769 | log.debug("consolidating target_entry {d} {}=>{}", .{ target_key, target_value, canon_mcv }); | 5809 | log.debug("consolidating target_entry {d} {}=>{}", .{ target_key, target_value, canon_mcv }); |
| 5770 | // TODO make sure the destination stack offset / register does not already have something | 5810 | // TODO handle the case where the destination stack offset / register has something |
| 5771 | // going on there. | 5811 | // going on there. |
| 5772 | try self.setRegOrMem(self.air.typeOfIndex(target_key), canon_mcv, target_value); | 5812 | assert(!hazard_map.contains(target_value)); |
| | 5813 | try hazard_map.putNoClobber(canon_mcv, {}); |
| | 5814 | if (set_values) { |
| | 5815 | try self.setRegOrMem(self.air.typeOfIndex(target_key), canon_mcv, target_value); |
| | 5816 | } else self.getValue(canon_mcv, target_key); |
| | 5817 | self.freeValue(target_value); |
| 5773 | // TODO track the new register / stack allocation | 5818 | // TODO track the new register / stack allocation |
| 5774 | } | 5819 | } |
| 5775 | try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, canon_branch.inst_table.count()); | 5820 | |
| 5776 | const canon_slice = canon_branch.inst_table.entries.slice(); | 5821 | if (update_parent) try self.ensureProcessDeathCapacity(canon_branch.inst_table.count()); |
| 5777 | for (canon_slice.items(.key), canon_slice.items(.value)) |canon_key, canon_value| { | 5822 | var canon_it = canon_branch.inst_table.iterator(); |
| | 5823 | while (canon_it.next()) |canon_entry| { |
| | 5824 | const canon_key = canon_entry.key_ptr.*; |
| | 5825 | const canon_value = canon_entry.value_ptr.*; |
| 5778 | // We already deleted the items from this table that matched the target_branch. | 5826 | // We already deleted the items from this table that matched the target_branch. |
| 5779 | // So these are all instructions that are only overridden in the canon branch. | 5827 | // So these are all instructions that are only overridden in the canon branch. |
| 5780 | parent_branch.inst_table.putAssumeCapacity(canon_key, canon_value); | 5828 | const parent_mcv = |
| 5781 | log.debug("canon_value = {}", .{canon_value}); | 5829 | if (canon_value != .dead) self.getResolvedInstValue(canon_key).?.* else undefined; |
| 5782 | if (canon_value == .dead) | 5830 | if (canon_value != .dead) { |
| 5783 | continue; | 5831 | log.debug("consolidating canon_entry {d} {}=>{}", .{ canon_key, parent_mcv, canon_value }); |
| 5784 | const parent_mcv = self.getResolvedInstValue(canon_key).?; | 5832 | // TODO handle the case where the destination stack offset / register has something |
| 5785 | log.debug("consolidating canon_entry {d} {}=>{}", .{ canon_key, parent_mcv, canon_value }); | 5833 | // going on there. |
| 5786 | // TODO make sure the destination stack offset / register does not already have something | 5834 | assert(!hazard_map.contains(parent_mcv)); |
| 5787 | // going on there. | 5835 | try hazard_map.putNoClobber(canon_value, {}); |
| 5788 | try self.setRegOrMem(self.air.typeOfIndex(canon_key), parent_mcv, canon_value); | 5836 | if (set_values) { |
| 5789 | // TODO track the new register / stack allocation | 5837 | try self.setRegOrMem(self.air.typeOfIndex(canon_key), canon_value, parent_mcv); |
| | 5838 | } else self.getValue(canon_value, canon_key); |
| | 5839 | self.freeValue(parent_mcv); |
| | 5840 | // TODO track the new register / stack allocation |
| | 5841 | } |
| | 5842 | if (update_parent) { |
| | 5843 | parent_branch.inst_table.putAssumeCapacity(canon_key, canon_value); |
| | 5844 | } |
| 5790 | } | 5845 | } |
| 5791 | } | 5846 | } |
| 5792 | | 5847 | |
| ... | @@ -5804,42 +5859,79 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void { | ... | @@ -5804,42 +5859,79 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void { |
| 5804 | } | 5859 | } |
| 5805 | | 5860 | |
| 5806 | fn airBr(self: *Self, inst: Air.Inst.Index) !void { | 5861 | fn airBr(self: *Self, inst: Air.Inst.Index) !void { |
| 5807 | const branch = self.air.instructions.items(.data)[inst].br; | 5862 | const br = self.air.instructions.items(.data)[inst].br; |
| 5808 | try self.br(branch.block_inst, branch.operand); | 5863 | const block = br.block_inst; |
| 5809 | return self.finishAir(inst, .dead, .{ branch.operand, .none, .none }); | 5864 | |
| 5810 | } | 5865 | // The first break instruction encounters `.none` here and chooses a |
| | 5866 | // machine code value for the block result, populating this field. |
| | 5867 | // Following break instructions encounter that value and use it for |
| | 5868 | // the location to store their block results. |
| | 5869 | if (self.getResolvedInstValue(block)) |dst_mcv| { |
| | 5870 | const src_mcv = try self.resolveInst(br.operand); |
| | 5871 | switch (dst_mcv.*) { |
| | 5872 | .none => { |
| | 5873 | const result = result: { |
| | 5874 | if (self.reuseOperand(inst, br.operand, 0, src_mcv)) break :result src_mcv; |
| 5811 | | 5875 | |
| 5812 | fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void { | | |
| 5813 | const block_data = self.blocks.getPtr(block).?; | | |
| 5814 | | | |
| 5815 | if (self.air.typeOf(operand).hasRuntimeBits()) { | | |
| 5816 | const operand_mcv = try self.resolveInst(operand); | | |
| 5817 | const block_mcv = block_data.mcv; | | |
| 5818 | if (block_mcv == .none) { | | |
| 5819 | block_data.mcv = switch (operand_mcv) { | | |
| 5820 | .none, .dead, .unreach => unreachable, | | |
| 5821 | .register, .stack_offset, .memory => operand_mcv, | | |
| 5822 | .eflags, .immediate, .ptr_stack_offset => blk: { | | |
| 5823 | const new_mcv = try self.allocRegOrMem(block, true); | 5876 | const new_mcv = try self.allocRegOrMem(block, true); |
| 5824 | try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv); | 5877 | try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, src_mcv); |
| 5825 | break :blk new_mcv; | 5878 | break :result new_mcv; |
| 5826 | }, | 5879 | }; |
| 5827 | else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}), | 5880 | dst_mcv.* = result; |
| 5828 | }; | 5881 | self.freeValue(result); |
| 5829 | } else { | 5882 | }, |
| 5830 | try self.setRegOrMem(self.air.typeOfIndex(block), block_mcv, operand_mcv); | 5883 | else => try self.setRegOrMem(self.air.typeOfIndex(block), dst_mcv.*, src_mcv), |
| 5831 | } | 5884 | } |
| 5832 | } | 5885 | } |
| 5833 | return self.brVoid(block); | | |
| 5834 | } | | |
| 5835 | | 5886 | |
| 5836 | fn brVoid(self: *Self, block: Air.Inst.Index) !void { | 5887 | // Process operand death early so that it is properly accounted for in the Branch below. |
| | 5888 | if (self.liveness.operandDies(inst, 0)) { |
| | 5889 | if (Air.refToIndex(br.operand)) |op_inst| self.processDeath(op_inst); |
| | 5890 | } |
| | 5891 | |
| 5837 | const block_data = self.blocks.getPtr(block).?; | 5892 | const block_data = self.blocks.getPtr(block).?; |
| | 5893 | { |
| | 5894 | var branch = Branch{}; |
| | 5895 | errdefer branch.deinit(self.gpa); |
| | 5896 | |
| | 5897 | var branch_i = self.branch_stack.items.len - 1; |
| | 5898 | while (branch_i >= block_data.branch_depth) : (branch_i -= 1) { |
| | 5899 | const table = &self.branch_stack.items[branch_i].inst_table; |
| | 5900 | try branch.inst_table.ensureUnusedCapacity(self.gpa, table.count()); |
| | 5901 | var it = table.iterator(); |
| | 5902 | while (it.next()) |entry| { |
| | 5903 | // This loop could be avoided by tracking inst depth, which |
| | 5904 | // will be needed later anyway for reusing loop deaths. |
| | 5905 | var parent_branch_i = block_data.branch_depth - 1; |
| | 5906 | while (parent_branch_i > 0) : (parent_branch_i -= 1) { |
| | 5907 | const parent_table = &self.branch_stack.items[parent_branch_i].inst_table; |
| | 5908 | if (parent_table.contains(entry.key_ptr.*)) break; |
| | 5909 | } else continue; |
| | 5910 | const gop = branch.inst_table.getOrPutAssumeCapacity(entry.key_ptr.*); |
| | 5911 | if (!gop.found_existing) gop.value_ptr.* = entry.value_ptr.*; |
| | 5912 | } |
| | 5913 | } |
| | 5914 | |
| | 5915 | log.debug("airBr: %{d}", .{inst}); |
| | 5916 | log.debug("Upper branches:", .{}); |
| | 5917 | for (self.branch_stack.items) |bs| { |
| | 5918 | log.debug("{}", .{bs.fmtDebug()}); |
| | 5919 | } |
| | 5920 | log.debug("Prev branch: {}", .{block_data.branch.fmtDebug()}); |
| | 5921 | log.debug("Cur branch: {}", .{branch.fmtDebug()}); |
| | 5922 | |
| | 5923 | try self.canonicaliseBranches(false, &block_data.branch, &branch, true, false); |
| | 5924 | block_data.branch.deinit(self.gpa); |
| | 5925 | block_data.branch = branch; |
| | 5926 | } |
| | 5927 | |
| 5838 | // Emit a jump with a relocation. It will be patched up after the block ends. | 5928 | // Emit a jump with a relocation. It will be patched up after the block ends. |
| 5839 | try block_data.relocs.ensureUnusedCapacity(self.gpa, 1); | 5929 | try block_data.relocs.ensureUnusedCapacity(self.gpa, 1); |
| 5840 | // Leave the jump offset undefined | 5930 | // Leave the jump offset undefined |
| 5841 | const jmp_reloc = try self.asmJmpReloc(undefined); | 5931 | const jmp_reloc = try self.asmJmpReloc(undefined); |
| 5842 | block_data.relocs.appendAssumeCapacity(jmp_reloc); | 5932 | block_data.relocs.appendAssumeCapacity(jmp_reloc); |
| | 5933 | |
| | 5934 | self.finishAirBookkeeping(); |
| 5843 | } | 5935 | } |
| 5844 | | 5936 | |
| 5845 | fn airAsm(self: *Self, inst: Air.Inst.Index) !void { | 5937 | fn airAsm(self: *Self, inst: Air.Inst.Index) !void { |
| ... | @@ -6916,7 +7008,8 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -6916,7 +7008,8 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void { |
| 6916 | const pl_op = self.air.instructions.items(.data)[inst].pl_op; | 7008 | const pl_op = self.air.instructions.items(.data)[inst].pl_op; |
| 6917 | const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; | 7009 | const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; |
| 6918 | | 7010 | |
| 6919 | const dst_reg = try self.register_manager.allocReg(inst, gp); | 7011 | const unused = self.liveness.isUnused(inst); |
| | 7012 | const dst_reg = try self.register_manager.allocReg(if (unused) null else inst, gp); |
| 6920 | | 7013 | |
| 6921 | const ptr_ty = self.air.typeOf(pl_op.operand); | 7014 | const ptr_ty = self.air.typeOf(pl_op.operand); |
| 6922 | const ptr_mcv = try self.resolveInst(pl_op.operand); | 7015 | const ptr_mcv = try self.resolveInst(pl_op.operand); |
| ... | @@ -6924,7 +7017,6 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -6924,7 +7017,6 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void { |
| 6924 | const val_ty = self.air.typeOf(extra.operand); | 7017 | const val_ty = self.air.typeOf(extra.operand); |
| 6925 | const val_mcv = try self.resolveInst(extra.operand); | 7018 | const val_mcv = try self.resolveInst(extra.operand); |
| 6926 | | 7019 | |
| 6927 | const unused = self.liveness.isUnused(inst); | | |
| 6928 | try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering()); | 7020 | try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering()); |
| 6929 | const result: MCValue = if (unused) .dead else .{ .register = dst_reg }; | 7021 | const result: MCValue = if (unused) .dead else .{ .register = dst_reg }; |
| 6930 | return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none }); | 7022 | return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none }); |
| ... | @@ -7205,17 +7297,17 @@ fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue { | ... | @@ -7205,17 +7297,17 @@ fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue { |
| 7205 | return gop.value_ptr.*; | 7297 | return gop.value_ptr.*; |
| 7206 | }, | 7298 | }, |
| 7207 | .const_ty => unreachable, | 7299 | .const_ty => unreachable, |
| 7208 | else => return self.getResolvedInstValue(inst_index).?, | 7300 | else => return self.getResolvedInstValue(inst_index).?.*, |
| 7209 | } | 7301 | } |
| 7210 | } | 7302 | } |
| 7211 | | 7303 | |
| 7212 | fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) ?MCValue { | 7304 | fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) ?*MCValue { |
| 7213 | // Treat each stack item as a "layer" on top of the previous one. | 7305 | // Treat each stack item as a "layer" on top of the previous one. |
| 7214 | var i: usize = self.branch_stack.items.len; | 7306 | var i: usize = self.branch_stack.items.len; |
| 7215 | while (true) { | 7307 | while (true) { |
| 7216 | i -= 1; | 7308 | i -= 1; |
| 7217 | if (self.branch_stack.items[i].inst_table.get(inst)) |mcv| { | 7309 | if (self.branch_stack.items[i].inst_table.getPtr(inst)) |mcv| { |
| 7218 | return if (mcv != .dead) mcv else null; | 7310 | return if (mcv.* != .dead) mcv else null; |
| 7219 | } | 7311 | } |
| 7220 | } | 7312 | } |
| 7221 | } | 7313 | } |
Jakub Konka <kubkon@jakubkonka.com> 2023-03-25 08:34:57+01:00
GitHub <noreply@github.com> 2023-03-25 08:34:57+01:00
Signed by PGP key 4AEE18F83AFDEB23
src