| ... | @@ -1969,12 +1969,46 @@ fn gen(self: *Self) InnerError!void { | ... | @@ -1969,12 +1969,46 @@ fn gen(self: *Self) InnerError!void { |
| 1969 | }); | 1969 | }); |
| 1970 | } | 1970 | } |
| 1971 | | 1971 | |
| | 1972 | fn checkInvariantsAfterAirInst(self: *Self, inst: Air.Inst.Index, old_air_bookkeeping: @TypeOf(air_bookkeeping_init)) void { |
| | 1973 | assert(!self.register_manager.lockedRegsExist()); |
| | 1974 | |
| | 1975 | if (std.debug.runtime_safety) { |
| | 1976 | if (self.air_bookkeeping < old_air_bookkeeping + 1) { |
| | 1977 | std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, self.air.instructions.items(.tag)[@intFromEnum(inst)] }); |
| | 1978 | } |
| | 1979 | |
| | 1980 | { // check consistency of tracked registers |
| | 1981 | var it = self.register_manager.free_registers.iterator(.{ .kind = .unset }); |
| | 1982 | while (it.next()) |index| { |
| | 1983 | const tracked_inst = self.register_manager.registers[index]; |
| | 1984 | const tracking = self.getResolvedInstValue(tracked_inst); |
| | 1985 | for (tracking.getRegs()) |reg| { |
| | 1986 | if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break; |
| | 1987 | } else unreachable; // tracked register not in use |
| | 1988 | } |
| | 1989 | } |
| | 1990 | } |
| | 1991 | } |
| | 1992 | |
| 1972 | fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { | 1993 | fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 1973 | const pt = self.pt; | 1994 | const pt = self.pt; |
| 1974 | const mod = pt.zcu; | 1995 | const mod = pt.zcu; |
| 1975 | const ip = &mod.intern_pool; | 1996 | const ip = &mod.intern_pool; |
| 1976 | const air_tags = self.air.instructions.items(.tag); | 1997 | const air_tags = self.air.instructions.items(.tag); |
| 1977 | | 1998 | |
| | 1999 | for (body) |inst| { |
| | 2000 | wip_mir_log.debug("{}", .{self.fmtAir(inst)}); |
| | 2001 | verbose_tracking_log.debug("{}", .{self.fmtTracking()}); |
| | 2002 | |
| | 2003 | const old_air_bookkeeping = self.air_bookkeeping; |
| | 2004 | try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1); |
| | 2005 | switch (air_tags[@intFromEnum(inst)]) { |
| | 2006 | .arg => try self.airArg(inst), |
| | 2007 | else => break, |
| | 2008 | } |
| | 2009 | self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping); |
| | 2010 | } |
| | 2011 | |
| 1978 | for (body) |inst| { | 2012 | for (body) |inst| { |
| 1979 | if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue; | 2013 | if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue; |
| 1980 | wip_mir_log.debug("{}", .{self.fmtAir(inst)}); | 2014 | wip_mir_log.debug("{}", .{self.fmtAir(inst)}); |
| ... | @@ -2054,7 +2088,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2054,7 +2088,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 2054 | | 2088 | |
| 2055 | .alloc => try self.airAlloc(inst), | 2089 | .alloc => try self.airAlloc(inst), |
| 2056 | .ret_ptr => try self.airRetPtr(inst), | 2090 | .ret_ptr => try self.airRetPtr(inst), |
| 2057 | .arg => try self.airArg(inst), | 2091 | .arg => try self.airDbgArg(inst), |
| 2058 | .assembly => try self.airAsm(inst), | 2092 | .assembly => try self.airAsm(inst), |
| 2059 | .bitcast => try self.airBitCast(inst), | 2093 | .bitcast => try self.airBitCast(inst), |
| 2060 | .block => try self.airBlock(inst), | 2094 | .block => try self.airBlock(inst), |
| ... | @@ -2218,25 +2252,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2218,25 +2252,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 2218 | .work_group_id => unreachable, | 2252 | .work_group_id => unreachable, |
| 2219 | // zig fmt: on | 2253 | // zig fmt: on |
| 2220 | } | 2254 | } |
| 2221 | | 2255 | self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping); |
| 2222 | assert(!self.register_manager.lockedRegsExist()); | | |
| 2223 | | | |
| 2224 | if (std.debug.runtime_safety) { | | |
| 2225 | if (self.air_bookkeeping < old_air_bookkeeping + 1) { | | |
| 2226 | std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@intFromEnum(inst)] }); | | |
| 2227 | } | | |
| 2228 | | | |
| 2229 | { // check consistency of tracked registers | | |
| 2230 | var it = self.register_manager.free_registers.iterator(.{ .kind = .unset }); | | |
| 2231 | while (it.next()) |index| { | | |
| 2232 | const tracked_inst = self.register_manager.registers[index]; | | |
| 2233 | const tracking = self.getResolvedInstValue(tracked_inst); | | |
| 2234 | for (tracking.getRegs()) |reg| { | | |
| 2235 | if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break; | | |
| 2236 | } else unreachable; // tracked register not in use | | |
| 2237 | } | | |
| 2238 | } | | |
| 2239 | } | | |
| 2240 | } | 2256 | } |
| 2241 | verbose_tracking_log.debug("{}", .{self.fmtTracking()}); | 2257 | verbose_tracking_log.debug("{}", .{self.fmtTracking()}); |
| 2242 | } | 2258 | } |
| ... | @@ -2351,7 +2367,7 @@ fn finishAirBookkeeping(self: *Self) void { | ... | @@ -2351,7 +2367,7 @@ fn finishAirBookkeeping(self: *Self) void { |
| 2351 | } | 2367 | } |
| 2352 | | 2368 | |
| 2353 | fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void { | 2369 | fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void { |
| 2354 | if (self.liveness.isUnused(inst)) switch (result) { | 2370 | if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) { |
| 2355 | .none, .dead, .unreach => {}, | 2371 | .none, .dead, .unreach => {}, |
| 2356 | else => unreachable, // Why didn't the result die? | 2372 | else => unreachable, // Why didn't the result die? |
| 2357 | } else { | 2373 | } else { |
| ... | @@ -11820,39 +11836,30 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M | ... | @@ -11820,39 +11836,30 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M |
| 11820 | | 11836 | |
| 11821 | fn airArg(self: *Self, inst: Air.Inst.Index) !void { | 11837 | fn airArg(self: *Self, inst: Air.Inst.Index) !void { |
| 11822 | const pt = self.pt; | 11838 | const pt = self.pt; |
| 11823 | const mod = pt.zcu; | 11839 | const zcu = pt.zcu; |
| 11824 | // skip zero-bit arguments as they don't have a corresponding arg instruction | 11840 | // skip zero-bit arguments as they don't have a corresponding arg instruction |
| 11825 | var arg_index = self.arg_index; | 11841 | var arg_index = self.arg_index; |
| 11826 | while (self.args[arg_index] == .none) arg_index += 1; | 11842 | while (self.args[arg_index] == .none) arg_index += 1; |
| 11827 | self.arg_index = arg_index + 1; | 11843 | self.arg_index = arg_index + 1; |
| 11828 | | 11844 | |
| 11829 | const result: MCValue = if (self.debug_output == .none and self.liveness.isUnused(inst)) .unreach else result: { | 11845 | const result: MCValue = if (self.debug_output == .none and self.liveness.isUnused(inst)) .unreach else result: { |
| 11830 | const name = switch (self.debug_output) { | | |
| 11831 | .none => "", | | |
| 11832 | else => name: { | | |
| 11833 | const name_nts = self.air.instructions.items(.data)[@intFromEnum(inst)].arg.name; | | |
| 11834 | break :name self.air.nullTerminatedString(@intFromEnum(name_nts)); | | |
| 11835 | }, | | |
| 11836 | }; | | |
| 11837 | if (name.len == 0 and self.liveness.isUnused(inst)) break :result .unreach; | | |
| 11838 | | | |
| 11839 | const arg_ty = self.typeOfIndex(inst); | 11846 | const arg_ty = self.typeOfIndex(inst); |
| 11840 | const src_mcv = self.args[arg_index]; | 11847 | const src_mcv = self.args[arg_index]; |
| 11841 | const dst_mcv = switch (src_mcv) { | 11848 | switch (src_mcv) { |
| 11842 | .register, .register_pair, .load_frame => dst: { | 11849 | .register, .register_pair, .load_frame => { |
| 11843 | for (src_mcv.getRegs()) |reg| self.register_manager.getRegAssumeFree(reg, inst); | 11850 | for (src_mcv.getRegs()) |reg| self.register_manager.getRegAssumeFree(reg, inst); |
| 11844 | break :dst src_mcv; | 11851 | break :result src_mcv; |
| 11845 | }, | 11852 | }, |
| 11846 | .indirect => |reg_off| dst: { | 11853 | .indirect => |reg_off| { |
| 11847 | self.register_manager.getRegAssumeFree(reg_off.reg, inst); | 11854 | self.register_manager.getRegAssumeFree(reg_off.reg, inst); |
| 11848 | const dst_mcv = try self.allocRegOrMem(inst, false); | 11855 | const dst_mcv = try self.allocRegOrMem(inst, false); |
| 11849 | try self.genCopy(arg_ty, dst_mcv, src_mcv, .{}); | 11856 | try self.genCopy(arg_ty, dst_mcv, src_mcv, .{}); |
| 11850 | break :dst dst_mcv; | 11857 | break :result dst_mcv; |
| 11851 | }, | 11858 | }, |
| 11852 | .elementwise_regs_then_frame => |regs_frame_addr| dst: { | 11859 | .elementwise_regs_then_frame => |regs_frame_addr| { |
| 11853 | try self.spillEflagsIfOccupied(); | 11860 | try self.spillEflagsIfOccupied(); |
| 11854 | | 11861 | |
| 11855 | const fn_info = mod.typeToFunc(self.fn_type).?; | 11862 | const fn_info = zcu.typeToFunc(self.fn_type).?; |
| 11856 | const cc = abi.resolveCallingConvention(fn_info.cc, self.target.*); | 11863 | const cc = abi.resolveCallingConvention(fn_info.cc, self.target.*); |
| 11857 | const param_int_regs = abi.getCAbiIntParamRegs(cc); | 11864 | const param_int_regs = abi.getCAbiIntParamRegs(cc); |
| 11858 | var prev_reg: Register = undefined; | 11865 | var prev_reg: Register = undefined; |
| ... | @@ -11929,27 +11936,31 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -11929,27 +11936,31 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void { |
| 11929 | try self.asmRegisterImmediate( | 11936 | try self.asmRegisterImmediate( |
| 11930 | .{ ._, .cmp }, | 11937 | .{ ._, .cmp }, |
| 11931 | index_reg.to32(), | 11938 | index_reg.to32(), |
| 11932 | Immediate.u(arg_ty.vectorLen(mod)), | 11939 | Immediate.u(arg_ty.vectorLen(zcu)), |
| 11933 | ); | 11940 | ); |
| 11934 | _ = try self.asmJccReloc(.b, loop); | 11941 | _ = try self.asmJccReloc(.b, loop); |
| 11935 | | 11942 | |
| 11936 | break :dst dst_mcv; | 11943 | break :result dst_mcv; |
| 11937 | }, | 11944 | }, |
| 11938 | else => return self.fail("TODO implement arg for {}", .{src_mcv}), | 11945 | else => return self.fail("TODO implement arg for {}", .{src_mcv}), |
| 11939 | }; | | |
| 11940 | | | |
| 11941 | if (name.len > 0) try self.genVarDebugInfo(.local_arg, .dbg_var_val, name, arg_ty, dst_mcv); | | |
| 11942 | | | |
| 11943 | if (self.liveness.isUnused(inst)) { | | |
| 11944 | assert(self.debug_output != .none and name.len > 0); | | |
| 11945 | try self.freeValue(dst_mcv); | | |
| 11946 | break :result .none; | | |
| 11947 | } | 11946 | } |
| 11948 | break :result dst_mcv; | | |
| 11949 | }; | 11947 | }; |
| 11950 | return self.finishAir(inst, result, .{ .none, .none, .none }); | 11948 | return self.finishAir(inst, result, .{ .none, .none, .none }); |
| 11951 | } | 11949 | } |
| 11952 | | 11950 | |
| | 11951 | fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void { |
| | 11952 | defer self.finishAirBookkeeping(); |
| | 11953 | if (self.debug_output == .none) return; |
| | 11954 | const name_nts = self.air.instructions.items(.data)[@intFromEnum(inst)].arg.name; |
| | 11955 | const name = self.air.nullTerminatedString(@intFromEnum(name_nts)); |
| | 11956 | if (name.len > 0) { |
| | 11957 | const arg_ty = self.typeOfIndex(inst); |
| | 11958 | const arg_mcv = self.getResolvedInstValue(inst).short; |
| | 11959 | try self.genVarDebugInfo(.local_arg, .dbg_var_val, name, arg_ty, arg_mcv); |
| | 11960 | } |
| | 11961 | if (self.liveness.isUnused(inst)) try self.processDeath(inst); |
| | 11962 | } |
| | 11963 | |
| 11953 | fn genVarDebugInfo( | 11964 | fn genVarDebugInfo( |
| 11954 | self: *Self, | 11965 | self: *Self, |
| 11955 | var_tag: link.File.Dwarf.WipNav.VarTag, | 11966 | var_tag: link.File.Dwarf.WipNav.VarTag, |
Jacob Young <jacobly@ziglang.org> 2024-08-16 15:20:05-04:00
src