diff --git a/src/link/Coff.zig b/src/link/Coff.zig index fcb4dcc2a50b7e4e00aa0526056e75ffbd77863b..756c4c8892b29f54be05456d12277aa49bc3a444 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -3602,13 +3602,13 @@ fn objectSectionMapIndex( const parent_alignment = parent_ni.alignment(&coff.mf); if (alignment.compare(.gt, parent_alignment)) { log.debug("realignParent({s}, {d}) {d}->{d}", .{ name.toSlice(coff), parent_ni, parent_alignment, alignment }); - try parent_ni.realign(&coff.mf, gpa, alignment, .{ .set_alignment = true }); + try parent_ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); } const old_alignment = sym.ni.alignment(&coff.mf); if (alignment.compare(.gt, old_alignment)) { log.debug("realignObject({s}) {d}->{d}", .{ name.toSlice(coff), old_alignment, alignment }); - try sym.ni.realign(&coff.mf, gpa, alignment, .{ .set_alignment = true }); + try sym.ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); } try coff.verifyParentSectionAttributes( diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 99f887d3e7871fc729d871d2d606f315415275fe..2c6aefe7bcbc8fc02fb9c46e3c12905e4d640826 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -4659,7 +4659,7 @@ fn mapInputSection(elf: *Elf, opts: struct { const new_alignment: std.mem.Alignment = .fromByteUnits( std.math.ceilPowerOfTwoAssert(usize, @intCast(opts.addralign)), ); - try existing_shndx.get(elf).ni.realign(&elf.mf, gpa, new_alignment, .{ .set_alignment = true }); + try existing_shndx.get(elf).ni.realign(&elf.mf, gpa, new_alignment, .{}); } // ...and update the shdr as needed. switch (elf.shdrPtr(existing_shndx)) { @@ -4796,7 +4796,7 @@ fn uavMapIndex( if (!uav_gop.found_existing) { const shndx: Section.Index = .data_rel_ro; // TODO: it would be better to use `.rodata` if the UAV value doesn't have relocs const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{ - .moved = true, // see assert at end of `flushUav` + .moved = true, // see assert at end of `genUav` .alignment = resolved_align.toStdMem(), }); var name_buf: [32]u8 = undefined; @@ -4822,7 +4822,7 @@ fn uavMapIndex( } else { const node = uav_gop.value_ptr.lsi.index().ptr(elf).node; if (resolved_align.toStdMem().order(node.alignment(&elf.mf)).compare(.gt)) { - try node.realign(&elf.mf, gpa, resolved_align.toStdMem(), .{ .set_alignment = true }); + try node.realign(&elf.mf, gpa, resolved_align.toStdMem(), .{}); } } return umi; @@ -5551,7 +5551,7 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadPars // We have a copy relocation for this global, but the amount of space we // reserved for it could be too small or underaligned! try copied_global.node.resize(&elf.mf, gpa, gop.value_ptr.size); - try copied_global.node.realign(&elf.mf, gpa, gop.value_ptr.alignment, .{ .set_alignment = true }); + try copied_global.node.realign(&elf.mf, gpa, gop.value_ptr.alignment, .{}); const global_ptr = elf.globalByName(name).?; switch (elf.symPtr(global_ptr.symtab_index)) { inline else => |sym_ptr| elf.targetStore(&sym_ptr.size, @intCast(gop.value_ptr.size)), @@ -7005,22 +7005,27 @@ fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) // called to apply the NAV's new relocations. try ni.moved(gpa, &elf.mf); - var nw: MappedFile.Node.Writer = undefined; - ni.writer(&elf.mf, gpa, &nw); - defer nw.deinit(); - codegen.generateSymbol( - &elf.base, - pt, - .fromInterned(nav.resolved.?.value), - &nw.interface, - .{ .atom_index = Node.toAtom(ni) }, - ) catch |err| switch (err) { - error.WriteFailed => return nw.err.?, - else => |e| return e, - }; - switch (elf.symPtr(nmi.symbol(elf).index())) { - inline else => |sym| elf.targetStore(&sym.size, @intCast(nw.interface.end)), + { + var nw: MappedFile.Node.Writer = undefined; + ni.writer(&elf.mf, gpa, &nw); + defer nw.deinit(); + codegen.generateSymbol( + &elf.base, + pt, + .fromInterned(nav.resolved.?.value), + &nw.interface, + .{ .atom_index = Node.toAtom(ni) }, + ) catch |err| switch (err) { + error.WriteFailed => return nw.err.?, + else => |e| return e, + }; + switch (elf.symPtr(nmi.symbol(elf).index())) { + inline else => |sym| elf.targetStore(&sym.size, @intCast(nw.interface.end)), + } } + + // The NAV's node is done---now generate any UAVs or lazy code/data which the NAV needs. + try elf.genPending(pt); } pub fn updateFunc( @@ -7056,29 +7061,34 @@ fn updateFuncInner( // called to apply the NAV's new relocations. try ni.moved(gpa, &elf.mf); - var nw: MappedFile.Node.Writer = undefined; - ni.writer(&elf.mf, gpa, &nw); - defer nw.deinit(); - codegen.emitFunction( - &elf.base, - pt, - func_index, - Node.toAtom(ni), - mir, - &nw.interface, - .none, - ) catch |err| switch (err) { - error.WriteFailed => return nw.err.?, - else => |e| return e, - }; - switch (elf.symPtr(nmi.symbol(elf).index())) { - inline else => |sym| elf.targetStore(&sym.size, @intCast(nw.interface.end)), + { + var nw: MappedFile.Node.Writer = undefined; + ni.writer(&elf.mf, gpa, &nw); + defer nw.deinit(); + codegen.emitFunction( + &elf.base, + pt, + func_index, + Node.toAtom(ni), + mir, + &nw.interface, + .none, + ) catch |err| switch (err) { + error.WriteFailed => return nw.err.?, + else => |e| return e, + }; + switch (elf.symPtr(nmi.symbol(elf).index())) { + inline else => |sym| elf.targetStore(&sym.size, @intCast(nw.interface.end)), + } } + + // The NAV's node is done---now generate any UAVs or lazy code/data which the NAV needs. + try elf.genPending(pt); } pub fn updateErrorData(elf: *Elf, pt: Zcu.PerThread) link.Error!void { const diags = &elf.base.comp.link_diags; - elf.flushLazy(pt, .{ + elf.genLazy(pt, .{ .kind = .const_data, .index = @intCast(elf.lazy.getPtr(.const_data).map.getIndex(.anyerror_type) orelse return), }) catch |err| switch (err) { @@ -7183,40 +7193,13 @@ fn updateDynamicTextrel(elf: *Elf) Error!void { pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) link.Error!bool { const comp = elf.base.comp; const diags = &comp.link_diags; + + assert(elf.pending_uavs.items.len == 0); + for (&elf.lazy.values) |*lazy| { + assert(lazy.pending_index == lazy.map.count()); + } + task: { - while (elf.pending_uavs.pop()) |umi| { - const sub_prog_node = elf.idleProgNode(tid, elf.const_prog_node, .{ .uav = umi }); - defer sub_prog_node.end(); - elf.flushUav(.{ .zcu = comp.zcu.?, .tid = tid }, umi) catch |err| switch (err) { - error.MappedFileIo => return diags.fail("failed to write output file: {t}", .{elf.mf.io_err.?}), - else => |e| return e, - }; - break :task; - } - var lazy_it = elf.lazy.iterator(); - while (lazy_it.next()) |lazy| if (lazy.value.pending_index < lazy.value.map.count()) { - const pt: Zcu.PerThread = .{ .zcu = comp.zcu.?, .tid = tid }; - const lmr: Node.LazyMapRef = .{ .kind = lazy.key, .index = lazy.value.pending_index }; - lazy.value.pending_index += 1; - const kind = switch (lmr.kind) { - .code => "code", - .const_data => "data", - }; - var name: [std.Progress.Node.max_name_len]u8 = undefined; - const sub_prog_node = elf.synth_prog_node.start( - std.fmt.bufPrint(&name, "lazy {s} for {f}", .{ - kind, - Type.fromInterned(lmr.lazySymbol(elf).ty).fmt(pt), - }) catch &name, - 0, - ); - defer sub_prog_node.end(); - elf.flushLazy(pt, lmr) catch |err| switch (err) { - error.MappedFileIo => return diags.fail("failed to write output file: {t}", .{elf.mf.io_err.?}), - else => |e| return e, - }; - break :task; - }; if (elf.input_section_pending_index < elf.input_sections.items.len) { const isi: InputSection.Index = @enumFromInt(elf.input_section_pending_index); elf.input_section_pending_index += 1; @@ -7315,8 +7298,6 @@ pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) link.Error!bool { } else elf.mf.update_prog_node.completeOne(); } } - if (elf.pending_uavs.items.len > 0) return true; - for (&elf.lazy.values) |lazy| if (lazy.map.count() > lazy.pending_index) return true; if (elf.input_sections.items.len > elf.input_section_pending_index) return true; if (elf.changed_symtab_index.count() > 0) return true; if (elf.mf.updates.items.len > 0) return true; @@ -7351,7 +7332,43 @@ fn idleProgNode( }, 0); } -fn flushUav( +fn genPending(elf: *Elf, pt: Zcu.PerThread) Error!void { + const zcu = elf.base.comp.zcu.?; + pending: while (true) { + if (elf.pending_uavs.pop()) |umi| { + var prog_name_buf: [std.Progress.Node.max_name_len]u8 = undefined; + const prog_name = std.mem.print(&prog_name_buf, "{f}", .{ + Value.fromInterned(umi.uavValue(elf)).fmtValue(pt), + }) catch &prog_name_buf; + const prog_node = elf.const_prog_node.start(prog_name, 0); + defer prog_node.end(); + try elf.genUav(pt, umi); + continue :pending; + } + var lazy_it = elf.lazy.iterator(); + while (lazy_it.next()) |lazy| if (lazy.value.pending_index < lazy.value.map.count()) { + const lmr: Node.LazyMapRef = .{ .kind = lazy.key, .index = lazy.value.pending_index }; + lazy.value.pending_index += 1; + const lazy_ty: Type = .fromInterned(lmr.lazySymbol(elf).ty); + var prog_name_buf: [std.Progress.Node.max_name_len]u8 = undefined; + const prog_name: []const u8 = switch (lazy_ty.zigTypeTag(zcu)) { + .@"enum" => std.mem.print(&prog_name_buf, "@tagName({f})", .{lazy_ty.fmt(pt)}) catch &prog_name_buf, + .error_set => switch (lmr.kind) { + .code => std.mem.print(&prog_name_buf, "@errorCast({f})", .{lazy_ty.fmt(pt)}) catch &prog_name_buf, + .const_data => "@errorName", + }, + else => unreachable, + }; + const prog_node = elf.synth_prog_node.start(prog_name, 0); + defer prog_node.end(); + try elf.genLazy(pt, lmr); + continue :pending; + }; + break; + } +} + +fn genUav( elf: *Elf, pt: Zcu.PerThread, umi: Node.UavMapIndex, @@ -7380,11 +7397,11 @@ fn flushUav( inline else => |sym| elf.targetStore(&sym.size, @intCast(nw.interface.end)), } // The UAV should already be considered to have moved, because it is created as moved and - // pending calls to `flushUav` always happen before pending calls to `flushMoved`. + // pending calls to `genUav` always happen before pending calls to `flushMoved`. assert(ni.hasMoved(&elf.mf)); } -fn flushLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void { +fn genLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void { const zcu = pt.zcu; const gpa = zcu.gpa; @@ -7497,6 +7514,10 @@ fn flushFileOffset(elf: *Elf, ni: MappedFile.Node.Index) void { fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void { const trace = tracy.trace(@src()); defer trace.end(); + + elf.mf.nodes_lock.lock(); + defer elf.mf.nodes_lock.unlock(); + switch (elf.getNode(ni)) { .file => unreachable, .ehdr, .shdr => elf.flushFileOffset(ni), @@ -7662,6 +7683,10 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void { const trace = tracy.trace(@src()); defer trace.end(); + + elf.mf.nodes_lock.lock(); + defer elf.mf.nodes_lock.unlock(); + _, const size = ni.location(&elf.mf).resolve(&elf.mf); switch (elf.getNode(ni)) { .file => {}, diff --git a/src/link/MappedFile.zig b/src/link/MappedFile.zig index a6000e0e620e6d3f97214732c1e3b869d837b145..004b3f3f409308abae8d27383e8a10f5300d9e4c 100644 --- a/src/link/MappedFile.zig +++ b/src/link/MappedFile.zig @@ -95,14 +95,11 @@ pub fn init(file: Io.File, gpa: std.mem.Allocator, io: Io) (Allocator.Error || I .fallocate_punch_hole_unsupported = false, }; try mf.nodes.ensureUnusedCapacity(gpa, 1); - const root_ni = mf.addNode(gpa, .{ .add_node = .{ + const root_ni = try mf.addNode(gpa, .{ .add_node = .{ .size = size, .alignment = mf.flags.block_size, .fixed = true, - } }) catch |err| switch (err) { - error.MappedFileIo => return mf.io_err.?, - else => |e| return e, - }; + } }); assert(root_ni == Node.Index.root); try mf.ensureTotalCapacityInner(@intCast(size)); return mf; @@ -355,9 +352,7 @@ pub const Node = extern struct { pub const RealignNodeOptions = struct { /// Shift the node backwards if possible - try_backwards: bool = true, - /// If `set, persists `new_alignment` as the node's alignment for future operations. - set_alignment: bool = true, + try_backwards: bool = false, }; /// Moves and expands a node such that its offset and size are aligned to `new_alignment`. @@ -378,11 +373,7 @@ pub const Node = extern struct { return error.MappedFileIo; }, }; - var writers_it = mf.writers.first; - while (writers_it) |writer_node| : (writers_it = writer_node.next) { - const w: *Node.Writer = @fieldParentPtr("writer_node", writer_node); - w.interface.buffer = w.ni.slice(mf); - } + mf.updateWriters(); } /// Shrink a node to `size`, exactly. @@ -398,11 +389,7 @@ pub const Node = extern struct { shift_next: bool, ) Error!void { try mf.shrinkNode(gpa, ni, size, shift_next); - var writers_it = mf.writers.first; - while (writers_it) |writer_node| : (writers_it = writer_node.next) { - const w: *Node.Writer = @fieldParentPtr("writer_node", writer_node); - w.interface.buffer = w.ni.slice(mf); - } + mf.updateWriters(); } pub fn writer(ni: Node.Index, mf: *MappedFile, gpa: std.mem.Allocator, w: *Writer) void { @@ -562,34 +549,15 @@ fn addNode(mf: *MappedFile, gpa: std.mem.Allocator, opts: struct { next: Node.Index = .none, offset: u64 = 0, add_node: AddNodeOptions, -}) Error!Node.Index { +}) (Allocator.Error || Io.Cancelable || IoError)!Node.Index { if (opts.add_node.moved or opts.add_node.resized) try mf.updates.ensureUnusedCapacity(gpa, 1); mf.nodes_lock.assertUnlocked(); - const offset = opts.add_node.alignment.forward(@intCast(opts.offset)); - if (opts.parent != .none) { - const new_end = offset + opts.add_node.size; - switch (opts.next) { - .none => { - _, const parent_size = opts.parent.location(mf).resolve(mf); - if (new_end > parent_size) - try opts.parent.resize(mf, gpa, new_end); - }, - else => |next_ni| { - const next_offset, _ = next_ni.location(mf).resolve(mf); - if (new_end > next_offset) - try next_ni.realign(mf, gpa, opts.add_node.alignment, .{ - .try_backwards = false, - .set_alignment = false, - }); - }, - } - } const location_tag: Node.Location.Tag, const location_payload: Node.Location.Payload = location: { - if (std.math.cast(u32, offset)) |small_offset| break :location .{ .small, .{ + if (std.math.cast(u32, opts.offset)) |small_offset| break :location .{ .small, .{ .small = .{ .offset = small_offset, .size = 0 }, } }; try mf.large.ensureUnusedCapacity(gpa, 2); - defer mf.large.appendSliceAssumeCapacity(&.{ offset, 0 }); + defer mf.large.appendSliceAssumeCapacity(&.{ opts.offset, 0 }); break :location .{ .large, .{ .large = .{ .index = mf.large.items.len } } }; }; const free_ni: Node.Index, const free_node = free: switch (mf.free_ni) { @@ -616,7 +584,7 @@ fn addNode(mf: *MappedFile, gpa: std.mem.Allocator, opts: struct { .last = .none, .flags = .{ .location_tag = location_tag, - .alignment = opts.add_node.alignment, + .alignment = .@"1", .fixed = opts.add_node.fixed, .moved = true, .resized = true, @@ -627,13 +595,15 @@ fn addNode(mf: *MappedFile, gpa: std.mem.Allocator, opts: struct { }; { - try free_ni.resize(mf, gpa, opts.add_node.size); + try mf.realignNode(gpa, free_ni, opts.add_node.alignment, .{}); + try mf.resizeNode(gpa, free_ni, opts.add_node.size); if (opts.add_node.moved or opts.add_node.resized) try mf.updates.ensureUnusedCapacity(gpa, 1); free_node.flags.moved = false; free_node.flags.resized = false; } if (opts.add_node.moved) free_ni.movedAssumeCapacity(mf); if (opts.add_node.resized) free_ni.resizedAssumeCapacity(mf); + mf.updateWriters(); return free_ni; } @@ -658,7 +628,15 @@ pub fn addOnlyChildNode( return mf.addNode(gpa, .{ .parent = parent_ni, .add_node = opts, - }); + }) catch |err| switch (err) { + error.OutOfMemory, + error.Canceled, + => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, + }; } pub fn addFirstChildNode( @@ -673,7 +651,15 @@ pub fn addFirstChildNode( .parent = parent_ni, .next = parent.first, .add_node = opts, - }); + }) catch |err| switch (err) { + error.OutOfMemory, + error.Canceled, + => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, + }; } pub fn addLastChildNode( @@ -695,7 +681,15 @@ pub fn addLastChildNode( }, }, .add_node = opts, - }); + }) catch |err| switch (err) { + error.OutOfMemory, + error.Canceled, + => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, + }; } pub fn addNodeAfter( @@ -714,7 +708,15 @@ pub fn addNodeAfter( .next = prev.next, .offset = prev_offset + prev_size, .add_node = opts, - }); + }) catch |err| switch (err) { + error.OutOfMemory, + error.Canceled, + => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, + }; } fn shrinkNode( @@ -1043,26 +1045,20 @@ fn realignNode( new_alignment: std.mem.Alignment, opts: Node.Index.RealignNodeOptions, ) (Allocator.Error || Io.Cancelable || IoError)!void { - assert(ni != Node.Index.root); // currently unsupported mf.nodes_lock.assertUnlocked(); const node = ni.get(mf); + { + const prev_alignment = node.flags.alignment; + node.flags.alignment = new_alignment; + if (new_alignment.compare(.lte, prev_alignment)) return; + } + const old_offset, const size = node.location().resolve(mf); + if (ni == Node.Index.root) return mf.resizeNode(gpa, ni, size); - assert(new_alignment.compare(.gt, node.flags.alignment)); - - const prev_alignment = node.flags.alignment; - node.flags.alignment = new_alignment; - defer { - // alignment needs to be temporarily set for the resizes below - if (!opts.set_alignment) node.flags.alignment = prev_alignment; - } - - const new_size = node.flags.alignment.forward(@intCast(size)); - if (new_alignment.check(@intCast(old_offset))) { - if (new_size > size) try mf.resizeNode(gpa, ni, new_size); - return; - } + const new_size = new_alignment.forward(@intCast(size)); + if (new_alignment.check(@intCast(old_offset))) return mf.resizeNode(gpa, ni, new_size); _, const parent_size = node.parent.location(mf).resolve(mf); const trailing_end = trailing_end: switch (node.next) { @@ -1122,7 +1118,7 @@ fn realignNode( ni.setLocationAssumeCapacity(mf, forward_offset, new_size); } else { - const temp_size = node.flags.alignment.forward(@intCast(new_size + 1)); + const temp_size = new_alignment.forward(@intCast(new_size + 1)); try mf.resizeNode(gpa, ni, temp_size); const new_offset, _ = ni.location(mf).resolve(mf); @@ -1148,6 +1144,14 @@ fn realignNode( } } +fn updateWriters(mf: *MappedFile) void { + var writers_it = mf.writers.first; + while (writers_it) |writer_node| : (writers_it = writer_node.next) { + const w: *Node.Writer = @fieldParentPtr("writer_node", writer_node); + w.interface.buffer = w.ni.slice(mf); + } +} + fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable || IoError)!void { // make a copy of this node at the new location try mf.copyRange(old_file_offset, new_file_offset, size);