From 56c1b0871c396155ca865b4f3531f6a7f17b35a0 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Thu, 13 Aug 2026 22:42:01 +0100 Subject: [PATCH 1/6] Elf2: keep the data segment at the end on SPARC SPARC has some strange relocations which are PC-relative, but generate an unsigned offset. These relocations are very frequently used to get a pointer to the GOT. As a result, SPARC generally requires that the GOT has a greater virtual address than all function code. Helpfully, this is a requirement which the old ABI neglects to actually define. The SPARC Compliance Definition 2.4.1 includes one "note" in its explanation of code models vaguely alluding to the fact that certain section orderings "may" be necessary; and later (when discussing the GOT) there is an *example* asm snippet which is stated to "assume" that the offset to the GOT is positive. Neither of these is phrased as to impose any particular requirement on the linker, and in fact, I believe both are intended to be non-normative text! Nonetheless, this requirement exists in practice, and we can't really get around it---before this patch, attempting to link any SPARC64 code with `Elf2` would just result in thousands of relocation errors. So let's follow this rule by forcing the "mutable data" segment, which holds the GOT, to be the last segment in the virtual address space, and therefore after all code (which is in the "text" segment). My spidey senses tell me that some other targets will probably end up having some stupid segment ordering requirements too, so I've pulled the check for whether to require this (currently just checking whether the target machine is `EM_SPARCV9`) into its own function returning an enum. --- src/link/Elf2.zig | 195 +++++++++++++++++++++++++++++++++------------- 1 file changed, 142 insertions(+), 53 deletions(-) diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 8ef617fad2e1cc2d4c7a67412a714502d4e8e2f6..982740dceb046166a05e5c1c29864325274bb7a2 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -3558,48 +3558,55 @@ fn initHeaders( .EXEC, .DYN => {}, } var phnum: u32 = 0; - break :ph .{ .{ - .phdr = phndx: { - defer phnum += 1; - break :phndx phnum; + break :ph .{ + .{ + .phdr = phndx: { + defer phnum += 1; + break :phndx phnum; + }, + .interp = if (maybe_interp) |_| phndx: { + defer phnum += 1; + break :phndx phnum; + } else undefined, + .rodata = phndx: { + defer phnum += 1; + break :phndx phnum; + }, + .text = phndx: { + defer phnum += 1; + break :phndx phnum; + }, + .plt = if (plt.got_plt == null) phndx: { + defer phnum += 1; + break :phndx phnum; + } else undefined, + // `data` must be assigned after all other loadable segments so that it has the greatest + // phndx of any loadable segment. This is so that `targetSegmentLoadAddressRestrictions` + // can be obeyed (specifically, the `.data_last` restriction, needed on SPARC). + .data = phndx: { + defer phnum += 1; + break :phndx phnum; + }, + .tls = if (comp.config.any_non_single_threaded) phndx: { + defer phnum += 1; + break :phndx phnum; + } else undefined, + .dynamic = if (have_dynamic_section) phndx: { + defer phnum += 1; + break :phndx phnum; + } else undefined, + .relro = phndx: { + defer phnum += 1; + break :phndx phnum; + }, + .gnu_stack = phndx: { + defer phnum += 1; + break :phndx phnum; + }, }, - .interp = if (maybe_interp) |_| phndx: { - defer phnum += 1; - break :phndx phnum; - } else undefined, - .rodata = phndx: { - defer phnum += 1; - break :phndx phnum; - }, - .text = phndx: { - defer phnum += 1; - break :phndx phnum; - }, - .data = phndx: { - defer phnum += 1; - break :phndx phnum; - }, - .plt = if (plt.got_plt == null) phndx: { - defer phnum += 1; - break :phndx phnum; - } else undefined, - .tls = if (comp.config.any_non_single_threaded) phndx: { - defer phnum += 1; - break :phndx phnum; - } else undefined, - .dynamic = if (have_dynamic_section) phndx: { - defer phnum += 1; - break :phndx phnum; - } else undefined, - .relro = phndx: { - defer phnum += 1; - break :phndx phnum; - }, - .gnu_stack = phndx: { - defer phnum += 1; - break :phndx phnum; - }, - }, phnum }; + // (I don't actually want the trailing comma below, but a `zig fmt` bug forces it.) + phnum, + }; }; const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_footer @@ -4539,6 +4546,22 @@ fn initHeaders( break :str try elf.string(.dynstr, slice); }, }; + + if (@"type" != .REL) switch (elf.targetSegmentLoadAddressRestrictions()) { + .none => {}, + .data_last => switch (elf.phdrSlice()) { + inline else => |phdr| { + // Ensure that the segment after `.data` (if any) is not a loadable segment. + const next_phndx = phndx.data + 1; + if (next_phndx < phdr.len) { + switch (elf.targetLoad(&phdr[next_phndx].type)) { + .NULL, .LOAD => unreachable, // data segment should be the last loadable segment + else => {}, + } + } + }, + }, + }; } pub fn startProgress(elf: *Elf, prog_node: std.Progress.Node) void { @@ -4888,7 +4911,31 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo { // TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`. }; } -pub fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { +/// Specifies any restrictions the current target has regarding how segments are ordered in the +/// virtual address space. Most targets do not have any such restrictions. +fn targetSegmentLoadAddressRestrictions(elf: *const Elf) enum { + none, + /// The "mutable data" segment must be the last loadable segment in the virtual address space. + data_last, +} { + return switch (elf.ehdrMachine()) { + .AARCH64, + .PPC64, + .RISCV, + .X86_64, + .LOONGARCH, + => .none, + + // SPARC uses `R_SPARC_PC{10,22}` relocations to construct pointers to the GOT, but these + // relocations write an *unsigned* PC-relative offset. This cannot even be worked around by + // using a larger code model, because the crt `_start` assembly always uses these specific + // relocations. Therefore, to avoid relocation errors, all code must appear before the GOT + // in the virtual address space. The easiest way for us to do that is to ensure that the + // "mutable data" segment, containing the GOT, is the last segment in the address space. + .SPARCV9 => .data_last, + }; +} +fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer; const Child = pointer_ty.child; const alignment = pointer_ty.attrs.@"align" orelse @alignOf(Child); @@ -8114,6 +8161,18 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro const page_align = elf.targetPageAlign(); const node_align = segment_ni.alignment(&elf.mf); const ph_align = page_align.max(node_align); + + // If we determine that the segment's virtual address needs to move, then it's a good idea to + // make it less likely that it needs to move *again* in the future, because it is expensive to + // change a segment's load address (a lot of re-flushing is necessary). To do that, we reserve + // more virtual address space than we need (multiplying the actual size by this value). That + // way, there will usually be padding between segments which they can grow into. + // + // TODO: we might want to decrease this multiplier, or even omit it entirely, in cases where + // virtual address space is constrained. For instance, 32-bit targets, or targets where short + // PC-relative relocations between segments are common. + const reserve_size_multiplier = 4; + switch (elf.phdrSlice()) { inline else => |phdr| { const offset = elf.targetLoad(&phdr[orig_phndx].offset); @@ -8172,15 +8231,46 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro // backwards to the start of the page. const next_page_vaddr = std.mem.alignBackward(u64, next_vaddr, page_align.toByteUnits()); - // If we're at the same vaddr we started at, then all we're worried about is the - // segment fitting here. However, if we've already changed our virtual address, then - // we might as well try to reserve a bit *more* virtual address space while we're at - // it, because changing virtual address is quite disruptive (we need to re-flush a - // lot of stuff!) and giving ourselves more space will make it less likely to happen - // again. - const target_size = if (vaddr == orig_vaddr) size else size * 4; - if (vaddr + target_size <= next_page_vaddr) { - break; // hooray, we fit here! + // Check if the segment fits here. We apply `reserve_size_multiplier`, but only if + // the segment is already known to be moving---making it easier to grow in-place is + // the whole point of the multiplier! + { + const target_size = if (vaddr == orig_vaddr) size else size * reserve_size_multiplier; + if (vaddr + target_size <= next_page_vaddr) { + break; // hooray, we fit here! + } + } + + const next_ni = elf.phdrs.items[next_phndx].unwrap().?; + + // This segment don't fit here, but before deciding how to proceed, we need to + // consider any target-specific restrictions we are subject to. + switch (elf.targetSegmentLoadAddressRestrictions()) { + .none => {}, + .data_last => if (next_ni == elf.ni.data) { + // We can't leapfrog over the data segment. Instead, that segment just needs + // to be shifted forwards to make space for us, and we'll then `break` with + // our current vaddr. + + if (next_phndx + 1 < phdr.len) switch (elf.targetLoad(&phdr[next_phndx + 1].type)) { + .NULL, .LOAD => unreachable, // data segment should be the last loadable segment + else => {}, + }; + + const free_vaddr = vaddr + size * reserve_size_multiplier; + + const next_align = page_align.max(next_ni.alignment(&elf.mf)); + const next_offset = elf.targetLoad(&next_ph.offset); + const next_new_vaddr = next_align.forward(free_vaddr) + next_offset % next_align.toByteUnits(); + + // This logic for updating the data segment's vaddr is identical to how we + // will update the vaddr of `phndx` when we break from the loop. + elf.targetStore(&next_ph.vaddr, @intCast(next_new_vaddr)); + elf.targetStore(&next_ph.paddr, @intCast(next_new_vaddr)); + try next_ni.childrenMoved(elf.base.comp.gpa, &elf.mf); + + break; + }, } // We don't fit here, so shift ourselves forward (i.e. swap with `next_phndx`). But @@ -8192,8 +8282,7 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro // Now just swap the phdrs and update our `phndx`. std.mem.swap(@TypeOf(next_ph.*), &phdr[phndx], next_ph); - const next_ni = elf.phdrs.items[next_phndx]; - elf.phdrs.items[phndx] = next_ni; + elf.phdrs.items[phndx] = .wrap(next_ni); elf.nodes.items(.data)[@backingInt(next_ni)] = .{ .segment = phndx }; elf.phdrs.items[next_phndx] = .wrap(segment_ni); elf.nodes.items(.data)[@backingInt(segment_ni)] = .{ .segment = @intCast(next_phndx) }; -- 2.54.0 From 3d4aabdf795edcb1e131ed3a2e3c2910d8343c85 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Wed, 26 Aug 2026 09:14:45 +0100 Subject: [PATCH 2/6] Elf2: align the initial PLT size I had kind of assumed that every target's PLT was no more aligned than the size of an entire PLT entry, but apparently SPARC was determined to prove me wrong on that front. --- src/link/Elf2.zig | 65 +++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 982740dceb046166a05e5c1c29864325274bb7a2..0f3656e6a13f7fa193b735361619e844526b7d26 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -4099,33 +4099,44 @@ fn initHeaders( .addralign = addr_align, .entsize = @intCast(addr_align.toByteUnits()), }); - if (plt.got_plt) |got_plt| { - const got_plt_segment_ni = if (elf.options.z_now) elf.ni.data_rel_ro else elf.ni.data; - elf.shndx.got_plt = try elf.addSection(got_plt_segment_ni, .{ - .name = ".got.plt", - .type = .PROGBITS, - .flags = .{ .WRITE = true, .ALLOC = true }, - .size = got_plt.header_entries * elf.targetPtrSize(), - .addralign = addr_align, - .entsize = @intCast(addr_align.toByteUnits()), - }); - elf.shndx.plt = try elf.addSection(elf.ni.text, .{ - .name = ".plt", - .type = .PROGBITS, - .flags = .{ .ALLOC = true, .EXECINSTR = true }, - .size = plt.entry_size * plt.header_entries, - .addralign = plt.@"align", - .node_align = node_block_align, - }); - } else { - elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{ - .name = ".plt", - .type = .PROGBITS, - .flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true }, - .size = plt.entry_size * plt.header_entries, - .addralign = plt.@"align", - .node_align = node_block_align, - }); + { + const init_plt_size = plt.entry_size * plt.header_entries; + if (plt.got_plt) |got_plt| { + const got_plt_segment_ni = if (elf.options.z_now) elf.ni.data_rel_ro else elf.ni.data; + elf.shndx.got_plt = try elf.addSection(got_plt_segment_ni, .{ + .name = ".got.plt", + .type = .PROGBITS, + .flags = .{ .WRITE = true, .ALLOC = true }, + .size = got_plt.header_entries * elf.targetPtrSize(), + .addralign = addr_align, + .entsize = @intCast(addr_align.toByteUnits()), + }); + elf.shndx.plt = try elf.addSection(elf.ni.text, .{ + .name = ".plt", + .type = .PROGBITS, + .flags = .{ .ALLOC = true, .EXECINSTR = true }, + .size = plt.@"align".forward(init_plt_size), + .addralign = plt.@"align", + .node_align = node_block_align, + }); + } else { + elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{ + .name = ".plt", + .type = .PROGBITS, + .flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true }, + .size = plt.@"align".forward(init_plt_size), + .addralign = plt.@"align", + .node_align = node_block_align, + }); + } + // And the award for most annoying PLT requirement goes to SPARC, which decided that the + // whole table should have a greater alignment than the size of the individual entries, + // hence this bullshit: + if (plt.@"align".forward(init_plt_size) != init_plt_size) { + switch (elf.shdrPtr(elf.shndx.plt)) { + inline else => |shdr| elf.targetStore(&shdr.size, init_plt_size), + } + } } if (plt.plt_sec != null) elf.shndx.plt_sec = try elf.addSection(elf.ni.text, .{ .name = ".plt.sec", -- 2.54.0 From 13f71bb636b3912952297c4b615edcc64c73a918 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Wed, 26 Aug 2026 09:53:34 +0100 Subject: [PATCH 3/6] Elf2: initialize GOT header entries properly Previously these weren't being written until flush, which meant we hadn't reserved capacity for them in `.rela.dyn`, so if they needed runtime relocations we might hit Illegal Behavior when adding that relocation in `updateGotEntry`. --- src/link/Elf2.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 0f3656e6a13f7fa193b735361619e844526b7d26..95c76025570db13d8374c5b18132d0e1ade52aae 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -4376,6 +4376,12 @@ fn initHeaders( assert(elf.targetLoad(&shdr.size) == elf.got.count() * @sizeOf(Addr)); }, } + if (elf.shndx.dynamic != .UNDEF) { + try elf.shndx.rela_dyn.relaEnsureAdditionalCapacity(elf, elf.got.count()); + } + for (0..elf.got.count()) |got_index| { + elf.updateGotEntry(got_index); + } // Create any always-provided linker-defined symbols. The symbols marking the `INIT_ARRAY`/ // `FINI_ARRAY`/`PREINIT_ARRAY` sections are instead created by `createInitFiniArraySection` @@ -7187,6 +7193,7 @@ fn addGotRelocAssumeCapacity( }); } fn updateGotEntry(elf: *Elf, got_index: usize) void { + assert(elf.ehdrType() != .REL); const entry_value: union(enum) { unsigned: u64, signed: i64, -- 2.54.0 From 463fb17320d7320f6f501a266cfd156329ee5cd4 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Tue, 25 Aug 2026 13:08:02 +0100 Subject: [PATCH 4/6] Elf2: rework archive generation Previously, `Elf2` implemented archives (static libraries) by allowing each member to grow to absorb padding following it. However, I didn't love that we would effectively adding padding bytes to link inputs, and there was no meaningful benefit to allowing reordering of archive members by the `MappedFile` implementation. Ideally, archive members would be densely packed, and their order fixed; and the generated ZCU object can be the last member of the object so that the entire rest of the archive is functionally one long header. Padding bytes in the archive itself (e.g. to align the ZCU object file contents to a block-aligned boundary) should be pulled into the long file names string table ("//"), since that member is generated by us and can be any size. The new `MappedFile` API makes it possible to express this layout fairly easily. The root node of the file has one header node, which contains the archive magic number, the file header for the "//" member, and the actual content of the "//" member; then, all archive *members* are in footer nodes, with the ZCU object being the very last footer (and being block-aligned). Most archive member headers can be written immediately, with the two exceptions being "//" (which updates its `ar_size` field in response to "next_moved" events on the archive header node) and the ZCU object file (which updates its `ar_size` field in response to "resized" events on the ZCU ELF node). Since I was reworking the implementation of archives anyway, I also took the opportunity to add support for long archive member names, and to add graceful error reporting for the archive-specific failure modes (thereby removing most possible calls to `panic` in this linker). Resolves: https://codeberg.org/ziglang/zig/issues/36496 --- src/link/Elf2.zig | 414 +++++++++++++++++++++++++++++++--------------- 1 file changed, 282 insertions(+), 132 deletions(-) diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 95c76025570db13d8374c5b18132d0e1ade52aae..4e0bdee194736cdff8716ab0ebdf764637a76b4a 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -24,6 +24,7 @@ base: link.File, options: link.File.OpenOptions, mf: MappedFile, ni: Node.Known, +archive: ?Archive, nodes: std.MultiArrayList(Node), /// Does not contain an item for `SHN_UNDEF`. shdrs: std.ArrayList(Section), @@ -200,19 +201,39 @@ input_prog_node: std.Progress.Node, const Error = link.Error || error{MappedFileIo}; const Node = union(enum) { + /// Only used when emitting a static library. + /// + /// Contains a header node which is an `.archive_header`. + /// + /// Contains the following footer nodes: + /// * One `.archive_input_member` for each external input in the archive + /// * One `.archive_elf_member_header` containing the `ar_hdr` for the ZCU + /// * One `.elf` containing the ZCU's actual ELF object + /// + /// Padding between the headers and footers is absorbed into the "//" member (whose actual + /// content is in the `.archive_header` node). archive, - /// This includes the archive magic and long file member. + /// Only used when emitting a static library. + /// + /// Contains the archive magic (`ARMAG`), as well as the `ar_hdr` and content for the long file + /// name string table member ("//"). archive_header, - /// This is a footer of the `.elf` node, and contains the next archive entry's file header. - archive_elf_footer, + /// Only used when emitting a static library. + /// + /// Contains the `ar_hdr` and content for one non-ZCU archive member (external link input). Also + /// includes the single byte '\n' padding at the end of this archive member, if necessary. + archive_input_member: InputIndex, + /// Only used when emitting a static library. + /// + /// Contains the `ar_hdr` for the `.elf` node. + archive_elf_member_header, + elf, ehdr, shdr, segment: u32, /// The section '.plt' may contain relocations via `elf.plt_first_symbol_reloc`. section: Section.Index, - /// Only valid for static libraries, represents one non-zcu archive member. - input_member: InputIndex, /// May contain relocations. input_section: InputSection.Index, /// Value is the name of a global which has an entry in `elf.copied_globals`, so, a global for @@ -404,6 +425,15 @@ const InputSection = struct { }; }; +const Archive = struct { + ni: MappedFile.Node.Index, + header_ni: MappedFile.Node.Index, + elf_member_header_ni: MappedFile.Node.Index, + + elf_member_too_big: bool, + strtab_member_too_big: bool, +}; + const Section = struct { /// The node corresponding to this section. ni: MappedFile.Node.Index, @@ -2954,13 +2984,13 @@ pub fn symbolForAtom(elf: *Elf, atom: link.File.AtomId) link.File.SymbolId { const lsi: Symbol.LocalIndex = switch (elf.getNode(Node.fromAtom(atom))) { .archive, .archive_header, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, .elf, .ehdr, .shdr, .segment, .section, - .input_member, .input_section, .copied_global, => unreachable, @@ -3364,6 +3394,7 @@ fn create( .data_rel_ro = undefined, .tls = .none, }, + .archive = null, .nodes = .empty, .shdrs = .empty, .phdrs = .empty, @@ -3609,7 +3640,7 @@ fn initHeaders( }; }; - const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_footer + const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_member_header 3 + // `.elf`, `.ehdr`, and `.shdr` nodes (shnum - 1) + // -1 because the SHN_UNDEF shdr does not have a `.section` node (phnum -| 1); // -1 because the GNU_STACK phdr does not have a `.segment` node @@ -3626,11 +3657,14 @@ fn initHeaders( const archive_ni: MappedFile.Node.Index = .root; const archive_header_ni = try archive_ni.addOnlyHeaderChild(&elf.mf, gpa, .{ - .size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr) * 2, - .alignment = .@"2", - .next_moved = true, - .bubbles_moved = false, + // We intentionally do not set `.alignment = .@"2"` here, because the string table data + // in this node does not need to have an aligned length. (This node's offset is aligned + // regardless by virtue of it being a header.) + .size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr), + // The archive header uses 'next_moved' events to resize the "//" member, so that it + // absorbs all padding between `archive_header_ni` and the actual object file members. .enable_next_moved = true, + .next_moved = true, }); elf.nodes.appendAssumeCapacity(.archive_header); const archive_header_slice = archive_header_ni.slice(&elf.mf); @@ -3642,23 +3676,47 @@ fn initHeaders( .ar_uid = @splat(' '), .ar_gid = @splat(' '), .ar_mode = @splat(' '), - .ar_size = @splat(' '), + .ar_size = undefined, // populated by `flushNextMoved` for `archive_header_ni` .ar_fmag = std.elf.ARFMAG.*, }; - elf.ni.elf = try archive_ni.addFloatingChild(&elf.mf, gpa, .{ + elf.ni.elf = try archive_ni.addOnlyFooterChild(&elf.mf, gpa, .{ .alignment = node_block_align.max(.@"2"), - .next_moved = true, .bubbles_moved = false, - .enable_next_moved = true, + .resized = true, // ensure that this node's `ar_hdr.ar_size` is updated at least once }); elf.nodes.appendAssumeCapacity(.elf); - _ = try elf.ni.elf.addOnlyFooterChild(&elf.mf, gpa, .{ + const elf_ar_hdr_ni = try archive_ni.addFooterChildBefore(&elf.mf, gpa, .wrap(elf.ni.elf), .{ .alignment = .@"2", .size = @sizeOf(std.elf.ar_hdr), }); - elf.nodes.appendAssumeCapacity(.archive_elf_footer); + elf.nodes.appendAssumeCapacity(.archive_elf_member_header); + + // Must be populated before we call `populateArchiveMemberName` below. + elf.archive = .{ + .ni = archive_ni, + .header_ni = archive_header_ni, + .elf_member_header_ni = elf_ar_hdr_ni, + + .elf_member_too_big = false, + .strtab_member_too_big = false, + }; + + const elf_ar_hdr: *std.elf.ar_hdr = @ptrCast(elf_ar_hdr_ni.slice(&elf.mf)); + elf_ar_hdr.* = .{ + .ar_name = undefined, // populated below + .ar_date = "0 ".*, + .ar_uid = "0 ".*, + .ar_gid = "0 ".*, + .ar_mode = "644 ".*, + .ar_size = undefined, // populated by `flushResized` for the `.elf` node + .ar_fmag = std.elf.ARFMAG.*, + }; + const zcu_member_name = try std.fmt.allocPrint(gpa, "{s}_zcu.o", .{comp.root_name}); + defer gpa.free(zcu_member_name); + // After this call returns, `elf_ar_hdr` is invalidated. + try elf.populateArchiveMemberName(elf_ar_hdr, zcu_member_name); } else { elf.ni.elf = .root; elf.nodes.appendAssumeCapacity(.elf); @@ -4613,12 +4671,12 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index { return switch (elf.getNode(ni)) { .archive, .archive_header, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, .elf, .ehdr, .shdr, .segment, - .input_member, => unreachable, .section => |shndx| shndx, .input_section, @@ -4634,12 +4692,12 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { return switch (elf.getNode(ni)) { .archive, .archive_header, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, .elf, .ehdr, .shdr, .segment, - .input_member, .copied_global, => unreachable, .section => |shndx| shndx.vaddr(elf), @@ -4653,14 +4711,18 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { } fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) { - .archive, .archive_header, .archive_elf_footer => unreachable, + .archive, + .archive_header, + .archive_input_member, + .archive_elf_member_header, + => unreachable, .elf => return 0, .ehdr, .shdr => unreachable, .segment => |phndx| switch (elf.phdrSlice()) { inline else => |phdr| elf.targetLoad(&phdr[phndx].vaddr), }, .section => |shndx| if (shndx == elf.shndx.tdata) 0 else shndx.vaddr(elf), - .input_member, .input_section, .copied_global => unreachable, + .input_section, .copied_global => unreachable, inline .nav, .uav, .lazy_code, .lazy_const_data => |i| Symbol.Id.local(i.symbol(elf)).value(elf), }; const offset, _ = ni.location(&elf.mf).resolve(&elf.mf); @@ -4679,12 +4741,12 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void { const symbol_relocs: *SymbolReloc.Index, const got_relocs: ?*GotReloc.Index = switch (elf.getNode(ni)) { .archive, .archive_header, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, .elf, .ehdr, .shdr, .segment, - .input_member, .copied_global, => unreachable, // cannot contain relocs .section => unreachable, // cannot contain relocs (.plt and .dynamic unsupported) @@ -4745,7 +4807,12 @@ fn flushMovedNodeRelocs( // changed, so update the `offset` field of the `ElfN.Rela` entry. reloc.relaSection(elf).relaSetOffset(elf, rela_index, node_vaddr + reloc.offset); } - reloc.apply(elf); + // This is not just the inverse of the above condition, because if `reloc` is relative + // to the base of this DSO, then `rela_index` is an `R_*_RELATIVE` relocation, but we + // still need to call `SymbolReloc.apply` to update that relocation's addend. + if (elf.ehdrType() != .REL) { + reloc.apply(elf); + } } } @@ -5035,16 +5102,6 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr { } } -fn arHdrPtr(elf: *Elf, ni: MappedFile.Node.Index) *align(2) std.elf.ar_hdr { - assert(elf.ni.elf != .root); - const file_offset = ni.fileLocation(&elf.mf, false).offset; - return @ptrCast(@alignCast(elf.mf.memory_map.memory[@intCast(switch (elf.getNode(ni)) { - else => unreachable, - .archive_header => file_offset + std.elf.ARMAG.len, - .elf, .input_member => file_offset - @sizeOf(std.elf.ar_hdr), - })..][0..@sizeOf(std.elf.ar_hdr)])); -} - const SymPtr = union(std.elf.CLASS) { NONE: noreturn, @"32": *std.elf.Elf32.Sym, @@ -5544,19 +5601,61 @@ fn loadObject( .member = if (member) |m| try gpa.dupe(u8, m) else null, .extra = undefined, }; - if (elf.ni.elf != .root) { - const archive_ni: MappedFile.Node.Index = .root; + if (elf.archive) |*archive| { + // We're creating a static library, so just add this input as an archive member. + assert(member == null); // don't try to put static library members into other static libraries + + const first_member_oni = archive.header_ni.next(&elf.mf); + + if (first_member_oni.unwrap()) |first_member_ni| switch (elf.getNode(first_member_ni)) { + .archive_input_member, .archive_elf_member_header => {}, + .elf => unreachable, // always preceded by `.archive_elf_member_header` + else => unreachable, // never a child of `.archive` + }; + try elf.nodes.ensureUnusedCapacity(gpa, 1); - input.extra = .{ .node = try archive_ni.addFloatingChild(&elf.mf, gpa, .{ - .size = Alignment.@"2".forward(fl.size + @sizeOf(std.elf.ar_hdr)), + const new_member_ni = try archive.ni.addFooterChildBefore(&elf.mf, gpa, first_member_oni, .{ + .size = Alignment.@"2".forward(@sizeOf(std.elf.ar_hdr) + fl.size), .alignment = .@"2", - .next_moved = true, - .bubbles_moved = false, - .enable_next_moved = true, - }) }; - elf.nodes.appendAssumeCapacity(.{ .input_member = input_index }); + }); + elf.nodes.appendAssumeCapacity(.{ .archive_input_member = input_index }); + input.extra = .{ .node = new_member_ni }; elf.input_prog_node.increaseEstimatedTotalItems(1); + // The contents of the input will be written to the file by an idle task (`flushInput`), but + // we do need to write the input's archive member header (`ar_hdr`) now, for two reasons: + // + // * If the input file has a long name, we need to add it to the archive member name string + // table, which must happen deterministically (i.e. not in an idle task). + // + // * `flushInput` needs to know the actual file size (before padding to the alignment). + const member_ar_hdr: *std.elf.ar_hdr = @ptrCast( + new_member_ni.slice(&elf.mf)[0..@sizeOf(std.elf.ar_hdr)], + ); + member_ar_hdr.* = .{ + .ar_name = undefined, // populated below + .ar_date = "0 ".*, + .ar_uid = "0 ".*, + .ar_gid = "0 ".*, + .ar_mode = "644 ".*, + .ar_size = undefined, // populated below + .ar_fmag = std.elf.ARFMAG.*, + }; + + if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{fl.size})) |size_str| { + @memset(member_ar_hdr.ar_size[size_str.len..], ' '); + } else |err| switch (err) { + error.NoSpaceLeft => return diags.failParse( + path, + "file size of {Bi} exceeds maximum size of archive member", + .{fl.size}, + ), + } + + const member_name = std.fs.path.basename(path.sub_path); + // After this call returns, `member_ar_hdr` is invalidated. + try elf.populateArchiveMemberName(member_ar_hdr, member_name); + // Since we are not emitting the archive symbol table (yet?) we do not need to parse // the symbols in this input. return; @@ -5969,6 +6068,46 @@ fn loadObject( }, } } +/// This function may resize the archive header, so therefore invalidates `member_ar_hdr`. +fn populateArchiveMemberName(elf: *Elf, member_ar_hdr: *std.elf.ar_hdr, member_name: []const u8) Error!void { + if (std.mem.print(&member_ar_hdr.ar_name, "{s}/", .{member_name})) |name_str| { + @memset(member_ar_hdr.ar_name[name_str.len..], ' '); + return; + } else |err| switch (err) { + error.NoSpaceLeft => {}, // handled below + } + + const gpa = elf.base.comp.gpa; + const archive_header_ni = elf.archive.?.header_ni; + + // The member's name is too big to put directly in the `ar_name` field, so it needs to go in the + // "long name" string table instead (in the special member named "//"). + + _, const old_archive_header_size = archive_header_ni.location(&elf.mf).resolve(&elf.mf); + + // We're going to add a new string at the end of the table. Update `member_ar_hdr` first, + // because resizing the string table will invalidate it. + const string_table_offset = old_archive_header_size - (std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr)); + if (std.mem.print(&member_ar_hdr.ar_name, "/{d}", .{string_table_offset})) |name_str| { + @memset(member_ar_hdr.ar_name[name_str.len..], ' '); + } else |inner_err| switch (inner_err) { + error.NoSpaceLeft => { + // The string table offset is itself too big to represent. This means the string table's + // *size* is definitely too big to represent (we only get 10 bytes for that whereas we + // get 16 here!), so as long as we still add the string, we're guaranteed to get a link + // error for that reason. Therefore, we can just ignore this error and carry on. + }, + } + + // We set the size of the archive header node exactly, because we want padding bytes to go into + // the root `.archive` node. That way, those bytes could still be used to grow the string table + // if necessary, but they could also be used for new archive members. + try archive_header_ni.resizeLeaf(&elf.mf, gpa, old_archive_header_size + member_name.len + 2); + + const dest_slice = archive_header_ni.slice(&elf.mf)[@intCast(old_archive_header_size)..]; + @memcpy(dest_slice[0 .. dest_slice.len - 2], member_name); + @memcpy(dest_slice[dest_slice.len - 2 ..], "/\n"); // yes, the terminator is weird +} fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadParseInputError || error{BadMagic})!void { const comp = elf.base.comp; const gpa = comp.gpa; @@ -7136,12 +7275,12 @@ fn addGotRelocAssumeCapacity( switch (elf.getNode(node)) { .archive, .archive_header, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, .elf, .ehdr, .shdr, .segment, - .input_member, .copied_global, => unreachable, // cannot contain relocs, .section, @@ -7579,6 +7718,17 @@ fn flushInner( diags.addError("failed to apply {d} relocations: misaligned value", .{elf.misaligned_reloc_count}); } + if (elf.archive) |*archive| { + if (archive.elf_member_too_big) diags.addError( + "file size of {Bi} exceeds maximum size of archive member", + .{elf.ni.elf.location(&elf.mf).resolve(&elf.mf)[1]}, + ); + if (archive.strtab_member_too_big) diags.addError( + "archive file name string table exceeds maximum size", + .{}, + ); + } + elf.flushDynamic(); const entry_addr: u64 = entry: { @@ -7610,6 +7760,9 @@ pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) link.Error!bool { const comp = elf.base.comp; const diags = &comp.link_diags; + elf.mf.nodes_lock.lock(); + defer elf.mf.nodes_lock.unlock(); + assert(elf.pending_uavs.items.len == 0); for (&elf.lazy.values) |*lazy| { assert(lazy.pending_index == lazy.map.count()); @@ -7763,7 +7916,7 @@ fn idleProgNode( return prog_node.start(name: switch (node) { else => |tag| @tagName(tag), .section => |shndx| shndx.name(elf).slice(elf), - .input_member => |ii| std.fmt.bufPrint(&name, "{f}{f}", .{ + .archive_input_member => |ii| std.fmt.bufPrint(&name, "{f}{f}", .{ ii.path(elf).fmtEscapeString(), fmtMemberString(ii.member(elf)), }) catch &name, @@ -7890,7 +8043,6 @@ fn genLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void { fn flushInput(elf: *Elf, ii: Node.InputIndex) Error!void { const comp = elf.base.comp; const io = comp.io; - const gpa = comp.gpa; const diags = &comp.link_diags; const path = ii.path(elf); const file = path.root_dir.handle.openFile(io, path.sub_path, .{}) catch |err| switch (err) { @@ -7898,23 +8050,40 @@ fn flushInput(elf: *Elf, ii: Node.InputIndex) Error!void { else => |e| return diags.fail("failed to open input file \"{f}\": {t}", .{ path.fmtEscapeString(), e }), }; defer file.close(io); + + const slice = ii.node(elf).slice(&elf.mf); + + const member_ar_hdr: *const std.elf.ar_hdr = @ptrCast(slice[0..@sizeOf(std.elf.ar_hdr)]); + const input_size: u32 = member_ar_hdr.size() catch |err| switch (err) { + // We wrote the `ar_hdr` ourselves (in `loadObject`), so it is definitely valid. + error.Overflow, error.InvalidCharacter => unreachable, + }; + + switch (slice.len - @sizeOf(std.elf.ar_hdr) - input_size) { + 0 => {}, + 1 => { + // Alignment added one padding byte, which the format requires to have value '\n'. + slice[slice.len - 1] = '\n'; + }, + else => unreachable, // node size should agree with the value we wrote into `ar_hdr.ar_size` + } + var fr = file.reader(io, &.{}); - var nw: MappedFile.Node.Writer = undefined; - ii.node(elf).writer(&elf.mf, gpa, &nw); - defer nw.deinit(); - const size = nw.interface.buffer.len - @sizeOf(std.elf.ar_hdr); - const n_bytes = nw.interface.sendFileAll(&fr, .limited(size)) catch |err| switch (err) { + var w: Io.Writer = .fixed(slice[@sizeOf(std.elf.ar_hdr)..]); + const n_bytes_read = w.sendFileAll(&fr, .limited(input_size)) catch |err| switch (err) { error.ReadFailed => return diags.fail("failed to read input \"{f}{f}\": {t}", .{ path.fmtEscapeString(), fmtMemberString(ii.member(elf)), fr.err orelse (fr.seek_err orelse fr.size_err.?), }), - error.WriteFailed => return nw.err.?, + error.WriteFailed => unreachable, // `.limited(input_size)` prevents us writing too many bytes }; - if (n_bytes + 1 < size) return diags.fail("failed to read input \"{f}{f}\": unexpected eof", .{ - path.fmtEscapeString(), - fmtMemberString(ii.member(elf)), - }); + if (n_bytes_read != input_size) { + return diags.fail("failed to load input \"{f}{f}\": file truncated during compilation", .{ + path.fmtEscapeString(), + fmtMemberString(ii.member(elf)), + }); + } } fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void { @@ -7996,12 +8165,18 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void const trace = tracy.trace(@src()); defer trace.end(); - elf.mf.nodes_lock.lock(); - defer elf.mf.nodes_lock.unlock(); - switch (elf.getNode(ni)) { - .archive, .archive_header => unreachable, - .archive_elf_footer, .elf => {}, + .archive => unreachable, + .archive_header => unreachable, + + .archive_input_member, + .archive_elf_member_header, + .elf, + => { + assert(elf.archive != null); + return; + }, + .ehdr, .shdr => elf.flushElfOffset(ni), .segment => |phndx| { elf.flushElfOffset(ni); @@ -8079,7 +8254,6 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void elf.flushMovedPltSection(.plt_sec, old_addr, addr); } }, - .input_member => {}, .input_section => |isi| { const old_section_addr = isi.ptr(elf).vaddr; const new_section_addr = elf.computeNodeVAddr(ni); @@ -8320,24 +8494,23 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo const trace = tracy.trace(@src()); defer trace.end(); - elf.mf.nodes_lock.lock(); - defer elf.mf.nodes_lock.unlock(); - _, const size = ni.location(&elf.mf).resolve(&elf.mf); switch (elf.getNode(ni)) { - .archive => { - if (ni.last(&elf.mf).unwrap()) |last_ni| { - if (last_ni.prev(&elf.mf).unwrap()) |prev_ni| { - if (prev_ni.hasNextMoved(&elf.mf)) return; - } - const offset, _ = last_ni.location(&elf.mf).resolve(&elf.mf); - _ = std.mem.print(&elf.arHdrPtr(last_ni).ar_size, "{d:<10}", .{ - size - offset, - }) catch @panic("archive member too large"); + .archive, .archive_header => {}, + .archive_input_member => unreachable, + .archive_elf_member_header => unreachable, + .elf => if (elf.archive) |*archive| { + const member_ar_hdr: *std.elf.ar_hdr = @ptrCast( + archive.elf_member_header_ni.slice(&elf.mf), + ); + if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{size})) |size_str| { + @memset(member_ar_hdr.ar_size[size_str.len..], ' '); + archive.elf_member_too_big = false; + } else |err| switch (err) { + error.NoSpaceLeft => archive.elf_member_too_big = true, } }, - .archive_header, .elf => {}, - .ehdr, .archive_elf_footer => unreachable, + .ehdr => unreachable, .shdr => {}, .segment => |phndx| switch (elf.phdrSlice()) { inline else => |phdr| { @@ -8409,7 +8582,7 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo } }, }, - .input_member, .input_section, .copied_global, .nav, .uav, .lazy_code, .lazy_const_data => {}, + .input_section, .copied_global, .nav, .uav, .lazy_code, .lazy_const_data => {}, } } @@ -8417,12 +8590,11 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error! const trace = tracy.trace(@src()); defer trace.end(); - elf.mf.nodes_lock.lock(); - defer elf.mf.nodes_lock.unlock(); - switch (elf.getNode(ni)) { .archive, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, + .elf, .ehdr, .shdr, .segment, @@ -8434,54 +8606,32 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error! .lazy_code, .lazy_const_data, => unreachable, - .archive_header, .elf, .input_member => |_, tag| { - const member_offset, const update_size = member_offset: { - const offset, _ = ni.location(&elf.mf).resolve(&elf.mf); - break :member_offset switch (tag) { - else => unreachable, - .archive_header => .{ offset + std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr), true }, - .elf, .input_member => .{ offset, !ni.prev(&elf.mf).unwrap().?.hasNextMoved(&elf.mf) }, - }; + + .archive_header => { + const archive = &elf.archive.?; + + // Because we can't just throw padding bytes in the middle of an archive file, we need + // the member name string table (the "//" member) to absorb all the padding bytes + // between it (in the `.archive_header` node) and the first actual member. + const next_member_ni = ni.next(&elf.mf).unwrap() orelse { + // I guess there are no link inputs yet? But there will be eventually! + return; }; - const member_size = if (ni.next(&elf.mf).unwrap()) |next_ni| member_size: { - const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf); - const next_member_size = if (next_ni.next(&elf.mf).unwrap()) |next_next_ni| next_member_size: { - const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf); - const next_member_end = next_next_offset - @sizeOf(std.elf.ar_hdr); - break :next_member_size next_member_end - next_offset; - } else next_member_size: { - _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf); - const next_member_end = parent_size; - break :next_member_size next_member_end - next_offset; - }; - const ar_hdr = elf.arHdrPtr(next_ni); - var name_buf: [16]u8 = undefined; - _ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{ - switch (elf.getNode(next_ni)) { - else => unreachable, - .elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}), - .input_member => |ii| std.mem.print(&name_buf, "{s}/", .{ - std.fs.path.basename(ii.path(elf).sub_path), - }), - } catch @panic("TODO: long archive member names"), - }) catch @panic("TODO: long archive member names"); - ar_hdr.ar_date = "0 ".*; - ar_hdr.ar_uid = "0 ".*; - ar_hdr.ar_gid = "0 ".*; - ar_hdr.ar_mode = "644 ".*; - _ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch - @panic("archive member too large"); - ar_hdr.ar_fmag = std.elf.ARFMAG.*; - const member_end = next_offset - @sizeOf(std.elf.ar_hdr); - break :member_size member_end - member_offset; - } else member_size: { - _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf); - const member_end = parent_size; - break :member_size member_end - member_offset; - }; - if (update_size) _ = std.mem.print(&elf.arHdrPtr(ni).ar_size, "{d:<10}", .{ - member_size, - }) catch @panic("archive member too large"); + const next_member_offset: u64, _ = next_member_ni.location(&elf.mf).resolve(&elf.mf); + const strtab_member_offset = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr); + assert(Alignment.@"2".check(next_member_offset)); + assert(Alignment.@"2".check(strtab_member_offset)); + const strtab_size = next_member_offset - strtab_member_offset; + + const member_ar_hdr: *std.elf.ar_hdr = @ptrCast( + archive.header_ni.slice(&elf.mf)[std.elf.ARMAG.len..][0..@sizeOf(std.elf.ar_hdr)], + ); + if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{strtab_size})) |size_str| { + @memset(member_ar_hdr.ar_size[size_str.len..], ' '); + archive.strtab_member_too_big = false; + } else |err| switch (err) { + error.NoSpaceLeft => archive.strtab_member_too_big = true, + } }, } } -- 2.54.0 From 78d44e41ef84f9968072914fe288a57e55c683bd Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Tue, 25 Aug 2026 14:20:48 +0100 Subject: [PATCH 5/6] MappedFile: clean up accidental duplicate code --- src/link/MappedFile.zig | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/src/link/MappedFile.zig b/src/link/MappedFile.zig index a790ecbd3b303fbe97ea860481e41913d2aa3806..0dfb6527f3f6768db0ba24e791ef560178d263b9 100644 --- a/src/link/MappedFile.zig +++ b/src/link/MappedFile.zig @@ -1544,25 +1544,6 @@ fn growNodeViaInsertRange( // We don't compute the size of the range yet, because depending on `grow_mode` we might want to // bump it based on our sibling and parent nodes' alignments. However, we can do an early check // for cases where we should obviously exit. - const requested_range_size = new_size - old_size; - if (!mf.flags.block_size.check(requested_range_size)) { - // The requested size isn't exactly aligned. - switch (grow_mode) { - .exact => return false, - .minimum => { - // We can still choose to allow it by increasing the size a bit, but we shouldn't do - // that if it would *significantly* increase the requested size. - const block_size = mf.flags.block_size.toByteUnits(); - if (requested_range_size < block_size * 2) { - // Bumping this size up to the next block boundary would be a quite significant - // increase; let's not do it. - return false; - } - }, - } - } - // If `grow_mode` is exact, we will use exactly this size, but if it is `.minimum`, we may bump - // the size a little more. const min_range_size: u64 = s: { const exact_size = new_size - old_size; if (mf.flags.block_size.check(exact_size)) { @@ -1649,16 +1630,16 @@ fn growNodeViaInsertRange( } // Traversal done. We didn't hit `max_moved_nodes`, so now we can use the computed alignment // requirement to figure out whether we're actually going to insert a range. - if (need_range_align.check(requested_range_size)) { - break :range_size requested_range_size; + if (need_range_align.check(min_range_size)) { + break :range_size min_range_size; } - // Perhaps we're allowed to grow by more than `requested_range_size`? + // Perhaps we're allowed to grow by more than `min_range_size`? switch (grow_mode) { .exact => return false, .minimum => { const candidate_range_size = need_range_align.forward(min_range_size); // Allow growing by up to 50% more than was requested. - if (candidate_range_size <= requested_range_size +| requested_range_size / 2) { + if (candidate_range_size <= min_range_size +| min_range_size / 2) { break :range_size candidate_range_size; } else { return false; -- 2.54.0 From a6ab86acd92b81dcdfa3f8192690de7a26fa13d7 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Sat, 29 Aug 2026 11:30:58 +0100 Subject: [PATCH 6/6] MappedFile: handle footers more efficiently A previous commit reworked how `Elf2` implements archives to make the main ZCU object file a footer of the root node. In theory this is fairly efficient, and this held in practice when `FALLOCATE_FL_INSERT_RANGE` was available, but when it wasn't (e.g. certain filesystems, non-Linux host), the behavior of growing a footer was very inefficient. Previously, to grow a footer, we would grow the parent (thereby moving all of its footers forwards in the file) to ensure free space before the footers, and then move the footer backwards to insert space where we need it. This meant that in the `Elf2` case, every single resize of the `.elf` node was effectively guaranteed to perform two `@memmove`s of the node's entire content. This is *extremely* inefficient, because the only operation which is actually necessary to grow the last footer node on the file itself is increasing the file size! I could have special-cased the situation `Elf2` finds itself in (a footer node which is the last child of the root), but instead, I opted to spend time on something a bit more generally applicable. We now have a new strategy for resizing footer nodes, where we ask the parent node to resize itself *without* moving its footers forwards (so the padding is inserted after its footers instead of between its floating nodes and footers), and move only any footers which actually need to move (any footers which follow the growing node, plus any nested footers). This strategy is not always possible, and it also has the downside of being incapable of reclaiming free space in the parent node, so in some cases the old strategy is still chosen. I tested this commit by disabling the `FALLOCATE_FL_INSERT_RANGE` code path and running the `MappedFile` fuzz test. Also, to verify that I had actually improved efficiency, I tried using `Elf2` to build some static libraries (the situation in which it uses footer nodes) and counted how many bytes' worth of `moveRange` occured before and after this patch. It seems that across the duration of such a compilation, the total number of bytes given to `moveRange` has decreased by around a factor of 10: for instance, the number when building compiler-rt (in debug mode) went from around 70.3 MB to 6.9 MB. --- src/link/MappedFile.zig | 622 ++++++++++++++++++++++++++-------------- 1 file changed, 405 insertions(+), 217 deletions(-) diff --git a/src/link/MappedFile.zig b/src/link/MappedFile.zig index 0dfb6527f3f6768db0ba24e791ef560178d263b9..937b0989a8d40dabcf74ea8c920ac47a2a5d1f67 100644 --- a/src/link/MappedFile.zig +++ b/src/link/MappedFile.zig @@ -648,7 +648,10 @@ pub const Node = extern struct { _, const current_size = ni.location(mf).resolve(mf); if (current_size >= min_size) return; const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor); - try mf.growNode(gpa, ni, new_size, .minimum); + try mf.growNode(gpa, ni, new_size, .{ + .exact_size = false, + .move_footers = true, + }); mf.updateWriters(); } @@ -664,7 +667,10 @@ pub const Node = extern struct { switch (std.math.order(size, old_size)) { .lt => try mf.shrinkLeafNode(gpa, ni, size), .eq => {}, // `old_size` must be well-aligned, so `size` is too - .gt => try mf.growNode(gpa, ni, size, .exact), + .gt => try mf.growNode(gpa, ni, size, .{ + .exact_size = true, + .move_footers = false, // irrelevant, since we have no footers + }), } mf.updateWriters(); } @@ -935,7 +941,10 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { try mf.realignNode(gpa, new_ni, opts.add_options.alignment); if (opts.add_options.size > 0) { - try mf.growNode(gpa, new_ni, opts.add_options.size, .exact); + try mf.growNode(gpa, new_ni, opts.add_options.size, .{ + .exact_size = true, + .move_footers = false, // irrelevant, since we have no footers + }); } mf.updateWriters(); @@ -1070,12 +1079,21 @@ fn shrinkLeafNode( } } -const GrowMode = enum { exact, minimum }; +const GrowOptions = struct { + /// If `true`, the node size must be set to exactly the given size. + /// + /// If `false`, the given size is a minimum, and the actual new node size may be larger. + exact_size: bool, + /// If `true`, footers within the resized node will be moved forwards to its new end. + /// + /// If `false`, footers will all remain at their current offsets (so the nodes are in a + /// temporarily invalid state), and moving them is the responsibility of the *caller*. + move_footers: bool, +}; -/// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`. -/// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`. +/// Increases the size of a node. /// -/// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!). +/// Asserts that `new_size` is aligned to `ni.alignment(mf)`, even if `!grow_options.exact_size`. /// /// Asserts that `new_size` is greater than the current size of `ni`. fn growNode( @@ -1083,7 +1101,7 @@ fn growNode( gpa: Allocator, ni: Node.Index, new_size: u64, - grow_mode: GrowMode, + grow_options: GrowOptions, ) Error!void { mf.nodes_lock.assertUnlocked(); @@ -1098,7 +1116,7 @@ fn growNode( const parent_ni = node.parent.unwrap() orelse { assert(ni == .root); - if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) { return; } @@ -1120,21 +1138,23 @@ fn growNode( }; try mf.ensureTotalCapacityPrecise(@intCast(new_size)); try ni.setLocation(mf, gpa, old_offset, new_size); - // We need to move any footers to be at the *new* end of the file. - if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { - const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); - const footers_size = old_size - old_footers_offset; - try mf.moveRange( - old_footers_offset, - old_footers_offset + (new_size - old_size), - footers_size, - ); - // Also update the footers' locations. - var cur_ni = first_footer_ni; - while (true) { - const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); - try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); - cur_ni = cur_ni.next(mf).unwrap() orelse break; + if (grow_options.move_footers) { + // We need to move any footers to be at the *new* end of the file. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = old_size - old_footers_offset; + try mf.moveRange( + old_footers_offset, + old_footers_offset + (new_size - old_size), + footers_size, + ); + // Also update the footers' locations. + var cur_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } } } return; @@ -1142,7 +1162,7 @@ fn growNode( switch (node.flags.position) { .header => { - if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) { return; } @@ -1163,7 +1183,13 @@ fn growNode( const old_headers_size = last_header_offset + last_header_size; // This is the first footer *inside* of `ni`. - const first_sub_footer_oni = ni.firstFooter(mf); + const first_sub_footer_oni: Node.Index.Optional = footer: { + if (!grow_options.move_footers) { + // Pretend there are no footers so as to not move them. + break :footer .none; + } + break :footer ni.firstFooter(mf); + }; const sub_footers_size = size: { const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); @@ -1215,92 +1241,264 @@ fn growNode( return; }, .floating => { - try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode); + try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_options); }, .footer => { - if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) { return; } - try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size); - - const first_footer_ni: Node.Index = first_footer: { - var footer_ni = ni; - while (true) { - const prev_ni = footer_ni.prev(mf).unwrap() orelse break; - if (prev_ni.position(mf) != .footer) break; - footer_ni = prev_ni; - } - break :first_footer footer_ni; - }; - // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself - // a footer within its parent). - const first_sub_footer_oni = ni.firstFooter(mf); - const sub_footers_size = size: { - const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; - const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); - break :size old_size - first_sub_footer_offset; + // a footer within its parent). We'll need this later in any case, so just find it now. + const first_sub_footer_oni: Node.Index.Optional = footer: { + if (!grow_options.move_footers) { + // Pretend there are no nested footers so as to not move them. + break :footer .none; + } + break :footer ni.firstFooter(mf); }; - _, const parent_size = parent_ni.location(mf).resolve(mf); - - const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0]; - const new_footers_size = old_footers_size - old_size + new_size; - - // Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node - // itself needs to shift its contents, because our offset was shifted backwards by - // `new_size - old_size`, and the added bytes should go at the end of this footer node. - // However, if we *contain* any footer nodes, they need to stay at the end of `ni`, so - // we *shouldn't* shift *that* data. - const old_footers_start = parent_size - old_footers_size; - const new_footers_start = parent_size - new_footers_size; - const end_offset = node.location().resolve(mf)[0] + old_size; - const parent_file_offset = parent_ni.fileLocation(mf, false).offset; - try mf.moveRange( - parent_file_offset + old_footers_start, - parent_file_offset + new_footers_start, - end_offset - old_footers_start - sub_footers_size, - ); - - // Update our own offset and size: - try ni.setLocation(mf, gpa, end_offset - new_size, new_size); - - // Any footers inside of us have had their offsets changed due to us growing: - if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { - var cur_ni = first_sub_footer_ni; - while (true) { - const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); - try cur_ni.setLocation( + // We have two different strategies for growing a footer node, with different advantages + // and disadvantages; so first we must decide which to use. + const strat: union(enum) { + /// Expand into pre-footer padding space in the parent node (growing the parent if + /// necessary). This strategy has the benefit that it can reclaim padding bytes in + /// the parent, but it has the disadvantage that it requires moving this node's + /// existing content backwards in the file, which may be expensive (particularly + /// since the src and dest ranges are likely to overlap). + grow_backwards, + + /// Grow the parent node with `GrowOptions.move_footers` set to `false`, and + /// implicitly grow ourselves into the newly available space. This usually requires + /// a lot less moving of bytes, but never reclaims unused space before the parent's + /// footers, and is sometimes straight-up impossible. + grow_parent_at_end: struct { + add_size: u64, + exact_size: bool, + }, + } = strat: { + // If this node is small, the move overhead is trivial, so prefer `.grow_backwards` + // to avoid unnecessary growth of the parent node. + if (old_size <= mf.flags.block_size.toByteUnits() * 2) { + break :strat .grow_backwards; + } + + // It may also be worth doing `.grow_backwards` if the parent has a *lot* of space + // we could grow into. More specifically, if "free space we can grow into" makes up + // a significant proportion of the parent's total size, then that implies the parent + // has quite poor utilization of space, *and* that we can significantly improve that + // statistic by growing into that space. + if (old_size + mf.availableFooterCapacity(parent_ni) >= new_size) { + break :strat .grow_backwards; + } + + if (grow_options.exact_size) { + const add_size = new_size - old_size; + if (parent_ni.alignment(mf).check(add_size)) { + break :strat .{ .grow_parent_at_end = .{ + .add_size = add_size, + .exact_size = true, + } }; + } else { + // We *can't* ask the parent to grow by this much, so we have no choice. + break :strat .grow_backwards; + } + } + + if (parent_ni.alignment(mf).compare(.lt, node.flags.alignment)) { + // Because the parent's alignment is less than our own, if we gave them the + // freedom to pick a size, they might choose one which results in *us* having a + // size incompatible with our alignment. Therefore, to prevent that, we need to + // request an *exact* size from the parent in this case. + break :strat .{ .grow_parent_at_end = .{ + .add_size = new_size - old_size, + .exact_size = true, + } }; + } + + // The parent's alignment is greater than or equal to our own, so we only need to + // give the parent a *minimum* size (although we need to ensure it matches their + // alignment since it could be greater than our own). + break :strat .{ .grow_parent_at_end = .{ + .add_size = parent_ni.alignment(mf).forward(new_size - old_size), + .exact_size = false, + } }; + }; + + switch (strat) { + .grow_backwards => { + // First, we might need to grow the parent to make enough space. + { + const available_size = mf.availableFooterCapacity(parent_ni); + if (old_size + available_size < new_size) { + _, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf); + const min_parent_size = old_parent_size + (new_size - old_size - available_size); + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, + ); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); + assert(old_size + mf.availableFooterCapacity(parent_ni) >= new_size); + } + } + + // Now we need to grow! To do that, we must move `ni` itself, and every footer + // before it in `parent_ni`, backwards. Unlike header nodes, `ni` is included in + // the shift, because the bytes we're adding need to go at the *end* of `ni` + // rather than its start. + + // This is the same as `parent_ni.firstFooter(mf)`, it's just more efficient to + // start at `ni` than to start at `parent_ni.last(mf)`. + const first_parent_footer_ni: Node.Index = first_footer: { + var footer_ni = ni; + while (true) { + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } + break :first_footer footer_ni; + }; + + const shift = new_size - old_size; + + // Update our own offset and size: + try ni.setLocation( mf, gpa, - old_sub_footer_offset + (new_size - old_size), - sub_footer_size, + node.location().resolve(mf)[0] - shift, + new_size, ); - cur_ni = cur_ni.next(mf).unwrap() orelse break; - } - } - // Finally, update the offsets of every footer before us: - if (node.prev.unwrap()) |prev_ni| { - var maybe_footer_ni = prev_ni; - while (true) { - switch (maybe_footer_ni.position(mf)) { - .header, .floating => break, - .footer => {}, + // Any footers *inside* of `ni` have had their offsets changed, because they are + // now positioned at the *new* end of `ni`: + { + var footer_oni = first_sub_footer_oni; + while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) { + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset + shift, footer_size); + } + } + + // Any footers *before* `ni` (in `parent_ni`) have been shifted backwards. We'll + // also be moving their actual bytes in a moment, so track whether they have + // content (if nothing does then we'll be able to skip the `moveRange`). That + // flag is initially whether `ni` has content because we're shifting our own + // bytes backwards too. + var moved_has_content: bool = node.flags.has_content; + { + var footer_ni = first_parent_footer_ni; + while (footer_ni != ni) : (footer_ni = footer_ni.next(mf).unwrap().?) { + moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset - shift, footer_size); + } + } + + if (moved_has_content) { + // We moved at least one thing containing initialized bytes, so we need to + // move the actual data. However, we should *not* move the bytes of any + // nested footers inside of `ni`, because they've been "moved" to the end + // of our new size, which is the same file location as before. + const sub_footers_size = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + break :size new_size - first_sub_footer_offset; + }; + const new_offset: u64, _ = node.location().resolve(mf); + const new_footers_offset: u64, _ = first_parent_footer_ni.location(mf).resolve(mf); + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + new_footers_offset + shift, + parent_file_offset + new_footers_offset, + (new_offset - new_footers_offset) + // accounts for every footer before `ni` + (old_size - sub_footers_size), // accounts for `ni` itself, excluding nested footers + ); } - const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf); - try maybe_footer_ni.setLocation( + }, + .grow_parent_at_end => |grow_parent| { + _, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf); + try mf.growNode(gpa, parent_ni, old_parent_size + grow_parent.add_size, .{ + .exact_size = grow_parent.exact_size, + .move_footers = false, + }); + _, const new_parent_size: u64 = parent_ni.location(mf).resolve(mf); + const shift = new_parent_size - old_parent_size; + + // Here's what we have left to do: + // + // * Increase our own size by `shift` to absorb the added space. + // + // * If there are any footers *inside* `ni`, increase their offsets by `shift`. + // + // * If there are any footers *after* `ni` (inside `parent_ni`), increase their + // offsets by `shift`. + // + // * Do a `moveRange` corresponding to those offset changes. This is a single + // range which starts at the footers *inside* `ni`. + + const actual_new_size = old_size + shift; + if (grow_options.exact_size) { + assert(actual_new_size == new_size); + } + + try ni.setLocation( mf, gpa, - moved_footer_offset + old_size - new_size, - moved_footer_size, + node.location().resolve(mf)[0], + actual_new_size, ); - maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break; - } - } - return; + // This will track whether any node with a changed offset actually contains + // initialized bytes. If not, there'll be no need to call `moveRange`. + var moved_has_content: bool = false; + + // Set any nested footers' offsets (and include them in `moved_has_content`). + { + var footer_oni = first_sub_footer_oni; + while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) { + assert(footer_ni.position(mf) == .footer); + moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content; + const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size); + } + } + + // Now set offsets for footers after `ni` inside of `parent_ni`. + { + var footer_oni = ni.next(mf); + while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) { + assert(footer_ni.position(mf) == .footer); + moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content; + const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size); + } + } + + if (moved_has_content) { + // We moved at least one footer containing initialized bytes, so we need to + // move the actual data. Compute how big the footers inside `ni` are... + const sub_footers_size: u64 = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + // `actual_new_size` is used here since we already updated the nested footers' offsets above. + break :size actual_new_size - first_sub_footer_offset; + }; + // ...and how big the footers *after* `ni`, inside `parent_ni`, are... + const post_footers_size: u64 = old_parent_size - (old_offset + old_size); + // ...and move them both. + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + const total_move_size = sub_footers_size + post_footers_size; + assert(total_move_size != 0); + try mf.moveRange( + parent_file_off + old_parent_size - total_move_size, + parent_file_off + new_parent_size - total_move_size, + total_move_size, + ); + } + }, + } }, } } @@ -1320,7 +1518,7 @@ fn growFloatingNodeWithAlignment( ni: Node.Index, new_alignment: ?Alignment, new_size: u64, - grow_mode: GrowMode, + grow_options: GrowOptions, ) Error!void { mf.nodes_lock.assertUnlocked(); @@ -1347,27 +1545,29 @@ fn growFloatingNodeWithAlignment( } // Great, we can grow this node without changing its offset or moving any siblings. try ni.setLocation(mf, gpa, old_offset, new_size); - // If we have any footers, we need to move them to the end of our new size, and update their - // offsets accordingly. - if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { - var cur_ni = first_footer_ni; - var footers_have_content = false; - while (true) { - footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; - const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); - try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); - cur_ni = cur_ni.next(mf).unwrap() orelse break; - } - if (footers_have_content) { - const parent_file_off = parent_ni.fileLocation(mf, false).offset; - // This gets the *new* offset because we already updated the offsets above. - const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); - const footers_size = new_size - new_footers_offset; - try mf.moveRange( - parent_file_off + old_offset + old_size - footers_size, - parent_file_off + old_offset + new_size - footers_size, - footers_size, - ); + if (grow_options.move_footers) { + // If we have any footers, we need to move them to the end of our new size, and update + // their offsets accordingly. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var cur_ni = first_footer_ni; + var footers_have_content = false; + while (true) { + footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + if (footers_have_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + // This gets the *new* offset because we already updated the offsets above. + const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = new_size - new_footers_offset; + try mf.moveRange( + parent_file_off + old_offset + old_size - footers_size, + parent_file_off + old_offset + new_size - footers_size, + footers_size, + ); + } } } return; @@ -1444,11 +1644,14 @@ fn growFloatingNodeWithAlignment( // that, let's first try the Linux "insert range" fast path. We didn't try it before now // because it would have been more efficient to just move ourselves into existing space. // - // If we were given a custom alignment, we cannot pass `grow_mode` directly into the + // If we were given a custom alignment, we need to set `GrowOptions.exact_size` for the // "insert range" path, because that function is unaware of `new_alignment`. - const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact; + const insert_range_grow_options: GrowOptions = .{ + .exact_size = grow_options.exact_size or new_alignment != null, + .move_footers = grow_options.move_footers, + }; if (alignment.check(old_offset) and - try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode)) + try mf.growNodeViaInsertRange(gpa, ni, new_size, insert_range_grow_options)) { // The Linux fast path did our job for us! return; @@ -1458,7 +1661,10 @@ fn growFloatingNodeWithAlignment( const new_parent_size = parent_ni.alignment(mf).forward( min_parent_size +| min_parent_size / growth_factor, ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); } break :new_loc .{ @@ -1471,6 +1677,10 @@ fn growFloatingNodeWithAlignment( // Footers need to move to a different place than the rest of our content. const footers_size: u64, const footers_have_content: bool = footers: { + if (!grow_options.move_footers) { + // Pretend there are no footers so as to not move them. + break :footers .{ 0, false }; + } const first_footer_ni = ni.firstFooter(mf).unwrap() orelse { break :footers .{ 0, false }; }; @@ -1525,15 +1735,14 @@ fn growFloatingNodeWithAlignment( /// If this strategy is inapplicable or unsuitable for this operation, this function returns `false` /// without changing any nodes' locations or invalidating any slices. /// -/// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node -/// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`, -/// the actual new size of `ni` may be greater than `new_size`. +/// Otherwise, this function grows `ni` to `new_size` (maybe larger if `!grow_options.exact_size`), +/// updates the location of `ni` and every node whose offset has changed, and returns `true`. fn growNodeViaInsertRange( mf: *MappedFile, gpa: Allocator, ni: Node.Index, new_size: u64, - grow_mode: GrowMode, + grow_options: GrowOptions, ) Error!bool { if (!is_linux or mf.flags.fallocate_insert_range_unsupported) { return false; @@ -1541,24 +1750,22 @@ fn growNodeViaInsertRange( _, const old_size = ni.location(mf).resolve(mf); - // We don't compute the size of the range yet, because depending on `grow_mode` we might want to - // bump it based on our sibling and parent nodes' alignments. However, we can do an early check - // for cases where we should obviously exit. + // We don't compute the size of the range yet, because depending on `grow_options` we might want + // to bump it based on our sibling and parent nodes' alignments. However, we can do an early + // check for cases where we should obviously exit. const min_range_size: u64 = s: { - const exact_size = new_size - old_size; - if (mf.flags.block_size.check(exact_size)) { - break :s exact_size; + const requested_size = new_size - old_size; + if (mf.flags.block_size.check(requested_size)) { + break :s requested_size; } - switch (grow_mode) { - .exact => return false, - .minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) { - // We're growing by at least a few blocks, so allow ourselves to bump the size - // slightly to give it the needed alignment. - break :s mf.flags.block_size.forward(exact_size); - } else { - return false; - }, + if (!grow_options.exact_size and + requested_size >= mf.flags.block_size.toByteUnits() * 2) + { + // We're growing by at least a few blocks, so allow ourselves to bump the size + // slightly to give it the needed alignment. + break :s mf.flags.block_size.forward(requested_size); } + return false; }; assert(min_range_size > 0); assert(mf.flags.block_size.check(min_range_size)); @@ -1573,14 +1780,17 @@ fn growNodeViaInsertRange( } break :range_file_offset range_file_offset; }; - const first_footer_oni = ni.firstFooter(mf); - const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| size: { + const pre_footer_oni: Node.Index.Optional, const footers_size: u64 = footers: { + if (!grow_options.move_footers) { + // Pretend there are no footers so as to not move them. + break :footers .{ .wrap(last_ni), 0 }; + } + const first_footer_ni = ni.firstFooter(mf).unwrap() orelse { + break :footers .{ .wrap(last_ni), 0 }; + }; const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); - break :size old_size - first_footer_offset; - } else 0; - const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) |first_footer_ni| pre_footer: { - break :pre_footer first_footer_ni.prev(mf); - } else .wrap(last_ni); + break :footers .{ first_footer_ni.prev(mf), old_size - first_footer_offset }; + }; const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: { const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf); break :end pre_footer_off + pre_footer_size; @@ -1634,18 +1844,14 @@ fn growNodeViaInsertRange( break :range_size min_range_size; } // Perhaps we're allowed to grow by more than `min_range_size`? - switch (grow_mode) { - .exact => return false, - .minimum => { - const candidate_range_size = need_range_align.forward(min_range_size); - // Allow growing by up to 50% more than was requested. - if (candidate_range_size <= min_range_size +| min_range_size / 2) { - break :range_size candidate_range_size; - } else { - return false; - } - }, + const candidate_range_size = need_range_align.forward(min_range_size); + if (!grow_options.exact_size and + // Allow growing by up to 50% more than was requested. + candidate_range_size <= min_range_size +| min_range_size / 2) + { + break :range_size candidate_range_size; } + return false; }; // This `range_size` is compatible with everyone's alignment requirements, and we won't move too @@ -1723,13 +1929,15 @@ fn growNodeViaInsertRange( cur_ni = cur_ni.parent(mf).unwrap() orelse break; } - // The only thing left is to update the offsets of any footers inside of `ni`. - if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { - var footer_ni = first_footer_ni; - while (true) { - const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); - try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size); - footer_ni = footer_ni.next(mf).unwrap() orelse break; + if (grow_options.move_footers) { + // The only thing left is to update the offsets of any footers inside of `ni`. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var footer_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size); + footer_ni = footer_ni.next(mf).unwrap() orelse break; + } } } @@ -1783,7 +1991,10 @@ fn ensureAdditionalHeaderCapacity( const new_parent_size = parent_ni.alignment(mf).forward( min_parent_size +| min_parent_size / growth_factor, ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); } return; }; @@ -1865,7 +2076,10 @@ fn ensureAdditionalHeaderCapacity( const new_parent_size = parent_ni.alignment(mf).forward( min_parent_size +| min_parent_size / growth_factor, ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); } if (moving_has_content) { @@ -1895,48 +2109,27 @@ fn ensureAdditionalHeaderCapacity( } } -/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current -/// footers, so that the footers can grow into that space. -fn ensureAdditionalFooterCapacity( - mf: *MappedFile, - gpa: Allocator, - parent_ni: Node.Index, - extra_capacity: u64, -) Error!void { - // This is way easier than the header case, because we don't need to actually move anything; we - // just need to expand the parent if there isn't space, and that will add padding after the - // parent's floating children, which is exactly where we want it. - +/// Returns how many padding bytes `parent_ni` currently has directly preceding its footers, which +/// footers can therefore grow into. +fn availableFooterCapacity(mf: *const MappedFile, parent_ni: Node.Index) u64 { const first_footer_oni = parent_ni.firstFooter(mf); - _, const parent_size = parent_ni.location(mf).resolve(mf); - - const footers_size: u64 = footers_size: { - const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; + const before_footers_oni: Node.Index.Optional, const footers_off: u64 = footers: { + const first_footer_ni = first_footer_oni.unwrap() orelse { + _, const parent_size = parent_ni.location(mf).resolve(mf); + break :footers .{ parent_ni.last(mf), parent_size }; + }; const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); - break :footers_size parent_size - first_footer_off; + break :footers .{ first_footer_ni.prev(mf), first_footer_off }; }; const header_and_floating_end: u64 = end: { - const before_footers_oni = if (first_footer_oni.unwrap()) |first_footer_ni| before_footers: { - break :before_footers first_footer_ni.prev(mf); - } else before_footers: { - break :before_footers parent_ni.last(mf); - }; const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0; const offset, const size = before_footers_ni.location(mf).resolve(mf); break :end offset + size; }; - assert(header_and_floating_end + footers_size <= parent_size); - - const min_parent_size = header_and_floating_end + footers_size + extra_capacity; - if (parent_size < min_parent_size) { - const new_parent_size = parent_ni.alignment(mf).forward( - min_parent_size +| min_parent_size / growth_factor, - ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); - } + return footers_off - header_and_floating_end; } fn removeNodesFromChildList( @@ -2030,7 +2223,7 @@ fn realignNode( mf: *MappedFile, gpa: Allocator, ni: Node.Index, - new_alignment: Alignment, + new_align: Alignment, ) Error!void { mf.nodes_lock.assertUnlocked(); @@ -2038,30 +2231,25 @@ fn realignNode( if (ni == .root or ni.position(mf) != .floating) { // Only this node's size is aligned, not its offset. - if (!new_alignment.check(old_size)) { - assert(new_alignment.compare(.gt, ni.alignment(mf))); - try mf.growNode( - gpa, - ni, - new_alignment.forward(old_size), - .exact, // because `growNode` is not aware that the size needs to match `new_alignment` - ); + if (!new_align.check(old_size)) { + assert(new_align.compare(.gt, ni.alignment(mf))); + try mf.growNode(gpa, ni, new_align.forward(old_size), .{ + .exact_size = true, // because `growNode` is not aware that the size needs to match `new_align` + .move_footers = true, + }); } } else { // This is a floating node, so its size and offset are both aligned. - if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) { - assert(new_alignment.compare(.gt, ni.alignment(mf))); - try mf.growFloatingNodeWithAlignment( - gpa, - ni, - new_alignment, - new_alignment.forward(old_size), - .minimum, - ); + if (!new_align.check(old_offset) or !new_align.check(old_size)) { + assert(new_align.compare(.gt, ni.alignment(mf))); + try mf.growFloatingNodeWithAlignment(gpa, ni, new_align, new_align.forward(old_size), .{ + .exact_size = false, + .move_footers = true, + }); } } - ni.get(mf).flags.alignment = new_alignment; + ni.get(mf).flags.alignment = new_align; } fn updateWriters(mf: *MappedFile) void { -- 2.54.0