diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 8ef617fad2e1cc2d4c7a67412a714502d4e8e2f6..4e0bdee194736cdff8716ab0ebdf764637a76b4a 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -24,6 +24,7 @@ base: link.File, options: link.File.OpenOptions, mf: MappedFile, ni: Node.Known, +archive: ?Archive, nodes: std.MultiArrayList(Node), /// Does not contain an item for `SHN_UNDEF`. shdrs: std.ArrayList(Section), @@ -200,19 +201,39 @@ input_prog_node: std.Progress.Node, const Error = link.Error || error{MappedFileIo}; const Node = union(enum) { + /// Only used when emitting a static library. + /// + /// Contains a header node which is an `.archive_header`. + /// + /// Contains the following footer nodes: + /// * One `.archive_input_member` for each external input in the archive + /// * One `.archive_elf_member_header` containing the `ar_hdr` for the ZCU + /// * One `.elf` containing the ZCU's actual ELF object + /// + /// Padding between the headers and footers is absorbed into the "//" member (whose actual + /// content is in the `.archive_header` node). archive, - /// This includes the archive magic and long file member. + /// Only used when emitting a static library. + /// + /// Contains the archive magic (`ARMAG`), as well as the `ar_hdr` and content for the long file + /// name string table member ("//"). archive_header, - /// This is a footer of the `.elf` node, and contains the next archive entry's file header. - archive_elf_footer, + /// Only used when emitting a static library. + /// + /// Contains the `ar_hdr` and content for one non-ZCU archive member (external link input). Also + /// includes the single byte '\n' padding at the end of this archive member, if necessary. + archive_input_member: InputIndex, + /// Only used when emitting a static library. + /// + /// Contains the `ar_hdr` for the `.elf` node. + archive_elf_member_header, + elf, ehdr, shdr, segment: u32, /// The section '.plt' may contain relocations via `elf.plt_first_symbol_reloc`. section: Section.Index, - /// Only valid for static libraries, represents one non-zcu archive member. - input_member: InputIndex, /// May contain relocations. input_section: InputSection.Index, /// Value is the name of a global which has an entry in `elf.copied_globals`, so, a global for @@ -404,6 +425,15 @@ const InputSection = struct { }; }; +const Archive = struct { + ni: MappedFile.Node.Index, + header_ni: MappedFile.Node.Index, + elf_member_header_ni: MappedFile.Node.Index, + + elf_member_too_big: bool, + strtab_member_too_big: bool, +}; + const Section = struct { /// The node corresponding to this section. ni: MappedFile.Node.Index, @@ -2954,13 +2984,13 @@ pub fn symbolForAtom(elf: *Elf, atom: link.File.AtomId) link.File.SymbolId { const lsi: Symbol.LocalIndex = switch (elf.getNode(Node.fromAtom(atom))) { .archive, .archive_header, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, .elf, .ehdr, .shdr, .segment, .section, - .input_member, .input_section, .copied_global, => unreachable, @@ -3364,6 +3394,7 @@ fn create( .data_rel_ro = undefined, .tls = .none, }, + .archive = null, .nodes = .empty, .shdrs = .empty, .phdrs = .empty, @@ -3558,51 +3589,58 @@ fn initHeaders( .EXEC, .DYN => {}, } var phnum: u32 = 0; - break :ph .{ .{ - .phdr = phndx: { - defer phnum += 1; - break :phndx phnum; + break :ph .{ + .{ + .phdr = phndx: { + defer phnum += 1; + break :phndx phnum; + }, + .interp = if (maybe_interp) |_| phndx: { + defer phnum += 1; + break :phndx phnum; + } else undefined, + .rodata = phndx: { + defer phnum += 1; + break :phndx phnum; + }, + .text = phndx: { + defer phnum += 1; + break :phndx phnum; + }, + .plt = if (plt.got_plt == null) phndx: { + defer phnum += 1; + break :phndx phnum; + } else undefined, + // `data` must be assigned after all other loadable segments so that it has the greatest + // phndx of any loadable segment. This is so that `targetSegmentLoadAddressRestrictions` + // can be obeyed (specifically, the `.data_last` restriction, needed on SPARC). + .data = phndx: { + defer phnum += 1; + break :phndx phnum; + }, + .tls = if (comp.config.any_non_single_threaded) phndx: { + defer phnum += 1; + break :phndx phnum; + } else undefined, + .dynamic = if (have_dynamic_section) phndx: { + defer phnum += 1; + break :phndx phnum; + } else undefined, + .relro = phndx: { + defer phnum += 1; + break :phndx phnum; + }, + .gnu_stack = phndx: { + defer phnum += 1; + break :phndx phnum; + }, }, - .interp = if (maybe_interp) |_| phndx: { - defer phnum += 1; - break :phndx phnum; - } else undefined, - .rodata = phndx: { - defer phnum += 1; - break :phndx phnum; - }, - .text = phndx: { - defer phnum += 1; - break :phndx phnum; - }, - .data = phndx: { - defer phnum += 1; - break :phndx phnum; - }, - .plt = if (plt.got_plt == null) phndx: { - defer phnum += 1; - break :phndx phnum; - } else undefined, - .tls = if (comp.config.any_non_single_threaded) phndx: { - defer phnum += 1; - break :phndx phnum; - } else undefined, - .dynamic = if (have_dynamic_section) phndx: { - defer phnum += 1; - break :phndx phnum; - } else undefined, - .relro = phndx: { - defer phnum += 1; - break :phndx phnum; - }, - .gnu_stack = phndx: { - defer phnum += 1; - break :phndx phnum; - }, - }, phnum }; + // (I don't actually want the trailing comma below, but a `zig fmt` bug forces it.) + phnum, + }; }; - const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_footer + const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_member_header 3 + // `.elf`, `.ehdr`, and `.shdr` nodes (shnum - 1) + // -1 because the SHN_UNDEF shdr does not have a `.section` node (phnum -| 1); // -1 because the GNU_STACK phdr does not have a `.segment` node @@ -3619,11 +3657,14 @@ fn initHeaders( const archive_ni: MappedFile.Node.Index = .root; const archive_header_ni = try archive_ni.addOnlyHeaderChild(&elf.mf, gpa, .{ - .size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr) * 2, - .alignment = .@"2", - .next_moved = true, - .bubbles_moved = false, + // We intentionally do not set `.alignment = .@"2"` here, because the string table data + // in this node does not need to have an aligned length. (This node's offset is aligned + // regardless by virtue of it being a header.) + .size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr), + // The archive header uses 'next_moved' events to resize the "//" member, so that it + // absorbs all padding between `archive_header_ni` and the actual object file members. .enable_next_moved = true, + .next_moved = true, }); elf.nodes.appendAssumeCapacity(.archive_header); const archive_header_slice = archive_header_ni.slice(&elf.mf); @@ -3635,23 +3676,47 @@ fn initHeaders( .ar_uid = @splat(' '), .ar_gid = @splat(' '), .ar_mode = @splat(' '), - .ar_size = @splat(' '), + .ar_size = undefined, // populated by `flushNextMoved` for `archive_header_ni` .ar_fmag = std.elf.ARFMAG.*, }; - elf.ni.elf = try archive_ni.addFloatingChild(&elf.mf, gpa, .{ + elf.ni.elf = try archive_ni.addOnlyFooterChild(&elf.mf, gpa, .{ .alignment = node_block_align.max(.@"2"), - .next_moved = true, .bubbles_moved = false, - .enable_next_moved = true, + .resized = true, // ensure that this node's `ar_hdr.ar_size` is updated at least once }); elf.nodes.appendAssumeCapacity(.elf); - _ = try elf.ni.elf.addOnlyFooterChild(&elf.mf, gpa, .{ + const elf_ar_hdr_ni = try archive_ni.addFooterChildBefore(&elf.mf, gpa, .wrap(elf.ni.elf), .{ .alignment = .@"2", .size = @sizeOf(std.elf.ar_hdr), }); - elf.nodes.appendAssumeCapacity(.archive_elf_footer); + elf.nodes.appendAssumeCapacity(.archive_elf_member_header); + + // Must be populated before we call `populateArchiveMemberName` below. + elf.archive = .{ + .ni = archive_ni, + .header_ni = archive_header_ni, + .elf_member_header_ni = elf_ar_hdr_ni, + + .elf_member_too_big = false, + .strtab_member_too_big = false, + }; + + const elf_ar_hdr: *std.elf.ar_hdr = @ptrCast(elf_ar_hdr_ni.slice(&elf.mf)); + elf_ar_hdr.* = .{ + .ar_name = undefined, // populated below + .ar_date = "0 ".*, + .ar_uid = "0 ".*, + .ar_gid = "0 ".*, + .ar_mode = "644 ".*, + .ar_size = undefined, // populated by `flushResized` for the `.elf` node + .ar_fmag = std.elf.ARFMAG.*, + }; + const zcu_member_name = try std.fmt.allocPrint(gpa, "{s}_zcu.o", .{comp.root_name}); + defer gpa.free(zcu_member_name); + // After this call returns, `elf_ar_hdr` is invalidated. + try elf.populateArchiveMemberName(elf_ar_hdr, zcu_member_name); } else { elf.ni.elf = .root; elf.nodes.appendAssumeCapacity(.elf); @@ -4092,33 +4157,44 @@ fn initHeaders( .addralign = addr_align, .entsize = @intCast(addr_align.toByteUnits()), }); - if (plt.got_plt) |got_plt| { - const got_plt_segment_ni = if (elf.options.z_now) elf.ni.data_rel_ro else elf.ni.data; - elf.shndx.got_plt = try elf.addSection(got_plt_segment_ni, .{ - .name = ".got.plt", - .type = .PROGBITS, - .flags = .{ .WRITE = true, .ALLOC = true }, - .size = got_plt.header_entries * elf.targetPtrSize(), - .addralign = addr_align, - .entsize = @intCast(addr_align.toByteUnits()), - }); - elf.shndx.plt = try elf.addSection(elf.ni.text, .{ - .name = ".plt", - .type = .PROGBITS, - .flags = .{ .ALLOC = true, .EXECINSTR = true }, - .size = plt.entry_size * plt.header_entries, - .addralign = plt.@"align", - .node_align = node_block_align, - }); - } else { - elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{ - .name = ".plt", - .type = .PROGBITS, - .flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true }, - .size = plt.entry_size * plt.header_entries, - .addralign = plt.@"align", - .node_align = node_block_align, - }); + { + const init_plt_size = plt.entry_size * plt.header_entries; + if (plt.got_plt) |got_plt| { + const got_plt_segment_ni = if (elf.options.z_now) elf.ni.data_rel_ro else elf.ni.data; + elf.shndx.got_plt = try elf.addSection(got_plt_segment_ni, .{ + .name = ".got.plt", + .type = .PROGBITS, + .flags = .{ .WRITE = true, .ALLOC = true }, + .size = got_plt.header_entries * elf.targetPtrSize(), + .addralign = addr_align, + .entsize = @intCast(addr_align.toByteUnits()), + }); + elf.shndx.plt = try elf.addSection(elf.ni.text, .{ + .name = ".plt", + .type = .PROGBITS, + .flags = .{ .ALLOC = true, .EXECINSTR = true }, + .size = plt.@"align".forward(init_plt_size), + .addralign = plt.@"align", + .node_align = node_block_align, + }); + } else { + elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{ + .name = ".plt", + .type = .PROGBITS, + .flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true }, + .size = plt.@"align".forward(init_plt_size), + .addralign = plt.@"align", + .node_align = node_block_align, + }); + } + // And the award for most annoying PLT requirement goes to SPARC, which decided that the + // whole table should have a greater alignment than the size of the individual entries, + // hence this bullshit: + if (plt.@"align".forward(init_plt_size) != init_plt_size) { + switch (elf.shdrPtr(elf.shndx.plt)) { + inline else => |shdr| elf.targetStore(&shdr.size, init_plt_size), + } + } } if (plt.plt_sec != null) elf.shndx.plt_sec = try elf.addSection(elf.ni.text, .{ .name = ".plt.sec", @@ -4358,6 +4434,12 @@ fn initHeaders( assert(elf.targetLoad(&shdr.size) == elf.got.count() * @sizeOf(Addr)); }, } + if (elf.shndx.dynamic != .UNDEF) { + try elf.shndx.rela_dyn.relaEnsureAdditionalCapacity(elf, elf.got.count()); + } + for (0..elf.got.count()) |got_index| { + elf.updateGotEntry(got_index); + } // Create any always-provided linker-defined symbols. The symbols marking the `INIT_ARRAY`/ // `FINI_ARRAY`/`PREINIT_ARRAY` sections are instead created by `createInitFiniArraySection` @@ -4539,6 +4621,22 @@ fn initHeaders( break :str try elf.string(.dynstr, slice); }, }; + + if (@"type" != .REL) switch (elf.targetSegmentLoadAddressRestrictions()) { + .none => {}, + .data_last => switch (elf.phdrSlice()) { + inline else => |phdr| { + // Ensure that the segment after `.data` (if any) is not a loadable segment. + const next_phndx = phndx.data + 1; + if (next_phndx < phdr.len) { + switch (elf.targetLoad(&phdr[next_phndx].type)) { + .NULL, .LOAD => unreachable, // data segment should be the last loadable segment + else => {}, + } + } + }, + }, + }; } pub fn startProgress(elf: *Elf, prog_node: std.Progress.Node) void { @@ -4573,12 +4671,12 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index { return switch (elf.getNode(ni)) { .archive, .archive_header, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, .elf, .ehdr, .shdr, .segment, - .input_member, => unreachable, .section => |shndx| shndx, .input_section, @@ -4594,12 +4692,12 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { return switch (elf.getNode(ni)) { .archive, .archive_header, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, .elf, .ehdr, .shdr, .segment, - .input_member, .copied_global, => unreachable, .section => |shndx| shndx.vaddr(elf), @@ -4613,14 +4711,18 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { } fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) { - .archive, .archive_header, .archive_elf_footer => unreachable, + .archive, + .archive_header, + .archive_input_member, + .archive_elf_member_header, + => unreachable, .elf => return 0, .ehdr, .shdr => unreachable, .segment => |phndx| switch (elf.phdrSlice()) { inline else => |phdr| elf.targetLoad(&phdr[phndx].vaddr), }, .section => |shndx| if (shndx == elf.shndx.tdata) 0 else shndx.vaddr(elf), - .input_member, .input_section, .copied_global => unreachable, + .input_section, .copied_global => unreachable, inline .nav, .uav, .lazy_code, .lazy_const_data => |i| Symbol.Id.local(i.symbol(elf)).value(elf), }; const offset, _ = ni.location(&elf.mf).resolve(&elf.mf); @@ -4639,12 +4741,12 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void { const symbol_relocs: *SymbolReloc.Index, const got_relocs: ?*GotReloc.Index = switch (elf.getNode(ni)) { .archive, .archive_header, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, .elf, .ehdr, .shdr, .segment, - .input_member, .copied_global, => unreachable, // cannot contain relocs .section => unreachable, // cannot contain relocs (.plt and .dynamic unsupported) @@ -4705,7 +4807,12 @@ fn flushMovedNodeRelocs( // changed, so update the `offset` field of the `ElfN.Rela` entry. reloc.relaSection(elf).relaSetOffset(elf, rela_index, node_vaddr + reloc.offset); } - reloc.apply(elf); + // This is not just the inverse of the above condition, because if `reloc` is relative + // to the base of this DSO, then `rela_index` is an `R_*_RELATIVE` relocation, but we + // still need to call `SymbolReloc.apply` to update that relocation's addend. + if (elf.ehdrType() != .REL) { + reloc.apply(elf); + } } } @@ -4888,7 +4995,31 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo { // TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`. }; } -pub fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { +/// Specifies any restrictions the current target has regarding how segments are ordered in the +/// virtual address space. Most targets do not have any such restrictions. +fn targetSegmentLoadAddressRestrictions(elf: *const Elf) enum { + none, + /// The "mutable data" segment must be the last loadable segment in the virtual address space. + data_last, +} { + return switch (elf.ehdrMachine()) { + .AARCH64, + .PPC64, + .RISCV, + .X86_64, + .LOONGARCH, + => .none, + + // SPARC uses `R_SPARC_PC{10,22}` relocations to construct pointers to the GOT, but these + // relocations write an *unsigned* PC-relative offset. This cannot even be worked around by + // using a larger code model, because the crt `_start` assembly always uses these specific + // relocations. Therefore, to avoid relocation errors, all code must appear before the GOT + // in the virtual address space. The easiest way for us to do that is to ensure that the + // "mutable data" segment, containing the GOT, is the last segment in the address space. + .SPARCV9 => .data_last, + }; +} +fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer; const Child = pointer_ty.child; const alignment = pointer_ty.attrs.@"align" orelse @alignOf(Child); @@ -4971,16 +5102,6 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr { } } -fn arHdrPtr(elf: *Elf, ni: MappedFile.Node.Index) *align(2) std.elf.ar_hdr { - assert(elf.ni.elf != .root); - const file_offset = ni.fileLocation(&elf.mf, false).offset; - return @ptrCast(@alignCast(elf.mf.memory_map.memory[@intCast(switch (elf.getNode(ni)) { - else => unreachable, - .archive_header => file_offset + std.elf.ARMAG.len, - .elf, .input_member => file_offset - @sizeOf(std.elf.ar_hdr), - })..][0..@sizeOf(std.elf.ar_hdr)])); -} - const SymPtr = union(std.elf.CLASS) { NONE: noreturn, @"32": *std.elf.Elf32.Sym, @@ -5480,19 +5601,61 @@ fn loadObject( .member = if (member) |m| try gpa.dupe(u8, m) else null, .extra = undefined, }; - if (elf.ni.elf != .root) { - const archive_ni: MappedFile.Node.Index = .root; + if (elf.archive) |*archive| { + // We're creating a static library, so just add this input as an archive member. + assert(member == null); // don't try to put static library members into other static libraries + + const first_member_oni = archive.header_ni.next(&elf.mf); + + if (first_member_oni.unwrap()) |first_member_ni| switch (elf.getNode(first_member_ni)) { + .archive_input_member, .archive_elf_member_header => {}, + .elf => unreachable, // always preceded by `.archive_elf_member_header` + else => unreachable, // never a child of `.archive` + }; + try elf.nodes.ensureUnusedCapacity(gpa, 1); - input.extra = .{ .node = try archive_ni.addFloatingChild(&elf.mf, gpa, .{ - .size = Alignment.@"2".forward(fl.size + @sizeOf(std.elf.ar_hdr)), + const new_member_ni = try archive.ni.addFooterChildBefore(&elf.mf, gpa, first_member_oni, .{ + .size = Alignment.@"2".forward(@sizeOf(std.elf.ar_hdr) + fl.size), .alignment = .@"2", - .next_moved = true, - .bubbles_moved = false, - .enable_next_moved = true, - }) }; - elf.nodes.appendAssumeCapacity(.{ .input_member = input_index }); + }); + elf.nodes.appendAssumeCapacity(.{ .archive_input_member = input_index }); + input.extra = .{ .node = new_member_ni }; elf.input_prog_node.increaseEstimatedTotalItems(1); + // The contents of the input will be written to the file by an idle task (`flushInput`), but + // we do need to write the input's archive member header (`ar_hdr`) now, for two reasons: + // + // * If the input file has a long name, we need to add it to the archive member name string + // table, which must happen deterministically (i.e. not in an idle task). + // + // * `flushInput` needs to know the actual file size (before padding to the alignment). + const member_ar_hdr: *std.elf.ar_hdr = @ptrCast( + new_member_ni.slice(&elf.mf)[0..@sizeOf(std.elf.ar_hdr)], + ); + member_ar_hdr.* = .{ + .ar_name = undefined, // populated below + .ar_date = "0 ".*, + .ar_uid = "0 ".*, + .ar_gid = "0 ".*, + .ar_mode = "644 ".*, + .ar_size = undefined, // populated below + .ar_fmag = std.elf.ARFMAG.*, + }; + + if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{fl.size})) |size_str| { + @memset(member_ar_hdr.ar_size[size_str.len..], ' '); + } else |err| switch (err) { + error.NoSpaceLeft => return diags.failParse( + path, + "file size of {Bi} exceeds maximum size of archive member", + .{fl.size}, + ), + } + + const member_name = std.fs.path.basename(path.sub_path); + // After this call returns, `member_ar_hdr` is invalidated. + try elf.populateArchiveMemberName(member_ar_hdr, member_name); + // Since we are not emitting the archive symbol table (yet?) we do not need to parse // the symbols in this input. return; @@ -5905,6 +6068,46 @@ fn loadObject( }, } } +/// This function may resize the archive header, so therefore invalidates `member_ar_hdr`. +fn populateArchiveMemberName(elf: *Elf, member_ar_hdr: *std.elf.ar_hdr, member_name: []const u8) Error!void { + if (std.mem.print(&member_ar_hdr.ar_name, "{s}/", .{member_name})) |name_str| { + @memset(member_ar_hdr.ar_name[name_str.len..], ' '); + return; + } else |err| switch (err) { + error.NoSpaceLeft => {}, // handled below + } + + const gpa = elf.base.comp.gpa; + const archive_header_ni = elf.archive.?.header_ni; + + // The member's name is too big to put directly in the `ar_name` field, so it needs to go in the + // "long name" string table instead (in the special member named "//"). + + _, const old_archive_header_size = archive_header_ni.location(&elf.mf).resolve(&elf.mf); + + // We're going to add a new string at the end of the table. Update `member_ar_hdr` first, + // because resizing the string table will invalidate it. + const string_table_offset = old_archive_header_size - (std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr)); + if (std.mem.print(&member_ar_hdr.ar_name, "/{d}", .{string_table_offset})) |name_str| { + @memset(member_ar_hdr.ar_name[name_str.len..], ' '); + } else |inner_err| switch (inner_err) { + error.NoSpaceLeft => { + // The string table offset is itself too big to represent. This means the string table's + // *size* is definitely too big to represent (we only get 10 bytes for that whereas we + // get 16 here!), so as long as we still add the string, we're guaranteed to get a link + // error for that reason. Therefore, we can just ignore this error and carry on. + }, + } + + // We set the size of the archive header node exactly, because we want padding bytes to go into + // the root `.archive` node. That way, those bytes could still be used to grow the string table + // if necessary, but they could also be used for new archive members. + try archive_header_ni.resizeLeaf(&elf.mf, gpa, old_archive_header_size + member_name.len + 2); + + const dest_slice = archive_header_ni.slice(&elf.mf)[@intCast(old_archive_header_size)..]; + @memcpy(dest_slice[0 .. dest_slice.len - 2], member_name); + @memcpy(dest_slice[dest_slice.len - 2 ..], "/\n"); // yes, the terminator is weird +} fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadParseInputError || error{BadMagic})!void { const comp = elf.base.comp; const gpa = comp.gpa; @@ -7072,12 +7275,12 @@ fn addGotRelocAssumeCapacity( switch (elf.getNode(node)) { .archive, .archive_header, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, .elf, .ehdr, .shdr, .segment, - .input_member, .copied_global, => unreachable, // cannot contain relocs, .section, @@ -7129,6 +7332,7 @@ fn addGotRelocAssumeCapacity( }); } fn updateGotEntry(elf: *Elf, got_index: usize) void { + assert(elf.ehdrType() != .REL); const entry_value: union(enum) { unsigned: u64, signed: i64, @@ -7514,6 +7718,17 @@ fn flushInner( diags.addError("failed to apply {d} relocations: misaligned value", .{elf.misaligned_reloc_count}); } + if (elf.archive) |*archive| { + if (archive.elf_member_too_big) diags.addError( + "file size of {Bi} exceeds maximum size of archive member", + .{elf.ni.elf.location(&elf.mf).resolve(&elf.mf)[1]}, + ); + if (archive.strtab_member_too_big) diags.addError( + "archive file name string table exceeds maximum size", + .{}, + ); + } + elf.flushDynamic(); const entry_addr: u64 = entry: { @@ -7545,6 +7760,9 @@ pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) link.Error!bool { const comp = elf.base.comp; const diags = &comp.link_diags; + elf.mf.nodes_lock.lock(); + defer elf.mf.nodes_lock.unlock(); + assert(elf.pending_uavs.items.len == 0); for (&elf.lazy.values) |*lazy| { assert(lazy.pending_index == lazy.map.count()); @@ -7698,7 +7916,7 @@ fn idleProgNode( return prog_node.start(name: switch (node) { else => |tag| @tagName(tag), .section => |shndx| shndx.name(elf).slice(elf), - .input_member => |ii| std.fmt.bufPrint(&name, "{f}{f}", .{ + .archive_input_member => |ii| std.fmt.bufPrint(&name, "{f}{f}", .{ ii.path(elf).fmtEscapeString(), fmtMemberString(ii.member(elf)), }) catch &name, @@ -7825,7 +8043,6 @@ fn genLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void { fn flushInput(elf: *Elf, ii: Node.InputIndex) Error!void { const comp = elf.base.comp; const io = comp.io; - const gpa = comp.gpa; const diags = &comp.link_diags; const path = ii.path(elf); const file = path.root_dir.handle.openFile(io, path.sub_path, .{}) catch |err| switch (err) { @@ -7833,23 +8050,40 @@ fn flushInput(elf: *Elf, ii: Node.InputIndex) Error!void { else => |e| return diags.fail("failed to open input file \"{f}\": {t}", .{ path.fmtEscapeString(), e }), }; defer file.close(io); + + const slice = ii.node(elf).slice(&elf.mf); + + const member_ar_hdr: *const std.elf.ar_hdr = @ptrCast(slice[0..@sizeOf(std.elf.ar_hdr)]); + const input_size: u32 = member_ar_hdr.size() catch |err| switch (err) { + // We wrote the `ar_hdr` ourselves (in `loadObject`), so it is definitely valid. + error.Overflow, error.InvalidCharacter => unreachable, + }; + + switch (slice.len - @sizeOf(std.elf.ar_hdr) - input_size) { + 0 => {}, + 1 => { + // Alignment added one padding byte, which the format requires to have value '\n'. + slice[slice.len - 1] = '\n'; + }, + else => unreachable, // node size should agree with the value we wrote into `ar_hdr.ar_size` + } + var fr = file.reader(io, &.{}); - var nw: MappedFile.Node.Writer = undefined; - ii.node(elf).writer(&elf.mf, gpa, &nw); - defer nw.deinit(); - const size = nw.interface.buffer.len - @sizeOf(std.elf.ar_hdr); - const n_bytes = nw.interface.sendFileAll(&fr, .limited(size)) catch |err| switch (err) { + var w: Io.Writer = .fixed(slice[@sizeOf(std.elf.ar_hdr)..]); + const n_bytes_read = w.sendFileAll(&fr, .limited(input_size)) catch |err| switch (err) { error.ReadFailed => return diags.fail("failed to read input \"{f}{f}\": {t}", .{ path.fmtEscapeString(), fmtMemberString(ii.member(elf)), fr.err orelse (fr.seek_err orelse fr.size_err.?), }), - error.WriteFailed => return nw.err.?, + error.WriteFailed => unreachable, // `.limited(input_size)` prevents us writing too many bytes }; - if (n_bytes + 1 < size) return diags.fail("failed to read input \"{f}{f}\": unexpected eof", .{ - path.fmtEscapeString(), - fmtMemberString(ii.member(elf)), - }); + if (n_bytes_read != input_size) { + return diags.fail("failed to load input \"{f}{f}\": file truncated during compilation", .{ + path.fmtEscapeString(), + fmtMemberString(ii.member(elf)), + }); + } } fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void { @@ -7931,12 +8165,18 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void const trace = tracy.trace(@src()); defer trace.end(); - elf.mf.nodes_lock.lock(); - defer elf.mf.nodes_lock.unlock(); - switch (elf.getNode(ni)) { - .archive, .archive_header => unreachable, - .archive_elf_footer, .elf => {}, + .archive => unreachable, + .archive_header => unreachable, + + .archive_input_member, + .archive_elf_member_header, + .elf, + => { + assert(elf.archive != null); + return; + }, + .ehdr, .shdr => elf.flushElfOffset(ni), .segment => |phndx| { elf.flushElfOffset(ni); @@ -8014,7 +8254,6 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void elf.flushMovedPltSection(.plt_sec, old_addr, addr); } }, - .input_member => {}, .input_section => |isi| { const old_section_addr = isi.ptr(elf).vaddr; const new_section_addr = elf.computeNodeVAddr(ni); @@ -8114,6 +8353,18 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro const page_align = elf.targetPageAlign(); const node_align = segment_ni.alignment(&elf.mf); const ph_align = page_align.max(node_align); + + // If we determine that the segment's virtual address needs to move, then it's a good idea to + // make it less likely that it needs to move *again* in the future, because it is expensive to + // change a segment's load address (a lot of re-flushing is necessary). To do that, we reserve + // more virtual address space than we need (multiplying the actual size by this value). That + // way, there will usually be padding between segments which they can grow into. + // + // TODO: we might want to decrease this multiplier, or even omit it entirely, in cases where + // virtual address space is constrained. For instance, 32-bit targets, or targets where short + // PC-relative relocations between segments are common. + const reserve_size_multiplier = 4; + switch (elf.phdrSlice()) { inline else => |phdr| { const offset = elf.targetLoad(&phdr[orig_phndx].offset); @@ -8172,15 +8423,46 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro // backwards to the start of the page. const next_page_vaddr = std.mem.alignBackward(u64, next_vaddr, page_align.toByteUnits()); - // If we're at the same vaddr we started at, then all we're worried about is the - // segment fitting here. However, if we've already changed our virtual address, then - // we might as well try to reserve a bit *more* virtual address space while we're at - // it, because changing virtual address is quite disruptive (we need to re-flush a - // lot of stuff!) and giving ourselves more space will make it less likely to happen - // again. - const target_size = if (vaddr == orig_vaddr) size else size * 4; - if (vaddr + target_size <= next_page_vaddr) { - break; // hooray, we fit here! + // Check if the segment fits here. We apply `reserve_size_multiplier`, but only if + // the segment is already known to be moving---making it easier to grow in-place is + // the whole point of the multiplier! + { + const target_size = if (vaddr == orig_vaddr) size else size * reserve_size_multiplier; + if (vaddr + target_size <= next_page_vaddr) { + break; // hooray, we fit here! + } + } + + const next_ni = elf.phdrs.items[next_phndx].unwrap().?; + + // This segment don't fit here, but before deciding how to proceed, we need to + // consider any target-specific restrictions we are subject to. + switch (elf.targetSegmentLoadAddressRestrictions()) { + .none => {}, + .data_last => if (next_ni == elf.ni.data) { + // We can't leapfrog over the data segment. Instead, that segment just needs + // to be shifted forwards to make space for us, and we'll then `break` with + // our current vaddr. + + if (next_phndx + 1 < phdr.len) switch (elf.targetLoad(&phdr[next_phndx + 1].type)) { + .NULL, .LOAD => unreachable, // data segment should be the last loadable segment + else => {}, + }; + + const free_vaddr = vaddr + size * reserve_size_multiplier; + + const next_align = page_align.max(next_ni.alignment(&elf.mf)); + const next_offset = elf.targetLoad(&next_ph.offset); + const next_new_vaddr = next_align.forward(free_vaddr) + next_offset % next_align.toByteUnits(); + + // This logic for updating the data segment's vaddr is identical to how we + // will update the vaddr of `phndx` when we break from the loop. + elf.targetStore(&next_ph.vaddr, @intCast(next_new_vaddr)); + elf.targetStore(&next_ph.paddr, @intCast(next_new_vaddr)); + try next_ni.childrenMoved(elf.base.comp.gpa, &elf.mf); + + break; + }, } // We don't fit here, so shift ourselves forward (i.e. swap with `next_phndx`). But @@ -8192,8 +8474,7 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro // Now just swap the phdrs and update our `phndx`. std.mem.swap(@TypeOf(next_ph.*), &phdr[phndx], next_ph); - const next_ni = elf.phdrs.items[next_phndx]; - elf.phdrs.items[phndx] = next_ni; + elf.phdrs.items[phndx] = .wrap(next_ni); elf.nodes.items(.data)[@backingInt(next_ni)] = .{ .segment = phndx }; elf.phdrs.items[next_phndx] = .wrap(segment_ni); elf.nodes.items(.data)[@backingInt(segment_ni)] = .{ .segment = @intCast(next_phndx) }; @@ -8213,24 +8494,23 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo const trace = tracy.trace(@src()); defer trace.end(); - elf.mf.nodes_lock.lock(); - defer elf.mf.nodes_lock.unlock(); - _, const size = ni.location(&elf.mf).resolve(&elf.mf); switch (elf.getNode(ni)) { - .archive => { - if (ni.last(&elf.mf).unwrap()) |last_ni| { - if (last_ni.prev(&elf.mf).unwrap()) |prev_ni| { - if (prev_ni.hasNextMoved(&elf.mf)) return; - } - const offset, _ = last_ni.location(&elf.mf).resolve(&elf.mf); - _ = std.mem.print(&elf.arHdrPtr(last_ni).ar_size, "{d:<10}", .{ - size - offset, - }) catch @panic("archive member too large"); + .archive, .archive_header => {}, + .archive_input_member => unreachable, + .archive_elf_member_header => unreachable, + .elf => if (elf.archive) |*archive| { + const member_ar_hdr: *std.elf.ar_hdr = @ptrCast( + archive.elf_member_header_ni.slice(&elf.mf), + ); + if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{size})) |size_str| { + @memset(member_ar_hdr.ar_size[size_str.len..], ' '); + archive.elf_member_too_big = false; + } else |err| switch (err) { + error.NoSpaceLeft => archive.elf_member_too_big = true, } }, - .archive_header, .elf => {}, - .ehdr, .archive_elf_footer => unreachable, + .ehdr => unreachable, .shdr => {}, .segment => |phndx| switch (elf.phdrSlice()) { inline else => |phdr| { @@ -8302,7 +8582,7 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo } }, }, - .input_member, .input_section, .copied_global, .nav, .uav, .lazy_code, .lazy_const_data => {}, + .input_section, .copied_global, .nav, .uav, .lazy_code, .lazy_const_data => {}, } } @@ -8310,12 +8590,11 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error! const trace = tracy.trace(@src()); defer trace.end(); - elf.mf.nodes_lock.lock(); - defer elf.mf.nodes_lock.unlock(); - switch (elf.getNode(ni)) { .archive, - .archive_elf_footer, + .archive_input_member, + .archive_elf_member_header, + .elf, .ehdr, .shdr, .segment, @@ -8327,54 +8606,32 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error! .lazy_code, .lazy_const_data, => unreachable, - .archive_header, .elf, .input_member => |_, tag| { - const member_offset, const update_size = member_offset: { - const offset, _ = ni.location(&elf.mf).resolve(&elf.mf); - break :member_offset switch (tag) { - else => unreachable, - .archive_header => .{ offset + std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr), true }, - .elf, .input_member => .{ offset, !ni.prev(&elf.mf).unwrap().?.hasNextMoved(&elf.mf) }, - }; + + .archive_header => { + const archive = &elf.archive.?; + + // Because we can't just throw padding bytes in the middle of an archive file, we need + // the member name string table (the "//" member) to absorb all the padding bytes + // between it (in the `.archive_header` node) and the first actual member. + const next_member_ni = ni.next(&elf.mf).unwrap() orelse { + // I guess there are no link inputs yet? But there will be eventually! + return; }; - const member_size = if (ni.next(&elf.mf).unwrap()) |next_ni| member_size: { - const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf); - const next_member_size = if (next_ni.next(&elf.mf).unwrap()) |next_next_ni| next_member_size: { - const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf); - const next_member_end = next_next_offset - @sizeOf(std.elf.ar_hdr); - break :next_member_size next_member_end - next_offset; - } else next_member_size: { - _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf); - const next_member_end = parent_size; - break :next_member_size next_member_end - next_offset; - }; - const ar_hdr = elf.arHdrPtr(next_ni); - var name_buf: [16]u8 = undefined; - _ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{ - switch (elf.getNode(next_ni)) { - else => unreachable, - .elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}), - .input_member => |ii| std.mem.print(&name_buf, "{s}/", .{ - std.fs.path.basename(ii.path(elf).sub_path), - }), - } catch @panic("TODO: long archive member names"), - }) catch @panic("TODO: long archive member names"); - ar_hdr.ar_date = "0 ".*; - ar_hdr.ar_uid = "0 ".*; - ar_hdr.ar_gid = "0 ".*; - ar_hdr.ar_mode = "644 ".*; - _ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch - @panic("archive member too large"); - ar_hdr.ar_fmag = std.elf.ARFMAG.*; - const member_end = next_offset - @sizeOf(std.elf.ar_hdr); - break :member_size member_end - member_offset; - } else member_size: { - _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf); - const member_end = parent_size; - break :member_size member_end - member_offset; - }; - if (update_size) _ = std.mem.print(&elf.arHdrPtr(ni).ar_size, "{d:<10}", .{ - member_size, - }) catch @panic("archive member too large"); + const next_member_offset: u64, _ = next_member_ni.location(&elf.mf).resolve(&elf.mf); + const strtab_member_offset = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr); + assert(Alignment.@"2".check(next_member_offset)); + assert(Alignment.@"2".check(strtab_member_offset)); + const strtab_size = next_member_offset - strtab_member_offset; + + const member_ar_hdr: *std.elf.ar_hdr = @ptrCast( + archive.header_ni.slice(&elf.mf)[std.elf.ARMAG.len..][0..@sizeOf(std.elf.ar_hdr)], + ); + if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{strtab_size})) |size_str| { + @memset(member_ar_hdr.ar_size[size_str.len..], ' '); + archive.strtab_member_too_big = false; + } else |err| switch (err) { + error.NoSpaceLeft => archive.strtab_member_too_big = true, + } }, } } diff --git a/src/link/MappedFile.zig b/src/link/MappedFile.zig index a790ecbd3b303fbe97ea860481e41913d2aa3806..937b0989a8d40dabcf74ea8c920ac47a2a5d1f67 100644 --- a/src/link/MappedFile.zig +++ b/src/link/MappedFile.zig @@ -648,7 +648,10 @@ pub const Node = extern struct { _, const current_size = ni.location(mf).resolve(mf); if (current_size >= min_size) return; const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor); - try mf.growNode(gpa, ni, new_size, .minimum); + try mf.growNode(gpa, ni, new_size, .{ + .exact_size = false, + .move_footers = true, + }); mf.updateWriters(); } @@ -664,7 +667,10 @@ pub const Node = extern struct { switch (std.math.order(size, old_size)) { .lt => try mf.shrinkLeafNode(gpa, ni, size), .eq => {}, // `old_size` must be well-aligned, so `size` is too - .gt => try mf.growNode(gpa, ni, size, .exact), + .gt => try mf.growNode(gpa, ni, size, .{ + .exact_size = true, + .move_footers = false, // irrelevant, since we have no footers + }), } mf.updateWriters(); } @@ -935,7 +941,10 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { try mf.realignNode(gpa, new_ni, opts.add_options.alignment); if (opts.add_options.size > 0) { - try mf.growNode(gpa, new_ni, opts.add_options.size, .exact); + try mf.growNode(gpa, new_ni, opts.add_options.size, .{ + .exact_size = true, + .move_footers = false, // irrelevant, since we have no footers + }); } mf.updateWriters(); @@ -1070,12 +1079,21 @@ fn shrinkLeafNode( } } -const GrowMode = enum { exact, minimum }; +const GrowOptions = struct { + /// If `true`, the node size must be set to exactly the given size. + /// + /// If `false`, the given size is a minimum, and the actual new node size may be larger. + exact_size: bool, + /// If `true`, footers within the resized node will be moved forwards to its new end. + /// + /// If `false`, footers will all remain at their current offsets (so the nodes are in a + /// temporarily invalid state), and moving them is the responsibility of the *caller*. + move_footers: bool, +}; -/// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`. -/// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`. +/// Increases the size of a node. /// -/// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!). +/// Asserts that `new_size` is aligned to `ni.alignment(mf)`, even if `!grow_options.exact_size`. /// /// Asserts that `new_size` is greater than the current size of `ni`. fn growNode( @@ -1083,7 +1101,7 @@ fn growNode( gpa: Allocator, ni: Node.Index, new_size: u64, - grow_mode: GrowMode, + grow_options: GrowOptions, ) Error!void { mf.nodes_lock.assertUnlocked(); @@ -1098,7 +1116,7 @@ fn growNode( const parent_ni = node.parent.unwrap() orelse { assert(ni == .root); - if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) { return; } @@ -1120,21 +1138,23 @@ fn growNode( }; try mf.ensureTotalCapacityPrecise(@intCast(new_size)); try ni.setLocation(mf, gpa, old_offset, new_size); - // We need to move any footers to be at the *new* end of the file. - if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { - const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); - const footers_size = old_size - old_footers_offset; - try mf.moveRange( - old_footers_offset, - old_footers_offset + (new_size - old_size), - footers_size, - ); - // Also update the footers' locations. - var cur_ni = first_footer_ni; - while (true) { - const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); - try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); - cur_ni = cur_ni.next(mf).unwrap() orelse break; + if (grow_options.move_footers) { + // We need to move any footers to be at the *new* end of the file. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = old_size - old_footers_offset; + try mf.moveRange( + old_footers_offset, + old_footers_offset + (new_size - old_size), + footers_size, + ); + // Also update the footers' locations. + var cur_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } } } return; @@ -1142,7 +1162,7 @@ fn growNode( switch (node.flags.position) { .header => { - if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) { return; } @@ -1163,7 +1183,13 @@ fn growNode( const old_headers_size = last_header_offset + last_header_size; // This is the first footer *inside* of `ni`. - const first_sub_footer_oni = ni.firstFooter(mf); + const first_sub_footer_oni: Node.Index.Optional = footer: { + if (!grow_options.move_footers) { + // Pretend there are no footers so as to not move them. + break :footer .none; + } + break :footer ni.firstFooter(mf); + }; const sub_footers_size = size: { const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); @@ -1215,92 +1241,264 @@ fn growNode( return; }, .floating => { - try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode); + try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_options); }, .footer => { - if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) { return; } - try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size); - - const first_footer_ni: Node.Index = first_footer: { - var footer_ni = ni; - while (true) { - const prev_ni = footer_ni.prev(mf).unwrap() orelse break; - if (prev_ni.position(mf) != .footer) break; - footer_ni = prev_ni; - } - break :first_footer footer_ni; - }; - // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself - // a footer within its parent). - const first_sub_footer_oni = ni.firstFooter(mf); - const sub_footers_size = size: { - const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; - const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); - break :size old_size - first_sub_footer_offset; + // a footer within its parent). We'll need this later in any case, so just find it now. + const first_sub_footer_oni: Node.Index.Optional = footer: { + if (!grow_options.move_footers) { + // Pretend there are no nested footers so as to not move them. + break :footer .none; + } + break :footer ni.firstFooter(mf); }; - _, const parent_size = parent_ni.location(mf).resolve(mf); - - const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0]; - const new_footers_size = old_footers_size - old_size + new_size; - - // Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node - // itself needs to shift its contents, because our offset was shifted backwards by - // `new_size - old_size`, and the added bytes should go at the end of this footer node. - // However, if we *contain* any footer nodes, they need to stay at the end of `ni`, so - // we *shouldn't* shift *that* data. - const old_footers_start = parent_size - old_footers_size; - const new_footers_start = parent_size - new_footers_size; - const end_offset = node.location().resolve(mf)[0] + old_size; - const parent_file_offset = parent_ni.fileLocation(mf, false).offset; - try mf.moveRange( - parent_file_offset + old_footers_start, - parent_file_offset + new_footers_start, - end_offset - old_footers_start - sub_footers_size, - ); - - // Update our own offset and size: - try ni.setLocation(mf, gpa, end_offset - new_size, new_size); - - // Any footers inside of us have had their offsets changed due to us growing: - if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { - var cur_ni = first_sub_footer_ni; - while (true) { - const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); - try cur_ni.setLocation( + // We have two different strategies for growing a footer node, with different advantages + // and disadvantages; so first we must decide which to use. + const strat: union(enum) { + /// Expand into pre-footer padding space in the parent node (growing the parent if + /// necessary). This strategy has the benefit that it can reclaim padding bytes in + /// the parent, but it has the disadvantage that it requires moving this node's + /// existing content backwards in the file, which may be expensive (particularly + /// since the src and dest ranges are likely to overlap). + grow_backwards, + + /// Grow the parent node with `GrowOptions.move_footers` set to `false`, and + /// implicitly grow ourselves into the newly available space. This usually requires + /// a lot less moving of bytes, but never reclaims unused space before the parent's + /// footers, and is sometimes straight-up impossible. + grow_parent_at_end: struct { + add_size: u64, + exact_size: bool, + }, + } = strat: { + // If this node is small, the move overhead is trivial, so prefer `.grow_backwards` + // to avoid unnecessary growth of the parent node. + if (old_size <= mf.flags.block_size.toByteUnits() * 2) { + break :strat .grow_backwards; + } + + // It may also be worth doing `.grow_backwards` if the parent has a *lot* of space + // we could grow into. More specifically, if "free space we can grow into" makes up + // a significant proportion of the parent's total size, then that implies the parent + // has quite poor utilization of space, *and* that we can significantly improve that + // statistic by growing into that space. + if (old_size + mf.availableFooterCapacity(parent_ni) >= new_size) { + break :strat .grow_backwards; + } + + if (grow_options.exact_size) { + const add_size = new_size - old_size; + if (parent_ni.alignment(mf).check(add_size)) { + break :strat .{ .grow_parent_at_end = .{ + .add_size = add_size, + .exact_size = true, + } }; + } else { + // We *can't* ask the parent to grow by this much, so we have no choice. + break :strat .grow_backwards; + } + } + + if (parent_ni.alignment(mf).compare(.lt, node.flags.alignment)) { + // Because the parent's alignment is less than our own, if we gave them the + // freedom to pick a size, they might choose one which results in *us* having a + // size incompatible with our alignment. Therefore, to prevent that, we need to + // request an *exact* size from the parent in this case. + break :strat .{ .grow_parent_at_end = .{ + .add_size = new_size - old_size, + .exact_size = true, + } }; + } + + // The parent's alignment is greater than or equal to our own, so we only need to + // give the parent a *minimum* size (although we need to ensure it matches their + // alignment since it could be greater than our own). + break :strat .{ .grow_parent_at_end = .{ + .add_size = parent_ni.alignment(mf).forward(new_size - old_size), + .exact_size = false, + } }; + }; + + switch (strat) { + .grow_backwards => { + // First, we might need to grow the parent to make enough space. + { + const available_size = mf.availableFooterCapacity(parent_ni); + if (old_size + available_size < new_size) { + _, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf); + const min_parent_size = old_parent_size + (new_size - old_size - available_size); + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, + ); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); + assert(old_size + mf.availableFooterCapacity(parent_ni) >= new_size); + } + } + + // Now we need to grow! To do that, we must move `ni` itself, and every footer + // before it in `parent_ni`, backwards. Unlike header nodes, `ni` is included in + // the shift, because the bytes we're adding need to go at the *end* of `ni` + // rather than its start. + + // This is the same as `parent_ni.firstFooter(mf)`, it's just more efficient to + // start at `ni` than to start at `parent_ni.last(mf)`. + const first_parent_footer_ni: Node.Index = first_footer: { + var footer_ni = ni; + while (true) { + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } + break :first_footer footer_ni; + }; + + const shift = new_size - old_size; + + // Update our own offset and size: + try ni.setLocation( mf, gpa, - old_sub_footer_offset + (new_size - old_size), - sub_footer_size, + node.location().resolve(mf)[0] - shift, + new_size, ); - cur_ni = cur_ni.next(mf).unwrap() orelse break; - } - } - // Finally, update the offsets of every footer before us: - if (node.prev.unwrap()) |prev_ni| { - var maybe_footer_ni = prev_ni; - while (true) { - switch (maybe_footer_ni.position(mf)) { - .header, .floating => break, - .footer => {}, + // Any footers *inside* of `ni` have had their offsets changed, because they are + // now positioned at the *new* end of `ni`: + { + var footer_oni = first_sub_footer_oni; + while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) { + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset + shift, footer_size); + } + } + + // Any footers *before* `ni` (in `parent_ni`) have been shifted backwards. We'll + // also be moving their actual bytes in a moment, so track whether they have + // content (if nothing does then we'll be able to skip the `moveRange`). That + // flag is initially whether `ni` has content because we're shifting our own + // bytes backwards too. + var moved_has_content: bool = node.flags.has_content; + { + var footer_ni = first_parent_footer_ni; + while (footer_ni != ni) : (footer_ni = footer_ni.next(mf).unwrap().?) { + moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset - shift, footer_size); + } + } + + if (moved_has_content) { + // We moved at least one thing containing initialized bytes, so we need to + // move the actual data. However, we should *not* move the bytes of any + // nested footers inside of `ni`, because they've been "moved" to the end + // of our new size, which is the same file location as before. + const sub_footers_size = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + break :size new_size - first_sub_footer_offset; + }; + const new_offset: u64, _ = node.location().resolve(mf); + const new_footers_offset: u64, _ = first_parent_footer_ni.location(mf).resolve(mf); + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + new_footers_offset + shift, + parent_file_offset + new_footers_offset, + (new_offset - new_footers_offset) + // accounts for every footer before `ni` + (old_size - sub_footers_size), // accounts for `ni` itself, excluding nested footers + ); } - const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf); - try maybe_footer_ni.setLocation( + }, + .grow_parent_at_end => |grow_parent| { + _, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf); + try mf.growNode(gpa, parent_ni, old_parent_size + grow_parent.add_size, .{ + .exact_size = grow_parent.exact_size, + .move_footers = false, + }); + _, const new_parent_size: u64 = parent_ni.location(mf).resolve(mf); + const shift = new_parent_size - old_parent_size; + + // Here's what we have left to do: + // + // * Increase our own size by `shift` to absorb the added space. + // + // * If there are any footers *inside* `ni`, increase their offsets by `shift`. + // + // * If there are any footers *after* `ni` (inside `parent_ni`), increase their + // offsets by `shift`. + // + // * Do a `moveRange` corresponding to those offset changes. This is a single + // range which starts at the footers *inside* `ni`. + + const actual_new_size = old_size + shift; + if (grow_options.exact_size) { + assert(actual_new_size == new_size); + } + + try ni.setLocation( mf, gpa, - moved_footer_offset + old_size - new_size, - moved_footer_size, + node.location().resolve(mf)[0], + actual_new_size, ); - maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break; - } - } - return; + // This will track whether any node with a changed offset actually contains + // initialized bytes. If not, there'll be no need to call `moveRange`. + var moved_has_content: bool = false; + + // Set any nested footers' offsets (and include them in `moved_has_content`). + { + var footer_oni = first_sub_footer_oni; + while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) { + assert(footer_ni.position(mf) == .footer); + moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content; + const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size); + } + } + + // Now set offsets for footers after `ni` inside of `parent_ni`. + { + var footer_oni = ni.next(mf); + while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) { + assert(footer_ni.position(mf) == .footer); + moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content; + const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size); + } + } + + if (moved_has_content) { + // We moved at least one footer containing initialized bytes, so we need to + // move the actual data. Compute how big the footers inside `ni` are... + const sub_footers_size: u64 = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + // `actual_new_size` is used here since we already updated the nested footers' offsets above. + break :size actual_new_size - first_sub_footer_offset; + }; + // ...and how big the footers *after* `ni`, inside `parent_ni`, are... + const post_footers_size: u64 = old_parent_size - (old_offset + old_size); + // ...and move them both. + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + const total_move_size = sub_footers_size + post_footers_size; + assert(total_move_size != 0); + try mf.moveRange( + parent_file_off + old_parent_size - total_move_size, + parent_file_off + new_parent_size - total_move_size, + total_move_size, + ); + } + }, + } }, } } @@ -1320,7 +1518,7 @@ fn growFloatingNodeWithAlignment( ni: Node.Index, new_alignment: ?Alignment, new_size: u64, - grow_mode: GrowMode, + grow_options: GrowOptions, ) Error!void { mf.nodes_lock.assertUnlocked(); @@ -1347,27 +1545,29 @@ fn growFloatingNodeWithAlignment( } // Great, we can grow this node without changing its offset or moving any siblings. try ni.setLocation(mf, gpa, old_offset, new_size); - // If we have any footers, we need to move them to the end of our new size, and update their - // offsets accordingly. - if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { - var cur_ni = first_footer_ni; - var footers_have_content = false; - while (true) { - footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; - const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); - try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); - cur_ni = cur_ni.next(mf).unwrap() orelse break; - } - if (footers_have_content) { - const parent_file_off = parent_ni.fileLocation(mf, false).offset; - // This gets the *new* offset because we already updated the offsets above. - const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); - const footers_size = new_size - new_footers_offset; - try mf.moveRange( - parent_file_off + old_offset + old_size - footers_size, - parent_file_off + old_offset + new_size - footers_size, - footers_size, - ); + if (grow_options.move_footers) { + // If we have any footers, we need to move them to the end of our new size, and update + // their offsets accordingly. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var cur_ni = first_footer_ni; + var footers_have_content = false; + while (true) { + footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + if (footers_have_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + // This gets the *new* offset because we already updated the offsets above. + const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = new_size - new_footers_offset; + try mf.moveRange( + parent_file_off + old_offset + old_size - footers_size, + parent_file_off + old_offset + new_size - footers_size, + footers_size, + ); + } } } return; @@ -1444,11 +1644,14 @@ fn growFloatingNodeWithAlignment( // that, let's first try the Linux "insert range" fast path. We didn't try it before now // because it would have been more efficient to just move ourselves into existing space. // - // If we were given a custom alignment, we cannot pass `grow_mode` directly into the + // If we were given a custom alignment, we need to set `GrowOptions.exact_size` for the // "insert range" path, because that function is unaware of `new_alignment`. - const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact; + const insert_range_grow_options: GrowOptions = .{ + .exact_size = grow_options.exact_size or new_alignment != null, + .move_footers = grow_options.move_footers, + }; if (alignment.check(old_offset) and - try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode)) + try mf.growNodeViaInsertRange(gpa, ni, new_size, insert_range_grow_options)) { // The Linux fast path did our job for us! return; @@ -1458,7 +1661,10 @@ fn growFloatingNodeWithAlignment( const new_parent_size = parent_ni.alignment(mf).forward( min_parent_size +| min_parent_size / growth_factor, ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); } break :new_loc .{ @@ -1471,6 +1677,10 @@ fn growFloatingNodeWithAlignment( // Footers need to move to a different place than the rest of our content. const footers_size: u64, const footers_have_content: bool = footers: { + if (!grow_options.move_footers) { + // Pretend there are no footers so as to not move them. + break :footers .{ 0, false }; + } const first_footer_ni = ni.firstFooter(mf).unwrap() orelse { break :footers .{ 0, false }; }; @@ -1525,15 +1735,14 @@ fn growFloatingNodeWithAlignment( /// If this strategy is inapplicable or unsuitable for this operation, this function returns `false` /// without changing any nodes' locations or invalidating any slices. /// -/// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node -/// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`, -/// the actual new size of `ni` may be greater than `new_size`. +/// Otherwise, this function grows `ni` to `new_size` (maybe larger if `!grow_options.exact_size`), +/// updates the location of `ni` and every node whose offset has changed, and returns `true`. fn growNodeViaInsertRange( mf: *MappedFile, gpa: Allocator, ni: Node.Index, new_size: u64, - grow_mode: GrowMode, + grow_options: GrowOptions, ) Error!bool { if (!is_linux or mf.flags.fallocate_insert_range_unsupported) { return false; @@ -1541,43 +1750,22 @@ fn growNodeViaInsertRange( _, const old_size = ni.location(mf).resolve(mf); - // We don't compute the size of the range yet, because depending on `grow_mode` we might want to - // bump it based on our sibling and parent nodes' alignments. However, we can do an early check - // for cases where we should obviously exit. - const requested_range_size = new_size - old_size; - if (!mf.flags.block_size.check(requested_range_size)) { - // The requested size isn't exactly aligned. - switch (grow_mode) { - .exact => return false, - .minimum => { - // We can still choose to allow it by increasing the size a bit, but we shouldn't do - // that if it would *significantly* increase the requested size. - const block_size = mf.flags.block_size.toByteUnits(); - if (requested_range_size < block_size * 2) { - // Bumping this size up to the next block boundary would be a quite significant - // increase; let's not do it. - return false; - } - }, - } - } - // If `grow_mode` is exact, we will use exactly this size, but if it is `.minimum`, we may bump - // the size a little more. + // We don't compute the size of the range yet, because depending on `grow_options` we might want + // to bump it based on our sibling and parent nodes' alignments. However, we can do an early + // check for cases where we should obviously exit. const min_range_size: u64 = s: { - const exact_size = new_size - old_size; - if (mf.flags.block_size.check(exact_size)) { - break :s exact_size; + const requested_size = new_size - old_size; + if (mf.flags.block_size.check(requested_size)) { + break :s requested_size; } - switch (grow_mode) { - .exact => return false, - .minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) { - // We're growing by at least a few blocks, so allow ourselves to bump the size - // slightly to give it the needed alignment. - break :s mf.flags.block_size.forward(exact_size); - } else { - return false; - }, + if (!grow_options.exact_size and + requested_size >= mf.flags.block_size.toByteUnits() * 2) + { + // We're growing by at least a few blocks, so allow ourselves to bump the size + // slightly to give it the needed alignment. + break :s mf.flags.block_size.forward(requested_size); } + return false; }; assert(min_range_size > 0); assert(mf.flags.block_size.check(min_range_size)); @@ -1592,14 +1780,17 @@ fn growNodeViaInsertRange( } break :range_file_offset range_file_offset; }; - const first_footer_oni = ni.firstFooter(mf); - const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| size: { + const pre_footer_oni: Node.Index.Optional, const footers_size: u64 = footers: { + if (!grow_options.move_footers) { + // Pretend there are no footers so as to not move them. + break :footers .{ .wrap(last_ni), 0 }; + } + const first_footer_ni = ni.firstFooter(mf).unwrap() orelse { + break :footers .{ .wrap(last_ni), 0 }; + }; const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); - break :size old_size - first_footer_offset; - } else 0; - const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) |first_footer_ni| pre_footer: { - break :pre_footer first_footer_ni.prev(mf); - } else .wrap(last_ni); + break :footers .{ first_footer_ni.prev(mf), old_size - first_footer_offset }; + }; const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: { const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf); break :end pre_footer_off + pre_footer_size; @@ -1649,22 +1840,18 @@ fn growNodeViaInsertRange( } // Traversal done. We didn't hit `max_moved_nodes`, so now we can use the computed alignment // requirement to figure out whether we're actually going to insert a range. - if (need_range_align.check(requested_range_size)) { - break :range_size requested_range_size; + if (need_range_align.check(min_range_size)) { + break :range_size min_range_size; } - // Perhaps we're allowed to grow by more than `requested_range_size`? - switch (grow_mode) { - .exact => return false, - .minimum => { - const candidate_range_size = need_range_align.forward(min_range_size); - // Allow growing by up to 50% more than was requested. - if (candidate_range_size <= requested_range_size +| requested_range_size / 2) { - break :range_size candidate_range_size; - } else { - return false; - } - }, + // Perhaps we're allowed to grow by more than `min_range_size`? + const candidate_range_size = need_range_align.forward(min_range_size); + if (!grow_options.exact_size and + // Allow growing by up to 50% more than was requested. + candidate_range_size <= min_range_size +| min_range_size / 2) + { + break :range_size candidate_range_size; } + return false; }; // This `range_size` is compatible with everyone's alignment requirements, and we won't move too @@ -1742,13 +1929,15 @@ fn growNodeViaInsertRange( cur_ni = cur_ni.parent(mf).unwrap() orelse break; } - // The only thing left is to update the offsets of any footers inside of `ni`. - if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { - var footer_ni = first_footer_ni; - while (true) { - const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); - try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size); - footer_ni = footer_ni.next(mf).unwrap() orelse break; + if (grow_options.move_footers) { + // The only thing left is to update the offsets of any footers inside of `ni`. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var footer_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size); + footer_ni = footer_ni.next(mf).unwrap() orelse break; + } } } @@ -1802,7 +1991,10 @@ fn ensureAdditionalHeaderCapacity( const new_parent_size = parent_ni.alignment(mf).forward( min_parent_size +| min_parent_size / growth_factor, ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); } return; }; @@ -1884,7 +2076,10 @@ fn ensureAdditionalHeaderCapacity( const new_parent_size = parent_ni.alignment(mf).forward( min_parent_size +| min_parent_size / growth_factor, ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); } if (moving_has_content) { @@ -1914,48 +2109,27 @@ fn ensureAdditionalHeaderCapacity( } } -/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current -/// footers, so that the footers can grow into that space. -fn ensureAdditionalFooterCapacity( - mf: *MappedFile, - gpa: Allocator, - parent_ni: Node.Index, - extra_capacity: u64, -) Error!void { - // This is way easier than the header case, because we don't need to actually move anything; we - // just need to expand the parent if there isn't space, and that will add padding after the - // parent's floating children, which is exactly where we want it. - +/// Returns how many padding bytes `parent_ni` currently has directly preceding its footers, which +/// footers can therefore grow into. +fn availableFooterCapacity(mf: *const MappedFile, parent_ni: Node.Index) u64 { const first_footer_oni = parent_ni.firstFooter(mf); - _, const parent_size = parent_ni.location(mf).resolve(mf); - - const footers_size: u64 = footers_size: { - const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; + const before_footers_oni: Node.Index.Optional, const footers_off: u64 = footers: { + const first_footer_ni = first_footer_oni.unwrap() orelse { + _, const parent_size = parent_ni.location(mf).resolve(mf); + break :footers .{ parent_ni.last(mf), parent_size }; + }; const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); - break :footers_size parent_size - first_footer_off; + break :footers .{ first_footer_ni.prev(mf), first_footer_off }; }; const header_and_floating_end: u64 = end: { - const before_footers_oni = if (first_footer_oni.unwrap()) |first_footer_ni| before_footers: { - break :before_footers first_footer_ni.prev(mf); - } else before_footers: { - break :before_footers parent_ni.last(mf); - }; const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0; const offset, const size = before_footers_ni.location(mf).resolve(mf); break :end offset + size; }; - assert(header_and_floating_end + footers_size <= parent_size); - - const min_parent_size = header_and_floating_end + footers_size + extra_capacity; - if (parent_size < min_parent_size) { - const new_parent_size = parent_ni.alignment(mf).forward( - min_parent_size +| min_parent_size / growth_factor, - ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); - } + return footers_off - header_and_floating_end; } fn removeNodesFromChildList( @@ -2049,7 +2223,7 @@ fn realignNode( mf: *MappedFile, gpa: Allocator, ni: Node.Index, - new_alignment: Alignment, + new_align: Alignment, ) Error!void { mf.nodes_lock.assertUnlocked(); @@ -2057,30 +2231,25 @@ fn realignNode( if (ni == .root or ni.position(mf) != .floating) { // Only this node's size is aligned, not its offset. - if (!new_alignment.check(old_size)) { - assert(new_alignment.compare(.gt, ni.alignment(mf))); - try mf.growNode( - gpa, - ni, - new_alignment.forward(old_size), - .exact, // because `growNode` is not aware that the size needs to match `new_alignment` - ); + if (!new_align.check(old_size)) { + assert(new_align.compare(.gt, ni.alignment(mf))); + try mf.growNode(gpa, ni, new_align.forward(old_size), .{ + .exact_size = true, // because `growNode` is not aware that the size needs to match `new_align` + .move_footers = true, + }); } } else { // This is a floating node, so its size and offset are both aligned. - if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) { - assert(new_alignment.compare(.gt, ni.alignment(mf))); - try mf.growFloatingNodeWithAlignment( - gpa, - ni, - new_alignment, - new_alignment.forward(old_size), - .minimum, - ); + if (!new_align.check(old_offset) or !new_align.check(old_size)) { + assert(new_align.compare(.gt, ni.alignment(mf))); + try mf.growFloatingNodeWithAlignment(gpa, ni, new_align, new_align.forward(old_size), .{ + .exact_size = false, + .move_footers = true, + }); } } - ni.get(mf).flags.alignment = new_alignment; + ni.get(mf).flags.alignment = new_align; } fn updateWriters(mf: *MappedFile) void {