From 96c9ff1c93532e7c2764449e0cd6f39d4dce2285 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Sun, 23 Aug 2026 11:21:31 +0100 Subject: [PATCH] link.MappedFile: rework node operations This commit is a refactor of the public `MappedFile` API, and a near-total rewrite of its implementation (or at least, the implementation of the node moving and resizing logic). Nodes can be "header", "footer", or "floating" nodes, which dictates how they are positioned relative to their parent; "header" nodes (similar to the old "fixed" nodes) are placed at the start of the parent node, "footer" nodes are placed at the end of the parent node, and "floating" nodes may appear anywhere in the parent. There are separate functions for adding each of these types of node. Notably, when adding a floating node, the API no longer permits the caller to specify *where* these nodes are placed, because floating nodes give the implementation the freedom to make this choice for itself. Another important property is that for header and footer nodes, only their size is aligned to the node's alignment. Their offsets are not necessarily aligned, because they are required to be placed at the start/end of the parent with no additional padding: this constraint already dictates their offset. The `Node.Index.resize` function is replaced with two functions. The first, `ensureMinimumSize`, is permitted for any node, and guarantees only that the node's size is *at least* a particular value, applying exponential growth (`growth_factor`) if necessary---it is essentially equivalent to the helper function `Elf2.ensureNodeSize` which the `Elf2` linker was already making frequent use of. The other, `resizeLeaf`, sets the size of a node *exactly*, but may only be used on leaf nodes. The logic for actually placing nodes in the file, as well as becoming slightly more involved due to handling the new semantics of headers and footers, has also been made more efficient. In particular, the amount of unused "padding" space is, broadly speaking, lower with this implementation than it previously was: empirically, binaries emitted by `Elf2` are about half the size as they were before (in terms of `stat` size, not size on disk), although binaries emitted by `Coff` are around the same size as before (perhaps slightly smaller). I have spoken with Casey about some potential enhancements to `Coff` which, as well as making it more performant, could also slightly improve its file sizes. At least one alignment-related bug, wherein the Linux-specific `FALLOCATE_FL_INSERT_RANGE` path did not respect neighbors' alignment requirements, has been fixed. In order to verify correctness of this new implementation (particularly since neither linker uses footer nodes yet), I wrote a small fuzz test which performs a random sequence of node operations (add, resize, realign), while writing content into (some) leaf nodes. After all operations are complete, it validates that the node structure is valid (headers are tightly packed against the start of the parent node, no two sibling nodes overlap, etc), and ensures that all leaves contain the expected content. Even with our alpha-quality fuzzer implementation, this fuzz test was surprisingly helpful in identifying bugs during development. I suspect `MappedFile` is unusually easy to fuzz, because most code paths can be hit with relatively few nodes, so even purely random fuzzing (as opposed to coverage-guided) is likely to discover any bugs fairly quickly. This fuzz test is in `src/link/MappedFile.zig`, and is referenced by the standard compiler unit tests, so `MappedFile` can be fuzzed at any time by running `zig build test-unit --fuzz`. --- src/link/Coff.zig | 290 ++--- src/link/Elf2.zig | 138 +- src/link/MappedFile.zig | 2688 ++++++++++++++++++++++++++------------- src/main.zig | 1 + 4 files changed, 1990 insertions(+), 1127 deletions(-) diff --git a/src/link/Coff.zig b/src/link/Coff.zig index f83074f558b19e84d803d17251a6d4014abdd2b2..09ea7b6a25e8dcba3cec1d1996266de05863b809 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -533,10 +533,10 @@ pub const Member = struct { errdefer _ = coff.export_table.entries.pop(); _, const old_size = Node.known.longnames_member.location(&coff.mf).resolve(&coff.mf); - const new_size = old_size + name.len + 1; + const new_size = Alignment.@"4".forward(old_size + name.len + 1); assert(new_size < comptime try std.math.powi(u64, 10, max_name_len - 1)); - try Node.known.longnames_member.resize(&coff.mf, gpa, new_size); + try Node.known.longnames_member.resizeLeaf(&coff.mf, gpa, new_size); const name_table_slice = Node.known.longnames_member.slice(&coff.mf); const name_slice = name_table_slice[@intCast(old_size)..][0 .. name.len + 1]; @memcpy(name_slice[0..name.len], name); @@ -1840,34 +1840,20 @@ fn initHeaders( coff.nodes.appendAssumeCapacity(.file); const header_ni = Node.known.header; - assert(header_ni == try coff.mf.addOnlyChildNode(gpa, Node.known.file, .{ + assert(header_ni == try Node.known.file.addOnlyHeaderChild(&coff.mf, gpa, .{ .alignment = coff.mf.flags.block_size, - .fixed = true, })); coff.nodes.appendAssumeCapacity(.header); - const signature_ni = Node.known.signature; - assert(signature_ni == try coff.mf.addLastChildNode(gpa, if (is_image or !is_archive) header_ni else Node.known.file, .{ - .size = if (is_image) - msdos_stub.len + std.coff.pe_signature.len - else if (is_archive) - std.coff.archive_signature.len - else - 0, - .alignment = .@"4", - .fixed = true, - })); - coff.nodes.appendAssumeCapacity(.signature); - - const signature_slice = signature_ni.slice(&coff.mf); - if (is_image) { - @memcpy(signature_slice[0..msdos_stub.len], &msdos_stub); - @memcpy(signature_slice[signature_slice.len - std.coff.pe_signature.len ..], std.coff.pe_signature); - } else if (is_archive) { + const coff_parent_ni: MappedFile.Node.Index = if (is_archive) parent: { + assert(try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .wrap(header_ni), .{ + .size = std.coff.archive_signature.len, + .alignment = .@"4", + }) == Node.known.signature); + coff.nodes.appendAssumeCapacity(.signature); + const signature_slice = Node.known.signature.slice(&coff.mf); @memcpy(signature_slice, std.coff.archive_signature); - } - const opt_coff_parent_ni = if (is_archive) parent: { const initial_member_count = Member.Index.known_count + @intFromBool(comp.zcu != null); try coff.members.ensureTotalCapacity(gpa, initial_member_count); @@ -1893,46 +1879,54 @@ fn initHeaders( const zcu_member = zcu_mi.get(coff); try zcu_member.initHeader(coff, zcu.main_mod.fully_qualified_name, timestamp); + assert(try zcu_member.content_ni.addOnlyHeaderChild(&coff.mf, gpa, .{ + .size = @sizeOf(std.coff.Header), + .alignment = .@"4", + }) == Node.known.coff_header); + coff.nodes.appendAssumeCapacity(.coff_header); + break :parent zcu_member.content_ni; } + // If we're not generating any code, no more known nodes are used + // These placeholder nodes are placed before the first member - if there are // no other members then the last linker member (longnames) needs to expand // to fill the padding at the end of the file. - assert(Node.known.zcu_member_header == try coff.mf.addNodeAfter(gpa, Node.known.header, .{})); - assert(Node.known.zcu_member == try coff.mf.addNodeAfter(gpa, Node.known.header, .{})); - coff.nodes.appendAssumeCapacity(.placeholder); - coff.nodes.appendAssumeCapacity(.placeholder); + while (coff.nodes.len < Node.known_count) { + _ = try Node.known.header.addHeaderChildAfter(&coff.mf, gpa, .none, .{}); + coff.nodes.appendAssumeCapacity(.placeholder); + } - break :parent null; + return; } else parent: { + assert(try header_ni.addOnlyHeaderChild(&coff.mf, gpa, .{ + .size = if (is_image) msdos_stub.len + std.coff.pe_signature.len else 0, + .alignment = .@"4", + }) == Node.known.signature); + coff.nodes.appendAssumeCapacity(.signature); + if (is_image) { + const signature_slice = Node.known.signature.slice(&coff.mf); + @memcpy(signature_slice[0..msdos_stub.len], &msdos_stub); + @memcpy(signature_slice[signature_slice.len - std.coff.pe_signature.len ..], std.coff.pe_signature); + } + // TODO: Not ideal to have this many placeholder nodes - use two distinct `Node.known` types? while (true) { - const placeholder_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{}); + const placeholder_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .none, .{}); coff.nodes.appendAssumeCapacity(.placeholder); if (placeholder_ni == Node.known.zcu_member) break; } - break :parent Node.known.header; - }; - - const coff_parent_ni = opt_coff_parent_ni orelse { - // If we're not generating any code, no more known nodes are used - while (coff.nodes.len < Node.known_count) { - _ = try coff.mf.addNodeAfter(gpa, Node.known.header, .{}); - coff.nodes.appendAssumeCapacity(.placeholder); - } + assert(try header_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(Node.known.signature), .{ + .size = @sizeOf(std.coff.Header), + .alignment = .@"4", + }) == Node.known.coff_header); + coff.nodes.appendAssumeCapacity(.coff_header); - return; + break :parent header_ni; }; - const coff_header_ni = Node.known.coff_header; - assert(coff_header_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ - .size = @sizeOf(std.coff.Header), - .alignment = .@"4", - .fixed = true, - })); - coff.nodes.appendAssumeCapacity(.coff_header); { const coff_header = coff.headerPtr(); coff_header.* = .{ @@ -1955,10 +1949,9 @@ fn initHeaders( } const optional_header_ni = Node.known.optional_header; - assert(optional_header_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + assert(optional_header_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(Node.known.coff_header), .{ .size = optional_header_size, .alignment = .@"4", - .fixed = true, })); coff.nodes.appendAssumeCapacity(.optional_header); if (is_image) { @@ -2067,10 +2060,9 @@ fn initHeaders( } const data_directories_ni = Node.known.data_directories; - assert(data_directories_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + assert(data_directories_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(optional_header_ni), .{ .size = data_directories_size, .alignment = .@"4", - .fixed = true, })); coff.nodes.appendAssumeCapacity(.data_directories); if (is_image) { @@ -2083,9 +2075,8 @@ fn initHeaders( } const section_table_ni = Node.known.section_table; - assert(section_table_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + assert(section_table_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(data_directories_ni), .{ .alignment = .@"4", - .fixed = true, })); coff.nodes.appendAssumeCapacity(.section_table); @@ -2093,16 +2084,14 @@ fn initHeaders( if (!is_image) { // TODO: These two nodes could be inside one movable node? - coff.symbol_table.ni = try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + coff.symbol_table.ni = try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(section_table_ni), .{ .alignment = .@"2", - .fixed = true, .moved = true, }); coff.nodes.appendAssumeCapacity(.symbol_table); - coff.symbol_table.strings_ni = try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + coff.symbol_table.strings_ni = try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(coff.symbol_table.ni), .{ .size = @sizeOf(u32), - .fixed = true, .resized = true, }); coff.nodes.appendAssumeCapacity(.string_table); @@ -2149,15 +2138,14 @@ fn initHeaders( } // TODO: Lazily initialize this instead, avoid the extra logic for this in flushMoved / flushResized - coff.import_table.ni = try coff.mf.addLastChildNode( - gpa, - (try coff.objectSectionMapIndex( - .@".idata", - coff.mf.flags.block_size, - .{ .read = true, .initialized = true }, - )).symbol(coff).node(coff), - .{ .alignment = .@"4" }, - ); + const import_table_parent_ni = (try coff.objectSectionMapIndex( + .@".idata", + coff.mf.flags.block_size, + .{ .read = true, .initialized = true }, + )).symbol(coff).node(coff); + coff.import_table.ni = try import_table_parent_ni.addFloatingChild(&coff.mf, gpa, .{ + .alignment = .@"4", + }); coff.nodes.appendAssumeCapacity(.import_directory_table); coff.export_table.ni = (try coff.pseudoSectionMapIndex( @@ -2166,15 +2154,10 @@ fn initHeaders( .{ .read = true, .initialized = true }, )).symbol(coff).node(coff); - coff.export_table.export_directory_table_ni = try coff.mf.addLastChildNode( - gpa, - coff.export_table.ni, - .{ - .size = @sizeOf(std.coff.ExportDirectoryTable) + file_name.len + 1, - .moved = true, - .fixed = true, - }, - ); + coff.export_table.export_directory_table_ni = try coff.export_table.ni.addHeaderChildAfter(&coff.mf, gpa, coff.export_table.ni.last(&coff.mf), .{ + .size = @sizeOf(std.coff.ExportDirectoryTable) + file_name.len + 1, + .moved = true, + }); coff.nodes.appendAssumeCapacity(.export_directory_table); const name_index = @sizeOf(std.coff.ExportDirectoryTable); @@ -2182,7 +2165,7 @@ fn initHeaders( @memcpy(table_slice[name_index..][0..file_name.len], file_name[0..file_name.len]); @memset(table_slice[name_index + file_name.len ..], 0); - const export_address_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + const export_address_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(std.coff.ExportAddressTableEntry), .moved = true, }); @@ -2198,19 +2181,19 @@ fn initHeaders( export_address_table_sym.section_number = coff.getNode(coff.export_table.ni).pseudo_section.symbol(coff).get(coff).section_number; - coff.export_table.name_pointer_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + coff.export_table.name_pointer_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(std.coff.ExportNamePointerTableEntry), .moved = true, }); coff.nodes.appendAssumeCapacity(.export_name_pointer_table); - coff.export_table.ordinal_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + coff.export_table.ordinal_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(std.coff.ExportOrdinalTableEntry), .moved = true, }); coff.nodes.appendAssumeCapacity(.export_ordinal_table); - coff.export_table.name_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + coff.export_table.name_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(u8), .moved = true, }); @@ -2303,9 +2286,8 @@ pub fn initBuiltins(coff: *Coff) !void { const list_len_si = try coff.globalSymbol(.{ .name = list.global, .type = .data }); const list_len_sym = list_len_si.get(coff); list_len_sym.setExtra(.{ .size = addr_info.size }); - list_len_sym.ni = .wrap(try coff.mf.addFirstChildNode(gpa, start_sym.ni.unwrap().?, .{ + list_len_sym.ni = .wrap(try start_sym.ni.unwrap().?.addHeaderChildAfter(&coff.mf, gpa, .none, .{ .size = addr_info.size, - .fixed = true, })); coff.nodes.appendAssumeCapacity(.{ .builtin = list_len_si }); list_len_sym.section_number = start_sym.section_number; @@ -2325,9 +2307,8 @@ pub fn initBuiltins(coff: *Coff) !void { const list_end_si = coff.addSymbolAssumeCapacity(); const list_end_sym = list_end_si.get(coff); list_end_sym.setExtra(.{ .size = addr_info.size }); - list_end_sym.ni = .wrap(try coff.mf.addFirstChildNode(gpa, end_sym.ni.unwrap().?, .{ + list_end_sym.ni = .wrap(try end_sym.ni.unwrap().?.addHeaderChildAfter(&coff.mf, gpa, .none, .{ .size = addr_info.size, - .fixed = true, })); coff.nodes.appendAssumeCapacity(.{ .builtin = list_end_si }); list_end_sym.section_number = start_sym.section_number; @@ -2742,7 +2723,7 @@ fn getOrPutSymbolName(coff: *Coff, name: []const u8, opt_string: ?String) !Symbo const string_index = coff.symbol_table.strings_ni.location(&coff.mf).resolve(&coff.mf)[1]; string_gop.value_ptr.* = @fromBackingInt(@intCast(string_index)); - try coff.symbol_table.strings_ni.resize(&coff.mf, gpa, string_index + name.len + 1); + try coff.symbol_table.strings_ni.resizeLeaf(&coff.mf, gpa, string_index + name.len + 1); const slice = coff.symbol_table.strings_ni.slice(&coff.mf); @memcpy(slice[@intCast(string_index)..][0..name.len], name); slice[@intCast(string_index + name.len)] = 0; @@ -2967,24 +2948,22 @@ fn addMemberAssumeCapacity(coff: *Coff, kind: std.coff.ArchiveMemberHeader.Kind, const comp = coff.base.comp; const gpa = comp.gpa; - // TODO: These two nodes could to be inside a movable node if kind == .coff|.import - const header_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{ + const header_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, Node.known.file.last(&coff.mf), .{ .size = @sizeOf(std.coff.ArchiveMemberHeader), .alignment = .@"2", - .fixed = true, .moved = true, }); - const content_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{ - // The actual alignment required by the spec is 2, but to allow aligned access to - // the various COFF data structures in-place during linking we overalign - .alignment = switch (kind) { - .coff => .@"4", - else => .@"2", - }, - .size = size, + // The actual alignment required by the spec is 2, but to allow aligned access to + // the various COFF data structures in-place during linking we overalign + const content_align: Alignment = switch (kind) { + .first_linker, .second_linker, .longnames, .coff => .@"4", + else => .@"2", + }; + const content_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .wrap(header_ni), .{ + .alignment = content_align, + .size = content_align.forward(size), .resized = size > 0, - .fixed = true, }); const mi: Member.Index = @fromBackingInt(@intCast(coff.members.items.len)); @@ -3010,7 +2989,7 @@ fn addMemberAssumeCapacity(coff: *Coff, kind: std.coff.ArchiveMemberHeader.Kind, const old_size = Node.known.second_linker_member.location(&coff.mf).resolve(&coff.mf)[1]; const old_header_size = new_num_members * @sizeOf(u32); const trailing_size: usize = @intCast(old_size - old_header_size); - try Node.known.second_linker_member.resize(&coff.mf, gpa, old_size + @sizeOf(u32)); + try Node.known.second_linker_member.resizeLeaf(&coff.mf, gpa, old_size + @sizeOf(u32)); const slice = Node.known.second_linker_member.slice(&coff.mf); @memmove( @@ -3048,7 +3027,7 @@ fn appendMemberSymbolString( name: []const u8, offset: u64, ) !void { - try strings_ni.resize(&coff.mf, coff.base.comp.gpa, new_size); + try strings_ni.resizeLeaf(&coff.mf, coff.base.comp.gpa, new_size); const name_slice = strings_ni.slice(&coff.mf)[offset..][0 .. name.len + 1]; @memcpy(name_slice[0..name.len], name); name_slice[name.len] = 0; @@ -3081,7 +3060,7 @@ fn ensureMemberSymbol(coff: *Coff, mi: Member.Index, name: String) !void { { const old_header_size: usize = @intCast(@sizeOf(u32) + @backingInt(mfli) * @sizeOf(u32)); const new_header_size: usize = @intCast(old_header_size + @sizeOf(u32)); - try Node.known.first_linker_member.resize(&coff.mf, gpa, new_header_size + new_string_table_size); + try Node.known.first_linker_member.resizeLeaf(&coff.mf, gpa, Alignment.@"4".forward(new_header_size + new_string_table_size)); const slice = Node.known.first_linker_member.slice(&coff.mf); @memmove(slice[new_header_size..][0..coff.lib_string_len], slice[old_header_size..][0..coff.lib_string_len]); @@ -3095,7 +3074,7 @@ fn ensureMemberSymbol(coff: *Coff, mi: Member.Index, name: String) !void { const num_members = coff.targetLoad(coff.secondLinkerMemberNumMembersPtr()); const old_header_size = 2 * @sizeOf(u32) + num_members * @sizeOf(u32) + @backingInt(mfli) * @sizeOf(u16); const new_header_size = old_header_size + @sizeOf(u16); - try Node.known.second_linker_member.resize(&coff.mf, gpa, new_header_size + new_string_table_size); + try Node.known.second_linker_member.resizeLeaf(&coff.mf, gpa, Alignment.@"4".forward(new_header_size + new_string_table_size)); const old_needs_sort = coff.pending_members.get(Member.Index.second) != null; const needs_sort = old_needs_sort or (if (coff.lib_string_table.items.len > 0) @@ -3202,7 +3181,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { const new_num_symbols = old_num_symbols + 1 + num_aux_symbols; coff.targetStore(&coff.headerPtr().number_of_symbols, new_num_symbols); - try coff.symbol_table.ni.resize(&coff.mf, gpa, new_num_symbols * std.coff.Symbol.sizeOf()); + try coff.symbol_table.ni.resizeLeaf(&coff.mf, gpa, new_num_symbols * std.coff.Symbol.sizeOf()); sti.* = .wrap(old_num_symbols); si.flushSymbolTableIndex(coff); @@ -3365,13 +3344,13 @@ fn addSection(coff: *Coff, name: String, flags: std.coff.SectionHeader.Flags) !S const section_index = coff.targetLoad(&coff_header.number_of_sections); const section_table_len = section_index + 1; coff.targetStore(&coff_header.number_of_sections, section_table_len); - try Node.known.section_table.resize( + try Node.known.section_table.resizeLeaf( &coff.mf, gpa, @sizeOf(std.coff.SectionHeader) * section_table_len, ); - const ni = try coff.mf.addLastChildNode(gpa, coff.sectionParent(), .{ + const ni = try coff.sectionParent().addFloatingChild(&coff.mf, gpa, .{ .alignment = coff.mf.flags.block_size, .moved = true, .bubbles_moved = false, @@ -3507,7 +3486,7 @@ fn pseudoSectionMapIndex( try coff.nodes.ensureUnusedCapacity(gpa, 1); try coff.symbols.ensureUnusedCapacity(gpa, 1); - const ni = try coff.mf.addLastChildNode(gpa, parent.node(coff), .{ .alignment = alignment }); + const ni = try parent.node(coff).addFloatingChild(&coff.mf, gpa, .{ .alignment = alignment }); const si = coff.addSymbolAssumeCapacity(); pseudo_section_gop.value_ptr.* = si; const sym = si.get(coff); @@ -3577,12 +3556,8 @@ fn objectSectionMapIndex( .eq => unreachable, .gt => prev_oni = .wrap(next_ni), }; - const ni = if (prev_oni.unwrap()) |prev_ni| try coff.mf.addNodeAfter(gpa, prev_ni, .{ + const ni = try parent_ni.addHeaderChildAfter(&coff.mf, gpa, prev_oni, .{ .alignment = alignment, - .fixed = true, - }) else try coff.mf.addFirstChildNode(gpa, parent_ni, .{ - .alignment = alignment, - .fixed = true, }); const si = coff.addSymbolAssumeCapacity(); object_section_gop.value_ptr.* = si; @@ -3600,13 +3575,13 @@ fn objectSectionMapIndex( const parent_alignment = parent_ni.alignment(&coff.mf); if (alignment.compare(.gt, parent_alignment)) { log.debug("realignParent({s}, {d}) {d}->{d}", .{ name.toSlice(coff), parent_ni, parent_alignment, alignment }); - try parent_ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); + try parent_ni.realign(&coff.mf, gpa, alignment); } const old_alignment = sym.ni.unwrap().?.alignment(&coff.mf); if (alignment.compare(.gt, old_alignment)) { log.debug("realignObject({s}) {d}->{d}", .{ name.toSlice(coff), old_alignment, alignment }); - try sym.ni.unwrap().?.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); + try sym.ni.unwrap().?.realign(&coff.mf, gpa, alignment); } try coff.verifyParentSectionAttributes( @@ -3763,18 +3738,14 @@ fn addRelocAssumeCapacity( coff.targetStore(&aux_ptr.number_of_relocations, new_num_relocations); if (section.relocation_table_ni.unwrap()) |relocation_table_ni| { - try relocation_table_ni.resize(&coff.mf, gpa, new_size); + try relocation_table_ni.resizeLeaf(&coff.mf, gpa, new_size); } else { - section.relocation_table_ni = .wrap(try coff.mf.addLastChildNode( - gpa, - coff.sectionParent(), - .{ - .size = new_size, - .alignment = .@"2", - .moved = true, - .resized = true, - }, - )); + section.relocation_table_ni = .wrap(try coff.sectionParent().addFloatingChild(&coff.mf, gpa, .{ + .size = new_size, + .alignment = .@"2", + .moved = true, + .resized = true, + })); coff.nodes.appendAssumeCapacity(.{ .relocation_table = loc_sn }); } @@ -4677,9 +4648,10 @@ fn loadObject( for (sections) |*section| { if (section.parent_si == .null) continue; - const ni = try coff.mf.addLastChildNode(gpa, section.parent_si.node(coff), .{ - .size = section.header.size_of_raw_data, - .alignment = .fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1), + const alignment: Alignment = .fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1); + const ni = try section.parent_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ + .size = alignment.forward(section.header.size_of_raw_data), + .alignment = alignment, .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .input_section = @fromBackingInt(@intCast(coff.input_sections.items.len)) }); @@ -5471,7 +5443,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde const sec_si = try coff.navSection(zcu, nav.resolved.?); try coff.nodes.ensureUnusedCapacity(gpa, 1); if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ .alignment = .fromIp(zcu.navAlignment(nav_index)), .moved = true, }); @@ -5510,21 +5482,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde } if (nav.resolved.?.@"linksection".unwrap()) |_| { - try ni.resize(&coff.mf, gpa, si.get(coff).extra.size); - var parent_ni = ni; - while (true) { - parent_ni = parent_ni.parent(&coff.mf).unwrap().?; - switch (coff.getNode(parent_ni)) { - else => unreachable, - .image_section, .pseudo_section => break, - .object_section => { - var child_it = parent_ni.reverseChildren(&coff.mf); - const last_offset, const last_size = - child_it.next().?.location(&coff.mf).resolve(&coff.mf); - try parent_ni.resize(&coff.mf, gpa, last_offset + last_size); - }, - } - } + try ni.resizeLeaf(&coff.mf, gpa, si.get(coff).extra.size); } } @@ -5596,7 +5554,7 @@ fn updateFuncInner( if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); const mod = zcu.navFileScope(func.owner_nav).mod.?; const target = &mod.resolved_target.result; - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ .alignment = switch (nav.resolved.?.@"align") { .none => switch (mod.optimize_mode) { .debug, @@ -5900,15 +5858,17 @@ pub fn flush( coff.symbol_table.pending_shrink = false; const number_of_symbols = coff.targetLoad(&coff.headerPtr().number_of_symbols); - coff.symbol_table.ni.shrink( + coff.symbol_table.ni.resizeLeaf( &coff.mf, comp.gpa, number_of_symbols * std.coff.Symbol.sizeOf(), - true, - ) catch |err| return comp.link_diags.fail( - "linker failed to compact symbol table: {t}", - .{err}, - ); + ) catch |err| switch (err) { + else => |e| return e, + error.MappedFileIo => return comp.link_diags.fail( + "linker failed to compact symbol table: {t}", + .{coff.mf.io_err.?}, + ), + }; } while (try coff.idle(tid)) {} @@ -6211,7 +6171,7 @@ fn flushUav( try coff.nodes.ensureUnusedCapacity(gpa, 1); if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); const sym = si.get(coff); - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ .alignment = .fromIp(uav_align), .moved = true, }); @@ -6503,7 +6463,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { const import_hint_name_align: Alignment = .@"2"; if (!gop.found_existing) { errdefer _ = coff.import_table.entries.pop(); - try coff.import_table.ni.resize( + try coff.import_table.ni.resizeLeaf( &coff.mf, gpa, @sizeOf(std.coff.ImportDirectoryEntry) * (gop.index + 2), @@ -6511,12 +6471,12 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { const import_hint_name_table_len = import_hint_name_align.forward(lib_name.len + ".dll".len + 1); const idata_section_ni = coff.import_table.ni.parent(&coff.mf).unwrap().?; - const import_lookup_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{ + const import_lookup_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{ .size = addr_info.size * 2, .alignment = addr_info.alignment, .moved = true, }); - const import_address_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{ + const import_address_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{ .size = addr_info.size * 2, .alignment = addr_info.alignment, .moved = true, @@ -6530,7 +6490,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { import_address_table_sym.section_number = coff.getNode(idata_section_ni).object_section.symbol(coff).get(coff).section_number; } - const import_hint_name_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{ + const import_hint_name_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{ .size = import_hint_name_table_len, .alignment = import_hint_name_align, .moved = true, @@ -6586,9 +6546,9 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { gop.value_ptr.len = import_symbol_index + 1; const new_symbol_table_size = addr_info.size * (import_symbol_index + 2); - try gop.value_ptr.import_lookup_table_ni.resize(&coff.mf, gpa, new_symbol_table_size); + try gop.value_ptr.import_lookup_table_ni.resizeLeaf(&coff.mf, gpa, new_symbol_table_size); const import_address_table_ni = gop.value_ptr.import_address_table_si.node(coff); - try import_address_table_ni.resize(&coff.mf, gpa, new_symbol_table_size); + try import_address_table_ni.resizeLeaf(&coff.mf, gpa, new_symbol_table_size); const opt_imp_name = import.name.toSlice(coff); const opt_import_hint_name_index = if (opt_imp_name) |imp_name| blk: { @@ -6596,7 +6556,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { gop.value_ptr.hint_name_len = @intCast( import_hint_name_align.forward(import_hint_name_index + 2 + imp_name.len + 1), ); - try gop.value_ptr.import_hint_name_table_ni.resize(&coff.mf, gpa, gop.value_ptr.hint_name_len); + try gop.value_ptr.import_hint_name_table_ni.resizeLeaf(&coff.mf, gpa, gop.value_ptr.hint_name_len); break :blk import_hint_name_index; } else null; @@ -6671,9 +6631,9 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { else => |tag| @panic(@tagName(tag)), .AMD64 => { const init = [_]u8{ 0xff, 0x25, 0x00, 0x00, 0x00, 0x00 }; - const ni = try coff.mf.addLastChildNode(gpa, parent_sym.ni.unwrap().?, .{ + const ni = try parent_sym.ni.unwrap().?.addFloatingChild(&coff.mf, gpa, .{ .alignment = alignment, - .size = init.len, + .size = alignment.forward(init.len), }); @memcpy(ni.slice(&coff.mf)[0..init.len], &init); sym.ni = .wrap(ni); @@ -6824,7 +6784,7 @@ fn flushLazy(coff: *Coff, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void { .code => .text, .const_data => .rdata, }; - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ .moved = true }); + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ .moved = true }); coff.nodes.appendAssumeCapacity(switch (lazy.kind) { .code => .{ .lazy_code = @fromBackingInt(@intCast(lmr.index)) }, .const_data => .{ .lazy_const_data = @fromBackingInt(@intCast(lmr.index)) }, @@ -7480,7 +7440,7 @@ fn updateExportInner( if (new_name_table_size > std.math.maxInt(@FieldType(ExportTable.Entry, "name_index"))) return coff.base.comp.link_diags.fail("exports name table limit reached", .{}); - try coff.export_table.name_table_ni.resize(&coff.mf, gpa, new_name_table_size); + try coff.export_table.name_table_ni.resizeLeaf(&coff.mf, gpa, new_name_table_size); const name_table_slice = coff.export_table.name_table_ni.slice(&coff.mf); @memcpy(name_table_slice[name_index..][0 .. name.len + 1], name[0 .. name.len + 1]); @@ -7503,19 +7463,19 @@ fn updateExportInner( // TODO: These should all be resized ahead of time to fit all exports // after https://github.com/ziglang/zig/issues/23616 - try coff.export_table.export_address_table_si.node(coff).resize( + try coff.export_table.export_address_table_si.node(coff).resizeLeaf( &coff.mf, gpa, export_count * @sizeOf(std.coff.ExportAddressTableEntry), ); - try coff.export_table.name_pointer_table_ni.resize( + try coff.export_table.name_pointer_table_ni.resizeLeaf( &coff.mf, gpa, export_count * @sizeOf(std.coff.ExportNamePointerTableEntry), ); - try coff.export_table.ordinal_table_ni.resize( + try coff.export_table.ordinal_table_ni.resizeLeaf( &coff.mf, gpa, export_count * @sizeOf(std.coff.ExportOrdinalTableEntry), @@ -7746,12 +7706,12 @@ pub fn printNode( { const mf_node = &coff.mf.nodes.items[@backingInt(ni)]; const off, const size = mf_node.location().resolve(&coff.mf); - try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x}{s}{s}{s}{s}\n", .{ + try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x} {t}{s}{s}{s}\n", .{ @backingInt(ni), off, size, mf_node.flags.alignment.toByteUnits(), - if (mf_node.flags.fixed) " fixed" else "", + mf_node.flags.position, if (mf_node.flags.moved) " moved" else "", if (mf_node.flags.resized) " resized" else "", if (mf_node.flags.has_content) " has_content" else "", diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 7f23494f4fe45d74c749f6b65b255b50a20781c7..0acc9a6fe86b17f887eb41a55f35ea199bb1dca7 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -550,7 +550,7 @@ const Section = struct { } const ni = shndx.get(elf).ni; if (min_align.compare(.gt, ni.alignment(&elf.mf))) { - try ni.realign(&elf.mf, elf.base.comp.gpa, min_align, .{}); + try ni.realign(&elf.mf, elf.base.comp.gpa, min_align); } switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) { .elf => {}, @@ -583,7 +583,7 @@ const Section = struct { break :need_size cur_size + need_additional * ent_size; }, }; - try elf.ensureNodeSize(node, need_size); + try node.ensureMinimumSize(&elf.mf, elf.base.comp.gpa, need_size); } /// Asserts that `rela_shndx` is a `SHT_RELA` section and deletes the `ElfN.Rela` entry at @@ -1787,6 +1787,8 @@ const SymbolReloc = struct { }; fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void { + const gpa = elf.base.comp.gpa; + const min_buckets = max_dynsym_count / 2; const cur_dynsym_count: u32 = switch (elf.shdrPtr(elf.shndx.dynsym)) { @@ -1807,7 +1809,7 @@ fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void { // We don't need to add any buckets, but we still need to make sure the section is large // enough to fit `max_dynsym_count` chains. const need_size = @sizeOf(info.Header()) + (nbucket + max_dynsym_count) * 4; - try elf.ensureNodeSize(elf.shndx.hash.get(elf).ni, need_size); + try elf.shndx.hash.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); return; } // We need more buckets, so we'll have to rebuild the hash table. @@ -1819,7 +1821,7 @@ fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void { { const need_size = @sizeOf(info.Header()) + (new_nbucket + max_dynsym_count) * 4; - try elf.ensureNodeSize(elf.shndx.hash.get(elf).ni, need_size); + try elf.shndx.hash.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } elf.mf.nodes_lock.lock(); @@ -1965,7 +1967,7 @@ fn ensureUnusedSymbolCapacity(elf: *Elf, len: u32, kind: enum { all_local, maybe const need_node_size: u64 = switch (elf.shdrPtr(.symtab)) { inline else => |shdr, class| elf.targetLoad(&shdr.size) + len * @sizeOf(class.ElfN().Sym), }; - try elf.ensureNodeSize(Section.Index.symtab.get(elf).ni, need_node_size); + try Section.Index.symtab.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_node_size); } switch (kind) { @@ -1988,7 +1990,7 @@ fn ensureUnusedSymbolCapacity(elf: *Elf, len: u32, kind: enum { all_local, maybe const dynsym_cur_len: u32 = @intCast(@divExact(dynsym_cur_size, dynsym_ent_size)); const dynsym_need_size: u64 = (dynsym_cur_len + len) * dynsym_ent_size; - try elf.ensureNodeSize(elf.shndx.dynsym.get(elf).ni, dynsym_need_size); + try elf.shndx.dynsym.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, dynsym_need_size); try elf.ensureDynsymHashCapacity(dynsym_cur_len + len); @@ -2010,19 +2012,19 @@ fn ensureUnusedPltCapacity(elf: *Elf, len: u32) Error!void { // Ensure the `.plt` section's node is big enough: { const need_size: usize = plt.entry_size * (1 + need_plt_count); - try elf.ensureNodeSize(elf.shndx.plt.get(elf).ni, need_size); + try elf.shndx.plt.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } // If there is a `.got.plt` section, ensure its node is big enough if (plt.got_plt) |got_plt| { const need_size: usize = elf.targetPtrSize() * (got_plt.header_entries + need_plt_count); - try elf.ensureNodeSize(elf.shndx.got_plt.get(elf).ni, need_size); + try elf.shndx.got_plt.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } // If there is a `.plt.sec` section, ensure its node is big enough if (plt.plt_sec) |plt_sec| { const need_size: usize = plt_sec.entry_size * need_plt_count; - try elf.ensureNodeSize(elf.shndx.plt_sec.get(elf).ni, need_size); + try elf.shndx.plt_sec.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } } /// Given an index into the PLT, returns whether that PLT entry is dead, meaning it may be reused at @@ -2989,7 +2991,7 @@ fn lazySymbolInner(elf: *Elf, lazy: link.File.LazySymbol) Error!link.File.Symbol .code => .{ .text, .FUNC }, .const_data => .{ .rodata, .OBJECT }, }; - const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{}); + const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{}); var name_buf: [64]u8 = undefined; const name = std.fmt.bufPrint( &name_buf, @@ -3248,7 +3250,7 @@ const StringTable = struct { break :size .{ old_size, new_size }; }, }; - try elf.ensureNodeSize(ni, new_size); + try ni.ensureMinimumSize(&elf.mf, gpa, new_size); const slice = ni.slice(&elf.mf)[old_size..]; @memcpy(slice[0..key.len], key); slice[key.len] = 0; @@ -3611,10 +3613,11 @@ fn initHeaders( if (is_archive) { elf.nodes.appendAssumeCapacity(.archive); - const archive_header_ni = try elf.mf.addOnlyChildNode(gpa, .root, .{ + const archive_ni: MappedFile.Node.Index = .root; + + const archive_header_ni = try archive_ni.addOnlyHeaderChild(&elf.mf, gpa, .{ .size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr) * 2, .alignment = .@"2", - .fixed = true, .next_moved = true, .bubbles_moved = false, .enable_next_moved = true, @@ -3633,7 +3636,7 @@ fn initHeaders( .ar_fmag = std.elf.ARFMAG.*, }; - elf.ni.elf = try elf.mf.addLastChildNode(gpa, .root, .{ + elf.ni.elf = try archive_ni.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align.max(.@"2"), .next_moved = true, .bubbles_moved = false, @@ -3657,19 +3660,18 @@ fn initHeaders( // the rodata segment. Although to my knowledge neither ELF nor any ELF-based OS strictly // requires this, it is highly conventional and therefore sometimes relied upon. if (@"type" != .REL) { - elf.ni.rodata = try elf.mf.addOnlyChildNode(gpa, elf.ni.elf, .{ + // This node will contain the ehdr, which must be at the start of the ELF file, so this + // node must itself be a header of the `.elf` node. + elf.ni.rodata = try elf.ni.elf.addOnlyHeaderChild(&elf.mf, gpa, .{ // Must be at least `addr_align` for `elf.ni.phdr` to be placed inside this node .alignment = node_block_align.max(addr_align), - // This node will contain the ehdr, which must be at the start of the ELF file, so this - // node must itself be fixed. - .fixed = true, .moved = true, .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.rodata }); elf.phdrs.items[phndx.rodata] = .wrap(elf.ni.rodata); - elf.ni.phdr = try elf.mf.addOnlyChildNode(gpa, elf.ni.rodata, .{ + elf.ni.phdr = try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{ .size = @as(u64, phnum) * entsize.ph, .alignment = addr_align, // keep in sync with `elf.ni.rodata` alignment above .moved = true, @@ -3679,7 +3681,7 @@ fn initHeaders( elf.nodes.appendAssumeCapacity(.{ .segment = phndx.phdr }); elf.phdrs.items[phndx.phdr] = .wrap(elf.ni.phdr); - elf.ni.text = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ + elf.ni.text = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align, .moved = true, .bubbles_moved = false, @@ -3687,7 +3689,7 @@ fn initHeaders( elf.nodes.appendAssumeCapacity(.{ .segment = phndx.text }); elf.phdrs.items[phndx.text] = .wrap(elf.ni.text); - elf.ni.data = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ + elf.ni.data = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ // Must be at least `addr_align` for `elf.ni.data_rel_ro` to be placed inside this node .alignment = node_block_align.max(addr_align), .moved = true, @@ -3697,7 +3699,7 @@ fn initHeaders( elf.phdrs.items[phndx.data] = .wrap(elf.ni.data); if (plt.got_plt == null) { - const plt_ni = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ + const plt_ni = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align, .moved = true, .bubbles_moved = false, @@ -3706,7 +3708,7 @@ fn initHeaders( elf.phdrs.items[phndx.plt] = .wrap(plt_ni); } - elf.ni.data_rel_ro = try elf.mf.addOnlyChildNode(gpa, elf.ni.data, .{ + elf.ni.data_rel_ro = try elf.ni.data.addFloatingChild(&elf.mf, gpa, .{ // Must be at least `addr_align` for the `PT_DYNAMIC` node to be placed inside this one // later (if `have_dynamic_section`). Keep in sync with `elf.ni.data` alignment above. .alignment = node_block_align.max(addr_align), @@ -3717,7 +3719,7 @@ fn initHeaders( elf.phdrs.items[phndx.relro] = .wrap(elf.ni.data_rel_ro); if (comp.config.any_non_single_threaded) { - elf.ni.tls = .wrap(try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{ + elf.ni.tls = .wrap(try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align, .moved = true, .bubbles_moved = false, @@ -3738,10 +3740,9 @@ fn initHeaders( .REL => elf.ni.elf, .DYN, .EXEC => elf.ni.rodata, }; - elf.ni.ehdr = try elf.mf.addFirstChildNode(gpa, parent_ni, .{ + elf.ni.ehdr = try parent_ni.addOnlyHeaderChild(&elf.mf, gpa, .{ .size = @sizeOf(ElfN.Ehdr), .alignment = addr_align, - .fixed = true, }); elf.nodes.appendAssumeCapacity(.ehdr); @@ -3793,8 +3794,8 @@ fn initHeaders( }, } - elf.ni.shdr = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ - .size = 1 * entsize.sh, // as above, only the SHN_UNDEF initially + elf.ni.shdr = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ + .size = node_block_align.forward(1 * entsize.sh), // as above, only the SHN_UNDEF initially .alignment = addr_align.max(node_block_align), .moved = true, .resized = true, @@ -4109,7 +4110,7 @@ fn initHeaders( .node_align = node_block_align, }); if (maybe_interp) |interp| { - const interp_ni = try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{ + const interp_ni = try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{ .size = interp.len + 1, .moved = true, .resized = true, @@ -4130,7 +4131,7 @@ fn initHeaders( } if (have_dynamic_section) { assert(elf.ni.data_rel_ro.alignment(&elf.mf).compare(.gte, addr_align)); - const dynamic_ni = try elf.mf.addLastChildNode(gpa, elf.ni.data_rel_ro, .{ + const dynamic_ni = try elf.ni.data_rel_ro.addFloatingChild(&elf.mf, gpa, .{ .alignment = addr_align, .moved = true, .bubbles_moved = false, @@ -4208,7 +4209,7 @@ fn initHeaders( .flags = .{ .ALLOC = true, .WRITE = true }, .link = dynstr_shndx.toSection().?, .entsize = @intCast(addr_align.toByteUnits() * 2), - .node_align = addr_align, + .addralign = addr_align, }); switch (elf.targetDynsymHashInfo()) { inline else => |info| { @@ -4869,7 +4870,7 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo { // TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`. }; } -fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { +pub fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer; const Child = pointer_ty.child; const alignment = pointer_ty.attrs.@"align" orelse @alignOf(Child); @@ -5051,7 +5052,7 @@ fn mapInputSection(elf: *Elf, opts: struct { const name_shstrtab = try elf.string(.shstrtab, name); const gop = try elf.section_by_name.getOrPut(gpa, name_shstrtab); if (gop.found_existing) { - break :existing @fromBackingInt(@intCast(gop.index)); + break :existing @fromBackingInt(@intCast(gop.index + 1)); // +1 to account for SHN_UDNEF } errdefer assert(elf.section_by_name.pop().?.key == name_shstrtab); const parent_node: MappedFile.Node.Index = parent: { @@ -5172,7 +5173,7 @@ fn navMapIndex(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) Error!Node }, }; try shndx.ensureAligned(elf, alignment); - const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{ + const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{ .alignment = alignment, }); nav_gop.value_ptr.* = .{ @@ -5216,7 +5217,7 @@ fn uavMapIndex( if (!uav_gop.found_existing) { const shndx: Section.Index = .data_rel_ro; // TODO: it would be better to use `.rodata` if the UAV value doesn't have relocs try shndx.ensureAligned(elf, resolved_align); - const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{ + const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{ .moved = true, // see assert at end of `genUav` .alignment = resolved_align, }); @@ -5245,7 +5246,7 @@ fn uavMapIndex( const shndx = elf.getNode(node.parent(&elf.mf).unwrap().?).section; try shndx.ensureAligned(elf, resolved_align); if (resolved_align.order(node.alignment(&elf.mf)).compare(.gt)) { - try node.realign(&elf.mf, gpa, resolved_align, .{}); + try node.realign(&elf.mf, gpa, resolved_align); } } return umi; @@ -5462,9 +5463,10 @@ fn loadObject( .extra = undefined, }; if (elf.ni.elf != .root) { + const archive_ni: MappedFile.Node.Index = .root; try elf.nodes.ensureUnusedCapacity(gpa, 1); - input.extra = .{ .node = try elf.mf.addLastChildNode(gpa, .root, .{ - .size = fl.size + @sizeOf(std.elf.ar_hdr), + input.extra = .{ .node = try archive_ni.addFloatingChild(&elf.mf, gpa, .{ + .size = Alignment.@"2".forward(fl.size + @sizeOf(std.elf.ar_hdr)), .alignment = .@"2", .next_moved = true, .bubbles_moved = false, @@ -5646,12 +5648,24 @@ fn loadObject( std.math.ceilPowerOfTwoAssert(usize, @intCast(@max(section.shdr.addralign, 1))), ); try opts.shndx.ensureAligned(elf, need_align); - const ni = try elf.mf.addLastChildNode(gpa, opts.shndx.get(elf).ni, .{ - .size = section.shdr.size, + const add_node_opts: MappedFile.Node.AddOptions = .{ + .size = need_align.forward(section.shdr.size), .alignment = need_align, .moved = true, // see assert at end of `flushInputSection` - .fixed = opts.node_fixed, - }); + }; + const ni = if (opts.node_fixed) ni: { + const shndx_ni = opts.shndx.get(elf).ni; + const after_oni: MappedFile.Node.Index.Optional = after: { + const last_ni = shndx_ni.last(&elf.mf).unwrap() orelse break :after .none; + break :after switch (last_ni.position(&elf.mf)) { + .header => .wrap(last_ni), + .footer, .floating => .none, + }; + }; + break :ni try shndx_ni.addHeaderChildAfter(&elf.mf, gpa, after_oni, add_node_opts); + } else ni: { + break :ni try opts.shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, add_node_opts); + }; elf.nodes.appendAssumeCapacity(.{ .input_section = @fromBackingInt(@intCast(elf.input_sections.items.len)), }); @@ -6019,8 +6033,8 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadPars // We have a copy relocation for this global, but the amount of space we // reserved for it could be too small or underaligned! try Section.Index.data.ensureAligned(elf, gop.value_ptr.alignment); - try copied_global.node.resize(&elf.mf, gpa, gop.value_ptr.size); - try copied_global.node.realign(&elf.mf, gpa, gop.value_ptr.alignment, .{}); + try copied_global.node.resizeLeaf(&elf.mf, gpa, gop.value_ptr.alignment.forward(gop.value_ptr.size)); + try copied_global.node.realign(&elf.mf, gpa, gop.value_ptr.alignment); const global_ptr = elf.globalByName(name).?; switch (elf.symPtr(global_ptr.symtab_index)) { inline else => |sym_ptr| elf.targetStore(&sym_ptr.size, @intCast(gop.value_ptr.size)), @@ -6267,7 +6281,7 @@ fn prepareDynamic(elf: *Elf) Error!void { const dynamic_size = dynamic_len * 2 * elf.targetPtrSize(); - try elf.shndx.dynamic.get(elf).ni.resize(&elf.mf, comp.gpa, dynamic_size); + try elf.shndx.dynamic.get(elf).ni.resizeLeaf(&elf.mf, comp.gpa, dynamic_size); switch (elf.shdrPtr(elf.shndx.dynamic)) { inline else => |shdr| elf.targetStore(&shdr.size, @intCast(dynamic_size)), } @@ -6393,7 +6407,6 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct { addralign: Alignment = .@"1", entsize: std.elf.Word = 0, node_align: Alignment = .@"1", - fixed: bool = false, }) Error!Section.Index { switch (opts.type) { .NULL => assert(opts.size == 0), @@ -6437,14 +6450,15 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct { break :shndx .{ @fromBackingInt(shndx), @as(u64, elf.targetLoad(&ehdr.shentsize)) * @as(u64, shnum) }; }, }; - try elf.ensureNodeSize(elf.ni.shdr, new_shdr_size); - const ni = try elf.mf.addLastChildNode(gpa, switch (elf.ehdrType()) { + try elf.ni.shdr.ensureMinimumSize(&elf.mf, gpa, new_shdr_size); + const parent_ni = switch (elf.ehdrType()) { .REL => elf.ni.elf, .EXEC, .DYN => segment_ni, - }, .{ - .size = opts.size, + }; + assert(opts.addralign.check(opts.size)); + const ni = try parent_ni.addFloatingChild(&elf.mf, gpa, .{ + .size = opts.node_align.forward(opts.size), .alignment = opts.addralign.max(opts.node_align), - .fixed = opts.fixed, .resized = opts.size > 0, }); const addr = elf.computeNodeVAddr(ni); @@ -6530,7 +6544,7 @@ fn ensureUnusedRelocCapacity(elf: *Elf, node: MappedFile.Node.Index, len: usize) .NONE, _ => unreachable, inline else => |ct_class| (elf.got.count() + new_got_entries) * @sizeOf(ct_class.ElfN().Addr), }; - try elf.ensureNodeSize(elf.shndx.got.get(elf).ni, need_got_size); + try elf.shndx.got.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_got_size); if (elf.shndx.dynamic != .UNDEF) { try elf.shndx.rela_dyn.relaEnsureAdditionalCapacity(elf, new_got_entries); @@ -7284,8 +7298,8 @@ fn maybeAddCopyRelocation(elf: *Elf, global_name: String(.strtab)) Error!bool { try Section.Index.data.ensureAligned(elf, dso_global.alignment); try elf.nodes.ensureUnusedCapacity(gpa, 1); - const node = try elf.mf.addLastChildNode(gpa, Section.Index.data.get(elf).ni, .{ - .size = dso_global.size, + const node = try Section.Index.data.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{ + .size = dso_global.alignment.forward(dso_global.size), .alignment = dso_global.alignment, }); errdefer comptime unreachable; @@ -8868,12 +8882,12 @@ pub fn printNode( { const mf_node = &elf.mf.nodes.items[@backingInt(ni)]; const off, const size = mf_node.location().resolve(&elf.mf); - try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x}{s}{s}{s}{s}{s}\n", .{ + try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x} {t}{s}{s}{s}{s}\n", .{ @backingInt(ni), off, size, mf_node.flags.alignment.toByteUnits(), - if (mf_node.flags.fixed) " fixed" else "", + mf_node.flags.position, if (mf_node.flags.moved) " moved" else "", if (mf_node.flags.next_moved) " next_moved" else "", if (mf_node.flags.resized) " resized" else "", @@ -8920,7 +8934,7 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: Alignment) Error // Align the actual node const seg_ni = elf.phdrs.items[phndx].unwrap().?; if (min_align.compare(.gt, seg_ni.alignment(&elf.mf))) { - try seg_ni.realign(&elf.mf, gpa, min_align, .{}); + try seg_ni.realign(&elf.mf, gpa, min_align); } // Update the phdr `@"align"` field if necessary switch (elf.phdrSlice()) { @@ -8960,15 +8974,7 @@ fn ensureElfNodeSize(elf: *Elf) MappedFile.Error!void { const last_offset, const last_size = last_ni.location(&elf.mf).resolve(&elf.mf); break :last_end last_offset + last_size; } else 0; - try elf.ensureNodeSize(elf.ni.elf, last_end + @sizeOf(std.elf.ar_hdr)); -} - -fn ensureNodeSize(elf: *Elf, node: MappedFile.Node.Index, need_size: u64) MappedFile.Error!void { - _, const node_size = node.location(&elf.mf).resolve(&elf.mf); - if (need_size <= node_size) return; - const gpa = elf.base.comp.gpa; - const new_size = need_size + need_size / MappedFile.growth_factor; - try node.resize(&elf.mf, gpa, new_size); + try elf.ni.elf.ensureMinimumSize(&elf.mf, elf.base.comp.gpa, last_end + @sizeOf(std.elf.ar_hdr)); } /// If `sym` has a PLT entry, returns the address of that entry (specifically, the address which a diff --git a/src/link/MappedFile.zig b/src/link/MappedFile.zig index b1da80973c756c2aab7b30d88b3605171b06362f..40af66dbf8c53093c492a79d69064879175cc4ef 100644 --- a/src/link/MappedFile.zig +++ b/src/link/MappedFile.zig @@ -183,14 +183,42 @@ pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error || Io.Cancel .fallocate_insert_range_unsupported = false, .fallocate_punch_hole_unsupported = false, }; - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const root_ni = try mf.addNode(gpa, .{ .add_node = .{ - .size = size, - .alignment = mf.flags.block_size, - .fixed = true, - } }); - assert(root_ni == .root); - try mf.ensureTotalCapacityInner(@intCast(size)); + + const root_location: Node.Location = l: { + if (std.math.cast(u32, size)) |small_size| { + break :l .{ .small = .{ .offset = 0, .size = small_size } }; + } + try mf.large.appendSlice(gpa, &.{ 0, size }); + break :l .{ .large = .{ .index = 0 } }; + }; + try mf.nodes.append(gpa, .{ + .parent = .none, + .prev = .none, + .next = .none, + .first = .none, + .last = .none, + .flags = .{ + .alignment = mf.flags.block_size, + .position = .floating, + .bubbles_moved = true, + .enable_next_moved = false, + .location_tag = root_location, + .moved = false, + .resized = false, + .next_moved = false, + .has_content = false, + }, + .location_payload = switch (root_location) { + .small => |small| .{ .small = small }, + .large => |large| .{ .large = large }, + }, + }); + + mf.ensureTotalCapacity(@intCast(size)) catch |err| switch (err) { + error.MappedFileIo => return mf.io_err.?, + else => |e| return e, + }; + return mf; } @@ -213,24 +241,53 @@ pub const Node = extern struct { flags: Flags, location_payload: Location.Payload, + /// Any non-leaf node may designate its first N children as "header" nodes. This means that its + /// first N children must be densely packed together and positioned at the start of the parent. + /// The implementation guarantees that it will never re-order these nodes, nor will it introduce + /// padding between them. + /// + /// Likewise, any non-leaf node may designate its *last* M children as "footer" nodes, which are + /// like header nodes except they are positioned at the *end* of the parent rather than the + /// start. + /// + /// Nodes which are neither headers nor footers are called "floating". The implementation is + /// always free to re-order floating nodes relative to one another, and to add or remove padding + /// between them. + pub const Position = enum(u2) { + header, + footer, + floating, + }; + pub const Flags = packed struct(u32) { - location_tag: Location.Tag, + /// While the number of header and footer nodes within a parent node is logically a part of + /// that parent, we actually store this information on the child nodes for efficiency: this + /// field indicates whether each child is a header node, a footer node, or a floating node. + /// + /// This value is meaningless for the root node, so is arbitrarily set to `.floating`. + position: Position, + /// For floating nodes, this node's offset into its parent will always be aligned to this + /// boundary. (This is not the case for header and footer nodes due to the requirement that + /// they be densely packed against the start/end of the parent node.) + /// + /// This node's size will also always be aligned to this boundary. (This applies regardless + /// of whether this is a floating node, a header node, or a footer node.) alignment: Alignment, - /// Whether this node can be moved. - fixed: bool, + /// Whether `moved` events on this node bubble down to children. + bubbles_moved: bool, + /// Whether `next_moved` events are reported in `updates`. + enable_next_moved: bool, + + location_tag: Location.Tag, /// Whether this node has been moved. moved: bool, /// Whether this node has been resized. resized: bool, /// Whether the next sibling has moved or is a different node. next_moved: bool, - /// Whether this node might contain non-zero bytes. + /// Whether this node might contain initialized bytes. has_content: bool, - /// Whether `moved` events on this node bubble down to children. - bubbles_moved: bool, - /// Whether `next_moved` events are reported in `updates`. - enable_next_moved: bool, - unused: u18 = 0, + unused: u17 = 0, }; pub const Location = union(enum(u1)) { @@ -267,6 +324,18 @@ pub const Node = extern struct { } }; + pub const AddOptions = struct { + /// Must be aligned to the given `alignment`. + size: u64 = 0, + alignment: Alignment = .@"1", + bubbles_moved: bool = true, + enable_next_moved: bool = false, + + moved: bool = false, + resized: bool = false, + next_moved: bool = false, + }; + pub const Index = enum(u32) { root, _, @@ -292,6 +361,70 @@ pub const Node = extern struct { return &mf.nodes.items[@backingInt(ni)]; } + /// Adds a floating child node to `parent_ni`. Returns the index of the new child. + pub fn addFloatingChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { + return mf.addNode(gpa, .{ + .add_options = opts, + .position = .floating, + .parent = parent_ni, + .prev = parent_ni.lastHeader(mf), + }); + } + /// Adds a header child node to `parent_ni`. Returns the index of the new child. + /// + /// Asserts that `parent_ni` has no existing header children. + pub fn addOnlyHeaderChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { + if (parent_ni.first(mf).unwrap()) |first_ni| { + assert(first_ni.position(mf) != .header); // `parent_ni` already has a header child + } + return parent_ni.addHeaderChildAfter(mf, gpa, .none, opts); + } + /// Adds a header child node to `parent_ni`. Returns the index of the new child. + /// + /// If `prev_oni` is `.none`, the new child is placed at the very start of the parent, + /// before any existing header nodes. + /// + /// Otherwise, asserts that `prev_oni` is a header node and a child of `parent_ni`, and + /// places the new child node immediately after `prev_oni`. + pub fn addHeaderChildAfter(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, prev_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index { + return mf.addNode(gpa, .{ + .add_options = opts, + .position = .header, + .parent = parent_ni, + .prev = prev_oni, + }); + } + /// Adds a footer child node to `parent_ni`. Returns the index of the new child. + /// + /// Asserts that `parent_ni` has no existing footer children. + pub fn addOnlyFooterChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { + if (parent_ni.last(mf).unwrap()) |last_ni| { + assert(last_ni.position(mf) != .footer); // `parent_ni` already has a footer child + } + return parent_ni.addFooterChildBefore(mf, gpa, .none, opts); + } + /// Adds a footer child node to `parent_ni`. Returns the index of the new child. + /// + /// If `next_oni` is `.none`, the new child is placed at the very end of the parent, after + /// any existing footer nodes. + /// + /// Otherwise, asserts that `next_oni` is a footer node and a child of `parent_ni`, and + /// places the new child node immediately before `next_oni`. + pub fn addFooterChildBefore(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, next_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index { + const prev_oni: Node.Index.Optional = prev: { + const next_ni = next_oni.unwrap() orelse { + break :prev parent_ni.last(mf); + }; + break :prev next_ni.prev(mf); + }; + return mf.addNode(gpa, .{ + .add_options = opts, + .position = .footer, + .parent = parent_ni, + .prev = prev_oni, + }); + } + /// Alias for `Optional.wrap`, provided for convenience when a result type is not available. pub const toOptional = Optional.wrap; @@ -299,19 +432,54 @@ pub const Node = extern struct { return ni.get(mf).parent; } + pub fn first(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + return ni.get(mf).first; + } + + pub fn last(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + return ni.get(mf).last; + } + + fn lastHeader(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + var header_ni = ni.first(mf).unwrap() orelse return .none; + if (header_ni.position(mf) != .header) return .none; + while (true) { + const next_ni = header_ni.next(mf).unwrap() orelse break; + if (next_ni.position(mf) != .header) break; + header_ni = next_ni; + } + return .wrap(header_ni); + } + fn firstFooter(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + var footer_ni = ni.last(mf).unwrap() orelse return .none; + if (footer_ni.position(mf) != .footer) return .none; + while (true) { + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } + return .wrap(footer_ni); + } + + /// Asserts that `ni` is not `.root`, because `Position` is meaningless for the root node. + pub fn position(ni: Node.Index, mf: *const MappedFile) Node.Position { + assert(ni != .root); + return ni.get(mf).flags.position; + } + pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { return ni.get(mf).next; } fn setNext( - prev_ni: Node.Index, + ni: Node.Index, gpa: Allocator, next_ni: Node.Index.Optional, mf: *MappedFile, ) Allocator.Error!void { - const prev_next = &prev_ni.get(mf).next; - if (prev_next.* == next_ni) return; - prev_next.* = next_ni; - try prev_ni.nextMoved(gpa, mf); + const next_ptr = &ni.get(mf).next; + if (next_ptr.* == next_ni) return; + next_ptr.* = next_ni; + try ni.nextMoved(gpa, mf); } pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { @@ -421,8 +589,14 @@ pub const Node = extern struct { return ni.get(mf).flags.alignment; } - fn setLocationAssumeCapacity(ni: Node.Index, mf: *MappedFile, offset: u64, size: u64) void { + fn setLocation(ni: Node.Index, mf: *MappedFile, gpa: Allocator, offset: u64, size: u64) Allocator.Error!void { + try mf.large.ensureUnusedCapacity(gpa, 2); + try mf.updates.ensureUnusedCapacity(gpa, 2); const node = ni.get(mf); + if (node.flags.position == .floating) { + assert(node.flags.alignment.check(offset)); + } + assert(node.flags.alignment.check(size)); if (size == 0) node.flags.has_content = false; switch (node.location()) { .small => |small| { @@ -485,62 +659,46 @@ pub const Node = extern struct { return mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)]; } - pub fn resize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void { - mf.resizeNode(gpa, ni, size) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; - var writers_it = mf.writers.first; - while (writers_it) |writer_node| : (writers_it = writer_node.next) { - const w: *Node.Writer = @fieldParentPtr("writer_node", writer_node); - w.interface.buffer = w.ni.slice(mf); + /// Ensures that the size of `ni` is at least `min_size`. Valid for any node. + /// + /// Applies `growth_factor` if necessary (so the caller should *not* apply `growth_factor`). + pub fn ensureMinimumSize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, min_size: u64) Error!void { + _, const current_size = ni.location(mf).resolve(mf); + if (current_size >= min_size) return; + const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor); + try mf.growNode(gpa, ni, new_size, .minimum); + mf.updateWriters(); + } + + /// Sets the size of `ni` to exactly `size`. + /// + /// Asserts that `ni` is a leaf node, i.e. has no children. + /// + /// Asserts that `size` is aligned to `ni.alignment(mf)`. + pub fn resizeLeaf(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void { + assert(ni.first(mf) == .none); + // The alignment of `size` is asserted by `shrinkLeafNode` and `growNode`. + _, const old_size = ni.location(mf).resolve(mf); + switch (std.math.order(size, old_size)) { + .lt => try mf.shrinkLeafNode(gpa, ni, size), + .eq => {}, // `old_size` must be well-aligned, so `size` is too + .gt => try mf.growNode(gpa, ni, size, .exact), } + mf.updateWriters(); } - pub const RealignNodeOptions = struct { - /// Shift the node backwards if possible - try_backwards: bool = false, - }; - - /// Moves and expands a node such that its offset and size are aligned to `new_alignment`. - /// Asserts that `ni` is not `.root`. + /// Updates a node's alignment to exactly `new_alignment`. Valid for any node. + /// + /// If the node's current offset or size is not sufficiently aligned, it will be moved + /// and/or resized to match the new alignment. The node's size may be increased by any + /// amount, as if `ensureMinimumSize` were used. pub fn realign( ni: Node.Index, mf: *MappedFile, gpa: Allocator, new_alignment: Alignment, - opts: RealignNodeOptions, ) Error!void { - mf.realignNode(gpa, ni, new_alignment, opts) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; - mf.updateWriters(); - } - - /// Shrink a node to `size`, exactly. - /// Asserts that the new size can contain all the children. - /// If `shift_next` is set, then the following node is shifted backwards into - /// the free space as much as alignment allows. - /// Asserts that `size` is >= the end of the last child node. - pub fn shrink( - ni: Node.Index, - mf: *MappedFile, - gpa: Allocator, - size: u64, - shift_next: bool, - ) Error!void { - try mf.shrinkNode(gpa, ni, size, shift_next); + try mf.realignNode(gpa, ni, new_alignment); mf.updateWriters(); } @@ -644,16 +802,9 @@ pub const Node = extern struct { file_reader.pos, w.ni.fileLocation(w.mf, true).offset + interface.end, limit.minInt(interface.unusedCapacityLen()), - ) catch |err| switch (err) { - error.Canceled => |e| { - w.err = e; - return error.WriteFailed; - }, - else => |e| { - w.mf.io_err = e; - w.err = error.MappedFileIo; - return error.WriteFailed; - }, + ) catch |err| { + w.err = err; + return error.WriteFailed; }); if (n == 0) return error.Unimplemented; file_reader.pos += n; @@ -680,10 +831,8 @@ pub const Node = extern struct { unused_capacity: usize, ) Io.Writer.Error!void { _ = preserve; - const total_capacity = interface.end + unused_capacity; - if (interface.buffer.len >= total_capacity) return; const w: *Writer = @fieldParentPtr("interface", interface); - w.ni.resize(w.mf, w.gpa, total_capacity +| total_capacity / growth_factor) catch |err| { + w.ni.ensureMinimumSize(w.mf, w.gpa, interface.end + unused_capacity) catch |err| { w.err = err; return error.WriteFailed; }; @@ -691,514 +840,1227 @@ pub const Node = extern struct { }; comptime { - if (!std.debug.runtime_safety) std.debug.assert(@sizeOf(Node) == 32); + if (!std.debug.runtime_safety) assert(@sizeOf(Node) == 32); } }; +/// Asserts that `opts.position` is compatible with `opts.prev` (i.e. that this addition will not +/// violate the requirement that header nodes come before floating nodes come before footer nodes). fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { - parent: Node.Index.Optional = .none, - prev: Node.Index.Optional = .none, - next: Node.Index.Optional = .none, - offset: u64 = 0, - add_node: AddNodeOptions, -}) (Allocator.Error || Io.Cancelable || IoError)!Node.Index { + add_options: Node.AddOptions, + position: Node.Position, + parent: Node.Index, + /// If `position == .floating`, this is just used as an initial value, and may be immediately + /// replaced when finding a location for this node. In this case, it is still necessary that + /// `prev` be compatible with `position` (so `prev` must be either a floating node or the last + /// header node in `parent`). + prev: Node.Index.Optional, +}) Error!Node.Index { mf.nodes_lock.assertUnlocked(); - const location_tag: Node.Location.Tag, const location_payload: Node.Location.Payload = location: { - if (std.math.cast(u32, opts.offset)) |small_offset| break :location .{ .small, .{ - .small = .{ .offset = small_offset, .size = 0 }, - } }; - try mf.large.ensureUnusedCapacity(gpa, 2); - defer mf.large.appendSliceAssumeCapacity(&.{ opts.offset, 0 }); - break :location .{ .large, .{ .large = .{ .index = mf.large.items.len } } }; - }; - const free_ni: Node.Index, const free_node: *Node = if (mf.free_ni.unwrap()) |free_ni| free: { - const free_node = free_ni.get(mf); - mf.free_ni = free_node.next; - break :free .{ free_ni, free_node }; - } else .{ - @fromBackingInt(@intCast(mf.nodes.items.len)), - mf.nodes.addOneAssumeCapacity(), + try mf.nodes.ensureUnusedCapacity(gpa, 1); + try mf.large.ensureUnusedCapacity(gpa, 2); + + const new_ni: Node.Index = new: { + if (mf.free_ni.unwrap()) |free_ni| { + mf.free_ni = free_ni.get(mf).next; + break :new free_ni; + } + const new_ni: Node.Index = @fromBackingInt(@intCast(mf.nodes.items.len)); + _ = mf.nodes.addOneAssumeCapacity(); + break :new new_ni; }; - if (opts.prev.unwrap()) |prev_ni| { - try prev_ni.setNext(gpa, .wrap(free_ni), mf); - } else if (opts.parent.unwrap()) |parent_ni| { - parent_ni.get(mf).first = .wrap(free_ni); - } else { - assert(free_ni == .root); - } + const next_oni: Node.Index.Optional = if (opts.prev.unwrap()) |prev_ni| next: { + assert(prev_ni.parent(mf) == opts.parent.toOptional()); // `prev` is not a child of `parent` + break :next prev_ni.get(mf).next; + } else opts.parent.first(mf); - if (opts.next.unwrap()) |next_ni| { - next_ni.get(mf).prev = .wrap(free_ni); - } else if (opts.parent.unwrap()) |parent_ni| { - parent_ni.get(mf).last = .wrap(free_ni); - } else { - assert(free_ni == .root); + // Validate node ordering + switch (opts.position) { + .floating => { + if (opts.prev.unwrap()) |prev_ni| { + assert(prev_ni.position(mf) != .footer); // tried to add floating node after footer node + } + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.position(mf) != .header); // tried to add floating node before header node + } + }, + .header => if (opts.prev.unwrap()) |prev_ni| { + switch (prev_ni.position(mf)) { + .header => {}, + .floating => unreachable, // tried to add header node after floating node + .footer => unreachable, // tried to add header node after footer node + } + }, + .footer => if (next_oni.unwrap()) |next_ni| { + switch (next_ni.position(mf)) { + .header => unreachable, // tried to add footer node before header node + .floating => unreachable, // tried to add footer node before floating node + .footer => {}, + } + }, } - free_node.* = .{ - .parent = opts.parent, - .prev = opts.prev, - .next = opts.next, + // Initialize the node as empty with alignment 1 + const location: Node.Location = loc: { + const offset: u64 = switch (opts.position) { + .header, .floating => offset: { + const prev_ni = opts.prev.unwrap() orelse break :offset 0; + const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); + break :offset prev_offset + prev_size; + }, + .footer => offset: { + const next_ni = next_oni.unwrap() orelse { + _, const parent_size = opts.parent.location(mf).resolve(mf); + break :offset parent_size; + }; + const next_offset, _ = next_ni.location(mf).resolve(mf); + break :offset next_offset; + }, + }; + if (std.math.cast(u32, offset)) |small_offset| { + break :loc .{ .small = .{ .offset = small_offset, .size = 0 } }; + } + const large_index = mf.large.items.len; + mf.large.appendSliceAssumeCapacity(&.{ offset, 0 }); + break :loc .{ .large = .{ .index = large_index } }; + }; + new_ni.get(mf).* = .{ + .parent = .wrap(opts.parent), + .prev = .none, + .next = .none, .first = .none, .last = .none, .flags = .{ - .location_tag = location_tag, + .position = opts.position, .alignment = .@"1", - .fixed = opts.add_node.fixed, - .moved = true, - .resized = true, - .next_moved = true, + .bubbles_moved = opts.add_options.bubbles_moved, + .enable_next_moved = opts.add_options.enable_next_moved, + .location_tag = location, + .moved = false, + .resized = false, + .next_moved = false, .has_content = false, - .bubbles_moved = opts.add_node.bubbles_moved, - .enable_next_moved = opts.add_node.enable_next_moved, }, - .location_payload = location_payload, + .location_payload = switch (location) { + .small => |small| .{ .small = small }, + .large => |large| .{ .large = large }, + }, }; - { - defer { - free_node.flags.moved = false; - free_node.flags.resized = false; - free_node.flags.next_moved = false; - } - try mf.realignNode(gpa, free_ni, opts.add_node.alignment, .{}); - try mf.resizeNode(gpa, free_ni, opts.add_node.size); + try mf.addNodesToChildListBefore(gpa, next_oni, new_ni, new_ni); + + try mf.realignNode(gpa, new_ni, opts.add_options.alignment); + if (opts.add_options.size > 0) { + try mf.growNode(gpa, new_ni, opts.add_options.size, .exact); } mf.updateWriters(); - if (opts.add_node.moved) try free_ni.moved(gpa, mf); - if (opts.add_node.resized) try free_ni.resized(gpa, mf); - if (opts.add_node.next_moved) try free_ni.nextMoved(gpa, mf); - return free_ni; -} -pub const AddNodeOptions = struct { - size: u64 = 0, - alignment: Alignment = .@"1", - fixed: bool = false, - moved: bool = false, - resized: bool = false, - next_moved: bool = false, - bubbles_moved: bool = true, - enable_next_moved: bool = false, -}; - -pub fn addOnlyChildNode( - mf: *MappedFile, - gpa: Allocator, - parent_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const parent = parent_ni.get(mf); - assert(parent.first == .none and parent.last == .none); - return mf.addNode(gpa, .{ - .parent = .wrap(parent_ni), - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; + new_ni.get(mf).flags.moved = false; + new_ni.get(mf).flags.resized = false; + new_ni.get(mf).flags.next_moved = false; + + if (opts.add_options.moved) try new_ni.moved(gpa, mf); + if (opts.add_options.resized) try new_ni.resized(gpa, mf); + if (opts.add_options.next_moved) try new_ni.nextMoved(gpa, mf); + + return new_ni; } -pub fn addFirstChildNode( +fn shrinkLeafNode( mf: *MappedFile, gpa: Allocator, - parent_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const parent = parent_ni.get(mf); - return mf.addNode(gpa, .{ - .parent = .wrap(parent_ni), - .next = parent.first, - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; + ni: Node.Index, + new_size: u64, +) Error!void { + mf.nodes_lock.assertUnlocked(); + + const old_offset, const old_size = ni.location(mf).resolve(mf); + + assert(new_size < old_size); + assert(ni.alignment(mf).check(new_size)); + assert(ni.first(mf) == .none); // `ni` must be a leaf node + + const parent_ni = ni.parent(mf).unwrap() orelse { + assert(ni == .root); + mf.memory_map.write(mf.io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; return error.MappedFileIo; - }, + }; + mf.memory_map.file.setLength(mf.io, new_size) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, + }; + try mf.ensureTotalCapacityPrecise(@intCast(new_size)); + try ni.setLocation(mf, gpa, old_offset, new_size); + return; }; -} -pub fn addLastChildNode( - mf: *MappedFile, - gpa: Allocator, - parent_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const parent = parent_ni.get(mf); - return mf.addNode(gpa, .{ - .parent = .wrap(parent_ni), - .prev = parent.last, - .offset = offset: { - const last_ni = parent.last.unwrap() orelse break :offset 0; - const last_offset, const last_size = last_ni.location(mf).resolve(mf); - break :offset last_offset + last_size; + switch (ni.position(mf)) { + .header => { + const shift = old_size - new_size; + + try ni.setLocation(mf, gpa, old_offset, new_size); + + // We need to shift backwards all header nodes following us. + const next_header_ni = ni.next(mf).unwrap() orelse return; + if (next_header_ni.position(mf) != .header) return; + + var header_ni = next_header_ni; + while (true) { + const old_header_off, const old_header_size = header_ni.location(mf).resolve(mf); + try header_ni.setLocation(mf, gpa, old_header_off - shift, old_header_size); + + const next_ni = header_ni.next(mf).unwrap() orelse break; + if (next_ni.position(mf) != .header) break; + header_ni = next_ni; + } + + // Now we must shift the actual header bytes of those nodes backwards. + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + const move_src_off = old_offset + old_size; + const move_dest_off = old_offset + new_size; + assert(next_header_ni.location(mf).resolve(mf)[0] == move_dest_off); // `move_dest_off` because we already updated the location + const move_size = size: { + // `header_ni` is the last header in the parent. + const last_off, const last_size = header_ni.location(mf).resolve(mf); + const move_end = last_off + last_size; + break :size move_end - move_dest_off; // `move_dest_off` because we already updated the location + }; + try mf.moveRange( + parent_file_off + move_src_off, + parent_file_off + move_dest_off, + move_size, + ); }, - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; + .floating => { + try ni.setLocation(mf, gpa, old_offset, new_size); }, - }; -} + .footer => { + const shift = old_size - new_size; -pub fn addNodeAfter( - mf: *MappedFile, - gpa: Allocator, - prev_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const prev = prev_ni.get(mf); - const prev_offset, const prev_size = prev.location().resolve(mf); - return mf.addNode(gpa, .{ - .parent = prev.parent, - .prev = .wrap(prev_ni), - .next = prev.next, - .offset = prev_offset + prev_size, - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; -} + const new_offset = old_offset + shift; + try ni.setLocation(mf, gpa, new_offset, new_size); -fn shrinkNode( - mf: *MappedFile, - gpa: Allocator, - ni: Node.Index, - size: u64, - shift_next: bool, -) !void { - mf.nodes_lock.assertUnlocked(); - const node = ni.get(mf); - const old_offset, _ = node.location().resolve(mf); + const prev_footers_size = prev_footers_size: { + // We need to shift forwards all footer nodes preceding us. + const prev_footer_ni = ni.prev(mf).unwrap() orelse { + break :prev_footers_size 0; + }; + if (prev_footer_ni.position(mf) != .footer) { + break :prev_footers_size 0; + } - // This would require unmapping first - assert(ni != .root); + var footer_ni = prev_footer_ni; + while (true) { + const old_footer_off, const old_footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_off + shift, old_footer_size); - if (node.last.unwrap()) |last_ni| { - const last = last_ni.get(mf); - const last_offset, const last_size = last.location().resolve(mf); - assert(last_offset + last_size > size); - } + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } - try mf.large.ensureUnusedCapacity(gpa, 4); - try mf.updates.ensureUnusedCapacity(gpa, 4); - - ni.setLocationAssumeCapacity(mf, old_offset, size); - if (!shift_next) return; - const next_ni = node.next.unwrap() orelse return; - - const next = next_ni.get(mf); - const old_next_offset, const next_size = next.location().resolve(mf); - const padding = old_next_offset - (old_offset + size); - const new_next_offset = next.flags.alignment.forward(@intCast(old_next_offset - padding)); - - if (next.flags.has_content and new_next_offset < old_next_offset) { - const old_file_offset = next_ni.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - old_next_offset) + new_next_offset; - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(next_size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(next_size)], - ); - @memset(mf.memory_map.memory[@intCast(new_file_offset + next_size)..@intCast(old_file_offset + next_size)], 0); - } + // `footer_ni` is the first footer in the parent. This expression gets its *new* + // offset because we already did the `setLocation` calls. + const first_footer_new_offset = footer_ni.location(mf).resolve(mf)[0]; + + break :prev_footers_size new_offset - first_footer_new_offset; + }; - next_ni.setLocationAssumeCapacity(mf, new_next_offset, next_size); + // Now we must shift the actual footer bytes forwards, including our own. + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + old_offset - prev_footers_size, + parent_file_offset + new_offset - prev_footers_size, + prev_footers_size + new_size, + ); + }, + } } -fn resizeNode( +const GrowMode = enum { exact, minimum }; + +/// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`. +/// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`. +/// +/// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!). +/// +/// Asserts that `new_size` is greater than the current size of `ni`. +fn growNode( mf: *MappedFile, gpa: Allocator, ni: Node.Index, - requested_size: u64, -) (Allocator.Error || Io.Cancelable || IoError)!void { + new_size: u64, + grow_mode: GrowMode, +) Error!void { mf.nodes_lock.assertUnlocked(); - const io = mf.io; + const node = ni.get(mf); + const old_offset, const old_size = node.location().resolve(mf); - const new_size = node.flags.alignment.forward(@intCast(requested_size)); - // Resize the entire file + assert(node.flags.alignment.check(old_size)); + assert(node.flags.alignment.check(new_size)); + assert(new_size > old_size); + const parent_ni = node.parent.unwrap() orelse { assert(ni == .root); - try mf.ensureCapacityForSetLocation(gpa); - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + return; + } + + mf.memory_map.write(mf.io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; + }; + mf.memory_map.file.setLength(mf.io, new_size) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, }; - try mf.memory_map.file.setLength(io, new_size); - try mf.ensureTotalCapacityInner(@intCast(new_size)); - ni.setLocationAssumeCapacity(mf, old_offset, new_size); + try mf.ensureTotalCapacityPrecise(@intCast(new_size)); + try ni.setLocation(mf, gpa, old_offset, new_size); + // We need to move any footers to be at the *new* end of the file. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = old_size - old_footers_offset; + try mf.moveRange( + old_footers_offset, + old_footers_offset + (new_size - old_size), + footers_size, + ); + // Also update the footers' locations. + var cur_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + } return; }; - const parent = parent_ni.get(mf); - _, var old_parent_size = parent.location().resolve(mf); - const trailing_end = trailing_end: { - const next_ni = node.next.unwrap() orelse break :trailing_end old_parent_size; - const next_offset, _ = next_ni.location(mf).resolve(mf); - break :trailing_end next_offset; - }; - assert(old_offset + old_size <= trailing_end); - if (old_offset + new_size <= trailing_end) { - // Expand the node into trailing free space - try mf.ensureCapacityForSetLocation(gpa); - ni.setLocationAssumeCapacity(mf, old_offset, new_size); - return; - } - insert_range: { - if (!is_linux) break :insert_range; - if (mf.flags.fallocate_insert_range_unsupported) break :insert_range; - - // We need the node to be aligned to `mf.flags.block_size` in the file in order to use this - // fast path. It is not sufficient to check `node.flags.alignment`, because that doesn't - // necessarily mean that all *parent* nodes are equally aligned; instead we must compute the - // actual file offset. - const range_file_offset = ni.fileLocation(mf, false).offset + old_size; - const range_size = node.flags.alignment.forward( - @intCast(requested_size +| requested_size / growth_factor), - ) - old_size; - if (!mf.flags.block_size.check(@intCast(range_file_offset))) break :insert_range; - if (!mf.flags.block_size.check(@intCast(range_size))) break :insert_range; - - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, - }; - // Ask the filesystem driver to insert extents into the file without copying any data - const last_offset, const last_size = parent.last.unwrap().?.location(mf).resolve(mf); - const last_end = last_offset + last_size; - assert(last_end <= old_parent_size); - _, const file_size = Node.Index.root.location(mf).resolve(mf); - while (true) switch (linux.errno(switch (std.math.order(range_file_offset, file_size)) { - .lt => linux.fallocate( - mf.memory_map.file.handle, - linux.FALLOC.FL_INSERT_RANGE, - @intCast(range_file_offset), - @intCast(range_size), - ), - .eq => linux.ftruncate(mf.memory_map.file.handle, @intCast(range_file_offset + range_size)), - .gt => unreachable, - })) { - .SUCCESS => { - var enclosing_ni = ni; + + switch (node.flags.position) { + .header => { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + return; + } + + try mf.ensureAdditionalHeaderCapacity(gpa, parent_ni, new_size - old_size); + + // `old_offset` is still valid because header nodes don't move when the parent resizes. + + const last_header_ni: Node.Index = last_header: { + var header_ni = ni; while (true) { - try mf.ensureCapacityForSetLocation(gpa); - const enclosing = enclosing_ni.get(mf); - const enclosing_offset, const old_enclosing_size = - enclosing.location().resolve(mf); - const new_enclosing_size = old_enclosing_size + range_size; - enclosing_ni.setLocationAssumeCapacity(mf, enclosing_offset, new_enclosing_size); - if (enclosing_ni == .root) { - assert(enclosing_offset == 0); - try mf.ensureTotalCapacityInner(@intCast(new_enclosing_size)); - break; - } - var after_oni = enclosing.next; - while (after_oni.unwrap()) |after_ni| { - try mf.ensureCapacityForSetLocation(gpa); - const after = after_ni.get(mf); - const after_offset, const after_size = after.location().resolve(mf); - after_ni.setLocationAssumeCapacity( - mf, - range_size + after_offset, - after_size, - ); - after_oni = after.next; - } - enclosing_ni = enclosing.parent.unwrap().?; + const next_ni = header_ni.next(mf).unwrap() orelse break; + if (next_ni.position(mf) != .header) break; + header_ni = next_ni; } - return; - }, - .INTR => continue, - .BADF, .FBIG, .INVAL => unreachable, - .IO => return error.InputOutput, - .NODEV => return error.NotFile, - .NOSPC => return error.NoSpaceLeft, - .NOSYS, .OPNOTSUPP => { - mf.flags.fallocate_insert_range_unsupported = true; - break :insert_range; - }, - .PERM => return error.PermissionDenied, - .SPIPE => return error.Unseekable, - .TXTBSY => return error.FileBusy, - else => |e| return std.posix.unexpectedErrno(e), - }; - } - if (node.next == .none) { - // As this is the last node, we simply need more space in the parent - const new_parent_size = old_offset + new_size; - try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); - try mf.ensureCapacityForSetLocation(gpa); - ni.setLocationAssumeCapacity(mf, old_offset, new_size); - return; - } - if (!node.flags.fixed) { - // Make space at the end of the parent for this floating node - const last = parent.last.unwrap().?.get(mf); - const last_offset, const last_size = last.location().resolve(mf); - const new_offset = node.flags.alignment.forward(@intCast(last_offset + last_size)); - const new_parent_size = new_offset + new_size; - if (new_parent_size > old_parent_size) - try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); - try mf.ensureCapacityForSetLocation(gpa); - const next_ni = node.next.unwrap().?; - next_ni.get(mf).prev = node.prev; - if (node.prev.unwrap()) |prev_ni| { - try prev_ni.setNext(gpa, .wrap(next_ni), mf); - } else { - parent.first = .wrap(next_ni); - } - try parent.last.unwrap().?.setNext(gpa, .wrap(ni), mf); - node.prev = parent.last; - try ni.setNext(gpa, .none, mf); - parent.last = .wrap(ni); - if (node.flags.has_content) { + break :last_header header_ni; + }; + const last_header_offset, const last_header_size = last_header_ni.location(mf).resolve(mf); + const old_headers_size = last_header_offset + last_header_size; + + // This is the first footer *inside* of `ni`. + const first_sub_footer_oni = ni.firstFooter(mf); + const sub_footers_size = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + break :size old_size - first_sub_footer_offset; + }; + + // We need to shift two things forwards; any header nodes which follow us, and any + // footer nodes *within* us (since they need to be at the end of our new size). const parent_file_offset = parent_ni.fileLocation(mf, false).offset; try mf.moveRange( - parent_file_offset + old_offset, - parent_file_offset + new_offset, - old_size, + parent_file_offset + old_offset + old_size - sub_footers_size, + parent_file_offset + old_offset + new_size - sub_footers_size, + old_headers_size - (old_offset + old_size - sub_footers_size), ); - } - ni.setLocationAssumeCapacity(mf, new_offset, new_size); - return; - } - // Search for the first floating node following this fixed node - var last_fixed_ni = ni; - var first_floating_oni = node.next; - var shift = new_size - old_size; - var max_shift_align: Alignment = .@"1"; - var direction: enum { forward, reverse } = .forward; - while (true) { - const last_fixed = last_fixed_ni.get(mf); - assert(last_fixed.flags.fixed); - const old_last_fixed_offset, const last_fixed_size = last_fixed.location().resolve(mf); - const new_last_fixed_offset = old_last_fixed_offset + shift; - if (first_floating_oni.unwrap()) |first_floating_ni| make_space: { - const first_floating = first_floating_ni.get(mf); - const old_first_floating_offset, const first_floating_size = - first_floating.location().resolve(mf); - assert(old_last_fixed_offset + last_fixed_size <= old_first_floating_offset); - if (new_last_fixed_offset + last_fixed_size <= old_first_floating_offset) - break :make_space; - assert(direction == .forward); - max_shift_align = max_shift_align.max(first_floating.flags.alignment.max(last_fixed.flags.alignment)); - if (first_floating.flags.fixed) { - shift = max_shift_align.forward(@intCast( - @max(shift, first_floating_size), - )); - - // Not enough space, try the next node - last_fixed_ni = first_floating_ni; - first_floating_oni = first_floating.next; - continue; + + // Any footers inside of us have had their offsets changed due to us growing: + if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { + var cur_ni = first_sub_footer_ni; + while (true) { + const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation( + mf, + gpa, + old_sub_footer_offset + (new_size - old_size), + sub_footer_size, + ); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } } - // Move the found floating node to make space for preceding fixed nodes - const last = parent.last.unwrap().?.get(mf); - const last_offset, const last_size = last.location().resolve(mf); - const new_first_floating_offset = max_shift_align.forward( - @intCast(@max(new_last_fixed_offset + last_fixed_size, last_offset + last_size)), + + // Update the offsets of all header nodes following us: + { + var moved_header_ni = last_header_ni; + while (moved_header_ni != ni) { + assert(moved_header_ni.position(mf) == .header); + const moved_header_offset, const moved_header_size = moved_header_ni.location(mf).resolve(mf); + try moved_header_ni.setLocation( + mf, + gpa, + moved_header_offset - old_size + new_size, + moved_header_size, + ); + moved_header_ni = moved_header_ni.prev(mf).unwrap().?; + } + } + + // Finally, update our own size: + try ni.setLocation(mf, gpa, old_offset, new_size); + return; + }, + .floating => { + try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode); + }, + .footer => { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + return; + } + + try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size); + + const first_footer_ni: Node.Index = first_footer: { + var footer_ni = ni; + while (true) { + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } + break :first_footer footer_ni; + }; + + // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself + // a footer within its parent). + const first_sub_footer_oni = ni.firstFooter(mf); + const sub_footers_size = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + break :size old_size - first_sub_footer_offset; + }; + + _, const parent_size = parent_ni.location(mf).resolve(mf); + + const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0]; + const new_footers_size = old_footers_size - old_size + new_size; + + // Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node + // itself needs to shift its contents, because our offset was shifted backwards by + // `new_size - old_size`, and the added bytes should go at the end of this footer node. + // However, if we *contain* any footer nodes, they need to stay at the end of `ni`, so + // we *shouldn't* shift *that* data. + const old_footers_start = parent_size - old_footers_size; + const new_footers_start = parent_size - new_footers_size; + const end_offset = node.location().resolve(mf)[0] + old_size; + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + old_footers_start, + parent_file_offset + new_footers_start, + end_offset - old_footers_start - sub_footers_size, ); - const new_parent_size = new_first_floating_offset + first_floating_size; - if (new_parent_size > old_parent_size) { - try mf.resizeNode( - gpa, - parent_ni, - new_parent_size +| new_parent_size / growth_factor, - ); - _, old_parent_size = parent.location().resolve(mf); + + // Update our own offset and size: + try ni.setLocation(mf, gpa, end_offset - new_size, new_size); + + // Any footers inside of us have had their offsets changed due to us growing: + if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { + var cur_ni = first_sub_footer_ni; + while (true) { + const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation( + mf, + gpa, + old_sub_footer_offset + (new_size - old_size), + sub_footer_size, + ); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } } - try mf.ensureCapacityForSetLocation(gpa); - if (parent.last.unwrap().? != first_floating_ni) { - const old_last = parent.last.unwrap().?; - first_floating.prev = .wrap(old_last); - parent.last = .wrap(first_floating_ni); - try old_last.setNext(gpa, .wrap(first_floating_ni), mf); - try last_fixed_ni.setNext(gpa, first_floating.next, mf); - if (first_floating.next.unwrap()) |next_ni| { - next_ni.get(mf).prev = .wrap(last_fixed_ni); + + // Finally, update the offsets of every footer before us: + if (node.prev.unwrap()) |prev_ni| { + var maybe_footer_ni = prev_ni; + while (true) { + switch (maybe_footer_ni.position(mf)) { + .header, .floating => break, + .footer => {}, + } + const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf); + try maybe_footer_ni.setLocation( + mf, + gpa, + moved_footer_offset + old_size - new_size, + moved_footer_size, + ); + maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break; } - try first_floating_ni.setNext(gpa, .none, mf); } - if (first_floating.flags.has_content) { - const parent_file_offset = - parent_ni.fileLocation(mf, false).offset; + + return; + }, + } +} + +/// Moves a floating node to an unused region with the given size, which may be greater than the +/// current size. If `new_alignment` is not `null`, then the offset and size of the new region will +/// have that alignment instead of `ni.alignment(mf)`. +/// +/// Asserts that `ni` is a floating node (and not `.root`). +/// +/// Asserts that `new_size` is aligned to `new_alignment orelse ni.alignment(mf)`. +/// +/// Asserts that `new_size` is greater than or equal to the current size of `ni`. +fn growFloatingNodeWithAlignment( + mf: *MappedFile, + gpa: Allocator, + ni: Node.Index, + new_alignment: ?Alignment, + new_size: u64, + grow_mode: GrowMode, +) Error!void { + mf.nodes_lock.assertUnlocked(); + + const parent_ni = ni.parent(mf).unwrap().?; // `ni` cannot be `.root` + const old_offset, const old_size = ni.location(mf).resolve(mf); + + const alignment = new_alignment orelse ni.alignment(mf); + + assert(new_size >= old_size); + assert(ni.position(mf) == .floating); + assert(alignment.check(new_size)); + + grow_in_place: { + if (!alignment.check(old_offset)) { + break :grow_in_place; + } + const limit: u64 = limit: { + const next_ni = ni.next(mf).unwrap() orelse break :limit parent_ni.location(mf).resolve(mf)[1]; + const next_offset, _ = next_ni.location(mf).resolve(mf); + break :limit next_offset; + }; + if (old_offset + new_size > limit) { + break :grow_in_place; // the parent is not big enough + } + // Great, we can grow this node without changing its offset or moving any siblings. + try ni.setLocation(mf, gpa, old_offset, new_size); + // If we have any footers, we need to move them to the end of our new size, and update their + // offsets accordingly. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var cur_ni = first_footer_ni; + var footers_have_content = false; + while (true) { + footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + if (footers_have_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + // This gets the *new* offset because we already updated the offsets above. + const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = new_size - new_footers_offset; try mf.moveRange( - parent_file_offset + old_first_floating_offset, - parent_file_offset + new_first_floating_offset, - first_floating_size, + parent_file_off + old_offset + old_size - footers_size, + parent_file_off + old_offset + new_size - footers_size, + footers_size, ); } - first_floating_ni.setLocationAssumeCapacity( - mf, - new_first_floating_offset, - first_floating_size, - ); - // Continue the search after the just-moved floating node - first_floating_oni = last_fixed.next; - continue; - } else { - assert(direction == .forward); - const new_parent_size = new_last_fixed_offset + last_fixed_size; - if (new_parent_size > old_parent_size) { - try mf.resizeNode( - gpa, - parent_ni, - new_parent_size +| new_parent_size / growth_factor, - ); - _, old_parent_size = parent.location().resolve(mf); + } + return; + } + + const new_loc: struct { + offset: u64, + prev: Node.Index.Optional, + } = new_loc: { + _, const parent_size = parent_ni.location(mf).resolve(mf); + + { + // See if there's space at the start of the parent. + const last_header_oni = parent_ni.lastHeader(mf); + const headers_end: u64 = if (last_header_oni.unwrap()) |last_header_ni| headers_end: { + const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf); + break :headers_end last_header_off + last_header_size; + } else 0; + const limit: u64 = limit: { + const after_header_oni: Node.Index.Optional = after_header: { + if (last_header_oni.unwrap()) |last_header_ni| { + break :after_header last_header_ni.next(mf); + } + break :after_header parent_ni.first(mf); + }; + if (after_header_oni.unwrap()) |after_header_ni| { + break :limit after_header_ni.location(mf).resolve(mf)[0]; + } else { + break :limit parent_size; + } + }; + if (alignment.forward(headers_end) + new_size <= limit) { + // There's space here! + break :new_loc .{ + // Put ourselves at the *end* of this range, so that the free space remains at the start of the parent. + .offset = alignment.backward(limit - new_size), + .prev = last_header_oni, + }; } } - try mf.ensureCapacityForSetLocation(gpa); - if (last_fixed_ni == ni) { - // The original fixed node now has enough space - last_fixed_ni.setLocationAssumeCapacity( - mf, - old_last_fixed_offset, - new_size, + + // Otherwise, use space at the end of the parent, or make space there if necessary. + + const first_footer_oni = parent_ni.firstFooter(mf); + + // We know there is a node before the footer[s], because `ni` itself is such a node. + const prev_ni: Node.Index = if (first_footer_oni.unwrap()) |first_footer_ni| prev: { + break :prev first_footer_ni.prev(mf).unwrap().?; + } else prev: { + break :prev parent_ni.last(mf).unwrap().?; + }; + + const result_offset: u64 = result_offset: { + if (prev_ni == ni and alignment.check(old_offset)) { + // We're already at the end of the parent, and our offset is already well-aligned. + // The only reason we didn't simply grow in place earlier is that the parent wasn't + // big enough---but now we're resizing the parent anyway, so growing in-place stops + // us from unnecessarily moving! + break :result_offset old_offset; + } + // Otherwise, just move after the last node. + const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); + break :result_offset alignment.forward(prev_offset + prev_size); + }; + + const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| footers_size: { + const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); + break :footers_size parent_size - first_footer_offset; + } else 0; + + const min_parent_size = result_offset + new_size + footers_size; + if (parent_size < min_parent_size) { + // Okay, at this point we're planning to expand the parent---so before we actually do + // that, let's first try the Linux "insert range" fast path. We didn't try it before now + // because it would have been more efficient to just move ourselves into existing space. + // + // If we were given a custom alignment, we cannot pass `grow_mode` directly into the + // "insert range" path, because that function is unaware of `new_alignment`. + const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact; + if (alignment.check(old_offset) and + try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode)) + { + // The Linux fast path did our job for us! + return; + } + + // Grow the parent and move to the end of the parent. + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, ); - return; - } - // Move a fixed node into trailing free space - if (last_fixed.flags.has_content) { - const parent_file_offset = parent_ni.fileLocation(mf, false).offset; - try mf.moveRange( - parent_file_offset + old_last_fixed_offset, - parent_file_offset + new_last_fixed_offset, - last_fixed_size, + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + } + + break :new_loc .{ + .offset = result_offset, + .prev = .wrap(prev_ni), + }; + }; + + // We've found our new location in `parent_ni`, now to actually move ourselves there. + + // Footers need to move to a different place than the rest of our content. + const footers_size: u64, const footers_have_content: bool = footers: { + const first_footer_ni = ni.firstFooter(mf).unwrap() orelse { + break :footers .{ 0, false }; + }; + + var cur_ni = first_footer_ni; + var footers_have_content = false; + while (true) { + footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + // Our footers' offsets must change to be at the end of our new size. + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + + // This is the *new* offset because we already updated the offsets above. + const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = new_size - new_footers_offset; + + break :footers .{ footers_size, footers_have_content }; + }; + + if (ni.get(mf).flags.has_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_off + old_offset, + parent_file_off + new_loc.offset, + old_size - footers_size, + ); + if (footers_have_content) try mf.moveRange( + parent_file_off + old_offset + old_size - footers_size, + parent_file_off + new_loc.offset + new_size - footers_size, + footers_size, + ); + } else { + assert(!footers_have_content); + } + + try ni.setLocation(mf, gpa, new_loc.offset, new_size); + + if (new_loc.prev != ni.toOptional()) { + // We're potentially in a different place in `parent_ni`'s child list, so remove and re-add ourselves. + try mf.removeNodesFromChildList(gpa, ni, ni); + try mf.addNodesToChildListAfter(gpa, new_loc.prev, ni, ni); + } +} + +/// Attempts to grow `ni` to `new_size` using `FALLOCATE_FL_INSERT_RANGE` on Linux. This strategy +/// has the advantage that it does not require manually moving any bytes in the file, but has the +/// disadvantages that it may increase the file size more than necessary, and that it changes the +/// offsets of all following nodes, recursively. +/// +/// If this strategy is inapplicable or unsuitable for this operation, this function returns `false` +/// without changing any nodes' locations or invalidating any slices. +/// +/// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node +/// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`, +/// the actual new size of `ni` may be greater than `new_size`. +fn growNodeViaInsertRange( + mf: *MappedFile, + gpa: Allocator, + ni: Node.Index, + new_size: u64, + grow_mode: GrowMode, +) Error!bool { + if (!is_linux or mf.flags.fallocate_insert_range_unsupported) { + return false; + } + + _, const old_size = ni.location(mf).resolve(mf); + + // We don't compute the size of the range yet, because depending on `grow_mode` we might want to + // bump it based on our sibling and parent nodes' alignments. However, we can do an early check + // for cases where we should obviously exit. + const requested_range_size = new_size - old_size; + if (!mf.flags.block_size.check(requested_range_size)) { + // The requested size isn't exactly aligned. + switch (grow_mode) { + .exact => return false, + .minimum => { + // We can still choose to allow it by increasing the size a bit, but we shouldn't do + // that if it would *significantly* increase the requested size. + const block_size = mf.flags.block_size.toByteUnits(); + if (requested_range_size < block_size * 2) { + // Bumping this size up to the next block boundary would be a quite significant + // increase; let's not do it. + return false; + } + }, + } + } + // If `grow_mode` is exact, we will use exactly this size, but if it is `.minimum`, we may bump + // the size a little more. + const min_range_size: u64 = s: { + const exact_size = new_size - old_size; + if (mf.flags.block_size.check(exact_size)) { + break :s exact_size; + } + switch (grow_mode) { + .exact => return false, + .minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) { + // We're growing by at least a few blocks, so allow ourselves to bump the size + // slightly to give it the needed alignment. + break :s mf.flags.block_size.forward(exact_size); + } else { + return false; + }, + } + }; + assert(min_range_size > 0); + assert(mf.flags.block_size.check(min_range_size)); + + const range_file_offset: u64 = range_file_offset: { + const node_file_offset = ni.fileLocation(mf, false).offset; + const last_ni = ni.last(mf).unwrap() orelse { + // If `ni` has no children (i.e. is a leaf node), we need to insert exactly at its end. + const range_file_offset = node_file_offset + old_size; + if (!mf.flags.block_size.check(range_file_offset)) { + return false; + } + break :range_file_offset range_file_offset; + }; + const first_footer_oni = ni.firstFooter(mf); + const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| size: { + const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); + break :size old_size - first_footer_offset; + } else 0; + const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) |first_footer_ni| pre_footer: { + break :pre_footer first_footer_ni.prev(mf); + } else .wrap(last_ni); + const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: { + const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf); + break :end pre_footer_off + pre_footer_size; + } else 0; + + const min_file_offset = node_file_offset + pre_footer_end; + const max_file_offset = node_file_offset + old_size - footers_size; + // We can go anywhere between `min_file_offset` and `max_file_offset`. + const candidate_file_offset = mf.flags.block_size.forward(min_file_offset); + if (candidate_file_offset > max_file_offset) { + return false; + } + break :range_file_offset candidate_file_offset; + }; + assert(mf.flags.block_size.check(range_file_offset)); + + const range_size: u64 = range_size: { + // For this strategy to be valid, the number of bytes we insert needs to be compatible with + // the alignments of all nodes following us (and following our parents, their parents, etc). + // We also probably don't want to trigger too many "node moved" events, since doing that + // repeatedly could result in a lot of extra work. Therefore, while we traverse parents and + // siblings to check their alignment requirements, we will also set an arbitrary limit on + // the number of nodes we can move, and give up if we walk more than that. + const max_moved_nodes = 32; + var num_moved: u32 = 0; + var cur_ni = ni; + // Alignment required for `range_size`: initially the block size (required for the syscall), + // then updated as we traverse based on how the operation would affect surrounding nodes. + var need_range_align: Alignment = mf.flags.block_size.max(ni.alignment(mf)); + while (true) { + // `cur_ni` will grow as a result of the range insertion. Its size must be well-aligned. + need_range_align = need_range_align.max(cur_ni.alignment(mf)); + + // Siblings following `cur_ni` don't get bigger, but their offsets change. + while (cur_ni.next(mf).unwrap()) |next_ni| { + // Only floating children need well-aligned offsets. + if (next_ni.position(mf) == .floating) { + need_range_align = need_range_align.max(next_ni.alignment(mf)); + } + num_moved += 1; + if (num_moved > max_moved_nodes) return false; + cur_ni = next_ni; + } + + // Move up to the parent. + cur_ni = cur_ni.parent(mf).unwrap() orelse break; + } + // Traversal done. We didn't hit `max_moved_nodes`, so now we can use the computed alignment + // requirement to figure out whether we're actually going to insert a range. + if (need_range_align.check(requested_range_size)) { + break :range_size requested_range_size; + } + // Perhaps we're allowed to grow by more than `requested_range_size`? + switch (grow_mode) { + .exact => return false, + .minimum => { + const candidate_range_size = need_range_align.forward(min_range_size); + // Allow growing by up to 50% more than was requested. + if (candidate_range_size <= requested_range_size +| requested_range_size / 2) { + break :range_size candidate_range_size; + } else { + return false; + } + }, + } + }; + + // This `range_size` is compatible with everyone's alignment requirements, and we won't move too + // many nodes, so let's do it! + + mf.memory_map.write(mf.io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; + }; + + // If we happen to be inserting at the very end of the file, we need to resize the file instead + // of using `FALLOCATE_FL_INSERT_RANGE`. + if (range_file_offset == Node.Index.root.location(mf).resolve(mf)[1]) { + mf.memory_map.file.setLength(mf.io, range_file_offset + range_size) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, + }; + } else { + while (true) switch (linux.errno(linux.fallocate( + mf.memory_map.file.handle, + linux.FALLOC.FL_INSERT_RANGE, + @intCast(range_file_offset), + @intCast(range_size), + ))) { + .SUCCESS => break, + .INTR => continue, + .NOSYS, .OPNOTSUPP => { + // After all that setup work, it turns out the operation is actually unsupported! + mf.flags.fallocate_insert_range_unsupported = true; + return false; + }, + else => |e| { + mf.io_err = switch (e) { + .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above + .BADF => unreachable, + .FBIG => unreachable, + .INVAL => unreachable, + .IO => error.InputOutput, + .NODEV => error.NotFile, + .NOSPC => error.NoSpaceLeft, + .PERM => error.PermissionDenied, + .SPIPE => error.Unseekable, + .TXTBSY => error.FileBusy, + else => std.posix.unexpectedErrno(e), + }; + return error.MappedFileIo; + }, + }; + } + + // We did it! Now to update all the sizes and offsets. This loop is exactly the same shape as + // above, except we're updating locations instead of checking alignments. + var cur_ni = ni; + while (true) { + const this_offset, const this_old_size = cur_ni.location(mf).resolve(mf); + if (cur_ni == .root) { + try mf.ensureTotalCapacityPrecise(@intCast(this_old_size + range_size)); + } + try cur_ni.setLocation(mf, gpa, this_offset, this_old_size + range_size); + + while (cur_ni.next(mf).unwrap()) |next_ni| { + const next_old_offset, const next_size = next_ni.location(mf).resolve(mf); + try next_ni.setLocation(mf, gpa, next_old_offset + range_size, next_size); + cur_ni = next_ni; + } + + cur_ni = cur_ni.parent(mf).unwrap() orelse break; + } + + // The only thing left is to update the offsets of any footers inside of `ni`. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var footer_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size); + footer_ni = footer_ni.next(mf).unwrap() orelse break; + } + } + + return true; +} + +/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes following its current +/// headers, so that the headers can grow into that space. +fn ensureAdditionalHeaderCapacity( + mf: *MappedFile, + gpa: Allocator, + parent_ni: Node.Index, + extra_capacity: u64, +) Error!void { + _, const parent_size = parent_ni.location(mf).resolve(mf); + + const last_header_oni = parent_ni.lastHeader(mf); + const first_footer_oni = parent_ni.firstFooter(mf); + + const headers_size: u64 = headers_size: { + const last_header_ni = last_header_oni.unwrap() orelse break :headers_size 0; + const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf); + break :headers_size last_header_off + last_header_size; + }; + + const footers_size: u64 = footers_size: { + const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; + const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); + break :footers_size parent_size - first_footer_off; + }; + + const first_floating_oni: Node.Index.Optional = if (last_header_oni.unwrap()) |last_header_ni| first_floating: { + const after_header_ni = last_header_ni.next(mf).unwrap() orelse break :first_floating .none; + break :first_floating switch (after_header_ni.position(mf)) { + .header => unreachable, + .floating => .wrap(after_header_ni), + .footer => .none, + }; + } else first_floating: { + const first_ni = parent_ni.first(mf).unwrap() orelse break :first_floating .none; + break :first_floating switch (first_ni.position(mf)) { + .header => unreachable, + .floating => .wrap(first_ni), + .footer => .none, + }; + }; + const first_floating_ni = first_floating_oni.unwrap() orelse { + // This node has only headers and footers. + const min_parent_size = headers_size + extra_capacity + footers_size; + if (parent_size < min_parent_size) { + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, ); + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + } + return; + }; + + const last_floating_ni = if (first_footer_oni.unwrap()) |first_footer_ni| last_floating: { + break :last_floating first_footer_ni.prev(mf).unwrap().?; + } else last_floating: { + break :last_floating parent_ni.last(mf).unwrap().?; + }; + assert(last_floating_ni.position(mf) == .floating); // we know `parent_ni` contains at least `first_floating_ni` + + // Find the first floating child, if any, which does not overlap the new header space. + const first_good_floating_oni: Node.Index.Optional = first_good_floating: { + var floating_ni = first_floating_ni; + while (true) { + const floating_offset, _ = floating_ni.location(mf).resolve(mf); + if (floating_offset >= headers_size + extra_capacity) { + break :first_good_floating .wrap(floating_ni); + } + const next_ni = floating_ni.next(mf).unwrap() orelse { + break :first_good_floating .none; + }; + switch (next_ni.position(mf)) { + .header => unreachable, // after the last header + .floating => floating_ni = next_ni, + .footer => break :first_good_floating .none, + } + } + }; + + if (first_good_floating_oni == first_floating_ni.toOptional()) { + // None of the floating children are in our way! That means there's already enough space. + return; + } + + const last_moving_ni = if (first_good_floating_oni.unwrap()) |first_good_floating_ni| last_moving: { + break :last_moving first_good_floating_ni.prev(mf).unwrap().?; + } else last_moving: { + break :last_moving last_floating_ni; + }; + + // We are going to move all nodes between `first_floating_ni` and `last_moving_ni` to the end of + // the parent. We'll move all the node data in one big block. + + const moving_offset: u64 = first_floating_ni.location(mf).resolve(mf)[0]; + const moving_size: u64 = size: { + const last_moving_off, const last_moving_size = last_moving_ni.location(mf).resolve(mf); + break :size last_moving_off + last_moving_size - moving_offset; + }; + + var moving_alignment: Alignment = .@"1"; + var moving_has_content = false; // optimization: no need to move data if it's all uninitialized + { + var cur_ni = first_floating_ni; + while (true) { + moving_alignment = moving_alignment.max(cur_ni.alignment(mf)); + moving_has_content = moving_has_content or cur_ni.get(mf).flags.has_content; + if (cur_ni == last_moving_ni) break; + cur_ni = cur_ni.next(mf).unwrap().?; } - last_fixed_ni.setLocationAssumeCapacity(mf, new_last_fixed_offset, last_fixed_size); - // Retry the previous nodes now that there is enough space - first_floating_oni = .wrap(last_fixed_ni); - last_fixed_ni = last_fixed.prev.unwrap().?; - direction = .reverse; } + + const first_free_offset = free_offset: { + const last_floating_off, const last_floating_size = last_floating_ni.location(mf).resolve(mf); + break :free_offset @max(last_floating_off + last_floating_size, headers_size + extra_capacity); + }; + // Alignment is a little tricky here. We don't necessarily want the new offset to be aligned to + // `moving_alignment` exactly, because if (e.g.) the first floating node is align(2) and the + // second is align(4), then the overall range we're moving may not be 4-byte aligned even though + // one of the nodes is. Instead, the old and new offsets must be congruent modulo the alignment. + const aligned_dest_offset = moving_alignment.forward(first_free_offset); + const dest_offset = aligned_dest_offset + (moving_offset - moving_alignment.backward(moving_offset)); + assert(dest_offset % moving_alignment.toByteUnits() == moving_offset % moving_alignment.toByteUnits()); + + // This expression is correct because `dest_offset` is after all floating nodes (except the ones + // we're moving there of course). + const min_parent_size = dest_offset + moving_size + footers_size; + if (parent_size < min_parent_size) { + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, + ); + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + } + + if (moving_has_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_off + moving_offset, + parent_file_off + dest_offset, + moving_size, + ); + } + + // Remove everything between `first_floating_ni` and `last_moving_ni` from the linked list, then + // re-insert them in their new position. + try mf.removeNodesFromChildList(gpa, first_floating_ni, last_moving_ni); + try mf.addNodesToChildListBefore(gpa, first_footer_oni, first_floating_ni, last_moving_ni); + + // Finally, we need to update the locations of all of those nodes. + var cur_ni = first_floating_ni; + while (true) { + assert(cur_ni.position(mf) == .floating); + const old_offset, const old_size = cur_ni.location(mf).resolve(mf); + const new_offset = old_offset - moving_offset + dest_offset; + assert(cur_ni.alignment(mf).check(new_offset)); + try cur_ni.setLocation(mf, gpa, new_offset, old_size); + if (cur_ni == last_moving_ni) break; + cur_ni = cur_ni.next(mf).unwrap().?; + } +} + +/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current +/// footers, so that the footers can grow into that space. +fn ensureAdditionalFooterCapacity( + mf: *MappedFile, + gpa: Allocator, + parent_ni: Node.Index, + extra_capacity: u64, +) Error!void { + // This is way easier than the header case, because we don't need to actually move anything; we + // just need to expand the parent if there isn't space, and that will add padding after the + // parent's floating children, which is exactly where we want it. + + const first_footer_oni = parent_ni.firstFooter(mf); + + _, const parent_size = parent_ni.location(mf).resolve(mf); + + const footers_size: u64 = footers_size: { + const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; + const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); + break :footers_size parent_size - first_footer_off; + }; + + const header_and_floating_end: u64 = end: { + const before_footers_oni = if (first_footer_oni.unwrap()) |first_footer_ni| before_footers: { + break :before_footers first_footer_ni.prev(mf); + } else before_footers: { + break :before_footers parent_ni.last(mf); + }; + const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0; + const offset, const size = before_footers_ni.location(mf).resolve(mf); + break :end offset + size; + }; + + assert(header_and_floating_end + footers_size <= parent_size); + + const min_parent_size = header_and_floating_end + footers_size + extra_capacity; + if (parent_size < min_parent_size) { + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, + ); + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + } +} + +fn removeNodesFromChildList( + mf: *MappedFile, + gpa: Allocator, + first_remove_ni: Node.Index, + last_remove_ni: Node.Index, +) Allocator.Error!void { + const parent_ni = first_remove_ni.parent(mf).unwrap().?; + assert(last_remove_ni.parent(mf).unwrap().? == parent_ni); + + const prev_oni = first_remove_ni.prev(mf); + const next_oni = last_remove_ni.next(mf); + + if (prev_oni.unwrap()) |prev_ni| { + assert(prev_ni.next(mf).unwrap().? == first_remove_ni); + try prev_ni.setNext(gpa, next_oni, mf); + } else { + assert(parent_ni.first(mf).unwrap().? == first_remove_ni); + parent_ni.get(mf).first = next_oni; + } + + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.prev(mf).unwrap().? == last_remove_ni); + next_ni.get(mf).prev = prev_oni; + } else { + assert(parent_ni.last(mf).unwrap().? == last_remove_ni); + parent_ni.get(mf).last = prev_oni; + } +} +/// Assumes `first_add_ni` and `last_add_ni` are connected, and that all nodes in between them +/// already have their `parent` field correctly populated. +/// +/// To add a single node, set `first_add_ni` equal to `last_add_ni`. +fn addNodesToChildListBefore( + mf: *MappedFile, + gpa: Allocator, + /// `null` means to add at the end of the parent. + next_oni: Node.Index.Optional, + first_add_ni: Node.Index, + last_add_ni: Node.Index, +) Allocator.Error!void { + const parent_ni = first_add_ni.parent(mf).unwrap().?; + assert(last_add_ni.parent(mf).unwrap().? == parent_ni); + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.parent(mf).unwrap().? == parent_ni); + } + + const prev_oni: Node.Index.Optional = if (next_oni.unwrap()) |next_ni| prev: { + break :prev next_ni.prev(mf); + } else prev: { + break :prev parent_ni.last(mf); + }; + + first_add_ni.get(mf).prev = prev_oni; + try last_add_ni.setNext(gpa, next_oni, mf); + + if (prev_oni.unwrap()) |prev_ni| { + assert(prev_ni.next(mf) == next_oni); + try prev_ni.setNext(gpa, .wrap(first_add_ni), mf); + } else { + assert(parent_ni.first(mf) == next_oni); + parent_ni.get(mf).first = .wrap(first_add_ni); + } + + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.prev(mf) == prev_oni); + next_ni.get(mf).prev = .wrap(last_add_ni); + } else { + assert(parent_ni.last(mf) == prev_oni); + parent_ni.get(mf).last = .wrap(last_add_ni); + } +} +fn addNodesToChildListAfter( + mf: *MappedFile, + gpa: Allocator, + /// `null` means to add at the start of the parent. + prev_oni: Node.Index.Optional, + first_add_ni: Node.Index, + last_add_ni: Node.Index, +) Allocator.Error!void { + const next_oni: Node.Index.Optional = next: { + if (prev_oni.unwrap()) |prev_ni| break :next prev_ni.next(mf); + const parent_ni = first_add_ni.parent(mf).unwrap().?; + break :next parent_ni.first(mf); + }; + return mf.addNodesToChildListBefore(gpa, next_oni, first_add_ni, last_add_ni); } fn realignNode( @@ -1206,107 +2068,37 @@ fn realignNode( gpa: Allocator, ni: Node.Index, new_alignment: Alignment, - opts: Node.Index.RealignNodeOptions, -) (Allocator.Error || Io.Cancelable || IoError)!void { +) Error!void { mf.nodes_lock.assertUnlocked(); - const node = ni.get(mf); - { - const prev_alignment = node.flags.alignment; - node.flags.alignment = new_alignment; - if (new_alignment.compare(.lte, prev_alignment)) return; - } - - const old_offset, const size = node.location().resolve(mf); - const parent_ni = node.parent.unwrap() orelse { - assert(ni == .root); - return mf.resizeNode(gpa, ni, size); - }; - - const new_size = new_alignment.forward(@intCast(size)); - if (new_alignment.check(@intCast(old_offset))) return mf.resizeNode(gpa, ni, new_size); - - _, const parent_size = parent_ni.location(mf).resolve(mf); - const trailing_end = trailing_end: { - const next_ni = node.next.unwrap() orelse break :trailing_end parent_size; - const next_offset, _ = next_ni.location(mf).resolve(mf); - break :trailing_end next_offset; - }; - - if (opts.try_backwards) { - const backward_offset = new_alignment.backward(@intCast(old_offset)); - const prev_end = prev_end: { - const prev_ni = node.prev.unwrap() orelse break :prev_end 0; - const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); - break :prev_end prev_offset + prev_size; - }; - - if (backward_offset >= prev_end) { - try mf.ensureCapacityForSetLocation(gpa); - - if (node.flags.has_content) { - const old_file_offset = ni.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - old_offset) + backward_offset; - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], - ); - @memset(mf.memory_map.memory[@intCast(new_file_offset + size)..@intCast(old_file_offset + size)], 0); - } - - if (backward_offset + new_size <= trailing_end) { - ni.setLocationAssumeCapacity(mf, backward_offset, new_size); - } else { - ni.setLocationAssumeCapacity(mf, backward_offset, size); - try mf.resizeNode(gpa, ni, new_size); - } - - return; + const old_offset, const old_size = ni.location(mf).resolve(mf); + + if (ni == .root or ni.position(mf) != .floating) { + // Only this node's size is aligned, not its offset. + if (!new_alignment.check(old_size)) { + assert(new_alignment.compare(.gt, ni.alignment(mf))); + try mf.growNode( + gpa, + ni, + new_alignment.forward(old_size), + .exact, // because `growNode` is not aware that the size needs to match `new_alignment` + ); } - } - - const forward_offset = new_alignment.forward(@intCast(old_offset)); - if (forward_offset + new_size <= trailing_end) { - // Shift into the free space if possible - try mf.ensureCapacityForSetLocation(gpa); - if (node.flags.has_content) { - const old_file_offset = ni.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - old_offset) + forward_offset; - if (new_file_offset < old_file_offset + size) { - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], - ); - } else try mf.moveRange(old_file_offset, new_file_offset, size); - @memset(mf.memory_map.memory[@intCast(new_file_offset + size)..][0..@intCast(new_size - size)], 0); - } - - ni.setLocationAssumeCapacity(mf, forward_offset, new_size); } else { - const temp_size = new_alignment.forward(@intCast(new_size + 1)); - try mf.resizeNode(gpa, ni, temp_size); - const new_offset, _ = ni.location(mf).resolve(mf); - - try mf.ensureCapacityForSetLocation(gpa); - - // Non-fixed nodes may now be aligned if the resize moved them - const new_forward_offset = new_alignment.forward(@intCast(new_offset)); - const final_offset = if (new_forward_offset != new_offset) final_offset: { - if (node.flags.has_content) { - const old_file_offset = ni.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - new_offset) + new_forward_offset; - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], - ); - @memset(mf.memory_map.memory[@intCast(old_file_offset)..@intCast(new_file_offset)], 0); - } - - break :final_offset new_forward_offset; - } else new_offset; - - ni.setLocationAssumeCapacity(mf, final_offset, new_size); + // This is a floating node, so its size and offset are both aligned. + if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) { + assert(new_alignment.compare(.gt, ni.alignment(mf))); + try mf.growFloatingNodeWithAlignment( + gpa, + ni, + new_alignment, + new_alignment.forward(old_size), + .minimum, + ); + } } + + ni.get(mf).flags.alignment = new_alignment; } fn updateWriters(mf: *MappedFile) void { @@ -1317,10 +2109,47 @@ fn updateWriters(mf: *MappedFile) void { } } -fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable || IoError)!void { - // make a copy of this node at the new location - try mf.copyRange(old_file_offset, new_file_offset, size); - // delete the copy of this node at the old location +fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) Error!void { + if (old_file_offset == new_file_offset) return; + + if (old_file_offset >= new_file_offset + size or + new_file_offset >= old_file_offset + size) + { + const n = try mf.copyFileRange( + mf.memory_map.file, + old_file_offset, + new_file_offset, + size, + ); + @memcpy( + mf.memory_map.memory[@intCast(new_file_offset + n)..][0..@intCast(size - n)], + mf.memory_map.memory[@intCast(old_file_offset + n)..][0..@intCast(size - n)], + ); + + try mf.zeroRange(old_file_offset, size); + + return; + } + + // TODO: if the non-overlapping region is greater than or equal to a filesystem block, is it + // ever worth doing multiple `copyFileRange` calls instead of a big `@memmove`? + + @memmove( + mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], + mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], + ); + + if (new_file_offset > old_file_offset) { + const clear_size = new_file_offset - old_file_offset; + assert(clear_size < size); + try mf.zeroRange(old_file_offset, clear_size); + } else { + const clear_size = old_file_offset - new_file_offset; + assert(clear_size < size); + try mf.zeroRange(new_file_offset + size, clear_size); + } +} +fn zeroRange(mf: *MappedFile, file_offset: u64, size: u64) Error!void { if (is_linux and !mf.flags.fallocate_punch_hole_unsupported and size >= mf.flags.block_size.toByteUnits() * 2 - 1) @@ -1328,147 +2157,149 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: while (true) switch (linux.errno(linux.fallocate( mf.memory_map.file.handle, linux.FALLOC.FL_PUNCH_HOLE | linux.FALLOC.FL_KEEP_SIZE, - @intCast(old_file_offset), + @intCast(file_offset), @intCast(size), ))) { .SUCCESS => return, .INTR => continue, - .BADF, .FBIG, .INVAL => unreachable, - .IO => return error.InputOutput, - .NODEV => return error.NotFile, - .NOSPC => return error.NoSpaceLeft, .NOSYS, .OPNOTSUPP => { mf.flags.fallocate_punch_hole_unsupported = true; break; // fall back to slow path }, - .PERM => return error.PermissionDenied, - .SPIPE => return error.Unseekable, - .TXTBSY => return error.FileBusy, - else => |e| return std.posix.unexpectedErrno(e), + else => |e| { + mf.io_err = switch (e) { + .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above + .BADF => unreachable, + .FBIG => unreachable, + .INVAL => unreachable, + .IO => error.InputOutput, + .NODEV => error.NotFile, + .NOSPC => error.NoSpaceLeft, + .PERM => error.PermissionDenied, + .SPIPE => error.Unseekable, + .TXTBSY => error.FileBusy, + else => std.posix.unexpectedErrno(e), + }; + return error.MappedFileIo; + }, }; } - @memset(mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], 0); + @memset(mf.memory_map.memory[@intCast(file_offset)..][0..@intCast(size)], 0); } - -fn copyRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable || IoError)!void { - const copy_size = try mf.copyFileRange(mf.memory_map.file, old_file_offset, new_file_offset, size); - if (copy_size < size) @memcpy( - mf.memory_map.memory[@intCast(new_file_offset + copy_size)..][0..@intCast(size - copy_size)], - mf.memory_map.memory[@intCast(old_file_offset + copy_size)..][0..@intCast(size - copy_size)], - ); -} - fn copyFileRange( mf: *MappedFile, old_file: Io.File, old_file_offset: u64, new_file_offset: u64, size: u64, -) (Io.Cancelable || IoError)!u64 { +) Error!u64 { + if (!is_linux or mf.flags.copy_file_range_unsupported) { + return 0; + } + + const min_size = mf.flags.block_size.toByteUnits() * 2 - 1; + if (size < min_size) return 0; + const io = mf.io; - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + mf.memory_map.write(io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; }; var remaining_size = size; - if (is_linux and !mf.flags.copy_file_range_unsupported) { - var old_file_offset_mut: i64 = @intCast(old_file_offset); - var new_file_offset_mut: i64 = @intCast(new_file_offset); - while (remaining_size >= mf.flags.block_size.toByteUnits() * 2 - 1) { - const copy_len = linux.copy_file_range( - old_file.handle, - &old_file_offset_mut, - mf.memory_map.file.handle, - &new_file_offset_mut, - @intCast(remaining_size), - 0, - ); - switch (linux.errno(copy_len)) { - .SUCCESS => { - if (copy_len == 0) break; - remaining_size -= copy_len; - if (remaining_size == 0) break; - }, - .INTR => continue, - .BADF, .FBIG, .INVAL, .OVERFLOW => unreachable, - .IO => return error.InputOutput, - .ISDIR => return error.IsDir, - .NOMEM => return error.SystemResources, - .NOSPC => return error.NoSpaceLeft, - .NOSYS, .OPNOTSUPP, .XDEV => { - mf.flags.copy_file_range_unsupported = true; - break; - }, - .PERM => return error.PermissionDenied, - .TXTBSY => return error.FileBusy, - else => |e| return std.posix.unexpectedErrno(e), - } + var old_file_offset_mut: i64 = @intCast(old_file_offset); + var new_file_offset_mut: i64 = @intCast(new_file_offset); + while (remaining_size >= min_size) { + const copy_len = linux.copy_file_range( + old_file.handle, + &old_file_offset_mut, + mf.memory_map.file.handle, + &new_file_offset_mut, + @intCast(remaining_size), + 0, + ); + switch (linux.errno(copy_len)) { + .SUCCESS => { + if (copy_len == 0) break; + remaining_size -= copy_len; + if (remaining_size == 0) break; + }, + .INTR => continue, + .NOSYS, .OPNOTSUPP, .XDEV => { + mf.flags.copy_file_range_unsupported = true; + break; + }, + else => |e| { + mf.io_err = switch (e) { + .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP, .XDEV => unreachable, // handled above + .BADF => unreachable, + .FBIG => unreachable, + .INVAL => unreachable, + .OVERFLOW => unreachable, + .IO => error.InputOutput, + .ISDIR => error.IsDir, + .NOMEM => error.SystemResources, + .NOSPC => error.NoSpaceLeft, + .PERM => error.PermissionDenied, + .TXTBSY => error.FileBusy, + else => std.posix.unexpectedErrno(e), + }; + return error.MappedFileIo; + }, } } return size - remaining_size; } -fn ensureCapacityForSetLocation(mf: *MappedFile, gpa: Allocator) Allocator.Error!void { - try mf.large.ensureUnusedCapacity(gpa, 2); - try mf.updates.ensureUnusedCapacity(gpa, 2); -} - pub fn ensureTotalCapacity(mf: *MappedFile, new_capacity: usize) Error!void { - mf.ensureTotalCapacityInner(new_capacity) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; -} -fn ensureTotalCapacityInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error || Io.Cancelable || IoError)!void { if (mf.memory_map.memory.len >= new_capacity) return; - try mf.ensureTotalCapacityPreciseInner(new_capacity +| new_capacity / growth_factor); + try mf.ensureTotalCapacityPrecise(new_capacity +| new_capacity / growth_factor); } pub fn ensureTotalCapacityPrecise(mf: *MappedFile, new_capacity: usize) Error!void { - mf.ensureTotalCapacityPreciseInner(new_capacity) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; -} -fn ensureTotalCapacityPreciseInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error || Io.Cancelable || IoError)!void { if (mf.memory_map.memory.len >= new_capacity) return; const io = mf.io; - const aligned_capacity = mf.flags.block_size.forward(new_capacity); + const aligned_capacity: usize = @intCast( + mf.flags.block_size.forward(new_capacity), + ); if (mf.memory_map.memory.len > 0) { if (mf.memory_map.setLength(io, aligned_capacity)) |_| { return; } else |err| switch (err) { error.OperationUnsupported => {}, - else => |e| return e, + error.OutOfMemory, error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, } - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + mf.memory_map.write(io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; }; unmap(mf); } const file = mf.memory_map.file; - mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForReading => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch |err| { + mf.io_err = switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForReading => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; }; } @@ -1487,7 +2318,7 @@ pub fn flush(mf: *MappedFile) (Io.Cancelable || error{MappedFileIo})!void { error.WouldBlock, // file was not opened as non-blocking error.NotOpenForWriting, // we definitely opened the file for writing - error.ReadOnlyFileSystem, + error.ReadOnlyFileSystem, // again, we opened the file for writing => { mf.io_err = error.Unexpected; return error.MappedFileIo; @@ -1512,211 +2343,276 @@ fn verify(mf: *MappedFile) void { assert(root.next == .none); mf.verifyNode(.root); } - fn verifyNode(mf: *MappedFile, parent_ni: Node.Index) void { const parent = parent_ni.get(mf); - const parent_offset, const parent_size = parent.location().resolve(mf); - var prev_ni: Node.Index = .none; + _, const parent_size = parent.location().resolve(mf); + + var prev_oni: Node.Index.Optional = .none; var prev_end: u64 = 0; - var ni = parent.first; - while (true) { - if (ni == .none) { - assert(parent.last == prev_ni); - return; - } + var prev_pos: Node.Position = .header; + var oni = parent.first; + while (oni.unwrap()) |ni| { const node = ni.get(mf); - assert(node.parent == parent_ni); + assert(node.parent == parent_ni.toOptional()); + assert(node.prev == prev_oni); + const offset, const size = node.location().resolve(mf); - assert(node.flags.alignment.check(@intCast(offset))); - assert(node.flags.alignment.check(@intCast(size))); const end = offset + size; - assert(end <= parent_offset + parent_size); + + assert(node.flags.alignment.check(size)); assert(offset >= prev_end); - assert(node.prev == prev_ni); + assert(end <= parent_size); + + switch (node.flags.position) { + .header => { + assert(prev_pos == .header); + assert(offset == prev_end); + }, + .floating => { + assert(prev_pos != .footer); + assert(node.flags.alignment.check(offset)); + }, + .footer => { + if (prev_pos == .footer) assert(offset == prev_end); + }, + } + mf.verifyNode(ni); - prev_ni = ni; + + prev_oni = .wrap(ni); prev_end = end; - ni = node.next; + prev_pos = ni.position(mf); + + oni = node.next; + } + assert(parent.last == prev_oni); + if (prev_pos == .footer) { + assert(prev_end == parent_size); } } -const testing = std.testing; -fn testVerifyContent(mf: *@This(), ni: Node.Index, value: u8, init_len: usize) !void { - // Not using std.mem.allEqual, so we can get useful output - const slice = ni.slice(mf); - var buf: [256]u8 = undefined; - @memset(buf[0..init_len], value); - @memset(buf[init_len..], 0); - try testing.expectEqualSlices(u8, buf[0..slice.len], slice); +test "fuzz node operations" { + try std.testing.fuzz({}, fuzzOneNodeOperations, .{}); } +fn fuzzOneNodeOperations(_: void, smith: *std.testing.Smith) anyerror!void { + const gpa = std.testing.allocator; + const io = std.testing.io; -test { - const gpa = testing.allocator; - - var tmp_dir = testing.tmpDir(.{}); + var tmp_dir = std.testing.tmpDir(.{}); defer tmp_dir.cleanup(); - var file = try tmp_dir.dir.createFile(testing.io, "test.mf", .{ .read = true }); - defer file.close(testing.io); + var tmp_file = try tmp_dir.dir.createFile(io, "test.mf", .{ .read = true }); + defer tmp_file.close(io); - var mf = try init(file, gpa, testing.io); + var mf: MappedFile = try .init(tmp_file, gpa, io); defer mf.deinit(gpa); - const a = try mf.addFirstChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" }); - const c = try mf.addLastChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" }); - const b = try mf.addNodeAfter(gpa, a, .{ .fixed = true, .alignment = .@"16" }); - const d = try mf.addNodeAfter(gpa, b, .{ .alignment = .@"4" }); - - const a_init_size = 8; - const b_init_size = 16; - const c_init_size = 24; - const d_init_size = 28; - - // Resize without content - { - // Verify size is aligned forward - try d.resize(&mf, gpa, d_init_size - 1); - try a.resize(&mf, gpa, a_init_size - 2); - try c.resize(&mf, gpa, c_init_size); - try b.resize(&mf, gpa, b_init_size); - mf.verify(); - - const a_loc, const a_size = a.location(&mf).resolve(&mf); - const b_loc, const b_size = b.location(&mf).resolve(&mf); - const c_loc, const c_size = c.location(&mf).resolve(&mf); - _, const d_size = d.location(&mf).resolve(&mf); - try testing.expect(a_size >= a_init_size); - try testing.expect(b_size >= b_init_size); - try testing.expect(c_size >= c_init_size); - try testing.expect(d_size >= d_init_size); - try testing.expect(b_loc >= a_loc + a_size); - try testing.expect(c_loc >= b_loc + b_size); - } - - const a_exp_size = 24; - const b_exp_size = 28; - const c_exp_size = 48; - const d_exp_size = 32; - - // Resize with content - { - @memset(a.slice(&mf)[0..a_init_size], 0xaa); - @memset(b.slice(&mf)[0..b_init_size], 0xbb); - @memset(c.slice(&mf)[0..c_init_size], 0xcc); - @memset(d.slice(&mf)[0..d_init_size], 0xdd); - - try a.resize(&mf, gpa, a_exp_size); - try b.resize(&mf, gpa, b_exp_size); - try c.resize(&mf, gpa, c_exp_size); - try d.resize(&mf, gpa, d_exp_size); - mf.verify(); - - const a_loc, const a_size = a.location(&mf).resolve(&mf); - const b_loc, const b_size = b.location(&mf).resolve(&mf); - const c_loc, const c_size = c.location(&mf).resolve(&mf); - _, const d_size = d.location(&mf).resolve(&mf); - try testing.expect(a_size >= a_exp_size); - try testing.expect(b_size >= b_exp_size); - try testing.expect(c_size >= c_exp_size); - try testing.expect(d_size >= d_exp_size); - try testing.expect(b_loc >= a_loc + a_size); - try testing.expect(c_loc >= b_loc + b_size); - - try testVerifyContent(&mf, a, 0xaa, a_init_size); - try testVerifyContent(&mf, b, 0xbb, b_init_size); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - try testVerifyContent(&mf, d, 0xdd, d_init_size); - } - - const child_init: []const struct { Alignment, usize } = &.{ - .{ .@"16", 16 }, - .{ .@"1", 1 }, - .{ .@"1", 19 }, - .{ .@"1", 3 }, - .{ .@"8", 30 }, - .{ .@"2", 5 }, - .{ .@"1", 60 }, - .{ .@"2", 2 }, - .{ .@"16", 32 }, + var nodes: std.array_hash_map.Auto(MappedFile.Node.Index, struct { + parent: MappedFile.Node.Index.Optional, + position: MappedFile.Node.Position, + num_headers: u32, + num_footers: u32, + /// For leaf nodes, this value is whether we have initialized the contents of the node or + /// not. For non-leaf nodes, this value is unspecified and should be ignored. + initialized: bool, + }) = .empty; + defer nodes.deinit(gpa); + + // When initializing a leaf node, we will place its 4-byte node index at the start of its range, + // and the bitwise NOT of its node index at the end of its range (both little-endian). This is + // just a simple way to put distinct values we can validate at all node boundaries. + + try nodes.putNoClobber(gpa, .root, .{ + .parent = .none, + .position = .floating, + .num_headers = 0, + .num_footers = 0, + .initialized = false, + }); + + // Allow a range of alignments, with most nodes having a small alignment of 1--32 bytes (most + // commonly 1 byte), but with a small chance for some large alignments too. + const alignment_weights: []const std.testing.Smith.Weight = comptime &.{ + .value(Alignment, .@"1", 20), + .value(Alignment, .@"2", 5), + .value(Alignment, .@"4", 5), + .value(Alignment, .@"8", 5), + .value(Alignment, .@"16", 5), + .value(Alignment, .@"32", 5), + .value(Alignment, .fromByteUnits(0x200), 1), + .value(Alignment, .fromByteUnits(0x400), 1), + .value(Alignment, .fromByteUnits(0x800), 1), + .value(Alignment, .fromByteUnits(0x1000), 1), + .value(Alignment, .fromByteUnits(0x2000), 1), + .value(Alignment, .fromByteUnits(0x4000), 1), + .value(Alignment, .fromByteUnits(0x8000), 1), }; - var children: [child_init.len]Node.Index = undefined; + const min_nonzero_size = 2 * @sizeOf(MappedFile.Node.Index); + const max_size = 0x10_000; + const initial_size_weights: []const std.testing.Smith.Weight = comptime &.{ + // initially, make nodes just as likely to be empty as non-empty + .value(u64, 0, max_size - min_nonzero_size + 1), + .rangeAtMost(u64, min_nonzero_size, max_size, 1), + }; + + while (!smith.eos()) switch (smith.value(enum { add, resize, realign })) { + .add => { + const parent_ni = nodes.keys()[smith.index(nodes.count())]; + + const alignment = smith.valueWeighted(Alignment, alignment_weights); + const size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); - // Differently-aligned fixed sibling nodes - { - for (children[0 .. children.len - 1], child_init[0 .. children.len - 1], 0..) |*ni, opts, i| { - ni.* = try mf.addLastChildNode(gpa, b, .{ - .alignment = opts.@"0", - .size = opts.@"1", - .fixed = true, + const position = smith.valueWeighted(Node.Position, comptime &.{ + // make floating nodes more common than header and footer nodes + .value(Node.Position, .header, 1), + .value(Node.Position, .footer, 1), + .value(Node.Position, .floating, 4), }); + const new_ni: Node.Index = switch (position) { + .header => new_ni: { + const parent_info = nodes.getPtr(parent_ni).?; + const prev_oni: Node.Index.Optional = prev_oni: { + const n = smith.valueRangeAtMost(u32, 0, parent_info.num_headers); + if (n == 0) break :prev_oni .none; + var cur_ni = parent_ni.first(&mf).unwrap().?; + for (1..n) |_| cur_ni = cur_ni.next(&mf).unwrap().?; + break :prev_oni .wrap(cur_ni); + }; + const new_ni = try parent_ni.addHeaderChildAfter(&mf, gpa, prev_oni, .{ + .size = size, + .alignment = alignment, + }); + parent_info.num_headers += 1; + break :new_ni new_ni; + }, + + .floating => try parent_ni.addFloatingChild(&mf, gpa, .{ + .size = size, + .alignment = alignment, + }), + + .footer => new_ni: { + const parent_info = nodes.getPtr(parent_ni).?; + const next_oni: Node.Index.Optional = next_oni: { + const n = smith.valueRangeAtMost(u32, 0, parent_info.num_footers); + if (n == 0) break :next_oni .none; + var cur_ni = parent_ni.last(&mf).unwrap().?; + for (1..n) |_| cur_ni = cur_ni.prev(&mf).unwrap().?; + break :next_oni .wrap(cur_ni); + }; + const new_ni = try parent_ni.addFooterChildBefore(&mf, gpa, next_oni, .{ + .size = size, + .alignment = alignment, + }); + parent_info.num_footers += 1; + break :new_ni new_ni; + }, + }; + + const initialize = size > 0 and smith.value(bool); + if (initialize) { + const slice = new_ni.slice(&mf); + std.mem.writeInt(u32, slice[0..4], @backingInt(new_ni), .little); + std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(new_ni), .little); + } + + try nodes.putNoClobber(gpa, new_ni, .{ + .parent = .wrap(parent_ni), + .position = position, + .num_headers = 0, + .num_footers = 0, + .initialized = initialize, + }); + }, + + .resize => { + const ni = nodes.keys()[smith.index(nodes.count())]; + const node_info = nodes.getPtr(ni).?; + + const alignment = ni.alignment(&mf); + + if (ni.first(&mf) == .none and smith.value(bool)) { + // Since this is a leaf node, we can use `resizeLeaf`. + const new_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); + try ni.resizeLeaf(&mf, gpa, new_size); + if (new_size == 0) { + node_info.initialized = false; + } + } else { + const min_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); + try ni.ensureMinimumSize(&mf, gpa, min_size); + } + + if (ni.first(&mf) == .none) { + // This is a leaf node, so it can contain data. + if (node_info.initialized) { + // It's already initialized, so we'll write the expected footer at the new end. + const slice = ni.slice(&mf); + std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little); + } else if (ni.location(&mf).resolve(&mf)[1] > 0) { + // It was uninitialized, but it has a non-zero size, so maybe we'd like to + // initialize it now? + if (smith.value(bool)) { + node_info.initialized = true; + const slice = ni.slice(&mf); + std.mem.writeInt(u32, slice[0..4], @backingInt(ni), .little); + std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little); + } + } + } + }, + .realign => { + const ni = nodes.keys()[smith.index(nodes.count())]; + const new_alignment = smith.valueWeighted(Alignment, alignment_weights); + if (new_alignment.compare(.gt, ni.alignment(&mf))) { + _, const old_size = ni.location(&mf).resolve(&mf); + try ni.realign(&mf, gpa, new_alignment); + if (ni.first(&mf) == .none and nodes.get(ni).?.initialized) { + const slice = ni.slice(&mf); + @memmove(slice[slice.len - 4 ..][0..4], slice[old_size - 4 ..][0..4]); + } + } + }, + }; - @memset(ni.slice(&mf)[0..opts.@"1"], @intCast(i + 1)); + mf.verify(); + + for (nodes.keys(), nodes.values()) |ni, expected| { + try std.testing.expectEqual(expected.parent, ni.parent(&mf)); + if (ni != .root) { + try std.testing.expectEqual(expected.position, ni.position(&mf)); } - // Shift differently-aligned nodes by inserting a node - children[children.len - 1] = try mf.addNodeAfter(gpa, children[3], .{ - .alignment = child_init[children.len - 1].@"0", - .size = child_init[children.len - 1].@"1", - .fixed = true, - }); - @memset(children[children.len - 1].slice(&mf), @intCast(children.len)); - - mf.verify(); - for (children, child_init, 0..) |ni, opts, i| { - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); + + { + var num_headers: u32 = 0; + var header_oni = ni.lastHeader(&mf); + while (header_oni.unwrap()) |header_ni| { + num_headers += 1; + header_oni = header_ni.prev(&mf); + } + try std.testing.expectEqual(expected.num_headers, num_headers); } - } - - // Shifting child nodes forward due via resize of parent.prev - { - try testing.expect(a.location(&mf).resolve(&mf)[1] < 64); - try a.resize(&mf, gpa, 64); - - try testVerifyContent(&mf, a, 0xaa, a_init_size); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - try testVerifyContent(&mf, d, 0xdd, d_init_size); - for (children, child_init, 0..) |ni, opts, i| { - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); + + { + var num_footers: u32 = 0; + var footer_oni = ni.firstFooter(&mf); + while (footer_oni.unwrap()) |footer_ni| { + num_footers += 1; + footer_oni = footer_ni.next(&mf); + } + try std.testing.expectEqual(expected.num_footers, num_footers); } - } - - // Re-align last node into trailing free space within parent - { - try b.resize(&mf, gpa, b.location(&mf).resolve(&mf)[1] + 64); - - const last = children[children.len - 2]; - try last.realign(&mf, gpa, .@"4", true); - mf.verify(); - - for (children, child_init, 0..) |ni, opts, i| - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - } - - // Re-align, shifting sibling nodes - { - try children[1].realign(&mf, gpa, .@"8", true); - mf.verify(); - - for (children, child_init, 0..) |ni, opts, i| - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - } - - // Shrink and shift start of trailing node into free space - { - try mf.shrinkNode(gpa, a, 16, true); - mf.verify(); - - const a_loc, const a_size = a.location(&mf).resolve(&mf); - const b_loc, _ = b.location(&mf).resolve(&mf); - try testing.expectEqual(b_loc, a_loc + a_size); - - try testVerifyContent(&mf, a, 0xaa, a_init_size); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - try testVerifyContent(&mf, d, 0xdd, d_init_size); - for (children, child_init, 0..) |ni, opts, i| { - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); + + if (ni.first(&mf) == .none and expected.initialized) { + const slice = ni.sliceConst(&mf); + if (slice.len > 0) { + try std.testing.expect(slice.len >= min_nonzero_size); + const header = std.mem.readInt(u32, slice[0..4], .little); + const footer = std.mem.readInt(u32, slice[slice.len - 4 ..][0..4], .little); + try std.testing.expectEqual(@backingInt(ni), header); + try std.testing.expectEqual(~@backingInt(ni), footer); + } } } } diff --git a/src/main.zig b/src/main.zig index 7e33a7fdf2d15ec9fa8e1417c0533fb702d22dfd..4584c950abd7855d781492de48e25204a8de10c1 100644 --- a/src/main.zig +++ b/src/main.zig @@ -36,6 +36,7 @@ const Module = @import("Module.zig"); test { _ = @import("codegen.zig"); + _ = @import("link/MappedFile.zig"); } const thread_stack_size = 60 << 20; -- 2.54.0