diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 9ecfa4d03951427c9b091f6e59d50c120d4ee7f1..0438a8d469f3678cedca64033aa1146886d98556 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -185,7 +185,6 @@ fn mainServer(init: std.process.Init.Minimal) !void { .environ = init.environ, }); defer io_instance.deinit(); - const io = io_instance.io(); const mode: fuzz_abi.LimitKind = @fromBackingInt(@intCast(try server.receiveBody_u8())); const amount_or_instance = try server.receiveBody_u64(); @@ -208,7 +207,7 @@ fn mainServer(init: std.process.Init.Minimal) !void { .indexes = test_indexes, .server = &server, .gpa = gpa, - .io = io, + .threaded_io = &io_instance, .input_poller = undefined, }; @@ -422,7 +421,7 @@ var fuzz_runner: if (builtin.fuzz) struct { indexes: []u32, server: *std.zig.Server, gpa: std.mem.Allocator, - io: Io, + threaded_io: *Io.Threaded, input_poller: Io.Future(Io.Cancelable!void), comptime { @@ -443,6 +442,12 @@ var fuzz_runner: if (builtin.fuzz) struct { defer if (testing.allocator_instance.deinit() != 0) std.process.exit(1); is_fuzz_test = false; + testing.io_instance = .init(testing.allocator, .{ + .argv0 = fuzz_runner.threaded_io.argv0, + .environ = fuzz_runner.threaded_io.environ.process_environ, + }); + defer testing.io_instance.deinit(); + builtin.test_functions[fuzz_runner.indexes[i]].func() catch |err| switch (err) { error.SkipZigTest => return, else => { @@ -473,7 +478,8 @@ var fuzz_runner: if (builtin.fuzz) struct { export fn runner_start_input_poller() void { @disableInstrumentation(); - const future = fuzz_runner.io.concurrent(inputPoller, .{}) catch |e| switch (e) { + const io = fuzz_runner.threaded_io.io(); + const future = io.concurrent(inputPoller, .{}) catch |e| switch (e) { error.ConcurrencyUnavailable => @panic("failed to spawn concurrent fuzz input poller"), }; fuzz_runner.input_poller = future; @@ -481,17 +487,20 @@ var fuzz_runner: if (builtin.fuzz) struct { export fn runner_stop_input_poller() void { @disableInstrumentation(); - assert(fuzz_runner.input_poller.cancel(fuzz_runner.io) == error.Canceled); + const io = fuzz_runner.threaded_io.io(); + assert(fuzz_runner.input_poller.cancel(io) == error.Canceled); } export fn runner_futex_wait(ptr: *const u32, expected: u32) bool { @disableInstrumentation(); - return fuzz_runner.io.futexWait(u32, ptr, expected) == error.Canceled; + const io = fuzz_runner.threaded_io.io(); + return io.futexWait(u32, ptr, expected) == error.Canceled; } export fn runner_futex_wake(ptr: *const u32, waiters: u32) void { @disableInstrumentation(); - fuzz_runner.io.futexWake(u32, ptr, waiters); + const io = fuzz_runner.threaded_io.io(); + io.futexWake(u32, ptr, waiters); } fn inputPoller() Io.Cancelable!void { diff --git a/src/InternPool.zig b/src/InternPool.zig index 96af077d99935e56e6290ca47b544b6e6bd076fc..94375c3f0ca7430d890588979d9409415c2054a6 100644 --- a/src/InternPool.zig +++ b/src/InternPool.zig @@ -5987,17 +5987,6 @@ pub const Alignment = enum(u6) { return n + 1; } - pub fn toStdMem(a: Alignment) std.mem.Alignment { - assert(a != .none); - return @fromBackingInt(@intCast(@backingInt(a))); - } - - pub fn fromStdMem(a: std.mem.Alignment) Alignment { - const r: Alignment = @fromBackingInt(@intCast(@backingInt(a))); - assert(r != .none); - return r; - } - pub fn toLlvm(a: Alignment) std.zig.llvm.Builder.Alignment { return @fromBackingInt(@intCast(@backingInt(a))); } diff --git a/src/link/Coff.zig b/src/link/Coff.zig index 7303c030011bd8935595353982ca2458b93ecae8..b7045cac54bb00945e2b9a6458d422151aca54ff 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -21,6 +21,7 @@ const Zcu = @import("../Zcu.zig"); const ModuleDefinition = @import("../libs/mingw/def.zig").ModuleDefinition; const implib = @import("../libs/mingw/implib.zig"); const Path = std.Build.Cache.Path; +const Alignment = MappedFile.Alignment; base: link.File, options: link.File.OpenOptions, @@ -532,10 +533,10 @@ pub const Member = struct { errdefer _ = coff.export_table.entries.pop(); _, const old_size = Node.known.longnames_member.location(&coff.mf).resolve(&coff.mf); - const new_size = old_size + name.len + 1; + const new_size = Alignment.@"4".forward(old_size + name.len + 1); assert(new_size < comptime try std.math.powi(u64, 10, max_name_len - 1)); - try Node.known.longnames_member.resize(&coff.mf, gpa, new_size); + try Node.known.longnames_member.resizeLeaf(&coff.mf, gpa, new_size); const name_table_slice = Node.known.longnames_member.slice(&coff.mf); const name_slice = name_table_slice[@intCast(old_size)..][0 .. name.len + 1]; @memcpy(name_slice[0..name.len], name); @@ -602,7 +603,7 @@ pub const Member = struct { }; pub const LongNamesTable = struct { - ni: MappedFile.Node.Index = .none, + ni: MappedFile.Node.Index.Optional = .none, entries: std.array_hash_map.Auto(void, Entry), pub const Entry = struct { @@ -832,7 +833,7 @@ pub const String = enum(u32) { pub const Section = struct { si: Symbol.Index, - relocation_table_ni: MappedFile.Node.Index, + relocation_table_ni: MappedFile.Node.Index.Optional, pub const RelocationIndex = enum(u16) { none, @@ -855,7 +856,7 @@ pub const Section = struct { sn: Symbol.SectionNumber, ) ?*align(2) std.coff.Relocation { if (sri == .none) return null; - const table_slice = sn.section(coff).relocation_table_ni.slice(&coff.mf); + const table_slice = sn.section(coff).relocation_table_ni.unwrap().?.slice(&coff.mf); return @ptrCast(@alignCast(&table_slice[@as(u32, sri.unwrap().?) * std.coff.Relocation.sizeOf()])); } }; @@ -891,7 +892,7 @@ const SpecialSymbol = enum { }; pub const Symbol = struct { - ni: MappedFile.Node.Index, + ni: MappedFile.Node.Index.Optional, rva: u32, value: std.meta.BareUnion(Symbol.Value), extra: std.meta.BareUnion(Symbol.Extra), @@ -986,7 +987,7 @@ pub const Symbol = struct { pub fn nodeOffset(sym: *const Symbol, coff: *Coff) u32 { return switch (sym.flags.value_tag) { .node_offset => offset: { - assert(switch (coff.getNode(sym.ni)) { + assert(switch (coff.getNode(sym.ni.unwrap().?)) { // Separate nodes are not created for these entries per-symbol .input_section, .import_address_table => true, else => false, @@ -1052,9 +1053,7 @@ pub const Symbol = struct { } pub fn node(si: Symbol.Index, coff: *Coff) MappedFile.Node.Index { - const ni = si.get(coff).ni; - assert(ni != .none); - return ni; + return si.get(coff).ni.unwrap().?; } pub fn sti(si: Symbol.Index, coff: *Coff) SymbolTable.Index { @@ -1075,7 +1074,7 @@ pub const Symbol = struct { pub fn flushMoved(si: Symbol.Index, coff: *Coff) !void { const sym = si.get(coff); - sym.rva = coff.computeNodeRva(sym.ni) + sym.nodeOffset(coff); + sym.rva = coff.computeNodeRva(sym.ni.unwrap().?) + sym.nodeOffset(coff); try si.applyLocationRelocs(coff); try si.applyTargetRelocs(coff, .none); @@ -1199,12 +1198,11 @@ pub const Reloc = extern struct { pub fn apply(reloc: *Reloc, coff: *Coff) !void { const loc_sym = reloc.loc.get(coff); - switch (loc_sym.ni) { - .none => return, - else => |ni| if (ni.hasMoved(&coff.mf)) return, - } - const loc_slice = loc_sym.ni.slice(&coff.mf)[@intCast(reloc.offset)..]; + const loc_sym_ni = loc_sym.ni.unwrap() orelse return; + if (loc_sym_ni.hasMoved(&coff.mf)) return; + + const loc_slice = loc_sym_ni.slice(&coff.mf)[@intCast(reloc.offset)..]; const target_endian = coff.targetEndian(); const target_machine = coff.targetLoad(&coff.headerPtr().machine); @@ -1331,9 +1329,12 @@ pub const Reloc = extern struct { } const target_sym = reloc.target.get(coff); - const is_abs = switch (target_sym.ni) { - .none => if (target_sym.section_number == .ABSOLUTE) true else return, - else => |ni| if (ni.hasMoved(&coff.mf)) return else false, + const is_abs = if (target_sym.ni.unwrap()) |ni| is_abs: { + if (ni.hasMoved(&coff.mf)) return; + break :is_abs false; + } else is_abs: { + if (target_sym.section_number != .ABSOLUTE) return; + break :is_abs true; }; const target_rva = target_sym.rva +% @as(u64, @bitCast(reloc.addend)); @@ -1573,7 +1574,7 @@ fn create( 33...64 => .@"PE32+", else => return error.UnsupportedCOFFArchitecture, }; - const section_align: std.mem.Alignment = switch (machine) { + const section_align: Alignment = switch (machine) { .AMD64, .I386 => @fromBackingInt(@intCast(12)), .SH3, .SH3DSP, .SH4, .SH5 => @fromBackingInt(@intCast(12)), .MIPS16, .MIPSFPU, .MIPSFPU16, .WCEMIPSV2 => @fromBackingInt(@intCast(12)), @@ -1617,22 +1618,22 @@ fn create( .entries = .empty, }, .import_table = .{ - .ni = .none, + .ni = undefined, .entries = .empty, .iat_symbol_indices = .empty, }, .export_table = .{ - .ni = .none, - .export_directory_table_ni = .none, + .ni = undefined, + .export_directory_table_ni = undefined, .export_address_table_si = .null, - .name_pointer_table_ni = .none, - .ordinal_table_ni = .none, - .name_table_ni = .none, + .name_pointer_table_ni = undefined, + .ordinal_table_ni = undefined, + .name_table_ni = undefined, .entries = .empty, }, .symbol_table = .{ - .ni = .none, - .strings_ni = .none, + .ni = undefined, + .strings_ni = undefined, .strings = .empty, .symbols = .empty, .pending_symbol_index = 0, @@ -1794,13 +1795,13 @@ fn initHeaders( minor_subsystem_version: u16, magic: std.coff.OptionalHeader.Magic, subsystem: std.coff.Subsystem, - section_align: std.mem.Alignment, + section_align: Alignment, file_name: []const u8, ) !void { const comp = coff.base.comp; const gpa = comp.gpa; const target_endian = coff.targetEndian(); - const file_align: std.mem.Alignment = comptime .fromByteUnits(default_file_alignment); + const file_align: Alignment = comptime .fromByteUnits(default_file_alignment); const is_image = coff.isImage(); const is_archive = coff.isArchive(); const target = &comp.root_mod.resolved_target.result; @@ -1839,34 +1840,20 @@ fn initHeaders( coff.nodes.appendAssumeCapacity(.file); const header_ni = Node.known.header; - assert(header_ni == try coff.mf.addOnlyChildNode(gpa, Node.known.file, .{ + assert(header_ni == try Node.known.file.addOnlyHeaderChild(&coff.mf, gpa, .{ .alignment = coff.mf.flags.block_size, - .fixed = true, })); coff.nodes.appendAssumeCapacity(.header); - const signature_ni = Node.known.signature; - assert(signature_ni == try coff.mf.addLastChildNode(gpa, if (is_image or !is_archive) header_ni else Node.known.file, .{ - .size = if (is_image) - msdos_stub.len + std.coff.pe_signature.len - else if (is_archive) - std.coff.archive_signature.len - else - 0, - .alignment = .@"4", - .fixed = true, - })); - coff.nodes.appendAssumeCapacity(.signature); - - const signature_slice = signature_ni.slice(&coff.mf); - if (is_image) { - @memcpy(signature_slice[0..msdos_stub.len], &msdos_stub); - @memcpy(signature_slice[signature_slice.len - std.coff.pe_signature.len ..], std.coff.pe_signature); - } else if (is_archive) { + const coff_parent_ni: MappedFile.Node.Index = if (is_archive) parent: { + assert(try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .wrap(header_ni), .{ + .size = std.coff.archive_signature.len, + .alignment = .@"4", + }) == Node.known.signature); + coff.nodes.appendAssumeCapacity(.signature); + const signature_slice = Node.known.signature.slice(&coff.mf); @memcpy(signature_slice, std.coff.archive_signature); - } - const opt_coff_parent_ni = if (is_archive) parent: { const initial_member_count = Member.Index.known_count + @intFromBool(comp.zcu != null); try coff.members.ensureTotalCapacity(gpa, initial_member_count); @@ -1892,46 +1879,54 @@ fn initHeaders( const zcu_member = zcu_mi.get(coff); try zcu_member.initHeader(coff, zcu.main_mod.fully_qualified_name, timestamp); + assert(try zcu_member.content_ni.addOnlyHeaderChild(&coff.mf, gpa, .{ + .size = @sizeOf(std.coff.Header), + .alignment = .@"4", + }) == Node.known.coff_header); + coff.nodes.appendAssumeCapacity(.coff_header); + break :parent zcu_member.content_ni; } + // If we're not generating any code, no more known nodes are used + // These placeholder nodes are placed before the first member - if there are // no other members then the last linker member (longnames) needs to expand // to fill the padding at the end of the file. - assert(Node.known.zcu_member_header == try coff.mf.addNodeAfter(gpa, Node.known.header, .{})); - assert(Node.known.zcu_member == try coff.mf.addNodeAfter(gpa, Node.known.header, .{})); - coff.nodes.appendAssumeCapacity(.placeholder); - coff.nodes.appendAssumeCapacity(.placeholder); + while (coff.nodes.len < Node.known_count) { + _ = try Node.known.header.addHeaderChildAfter(&coff.mf, gpa, .none, .{}); + coff.nodes.appendAssumeCapacity(.placeholder); + } - break :parent null; + return; } else parent: { + assert(try header_ni.addOnlyHeaderChild(&coff.mf, gpa, .{ + .size = if (is_image) msdos_stub.len + std.coff.pe_signature.len else 0, + .alignment = .@"4", + }) == Node.known.signature); + coff.nodes.appendAssumeCapacity(.signature); + if (is_image) { + const signature_slice = Node.known.signature.slice(&coff.mf); + @memcpy(signature_slice[0..msdos_stub.len], &msdos_stub); + @memcpy(signature_slice[signature_slice.len - std.coff.pe_signature.len ..], std.coff.pe_signature); + } + // TODO: Not ideal to have this many placeholder nodes - use two distinct `Node.known` types? while (true) { - const placeholder_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{}); + const placeholder_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .none, .{}); coff.nodes.appendAssumeCapacity(.placeholder); if (placeholder_ni == Node.known.zcu_member) break; } - break :parent Node.known.header; - }; - - const coff_parent_ni = opt_coff_parent_ni orelse { - // If we're not generating any code, no more known nodes are used - while (coff.nodes.len < Node.known_count) { - _ = try coff.mf.addNodeAfter(gpa, Node.known.header, .{}); - coff.nodes.appendAssumeCapacity(.placeholder); - } + assert(try header_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(Node.known.signature), .{ + .size = @sizeOf(std.coff.Header), + .alignment = .@"4", + }) == Node.known.coff_header); + coff.nodes.appendAssumeCapacity(.coff_header); - return; + break :parent header_ni; }; - const coff_header_ni = Node.known.coff_header; - assert(coff_header_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ - .size = @sizeOf(std.coff.Header), - .alignment = .@"4", - .fixed = true, - })); - coff.nodes.appendAssumeCapacity(.coff_header); { const coff_header = coff.headerPtr(); coff_header.* = .{ @@ -1954,10 +1949,9 @@ fn initHeaders( } const optional_header_ni = Node.known.optional_header; - assert(optional_header_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + assert(optional_header_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(Node.known.coff_header), .{ .size = optional_header_size, .alignment = .@"4", - .fixed = true, })); coff.nodes.appendAssumeCapacity(.optional_header); if (is_image) { @@ -2066,10 +2060,9 @@ fn initHeaders( } const data_directories_ni = Node.known.data_directories; - assert(data_directories_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + assert(data_directories_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(optional_header_ni), .{ .size = data_directories_size, .alignment = .@"4", - .fixed = true, })); coff.nodes.appendAssumeCapacity(.data_directories); if (is_image) { @@ -2082,9 +2075,8 @@ fn initHeaders( } const section_table_ni = Node.known.section_table; - assert(section_table_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + assert(section_table_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(data_directories_ni), .{ .alignment = .@"4", - .fixed = true, })); coff.nodes.appendAssumeCapacity(.section_table); @@ -2092,16 +2084,14 @@ fn initHeaders( if (!is_image) { // TODO: These two nodes could be inside one movable node? - coff.symbol_table.ni = try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + coff.symbol_table.ni = try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(section_table_ni), .{ .alignment = .@"2", - .fixed = true, .moved = true, }); coff.nodes.appendAssumeCapacity(.symbol_table); - coff.symbol_table.strings_ni = try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + coff.symbol_table.strings_ni = try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(coff.symbol_table.ni), .{ .size = @sizeOf(u32), - .fixed = true, .resized = true, }); coff.nodes.appendAssumeCapacity(.string_table); @@ -2148,15 +2138,14 @@ fn initHeaders( } // TODO: Lazily initialize this instead, avoid the extra logic for this in flushMoved / flushResized - coff.import_table.ni = try coff.mf.addLastChildNode( - gpa, - (try coff.objectSectionMapIndex( - .@".idata", - coff.mf.flags.block_size, - .{ .read = true, .initialized = true }, - )).symbol(coff).node(coff), - .{ .alignment = .@"4" }, - ); + const import_table_parent_ni = (try coff.objectSectionMapIndex( + .@".idata", + coff.mf.flags.block_size, + .{ .read = true, .initialized = true }, + )).symbol(coff).node(coff); + coff.import_table.ni = try import_table_parent_ni.addFloatingChild(&coff.mf, gpa, .{ + .alignment = .@"4", + }); coff.nodes.appendAssumeCapacity(.import_directory_table); coff.export_table.ni = (try coff.pseudoSectionMapIndex( @@ -2165,15 +2154,10 @@ fn initHeaders( .{ .read = true, .initialized = true }, )).symbol(coff).node(coff); - coff.export_table.export_directory_table_ni = try coff.mf.addLastChildNode( - gpa, - coff.export_table.ni, - .{ - .size = @sizeOf(std.coff.ExportDirectoryTable) + file_name.len + 1, - .moved = true, - .fixed = true, - }, - ); + coff.export_table.export_directory_table_ni = try coff.export_table.ni.addHeaderChildAfter(&coff.mf, gpa, coff.export_table.ni.last(&coff.mf), .{ + .size = @sizeOf(std.coff.ExportDirectoryTable) + file_name.len + 1, + .moved = true, + }); coff.nodes.appendAssumeCapacity(.export_directory_table); const name_index = @sizeOf(std.coff.ExportDirectoryTable); @@ -2181,7 +2165,7 @@ fn initHeaders( @memcpy(table_slice[name_index..][0..file_name.len], file_name[0..file_name.len]); @memset(table_slice[name_index + file_name.len ..], 0); - const export_address_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + const export_address_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(std.coff.ExportAddressTableEntry), .moved = true, }); @@ -2191,25 +2175,25 @@ fn initHeaders( coff.export_table.export_address_table_si = coff.addSymbolAssumeCapacity(); const export_address_table_sym = coff.export_table.export_address_table_si.get(coff); - export_address_table_sym.ni = export_address_table_ni; + export_address_table_sym.ni = .wrap(export_address_table_ni); assert(export_address_table_sym.loc_relocs == .none); export_address_table_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); export_address_table_sym.section_number = coff.getNode(coff.export_table.ni).pseudo_section.symbol(coff).get(coff).section_number; - coff.export_table.name_pointer_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + coff.export_table.name_pointer_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(std.coff.ExportNamePointerTableEntry), .moved = true, }); coff.nodes.appendAssumeCapacity(.export_name_pointer_table); - coff.export_table.ordinal_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + coff.export_table.ordinal_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(std.coff.ExportOrdinalTableEntry), .moved = true, }); coff.nodes.appendAssumeCapacity(.export_ordinal_table); - coff.export_table.name_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + coff.export_table.name_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(u8), .moved = true, }); @@ -2260,7 +2244,7 @@ pub fn initBuiltins(coff: *Coff) !void { if (coff.isImage()) { const si = try coff.globalSymbol(.{ .name = "__ImageBase", .type = .data }); const sym = si.get(coff); - sym.ni = Node.known.header; + sym.ni = .wrap(Node.known.header); } defer coff.flushSectionMerges() catch unreachable; @@ -2302,14 +2286,13 @@ pub fn initBuiltins(coff: *Coff) !void { const list_len_si = try coff.globalSymbol(.{ .name = list.global, .type = .data }); const list_len_sym = list_len_si.get(coff); list_len_sym.setExtra(.{ .size = addr_info.size }); - list_len_sym.ni = try coff.mf.addFirstChildNode(gpa, start_sym.ni, .{ + list_len_sym.ni = .wrap(try start_sym.ni.unwrap().?.addHeaderChildAfter(&coff.mf, gpa, .none, .{ .size = addr_info.size, - .fixed = true, - }); + })); coff.nodes.appendAssumeCapacity(.{ .builtin = list_len_si }); list_len_sym.section_number = start_sym.section_number; - const start_slice = list_len_sym.ni.slice(&coff.mf); + const start_slice = list_len_sym.ni.unwrap().?.slice(&coff.mf); switch (addr_info.magic) { _ => unreachable, inline .PE32, .@"PE32+" => |t| { @@ -2324,14 +2307,13 @@ pub fn initBuiltins(coff: *Coff) !void { const list_end_si = coff.addSymbolAssumeCapacity(); const list_end_sym = list_end_si.get(coff); list_end_sym.setExtra(.{ .size = addr_info.size }); - list_end_sym.ni = try coff.mf.addFirstChildNode(gpa, end_sym.ni, .{ + list_end_sym.ni = .wrap(try end_sym.ni.unwrap().?.addHeaderChildAfter(&coff.mf, gpa, .none, .{ .size = addr_info.size, - .fixed = true, - }); + })); coff.nodes.appendAssumeCapacity(.{ .builtin = list_end_si }); list_end_sym.section_number = start_sym.section_number; - @memset(list_end_sym.ni.slice(&coff.mf), 0); + @memset(list_end_sym.ni.unwrap().?.slice(&coff.mf), 0); try list_len_si.flushMoved(coff); try list_end_si.flushMoved(coff); @@ -2387,7 +2369,7 @@ fn getNode(coff: *const Coff, ni: MappedFile.Node.Index) Node { } fn computeNodeRva(coff: *Coff, ni: MappedFile.Node.Index) u32 { const parent_rva = parent_rva: { - const parent_si = switch (coff.getNode(ni.parent(&coff.mf))) { + const parent_si = switch (coff.getNode(ni.parent(&coff.mf).unwrap().?)) { .file, .header, .signature, @@ -2452,11 +2434,11 @@ fn computeSymbolSectionOffset( relative_to: enum { image, pseudo }, ) u32 { var section_offset: u32 = sym.nodeOffset(coff); - var parent_ni = sym.ni; + var parent_ni = sym.ni.unwrap().?; while (true) { const offset, _ = parent_ni.location(&coff.mf).resolve(&coff.mf); section_offset += @intCast(offset); - parent_ni = parent_ni.parent(&coff.mf); + parent_ni = parent_ni.parent(&coff.mf).unwrap().?; switch (coff.getNode(parent_ni)) { else => unreachable, .image_section => break, @@ -2475,7 +2457,7 @@ pub inline fn targetEndian(_: *const Coff) std.lang.Endian { fn targetAddrInfo(coff: *Coff) struct { size: u8, - alignment: std.mem.Alignment, + alignment: Alignment, magic: std.coff.OptionalHeader.Magic, } { const magic = coff.targetLoad(&coff.optionalHeaderStandardPtr().magic); @@ -2741,7 +2723,7 @@ fn getOrPutSymbolName(coff: *Coff, name: []const u8, opt_string: ?String) !Symbo const string_index = coff.symbol_table.strings_ni.location(&coff.mf).resolve(&coff.mf)[1]; string_gop.value_ptr.* = @fromBackingInt(@intCast(string_index)); - try coff.symbol_table.strings_ni.resize(&coff.mf, gpa, string_index + name.len + 1); + try coff.symbol_table.strings_ni.resizeLeaf(&coff.mf, gpa, string_index + name.len + 1); const slice = coff.symbol_table.strings_ni.slice(&coff.mf); @memcpy(slice[@intCast(string_index)..][0..name.len], name); slice[@intCast(string_index + name.len)] = 0; @@ -2875,9 +2857,9 @@ fn navSection( switch (nav_resolved.@"linksection") { .none => coff.mf.flags.block_size, else => switch (nav_resolved.@"align") { - .none => Type.fromInterned(ip.typeOf(nav_resolved.value)).abiAlignment(zcu), - else => |alignment| alignment, - }.toStdMem(), + .none => .fromIp(Type.fromInterned(ip.typeOf(nav_resolved.value)).abiAlignment(zcu)), + else => |a| .fromIp(a), + }, }, attributes, )).symbol(coff); @@ -2966,24 +2948,22 @@ fn addMemberAssumeCapacity(coff: *Coff, kind: std.coff.ArchiveMemberHeader.Kind, const comp = coff.base.comp; const gpa = comp.gpa; - // TODO: These two nodes could to be inside a movable node if kind == .coff|.import - const header_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{ + const header_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, Node.known.file.last(&coff.mf), .{ .size = @sizeOf(std.coff.ArchiveMemberHeader), .alignment = .@"2", - .fixed = true, .moved = true, }); - const content_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{ - // The actual alignment required by the spec is 2, but to allow aligned access to - // the various COFF data structures in-place during linking we overalign - .alignment = switch (kind) { - .coff => .@"4", - else => .@"2", - }, - .size = size, + // The actual alignment required by the spec is 2, but to allow aligned access to + // the various COFF data structures in-place during linking we overalign + const content_align: Alignment = switch (kind) { + .first_linker, .second_linker, .longnames, .coff => .@"4", + else => .@"2", + }; + const content_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .wrap(header_ni), .{ + .alignment = content_align, + .size = content_align.forward(size), .resized = size > 0, - .fixed = true, }); const mi: Member.Index = @fromBackingInt(@intCast(coff.members.items.len)); @@ -3009,7 +2989,7 @@ fn addMemberAssumeCapacity(coff: *Coff, kind: std.coff.ArchiveMemberHeader.Kind, const old_size = Node.known.second_linker_member.location(&coff.mf).resolve(&coff.mf)[1]; const old_header_size = new_num_members * @sizeOf(u32); const trailing_size: usize = @intCast(old_size - old_header_size); - try Node.known.second_linker_member.resize(&coff.mf, gpa, old_size + @sizeOf(u32)); + try Node.known.second_linker_member.resizeLeaf(&coff.mf, gpa, old_size + @sizeOf(u32)); const slice = Node.known.second_linker_member.slice(&coff.mf); @memmove( @@ -3047,7 +3027,7 @@ fn appendMemberSymbolString( name: []const u8, offset: u64, ) !void { - try strings_ni.resize(&coff.mf, coff.base.comp.gpa, new_size); + try strings_ni.resizeLeaf(&coff.mf, coff.base.comp.gpa, new_size); const name_slice = strings_ni.slice(&coff.mf)[offset..][0 .. name.len + 1]; @memcpy(name_slice[0..name.len], name); name_slice[name.len] = 0; @@ -3080,7 +3060,7 @@ fn ensureMemberSymbol(coff: *Coff, mi: Member.Index, name: String) !void { { const old_header_size: usize = @intCast(@sizeOf(u32) + @backingInt(mfli) * @sizeOf(u32)); const new_header_size: usize = @intCast(old_header_size + @sizeOf(u32)); - try Node.known.first_linker_member.resize(&coff.mf, gpa, new_header_size + new_string_table_size); + try Node.known.first_linker_member.resizeLeaf(&coff.mf, gpa, Alignment.@"4".forward(new_header_size + new_string_table_size)); const slice = Node.known.first_linker_member.slice(&coff.mf); @memmove(slice[new_header_size..][0..coff.lib_string_len], slice[old_header_size..][0..coff.lib_string_len]); @@ -3094,7 +3074,7 @@ fn ensureMemberSymbol(coff: *Coff, mi: Member.Index, name: String) !void { const num_members = coff.targetLoad(coff.secondLinkerMemberNumMembersPtr()); const old_header_size = 2 * @sizeOf(u32) + num_members * @sizeOf(u32) + @backingInt(mfli) * @sizeOf(u16); const new_header_size = old_header_size + @sizeOf(u16); - try Node.known.second_linker_member.resize(&coff.mf, gpa, new_header_size + new_string_table_size); + try Node.known.second_linker_member.resizeLeaf(&coff.mf, gpa, Alignment.@"4".forward(new_header_size + new_string_table_size)); const old_needs_sort = coff.pending_members.get(Member.Index.second) != null; const needs_sort = old_needs_sort or (if (coff.lib_string_table.items.len > 0) @@ -3151,7 +3131,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { else .NULL, }; - } else blk: switch (coff.getNode(sym.ni)) { + } else blk: switch (coff.getNode(sym.ni.unwrap().?)) { .image_section => .{ try coff.getOrPutSymbolName(&sym.section_number.header(coff).name, null), 1, @@ -3192,7 +3172,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { }; }, else => { - log.err("TODO implement symbol table init for {s} ({d})", .{ @tagName(coff.getNode(sym.ni)), si }); + log.err("TODO implement symbol table init for {s} ({d})", .{ @tagName(coff.getNode(sym.ni.unwrap().?)), si }); unreachable; }, }; @@ -3201,7 +3181,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { const new_num_symbols = old_num_symbols + 1 + num_aux_symbols; coff.targetStore(&coff.headerPtr().number_of_symbols, new_num_symbols); - try coff.symbol_table.ni.resize(&coff.mf, gpa, new_num_symbols * std.coff.Symbol.sizeOf()); + try coff.symbol_table.ni.resizeLeaf(&coff.mf, gpa, new_num_symbols * std.coff.Symbol.sizeOf()); sti.* = .wrap(old_num_symbols); si.flushSymbolTableIndex(coff); @@ -3255,13 +3235,13 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { std.mem.byteSwapAllFieldsAligned(std.coff.WeakExternalDefinition, .@"2", aux_ptr); break :aux_init; - } else switch (coff.getNode(sym.ni)) { + } else switch (coff.getNode(sym.ni.unwrap().?)) { .image_section => |sec_si| { assert(si == sec_si); const header = sym.section_number.header(coff); const aux_ptr = coff.symbolTableSectionAuxEntryPtr(sti.*).?; aux_ptr.* = .{ - .length = @intCast(sym.ni.location(&coff.mf).resolve(&coff.mf)[1]), + .length = @intCast(sym.ni.unwrap().?.location(&coff.mf).resolve(&coff.mf)[1]), .number_of_relocations = header.number_of_relocations, .number_of_linenumbers = header.number_of_linenumbers, .checksum = 0, @@ -3288,7 +3268,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { .ABSOLUTE, .DEBUG, => unreachable, - else => switch (coff.getNode(sym.ni)) { + else => switch (coff.getNode(sym.ni.unwrap().?)) { .image_section => 0, else => coff.computeSymbolSectionOffset(sym, .image), }, @@ -3364,13 +3344,13 @@ fn addSection(coff: *Coff, name: String, flags: std.coff.SectionHeader.Flags) !S const section_index = coff.targetLoad(&coff_header.number_of_sections); const section_table_len = section_index + 1; coff.targetStore(&coff_header.number_of_sections, section_table_len); - try Node.known.section_table.resize( + try Node.known.section_table.resizeLeaf( &coff.mf, gpa, @sizeOf(std.coff.SectionHeader) * section_table_len, ); - const ni = try coff.mf.addLastChildNode(gpa, coff.sectionParent(), .{ + const ni = try coff.sectionParent().addFloatingChild(&coff.mf, gpa, .{ .alignment = coff.mf.flags.block_size, .moved = true, .bubbles_moved = false, @@ -3397,7 +3377,7 @@ fn addSection(coff: *Coff, name: String, flags: std.coff.SectionHeader.Flags) !S { const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.rva = rva; sym.section_number = @fromBackingInt(@intCast(section_table_len)); } @@ -3481,7 +3461,7 @@ const ObjectSectionAttributes = packed struct { fn pseudoSectionMapIndex( coff: *Coff, name: String, - alignment: std.mem.Alignment, + alignment: Alignment, attributes: ObjectSectionAttributes, ) !Node.PseudoSectionMapIndex { const gpa = coff.base.comp.gpa; @@ -3506,11 +3486,11 @@ fn pseudoSectionMapIndex( try coff.nodes.ensureUnusedCapacity(gpa, 1); try coff.symbols.ensureUnusedCapacity(gpa, 1); - const ni = try coff.mf.addLastChildNode(gpa, parent.node(coff), .{ .alignment = alignment }); + const ni = try parent.node(coff).addFloatingChild(&coff.mf, gpa, .{ .alignment = alignment }); const si = coff.addSymbolAssumeCapacity(); pseudo_section_gop.value_ptr.* = si; const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.rva = coff.computeNodeRva(ni); sym.section_number = parent.get(coff).section_number; assert(sym.loc_relocs == .none); @@ -3543,7 +3523,7 @@ fn objectSectionParentName(coff: *Coff, name: []const u8) []const u8 { fn objectSectionMapIndex( coff: *Coff, name: String, - alignment: std.mem.Alignment, + alignment: Alignment, attributes: ObjectSectionAttributes, ) !Node.ObjectSectionMapIndex { const gpa = coff.base.comp.gpa; @@ -3565,31 +3545,28 @@ fn objectSectionMapIndex( try coff.nodes.ensureUnusedCapacity(gpa, 1); try coff.symbols.ensureUnusedCapacity(gpa, 1); const parent_ni = parent.node(coff); - var prev_ni: MappedFile.Node.Index = .none; - var next_it = parent_ni.children(&coff.mf); - while (next_it.next()) |next_ni| switch (std.mem.order( - u8, - name_slice, - coff.getNode(next_ni).object_section.name(coff).toSlice(coff), - )) { - .lt => break, - .eq => unreachable, - .gt => prev_ni = next_ni, - }; - const ni = switch (prev_ni) { - .none => try coff.mf.addFirstChildNode(gpa, parent_ni, .{ - .alignment = alignment, - .fixed = true, - }), - else => try coff.mf.addNodeAfter(gpa, prev_ni, .{ - .alignment = alignment, - .fixed = true, - }), - }; + var prev_oni: MappedFile.Node.Index.Optional = .none; + { + var child_oni = parent_ni.first(&coff.mf); + while (child_oni.unwrap()) |child_ni| : (child_oni = child_ni.next(&coff.mf)) { + switch (std.mem.order( + u8, + name_slice, + coff.getNode(child_ni).object_section.name(coff).toSlice(coff), + )) { + .lt => break, + .eq => unreachable, + .gt => prev_oni = .wrap(child_ni), + } + } + } + const ni = try parent_ni.addHeaderChildAfter(&coff.mf, gpa, prev_oni, .{ + .alignment = alignment, + }); const si = coff.addSymbolAssumeCapacity(); object_section_gop.value_ptr.* = si; const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.rva = coff.computeNodeRva(ni); sym.section_number = parent.get(coff).section_number; assert(sym.loc_relocs == .none); @@ -3598,17 +3575,17 @@ fn objectSectionMapIndex( break :sym sym; } else object_section_gop.value_ptr.get(coff); - const parent_ni = sym.ni.parent(&coff.mf); + const parent_ni = sym.ni.unwrap().?.parent(&coff.mf).unwrap().?; const parent_alignment = parent_ni.alignment(&coff.mf); if (alignment.compare(.gt, parent_alignment)) { log.debug("realignParent({s}, {d}) {d}->{d}", .{ name.toSlice(coff), parent_ni, parent_alignment, alignment }); - try parent_ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); + try parent_ni.realign(&coff.mf, gpa, alignment); } - const old_alignment = sym.ni.alignment(&coff.mf); + const old_alignment = sym.ni.unwrap().?.alignment(&coff.mf); if (alignment.compare(.gt, old_alignment)) { log.debug("realignObject({s}) {d}->{d}", .{ name.toSlice(coff), old_alignment, alignment }); - try sym.ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); + try sym.ni.unwrap().?.realign(&coff.mf, gpa, alignment); } try coff.verifyParentSectionAttributes( @@ -3764,20 +3741,16 @@ fn addRelocAssumeCapacity( if (coff.symbolTableSectionAuxEntryPtr(loc_sn.symbol(coff).sti(coff))) |aux_ptr| coff.targetStore(&aux_ptr.number_of_relocations, new_num_relocations); - if (section.relocation_table_ni == .none) { - section.relocation_table_ni = try coff.mf.addLastChildNode( - gpa, - coff.sectionParent(), - .{ - .size = new_size, - .alignment = .@"2", - .moved = true, - .resized = true, - }, - ); - coff.nodes.appendAssumeCapacity(.{ .relocation_table = loc_sn }); + if (section.relocation_table_ni.unwrap()) |relocation_table_ni| { + try relocation_table_ni.resizeLeaf(&coff.mf, gpa, new_size); } else { - try section.relocation_table_ni.resize(&coff.mf, gpa, new_size); + section.relocation_table_ni = .wrap(try coff.sectionParent().addFloatingChild(&coff.mf, gpa, .{ + .size = new_size, + .alignment = .@"2", + .moved = true, + .resized = true, + })); + coff.nodes.appendAssumeCapacity(.{ .relocation_table = loc_sn }); } // TODO: These need to allocate from a free list, once deleting relocs from the table is supported @@ -4581,7 +4554,7 @@ fn loadObject( }, .SAME_SIZE => { // TODO: Verify that this node isn't resized after creation - _, const size = si.get(coff).ni.location(&coff.mf).resolve(&coff.mf); + _, const size = si.get(coff).ni.unwrap().?.location(&coff.mf).resolve(&coff.mf); if (size == section.header.size_of_raw_data) { symbol.si = si; break :comdat .skip; @@ -4598,9 +4571,9 @@ fn loadObject( }, .EXACT_MATCH => { const sym = si.get(coff); - const existing_crc = switch (coff.getNode(sym.ni)) { + const existing_crc = switch (coff.getNode(sym.ni.unwrap().?)) { .input_section => |isi| isi.inputSection(coff).crc, - else => Crc32.hash(sym.ni.sliceConst(&coff.mf)), + else => Crc32.hash(sym.ni.unwrap().?.sliceConst(&coff.mf)), }; if (existing_crc == section.comdat_crc) { @@ -4666,7 +4639,7 @@ fn loadObject( section.parent_si = (try coff.objectSectionMapIndex( section.name, - section.header.flags.ALIGN.alignment() orelse .@"1", + .fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1), .fromFlags(section.header.flags), )).symbol(coff); } @@ -4679,9 +4652,10 @@ fn loadObject( for (sections) |*section| { if (section.parent_si == .null) continue; - const ni = try coff.mf.addLastChildNode(gpa, section.parent_si.node(coff), .{ - .size = section.header.size_of_raw_data, - .alignment = section.header.flags.ALIGN.alignment() orelse .@"1", + const alignment: Alignment = .fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1); + const ni = try section.parent_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ + .size = alignment.forward(section.header.size_of_raw_data), + .alignment = alignment, .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .input_section = @fromBackingInt(@intCast(coff.input_sections.items.len)) }); @@ -4691,7 +4665,7 @@ fn loadObject( pending_symbols.values()[psi].si = section.si; const sym = section.si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = section.parent_si.get(coff).section_number; coff.input_sections.addOneAssumeCapacity().* = .{ @@ -4852,7 +4826,7 @@ fn loadObject( } if (section.comdat_psi.unwrap() == @as(u32, @intCast(i))) - coff.getNode(section.si.get(coff).ni).input_section.inputSection(coff).comdat_si = symbol.si; + coff.getNode(section.si.get(coff).ni.unwrap().?).input_section.inputSection(coff).comdat_si = symbol.si; } if (symbol.weak_external_psi.unwrap()) |weak_external_i| { @@ -4967,14 +4941,14 @@ fn loadObject( const section = §ions[symbol.section_number.toIndex()]; include_section = section.comdat_result == .include; if (include_section) { - const isi = coff.getNode(section.si.get(coff).ni).input_section; + const isi = coff.getNode(section.si.get(coff).ni.unwrap().?).input_section; isi.inputSection(coff).first_li = @fromBackingInt(@intCast(coff.input_symbols.items.len)); } } } if (include_section) { - assert(coff.getNode(symbol.si.get(coff).ni) == .input_section); + assert(coff.getNode(symbol.si.get(coff).ni.unwrap().?) == .input_section); symbol.si.get(coff).setExtra(.{ .isli = @fromBackingInt(@intCast(coff.input_symbols.items.len)) }); coff.input_symbols.addOneAssumeCapacity().* = .{ .si = symbol.si, @@ -5002,7 +4976,7 @@ fn failMultipleDefinitions( var err = try coff.base.comp.link_diags.addErrorWithNotes(num_notes); try err.addMsg("multiple definitions of '{s}'", .{name.toSlice(coff)}); - switch (coff.getNode(existing_si.get(coff).ni)) { + switch (coff.getNode(existing_si.get(coff).ni.unwrap().?)) { .input_section => |isi| { const other_ioi = isi.input(coff); err.addNote("first seen in input '{f}{f}'", .{ @@ -5473,13 +5447,13 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde const sec_si = try coff.navSection(zcu, nav.resolved.?); try coff.nodes.ensureUnusedCapacity(gpa, 1); if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ - .alignment = zcu.navAlignment(nav_index).toStdMem(), + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ + .alignment = .fromIp(zcu.navAlignment(nav_index)), .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .nav = nmi }); const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => si.deleteLocationRelocs(coff), @@ -5490,7 +5464,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; { @@ -5512,21 +5486,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde } if (nav.resolved.?.@"linksection".unwrap()) |_| { - try ni.resize(&coff.mf, gpa, si.get(coff).extra.size); - var parent_ni = ni; - while (true) { - parent_ni = parent_ni.parent(&coff.mf); - switch (coff.getNode(parent_ni)) { - else => unreachable, - .image_section, .pseudo_section => break, - .object_section => { - var child_it = parent_ni.reverseChildren(&coff.mf); - const last_offset, const last_size = - child_it.next().?.location(&coff.mf).resolve(&coff.mf); - try parent_ni.resize(&coff.mf, gpa, last_offset + last_size); - }, - } - } + try ni.resizeLeaf(&coff.mf, gpa, si.get(coff).extra.size); } } @@ -5542,10 +5502,11 @@ pub fn lowerUav( try coff.pending_uavs.ensureUnusedCapacity(gpa, 1); const umi = try coff.uavMapIndex(uav_val); const si = umi.symbol(coff); - if (switch (si.get(coff).ni) { - .none => true, - else => |ni| uav_align.toStdMem().order(ni.alignment(&coff.mf)).compare(.gt), - }) { + const need_update: bool = update: { + const existing_ni = si.get(coff).ni.unwrap() orelse break :update true; + break :update Alignment.compare(.fromIp(uav_align), .gt, existing_ni.alignment(&coff.mf)); + }; + if (need_update) { const gop = coff.pending_uavs.getOrPutAssumeCapacity(umi); if (gop.found_existing) { gop.value_ptr.alignment = gop.value_ptr.alignment.max(uav_align); @@ -5597,22 +5558,22 @@ fn updateFuncInner( if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); const mod = zcu.navFileScope(func.owner_nav).mod.?; const target = &mod.resolved_target.result; - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ .alignment = switch (nav.resolved.?.@"align") { .none => switch (mod.optimize_mode) { .debug, .safe, .fast, - => target_util.defaultFunctionAlignment(target), - .small => target_util.minFunctionAlignment(target), + => .fromIp(target_util.defaultFunctionAlignment(target)), + .small => .fromIp(target_util.minFunctionAlignment(target)), }, - else => |a| a.maxStrict(target_util.minFunctionAlignment(target)), - }.toStdMem(), + else => |a| .fromIp(a.maxStrict(target_util.minFunctionAlignment(target))), + }, .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .nav = nmi }); const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => si.deleteLocationRelocs(coff), @@ -5622,7 +5583,7 @@ fn updateFuncInner( sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; var nw: MappedFile.Node.Writer = undefined; @@ -5662,7 +5623,6 @@ fn flushImplib( implib_file: []const u8, ) !void { // Emitting implibs is only valid for images - assert(coff.export_table.ni != .none); const comp = coff.base.comp; const gpa = comp.gpa; @@ -5797,7 +5757,7 @@ fn reportUndefs(coff: *Coff, tid: Zcu.PerThread.Id) !void { const loc_sym = loc_si.get(coff); // TODO: Make this a helper for anything that needs to report "referenced by" notes - switch (coff.getNode(loc_sym.ni)) { + switch (coff.getNode(loc_sym.ni.unwrap().?)) { .data_directories => { const dir: std.coff.IMAGE.DIRECTORY_ENTRY = @fromBackingInt(@intCast(reloc.offset / @sizeOf(std.coff.ImageDataDirectory))); @@ -5808,7 +5768,7 @@ fn reportUndefs(coff: *Coff, tid: Zcu.PerThread.Id) !void { const other_ioi = isi.input(coff); if (loc_sym.gmi == .none) { const section = isi.inputSection(coff); - const section_name = coff.getNode(loc_sym.ni.parent(&coff.mf)) + const section_name = coff.getNode(loc_sym.ni.unwrap().?.parent(&coff.mf).unwrap().?) .object_section.name(coff).toSlice(coff); if (section.comdat_si != .null) { @@ -5902,15 +5862,17 @@ pub fn flush( coff.symbol_table.pending_shrink = false; const number_of_symbols = coff.targetLoad(&coff.headerPtr().number_of_symbols); - coff.symbol_table.ni.shrink( + coff.symbol_table.ni.resizeLeaf( &coff.mf, comp.gpa, number_of_symbols * std.coff.Symbol.sizeOf(), - true, - ) catch |err| return comp.link_diags.fail( - "linker failed to compact symbol table: {t}", - .{err}, - ); + ) catch |err| switch (err) { + else => |e| return e, + error.MappedFileIo => return comp.link_diags.fail( + "linker failed to compact symbol table: {t}", + .{coff.mf.io_err.?}, + ), + }; } while (try coff.idle(tid)) {} @@ -6055,8 +6017,8 @@ fn resolve(coff: *Coff, tid: Zcu.PerThread.Id) !bool { const sub_prog_node = coff.idleProgNode( tid, coff.symbol_prog_node, - if (sym.ni != .none) - coff.getNode(sym.ni) + if (sym.ni.unwrap()) |sym_ni| + coff.getNode(sym_ni) else .{ .import_thunk = sym.gmi }, ); @@ -6173,7 +6135,7 @@ fn idleProgNode( break :name std.fmt.bufPrint(&name, "{f}{f} {s}", .{ ioi.path(coff).fmtEscapeString(), fmtMemberNameString(ioi.memberName(coff)), - coff.getNode(isi.symbol(coff).node(coff).parent(&coff.mf)).object_section.name(coff).toSlice(coff), + coff.getNode(isi.symbol(coff).node(coff).parent(&coff.mf).unwrap().?).object_section.name(coff).toSlice(coff), }) catch &name; }, .import_thunk => |gmi| gmi.name(coff).toSlice(coff), @@ -6213,17 +6175,22 @@ fn flushUav( try coff.nodes.ensureUnusedCapacity(gpa, 1); if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); const sym = si.get(coff); - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ - .alignment = uav_align.toStdMem(), + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ + .alignment = .fromIp(uav_align), .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .uav = umi }); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => { - if (si.get(coff).ni.alignment(&coff.mf).order(uav_align.toStdMem()).compare(.gte)) + if (Alignment.compare( + si.get(coff).ni.unwrap().?.alignment(&coff.mf), + .gte, + .fromIp(uav_align), + )) { return; + } si.deleteLocationRelocs(coff); }, } @@ -6233,7 +6200,7 @@ fn flushUav( if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; var nw: MappedFile.Node.Writer = undefined; @@ -6497,23 +6464,23 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { lib_name, ImportTable.Adapter{ .coff = coff }, ); - const import_hint_name_align: std.mem.Alignment = .@"2"; + const import_hint_name_align: Alignment = .@"2"; if (!gop.found_existing) { errdefer _ = coff.import_table.entries.pop(); - try coff.import_table.ni.resize( + try coff.import_table.ni.resizeLeaf( &coff.mf, gpa, @sizeOf(std.coff.ImportDirectoryEntry) * (gop.index + 2), ); const import_hint_name_table_len = import_hint_name_align.forward(lib_name.len + ".dll".len + 1); - const idata_section_ni = coff.import_table.ni.parent(&coff.mf); - const import_lookup_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{ + const idata_section_ni = coff.import_table.ni.parent(&coff.mf).unwrap().?; + const import_lookup_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{ .size = addr_info.size * 2, .alignment = addr_info.alignment, .moved = true, }); - const import_address_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{ + const import_address_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{ .size = addr_info.size * 2, .alignment = addr_info.alignment, .moved = true, @@ -6521,13 +6488,13 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { const import_address_table_si = coff.addSymbolAssumeCapacity(); { const import_address_table_sym = import_address_table_si.get(coff); - import_address_table_sym.ni = import_address_table_ni; + import_address_table_sym.ni = .wrap(import_address_table_ni); assert(import_address_table_sym.loc_relocs == .none); import_address_table_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); import_address_table_sym.section_number = coff.getNode(idata_section_ni).object_section.symbol(coff).get(coff).section_number; } - const import_hint_name_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{ + const import_hint_name_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{ .size = import_hint_name_table_len, .alignment = import_hint_name_align, .moved = true, @@ -6583,9 +6550,9 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { gop.value_ptr.len = import_symbol_index + 1; const new_symbol_table_size = addr_info.size * (import_symbol_index + 2); - try gop.value_ptr.import_lookup_table_ni.resize(&coff.mf, gpa, new_symbol_table_size); + try gop.value_ptr.import_lookup_table_ni.resizeLeaf(&coff.mf, gpa, new_symbol_table_size); const import_address_table_ni = gop.value_ptr.import_address_table_si.node(coff); - try import_address_table_ni.resize(&coff.mf, gpa, new_symbol_table_size); + try import_address_table_ni.resizeLeaf(&coff.mf, gpa, new_symbol_table_size); const opt_imp_name = import.name.toSlice(coff); const opt_import_hint_name_index = if (opt_imp_name) |imp_name| blk: { @@ -6593,7 +6560,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { gop.value_ptr.hint_name_len = @intCast( import_hint_name_align.forward(import_hint_name_index + 2 + imp_name.len + 1), ); - try gop.value_ptr.import_hint_name_table_ni.resize(&coff.mf, gpa, gop.value_ptr.hint_name_len); + try gop.value_ptr.import_hint_name_table_ni.resizeLeaf(&coff.mf, gpa, gop.value_ptr.hint_name_len); break :blk import_hint_name_index; } else null; @@ -6648,13 +6615,13 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); const target = &comp.root_mod.resolved_target.result; - const alignment = switch (comp.root_mod.optimize_mode) { + const alignment: Alignment = switch (comp.root_mod.optimize_mode) { .debug, .safe, .fast, - => target_util.defaultFunctionAlignment(target), - .small => target_util.minFunctionAlignment(target), - }.toStdMem(); + => .fromIp(target_util.defaultFunctionAlignment(target)), + .small => .fromIp(target_util.minFunctionAlignment(target)), + }; const parent_si = (try coff.pseudoSectionMapIndex( .@".thunks", alignment, @@ -6668,12 +6635,12 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { else => |tag| @panic(@tagName(tag)), .AMD64 => { const init = [_]u8{ 0xff, 0x25, 0x00, 0x00, 0x00, 0x00 }; - const ni = try coff.mf.addLastChildNode(gpa, parent_sym.ni, .{ + const ni = try parent_sym.ni.unwrap().?.addFloatingChild(&coff.mf, gpa, .{ .alignment = alignment, - .size = init.len, + .size = alignment.forward(init.len), }); @memcpy(ni.slice(&coff.mf)[0..init.len], &init); - sym.ni = ni; + sym.ni = .wrap(ni); sym.extra.size = init.len; try coff.addReloc( si, @@ -6736,7 +6703,7 @@ fn flushSpecialSymbol(coff: *Coff, pending: SpecialSymbol) !SpecialSymbol { try coff.symbols.ensureUnusedCapacity(gpa, 1); const optional_hdr_si = coff.addSymbolAssumeCapacity(); const optional_hdr_sym = optional_hdr_si.get(coff); - optional_hdr_sym.ni = Node.known.optional_header; + optional_hdr_sym.ni = .wrap(Node.known.optional_header); assert(optional_hdr_sym.loc_relocs == .none); optional_hdr_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); @@ -6783,7 +6750,7 @@ fn flushSpecialSymbol(coff: *Coff, pending: SpecialSymbol) !SpecialSymbol { try coff.symbols.ensureUnusedCapacity(gpa, 1); const data_dir_si = coff.addSymbolAssumeCapacity(); const data_dir_sym = data_dir_si.get(coff); - data_dir_sym.ni = Node.known.data_directories; + data_dir_sym.ni = .wrap(Node.known.data_directories); assert(data_dir_sym.loc_relocs == .none); data_dir_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); @@ -6821,12 +6788,12 @@ fn flushLazy(coff: *Coff, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void { .code => .text, .const_data => .rdata, }; - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ .moved = true }); + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ .moved = true }); coff.nodes.appendAssumeCapacity(switch (lazy.kind) { .code => .{ .lazy_code = @fromBackingInt(@intCast(lmr.index)) }, .const_data => .{ .lazy_const_data = @fromBackingInt(@intCast(lmr.index)) }, }); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => si.deleteLocationRelocs(coff), @@ -6836,7 +6803,7 @@ fn flushLazy(coff: *Coff, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void { if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; var required_alignment: InternPool.Alignment = .none; @@ -6914,7 +6881,7 @@ fn flushMoved(coff: *Coff, ni: MappedFile.Node.Index) !void { const flags = coff.targetLoad(&sym.section_number.header(coff).flags); if (!flags.CNT_UNINITIALIZED_DATA) { const file_offset = if (isArchive(coff)) - sym.ni.location(&coff.mf).resolve(&coff.mf)[0] + sym.ni.unwrap().?.location(&coff.mf).resolve(&coff.mf)[0] else ni.fileLocation(&coff.mf, false).offset; @@ -6927,7 +6894,7 @@ fn flushMoved(coff: *Coff, ni: MappedFile.Node.Index) !void { .input_section => |isi| { try isi.symbol(coff).flushMoved(coff); for (coff.input_symbols.items[@backingInt(isi.firstSymbol(coff))..]) |input_symbol| { - if (input_symbol.si.get(coff).ni != ni) break; + if (input_symbol.si.get(coff).ni != ni.toOptional()) break; try input_symbol.si.flushMoved(coff); } }, @@ -7062,7 +7029,7 @@ fn flushResized(coff: *Coff, ni: MappedFile.Node.Index) !void { if (coff.isArchive() and coff.members.items.len > 0) { const last_member = coff.members.items[coff.members.items.len - 1]; // See .archive_member branch for reasoning - assert(Node.known.file.reverseChildren(&coff.mf).ni == last_member.content_ni); + assert(Node.known.file.last(&coff.mf).unwrap().? == last_member.content_ni); try coff.flushResized(last_member.content_ni); } }, @@ -7090,19 +7057,15 @@ fn flushResized(coff: *Coff, ni: MappedFile.Node.Index) !void { => unreachable, .archive_member => |mi| { const content_ni = mi.get(coff).content_ni; - const next_ni = content_ni.next(&coff.mf); const content_offset, _ = content_ni.location(&coff.mf).resolve(&coff.mf); - const next_offset = switch (next_ni) { - .none => offset: { - assert(content_ni.parent(&coff.mf) == Node.known.file); - // This must take into account the final file size. If there are trailing - // bytes, they will be expected to contain another valid member header - break :offset coff.mf.memory_map.memory.len; - }, - else => offset: { - assert(coff.getNode(next_ni) == .archive_member_header); - break :offset next_ni.location(&coff.mf).resolve(&coff.mf)[0]; - }, + const next_offset = if (content_ni.next(&coff.mf).unwrap()) |next_ni| offset: { + assert(coff.getNode(next_ni) == .archive_member_header); + break :offset next_ni.location(&coff.mf).resolve(&coff.mf)[0]; + } else offset: { + assert(content_ni.parent(&coff.mf) == Node.known.file.toOptional()); + // This must take into account the final file size. If there are trailing + // bytes, they will be expected to contain another valid member header + break :offset coff.mf.memory_map.memory.len; }; // Not inserting IMAGE_ARCHIVE_PAD `\n` byte here, because we are expanding to full size @@ -7356,7 +7319,7 @@ fn virtualSlide(coff: *Coff, start_section_index: usize, start_rva: u32) !void { const section_sym = section.si.get(coff); section_sym.rva = rva; coff.targetStore(&header.virtual_address, rva); - try section_sym.ni.childrenMoved(coff.base.comp.gpa, &coff.mf); + try section_sym.ni.unwrap().?.childrenMoved(coff.base.comp.gpa, &coff.mf); rva += coff.targetLoad(&header.virtual_size); } switch (coff.optionalHeaderPtr()) { @@ -7430,7 +7393,7 @@ fn updateExportInner( // TODO: add an errMsg if this conflicts with an existing symbol const export_si = try coff.globalSymbol(.{ .name = name }); const export_sym = export_si.get(coff); - export_sym.ni = exported_ni; + export_sym.ni = .wrap(exported_ni); export_sym.rva = exported_sym.rva; export_sym.section_number = exported_sym.section_number; if (@"export".opts.linkage == .weak and !coff.isImage()) { @@ -7481,7 +7444,7 @@ fn updateExportInner( if (new_name_table_size > std.math.maxInt(@FieldType(ExportTable.Entry, "name_index"))) return coff.base.comp.link_diags.fail("exports name table limit reached", .{}); - try coff.export_table.name_table_ni.resize(&coff.mf, gpa, new_name_table_size); + try coff.export_table.name_table_ni.resizeLeaf(&coff.mf, gpa, new_name_table_size); const name_table_slice = coff.export_table.name_table_ni.slice(&coff.mf); @memcpy(name_table_slice[name_index..][0 .. name.len + 1], name[0 .. name.len + 1]); @@ -7504,19 +7467,19 @@ fn updateExportInner( // TODO: These should all be resized ahead of time to fit all exports // after https://github.com/ziglang/zig/issues/23616 - try coff.export_table.export_address_table_si.node(coff).resize( + try coff.export_table.export_address_table_si.node(coff).resizeLeaf( &coff.mf, gpa, export_count * @sizeOf(std.coff.ExportAddressTableEntry), ); - try coff.export_table.name_pointer_table_ni.resize( + try coff.export_table.name_pointer_table_ni.resizeLeaf( &coff.mf, gpa, export_count * @sizeOf(std.coff.ExportNamePointerTableEntry), ); - try coff.export_table.ordinal_table_ni.resize( + try coff.export_table.ordinal_table_ni.resizeLeaf( &coff.mf, gpa, export_count * @sizeOf(std.coff.ExportOrdinalTableEntry), @@ -7599,14 +7562,13 @@ fn printSymbol( si: Symbol.Index, ) !void { const sym = si.get(coff); - const node = coff.getNode(sym.ni); - try w.print("{d:0>6}@{d:0>2} {x:08} {s} {s} {s} n{d:0>8}+{x:08}:{t: <26} | {x:08} ", .{ + try w.print("{d:0>6}@{d:0>2} {x:08} {s} {s} {s} n{d:0>8}+{x:08}:{s: <26} | {x:08} ", .{ si, sym.section_number, if (sym.flags.extra_tag == .size) @as(u64, sym.extra.size) - else if (sym.ni != .none) - sym.ni.location(&coff.mf).resolve(&coff.mf)[1] + else if (sym.ni.unwrap()) |ni| + ni.location(&coff.mf).resolve(&coff.mf)[1] else 0, switch (sym.flags.value_tag) { @@ -7627,7 +7589,7 @@ fn printSymbol( }, sym.ni, if (sym.flags.value_tag == .node_offset) sym.value.node_offset else 0, - node, + if (sym.ni.unwrap()) |ni| @tagName(coff.getNode(ni)) else "", sym.rva, }); @@ -7635,7 +7597,7 @@ fn printSymbol( try w.print("G {f}\n", .{fmtGlobalName(coff, sym.gmi)}); } else { try w.writeAll("| "); - try coff.printNodeName(w, tid, node); + try coff.printNodeName(w, tid, coff.getNode(sym.ni.unwrap().?)); if (sym.flags.extra_tag == .isli) try w.print(" | {s}", .{sym.extra.isli.name(coff).toSlice(coff)}); try w.writeByte('\n'); @@ -7672,7 +7634,7 @@ fn printNodeName( try w.print("({f}{f}, {s}", .{ ioi.path(coff).fmtEscapeString(), fmtMemberNameString(ioi.memberName(coff)), - coff.getNode(is.si.node(coff).parent(&coff.mf)).object_section.name(coff).toSlice(coff), + coff.getNode(is.si.node(coff).parent(&coff.mf).unwrap().?).object_section.name(coff).toSlice(coff), }); if (is.comdat_si != .null) { const comdat_sym = is.comdat_si.get(coff); @@ -7748,41 +7710,42 @@ pub fn printNode( { const mf_node = &coff.mf.nodes.items[@backingInt(ni)]; const off, const size = mf_node.location().resolve(&coff.mf); - try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x}{s}{s}{s}{s}\n", .{ + try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x} {t}{s}{s}{s}\n", .{ @backingInt(ni), off, size, mf_node.flags.alignment.toByteUnits(), - if (mf_node.flags.fixed) " fixed" else "", + mf_node.flags.position, if (mf_node.flags.moved) " moved" else "", if (mf_node.flags.resized) " resized" else "", if (mf_node.flags.has_content) " has_content" else "", }); } - var leaf = true; - var child_it = ni.children(&coff.mf); - while (child_it.next()) |child_ni| { - leaf = false; - try coff.printNode(tid, w, child_ni, indent + 1); - } - if (leaf) { - const file_loc = ni.fileLocation(&coff.mf, false); - if (file_loc.size == 0) return; - var address = file_loc.offset; - const line_len = 0x10; - var line_it = std.mem.window( - u8, - coff.mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)], - line_len, - line_len, - ); - while (line_it.next()) |line_bytes| : (address += line_len) { - try w.splatByteAll(' ', indent + 1); - try w.print("{x:0>8} ", .{address}); - for (line_bytes) |byte| try w.print("{x:0>2} ", .{byte}); - try w.splatByteAll(' ', 3 * (line_len - line_bytes.len) + 1); - for (line_bytes) |byte| try w.writeByte(if (std.ascii.isPrint(byte)) byte else '.'); - try w.writeByte('\n'); + if (ni.first(&coff.mf).unwrap()) |first_ni| { + // non-leaf, just print children + var child_ni = first_ni; + while (true) { + try coff.printNode(tid, w, child_ni, indent + 1); + child_ni = child_ni.next(&coff.mf).unwrap() orelse break; } + return; + } + const file_loc = ni.fileLocation(&coff.mf, false); + if (file_loc.size == 0) return; + var address = file_loc.offset; + const line_len = 0x10; + var line_it = std.mem.window( + u8, + coff.mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)], + line_len, + line_len, + ); + while (line_it.next()) |line_bytes| : (address += line_len) { + try w.splatByteAll(' ', indent + 1); + try w.print("{x:0>8} ", .{address}); + for (line_bytes) |byte| try w.print("{x:0>2} ", .{byte}); + try w.splatByteAll(' ', 3 * (line_len - line_bytes.len) + 1); + for (line_bytes) |byte| try w.writeByte(if (std.ascii.isPrint(byte)) byte else '.'); + try w.writeByte('\n'); } } diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 1400d0174815615d4f853ebb53ff2bc73dcd7915..8ef617fad2e1cc2d4c7a67412a714502d4e8e2f6 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -18,14 +18,16 @@ const tracy = @import("../tracy.zig"); const Type = @import("../Type.zig"); const Value = @import("../Value.zig"); const Zcu = @import("../Zcu.zig"); +const Alignment = MappedFile.Alignment; base: link.File, options: link.File.OpenOptions, mf: MappedFile, ni: Node.Known, nodes: std.MultiArrayList(Node), +/// Does not contain an item for `SHN_UNDEF`. shdrs: std.ArrayList(Section), -phdrs: std.ArrayList(MappedFile.Node.Index), +phdrs: std.ArrayList(MappedFile.Node.Index.Optional), shndx: struct { got: Section.Index, /// Always `.UNDEF` on some targets (e.g. SPARC). @@ -99,7 +101,7 @@ dso_globals: std.array_hash_map.Auto(String(.strtab), struct { /// the section containing the symbol, and the symbol's offset within the section. I know this /// sounds like a terrible hack, but it is *genuinely* how you're supposed to do this. Copy /// relocations suck. - alignment: std.mem.Alignment, + alignment: Alignment, }), shstrtab: StringTable, strtab: StringTable, @@ -175,7 +177,7 @@ symbol_relocs: std.ArrayList(SymbolReloc), got_relocs: std.ArrayList(GotReloc), /// Set of relocations which must be re-applied if the size of the TLS segment changes. tls_size_symbol_relocs: std.array_hash_map.Auto(SymbolReloc.Index, void), -/// Index matches the index into `shdrs`. +/// Index matches the index into `shdrs`. Like `shdrs`, this map excludes `SHN_UNDEF`. section_by_name: std.array_hash_map.Auto(String(.shstrtab), void), /// Key is the name of a global symbol which has been moved to a new symtab index. Any relocation /// entries which target that symbol must be updated to reference the correct symbol index. @@ -201,6 +203,8 @@ const Node = union(enum) { archive, /// This includes the archive magic and long file member. archive_header, + /// This is a footer of the `.elf` node, and contains the next archive entry's file header. + archive_elf_footer, elf, ehdr, shdr, @@ -339,8 +343,6 @@ const Node = union(enum) { }; pub const Known = struct { - archive: MappedFile.Node.Index, - archive_header: MappedFile.Node.Index, elf: MappedFile.Node.Index, ehdr: MappedFile.Node.Index, shdr: MappedFile.Node.Index, @@ -349,7 +351,7 @@ const Node = union(enum) { text: MappedFile.Node.Index, data: MappedFile.Node.Index, data_rel_ro: MappedFile.Node.Index, - tls: MappedFile.Node.Index, + tls: MappedFile.Node.Index.Optional, }; comptime { @@ -505,7 +507,7 @@ const Section = struct { } fn get(s: Index, elf: *Elf) *Section { - return &elf.shdrs.items[@backingInt(s)]; + return &elf.shdrs.items[@backingInt(s) - 1]; // overflow means you tried to get the `.UNDEF` section } fn name(s: Index, elf: *Elf) String(.shstrtab) { @@ -539,7 +541,7 @@ const Section = struct { } } - fn ensureAligned(shndx: Index, elf: *Elf, min_align: std.mem.Alignment) Error!void { + fn ensureAligned(shndx: Index, elf: *Elf, min_align: Alignment) Error!void { switch (elf.shdrPtr(shndx)) { inline else => |shdr| { if (elf.targetLoad(&shdr.addralign) >= min_align.toByteUnits()) { @@ -550,9 +552,9 @@ const Section = struct { } const ni = shndx.get(elf).ni; if (min_align.compare(.gt, ni.alignment(&elf.mf))) { - try ni.realign(&elf.mf, elf.base.comp.gpa, min_align, .{}); + try ni.realign(&elf.mf, elf.base.comp.gpa, min_align); } - switch (elf.getNode(ni.parent(&elf.mf))) { + switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) { .elf => {}, .segment => |phndx| try elf.ensureSegmentAligned(phndx, min_align), else => unreachable, @@ -583,7 +585,7 @@ const Section = struct { break :need_size cur_size + need_additional * ent_size; }, }; - try elf.ensureNodeSize(node, need_size); + try node.ensureMinimumSize(&elf.mf, elf.base.comp.gpa, need_size); } /// Asserts that `rela_shndx` is a `SHT_RELA` section and deletes the `ElfN.Rela` entry at @@ -818,7 +820,7 @@ const GotReloc = struct { /// * A section /// * A NAV, UAV, or lazy code/data /// * `.none`, if this relocation was deleted (in which case it should be ignored) - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, /// The offset of the relocation inside of `node`. offset: u64, target: GotKey, @@ -942,8 +944,10 @@ const GotReloc = struct { fn apply(reloc: *GotReloc, elf: *Elf) void { assert(elf.ehdrType() != .REL); - if (reloc.node == .none) return; // deleted - if (reloc.node.hasMoved(&elf.mf) or elf.shndx.got.get(elf).ni.hasMoved(&elf.mf)) { + const node = reloc.node.unwrap() orelse { + return; // deleted + }; + if (node.hasMoved(&elf.mf) or elf.shndx.got.get(elf).ni.hasMoved(&elf.mf)) { // There's no point applying the relocation now, because it will be re-applied by // `flushMoved` at some point anyway. return; @@ -968,8 +972,9 @@ const GotReloc = struct { } } fn applyInner(reloc: *const GotReloc, elf: *Elf) error{ RelocationOverflow, RelocationMisaligned }!void { - const dest_vaddr = elf.getNodeVAddr(reloc.node) + reloc.offset; - const dest_slice = reloc.node.slice(&elf.mf)[@intCast(reloc.offset)..]; + const node = reloc.node.unwrap().?; + const dest_vaddr = elf.getNodeVAddr(node) + reloc.offset; + const dest_slice = node.slice(&elf.mf)[@intCast(reloc.offset)..]; const got_vaddr = elf.shndx.got.vaddr(elf); const got_index: u64 = elf.got.getIndex(reloc.target).?; @@ -1587,7 +1592,7 @@ const SymbolReloc = struct { } }, .sparc_le_hix22 => { - const tls_phndx = elf.getNode(elf.ni.tls).segment; + const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment; const tls_size: u64 = switch (elf.phdrSlice()) { inline else => |phdr| tls_size: { assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS); @@ -1646,7 +1651,6 @@ const SymbolReloc = struct { fn apply(reloc: *SymbolReloc, elf: *Elf) void { assert(elf.ehdrType() != .REL); - assert(reloc.node != .none); if (reloc.node.hasMoved(&elf.mf) or reloc.target.hasMoved(elf)) { // There's no point applying the relocation now, because it will be re-applied by // `flushMoved` at some point anyway. @@ -1692,7 +1696,7 @@ const SymbolReloc = struct { .I_original => |tls| tls.tcb_size +% reloc.target.value(elf) +% addend, .I_modified => |tls| 0 -% tls.tp_off +% reloc.target.value(elf) +% addend, .II => { - const tls_phndx = elf.getNode(elf.ni.tls).segment; + const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment; const tls_size: u64 = switch (elf.phdrSlice()) { inline else => |phdr| tls_size: { assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS); @@ -1785,6 +1789,8 @@ const SymbolReloc = struct { }; fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void { + const gpa = elf.base.comp.gpa; + const min_buckets = max_dynsym_count / 2; const cur_dynsym_count: u32 = switch (elf.shdrPtr(elf.shndx.dynsym)) { @@ -1805,7 +1811,7 @@ fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void { // We don't need to add any buckets, but we still need to make sure the section is large // enough to fit `max_dynsym_count` chains. const need_size = @sizeOf(info.Header()) + (nbucket + max_dynsym_count) * 4; - try elf.ensureNodeSize(elf.shndx.hash.get(elf).ni, need_size); + try elf.shndx.hash.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); return; } // We need more buckets, so we'll have to rebuild the hash table. @@ -1817,7 +1823,7 @@ fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void { { const need_size = @sizeOf(info.Header()) + (new_nbucket + max_dynsym_count) * 4; - try elf.ensureNodeSize(elf.shndx.hash.get(elf).ni, need_size); + try elf.shndx.hash.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } elf.mf.nodes_lock.lock(); @@ -1963,7 +1969,7 @@ fn ensureUnusedSymbolCapacity(elf: *Elf, len: u32, kind: enum { all_local, maybe const need_node_size: u64 = switch (elf.shdrPtr(.symtab)) { inline else => |shdr, class| elf.targetLoad(&shdr.size) + len * @sizeOf(class.ElfN().Sym), }; - try elf.ensureNodeSize(Section.Index.symtab.get(elf).ni, need_node_size); + try Section.Index.symtab.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_node_size); } switch (kind) { @@ -1986,7 +1992,7 @@ fn ensureUnusedSymbolCapacity(elf: *Elf, len: u32, kind: enum { all_local, maybe const dynsym_cur_len: u32 = @intCast(@divExact(dynsym_cur_size, dynsym_ent_size)); const dynsym_need_size: u64 = (dynsym_cur_len + len) * dynsym_ent_size; - try elf.ensureNodeSize(elf.shndx.dynsym.get(elf).ni, dynsym_need_size); + try elf.shndx.dynsym.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, dynsym_need_size); try elf.ensureDynsymHashCapacity(dynsym_cur_len + len); @@ -2008,19 +2014,19 @@ fn ensureUnusedPltCapacity(elf: *Elf, len: u32) Error!void { // Ensure the `.plt` section's node is big enough: { const need_size: usize = plt.entry_size * (1 + need_plt_count); - try elf.ensureNodeSize(elf.shndx.plt.get(elf).ni, need_size); + try elf.shndx.plt.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } // If there is a `.got.plt` section, ensure its node is big enough if (plt.got_plt) |got_plt| { const need_size: usize = elf.targetPtrSize() * (got_plt.header_entries + need_plt_count); - try elf.ensureNodeSize(elf.shndx.got_plt.get(elf).ni, need_size); + try elf.shndx.got_plt.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } // If there is a `.plt.sec` section, ensure its node is big enough if (plt.plt_sec) |plt_sec| { const need_size: usize = plt_sec.entry_size * need_plt_count; - try elf.ensureNodeSize(elf.shndx.plt_sec.get(elf).ni, need_size); + try elf.shndx.plt_sec.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } } /// Given an index into the PLT, returns whether that PLT entry is dead, meaning it may be reused at @@ -2044,7 +2050,7 @@ fn pltEntryIsDead(elf: *Elf, plt_index: usize) bool { } const AddLocalSymbolOptions = struct { - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, name: String(.strtab), value: u64, size: u64, @@ -2126,7 +2132,7 @@ const AddGlobalSymbolOptions = struct { } }; - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, name: Name, lib_name: ?[]const u8 = null, value: u64, @@ -2294,8 +2300,8 @@ fn addGlobalSymbolAssumeCapacity(elf: *Elf, opts: AddGlobalSymbolOptions) error{ } const old_head: String(.strtab) = old_head: { - if (opts.node == .none) break :old_head .empty; - const gop = elf.node_global_symbols.getOrPutAssumeCapacity(opts.node); + const node = opts.node.unwrap() orelse break :old_head .empty; + const gop = elf.node_global_symbols.getOrPutAssumeCapacity(node); const old_head: String(.strtab) = if (gop.found_existing) gop.value_ptr.* else .empty; gop.value_ptr.* = opts.name.strtab; break :old_head old_head; @@ -2363,7 +2369,7 @@ fn setGlobalSymbolValue( global_name: String(.strtab), global_ptr: *Symbol.Global, new: struct { - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, value: u64, size: u64, type: std.elf.STT, @@ -2371,18 +2377,17 @@ fn setGlobalSymbolValue( }, ) void { assert(new.shndx != .UNDEF); - const old_node = global_ptr.symtab_index.ptr(elf).node; - if (old_node != .none) { + if (global_ptr.symtab_index.ptr(elf).node.unwrap()) |old_node| { if (global_ptr.next_in_node != .empty) { const next = elf.globalByName(global_ptr.next_in_node).?; assert(next.prev_in_node == global_name); - assert(next.symtab_index.ptr(elf).node == old_node); + assert(next.symtab_index.ptr(elf).node.unwrap().? == old_node); next.prev_in_node = global_ptr.prev_in_node; } if (global_ptr.prev_in_node != .empty) { const prev = elf.globalByName(global_ptr.prev_in_node).?; assert(prev.next_in_node == global_name); - assert(prev.symtab_index.ptr(elf).node == old_node); + assert(prev.symtab_index.ptr(elf).node.unwrap().? == old_node); prev.next_in_node = global_ptr.next_in_node; } else { // We're the start of the linked list, so we need to change the head. @@ -2417,8 +2422,8 @@ fn setGlobalSymbolValue( global_ptr.symtab_index.ptr(elf).node = new.node; const old_head: String(.strtab) = old_head: { - if (new.node == .none) break :old_head .empty; - const gop = elf.node_global_symbols.getOrPutAssumeCapacity(new.node); + const new_node = new.node.unwrap() orelse break :old_head .empty; + const gop = elf.node_global_symbols.getOrPutAssumeCapacity(new_node); const old_head: String(.strtab) = if (gop.found_existing) gop.value_ptr.* else .empty; gop.value_ptr.* = global_name; break :old_head old_head; @@ -2644,7 +2649,7 @@ const Symbol = struct { /// * A section (the symbol's value is some vaddr in that section) /// * An input section (the symbol's value is some vaddr in that input section) /// * A NAV, UAV, or lazy code/data (the symbol's value is exactly the vaddr of that node) - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, /// The head of a linked list of relocations targeting this symbol. first_target_reloc: SymbolReloc.Index, @@ -2852,8 +2857,7 @@ const Symbol = struct { /// Returns `true` if the target of `s` has moved, meaning the symbol's value will change at /// some point due to a call to `flushMoved`. fn hasMoved(s: Symbol.Id, elf: *Elf) bool { - const node = s.index(elf).ptr(elf).node; - if (node != .none) { + if (s.index(elf).ptr(elf).node.unwrap()) |node| { return node.hasMoved(&elf.mf); } switch (s.unwrap()) { @@ -2950,6 +2954,7 @@ pub fn symbolForAtom(elf: *Elf, atom: link.File.AtomId) link.File.SymbolId { const lsi: Symbol.LocalIndex = switch (elf.getNode(Node.fromAtom(atom))) { .archive, .archive_header, + .archive_elf_footer, .elf, .ehdr, .shdr, @@ -2989,7 +2994,7 @@ fn lazySymbolInner(elf: *Elf, lazy: link.File.LazySymbol) Error!link.File.Symbol .code => .{ .text, .FUNC }, .const_data => .{ .rodata, .OBJECT }, }; - const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{}); + const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{}); var name_buf: [64]u8 = undefined; const name = std.fmt.bufPrint( &name_buf, @@ -2998,7 +3003,7 @@ fn lazySymbolInner(elf: *Elf, lazy: link.File.LazySymbol) Error!link.File.Symbol ) catch unreachable; gop.value_ptr.* = .{ .lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .name = try elf.string(.strtab, name), .value = 0, .size = 0, @@ -3248,7 +3253,7 @@ const StringTable = struct { break :size .{ old_size, new_size }; }, }; - try elf.ensureNodeSize(ni, new_size); + try ni.ensureMinimumSize(&elf.mf, gpa, new_size); const slice = ni.slice(&elf.mf)[old_size..]; @memcpy(slice[0..key.len], key); slice[key.len] = 0; @@ -3349,16 +3354,14 @@ fn create( .options = options, .mf = try .init(file, comp.gpa, io), .ni = .{ - .archive = .root, - .archive_header = .none, - .elf = .root, - .ehdr = .none, - .shdr = .none, - .rodata = .none, - .phdr = .none, - .text = .none, - .data = .none, - .data_rel_ro = .none, + .elf = undefined, + .ehdr = undefined, + .shdr = undefined, + .rodata = undefined, + .phdr = undefined, + .text = undefined, + .data = undefined, + .data_rel_ro = undefined, .tls = .none, }, .nodes = .empty, @@ -3489,7 +3492,7 @@ fn initHeaders( .EXEC => comp.config.link_mode == .dynamic, .DYN => true, }; - const addr_align: std.mem.Alignment = switch (class) { + const addr_align: Alignment = switch (class) { .NONE, _ => unreachable, .@"32" => .@"4", .@"64" => .@"8", @@ -3503,7 +3506,7 @@ fn initHeaders( // // It can be handy to temporarily set this to `.@"1"` when working on the linker, because it // prevents alignment bugs from being hidden by your filesystem's block alignment. - const node_block_align: std.mem.Alignment = elf.mf.flags.block_size; + const node_block_align: Alignment = elf.mf.flags.block_size; const plt: PltInfo = .fromMachine(machine); @@ -3599,28 +3602,31 @@ fn initHeaders( }, phnum }; }; - const expected_nodes_len = @as(usize, if (is_archive) 2 else 0) + // .archive, .archive_header - 3 + // `.file`, `.ehdr`, and `.shdr` nodes - (shnum - 1) + // -1 because the null shdr does not have a `.section` node + const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_footer + 3 + // `.elf`, `.ehdr`, and `.shdr` nodes + (shnum - 1) + // -1 because the SHN_UNDEF shdr does not have a `.section` node (phnum -| 1); // -1 because the GNU_STACK phdr does not have a `.segment` node try elf.nodes.ensureTotalCapacity(gpa, expected_nodes_len); - try elf.shdrs.ensureTotalCapacity(gpa, shnum); - try elf.section_by_name.ensureUnusedCapacity(gpa, shnum); + try elf.shdrs.ensureTotalCapacity(gpa, shnum - 1); // -1 to exclude SHN_UNDEF + try elf.section_by_name.ensureUnusedCapacity(gpa, shnum - 1); // -1 to exclude SHN_UNDEF try elf.phdrs.resize(gpa, phnum); try elf.symtab.ensureTotalCapacity(gpa, 1); if (is_archive) { elf.nodes.appendAssumeCapacity(.archive); - elf.ni.archive_header = try elf.mf.addOnlyChildNode(gpa, elf.ni.archive, .{ + + const archive_ni: MappedFile.Node.Index = .root; + + const archive_header_ni = try archive_ni.addOnlyHeaderChild(&elf.mf, gpa, .{ .size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr) * 2, .alignment = .@"2", - .fixed = true, .next_moved = true, .bubbles_moved = false, .enable_next_moved = true, }); - const archive_header_slice = elf.ni.archive_header.slice(&elf.mf); + elf.nodes.appendAssumeCapacity(.archive_header); + const archive_header_slice = archive_header_ni.slice(&elf.mf); @memcpy(archive_header_slice[0..std.elf.ARMAG.len], std.elf.ARMAG); const strtab_ar_hdr: *std.elf.ar_hdr = @ptrCast(archive_header_slice[std.elf.ARMAG.len..]); strtab_ar_hdr.* = .{ @@ -3633,15 +3639,23 @@ fn initHeaders( .ar_fmag = std.elf.ARFMAG.*, }; - elf.nodes.appendAssumeCapacity(.archive_header); - elf.ni.elf = try elf.mf.addLastChildNode(gpa, elf.ni.archive, .{ + elf.ni.elf = try archive_ni.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align.max(.@"2"), .next_moved = true, .bubbles_moved = false, .enable_next_moved = true, }); + elf.nodes.appendAssumeCapacity(.elf); + + _ = try elf.ni.elf.addOnlyFooterChild(&elf.mf, gpa, .{ + .alignment = .@"2", + .size = @sizeOf(std.elf.ar_hdr), + }); + elf.nodes.appendAssumeCapacity(.archive_elf_footer); + } else { + elf.ni.elf = .root; + elf.nodes.appendAssumeCapacity(.elf); } - elf.nodes.appendAssumeCapacity(.elf); const entsize: struct { ph: u32, sh: u32 } = switch (class) { .NONE, _ => unreachable, @@ -3655,19 +3669,18 @@ fn initHeaders( // the rodata segment. Although to my knowledge neither ELF nor any ELF-based OS strictly // requires this, it is highly conventional and therefore sometimes relied upon. if (@"type" != .REL) { - elf.ni.rodata = try elf.mf.addOnlyChildNode(gpa, elf.ni.elf, .{ + // This node will contain the ehdr, which must be at the start of the ELF file, so this + // node must itself be a header of the `.elf` node. + elf.ni.rodata = try elf.ni.elf.addOnlyHeaderChild(&elf.mf, gpa, .{ // Must be at least `addr_align` for `elf.ni.phdr` to be placed inside this node .alignment = node_block_align.max(addr_align), - // This node will contain the ehdr, which must be at the start of the ELF file, so this - // node must itself be fixed. - .fixed = true, .moved = true, .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.rodata }); - elf.phdrs.items[phndx.rodata] = elf.ni.rodata; + elf.phdrs.items[phndx.rodata] = .wrap(elf.ni.rodata); - elf.ni.phdr = try elf.mf.addOnlyChildNode(gpa, elf.ni.rodata, .{ + elf.ni.phdr = try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{ .size = @as(u64, phnum) * entsize.ph, .alignment = addr_align, // keep in sync with `elf.ni.rodata` alignment above .moved = true, @@ -3675,36 +3688,36 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.phdr }); - elf.phdrs.items[phndx.phdr] = elf.ni.phdr; + elf.phdrs.items[phndx.phdr] = .wrap(elf.ni.phdr); - elf.ni.text = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ + elf.ni.text = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align, .moved = true, .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.text }); - elf.phdrs.items[phndx.text] = elf.ni.text; + elf.phdrs.items[phndx.text] = .wrap(elf.ni.text); - elf.ni.data = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ + elf.ni.data = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ // Must be at least `addr_align` for `elf.ni.data_rel_ro` to be placed inside this node .alignment = node_block_align.max(addr_align), .moved = true, .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.data }); - elf.phdrs.items[phndx.data] = elf.ni.data; + elf.phdrs.items[phndx.data] = .wrap(elf.ni.data); if (plt.got_plt == null) { - const plt_ni = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ + const plt_ni = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align, .moved = true, .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.plt }); - elf.phdrs.items[phndx.plt] = plt_ni; + elf.phdrs.items[phndx.plt] = .wrap(plt_ni); } - elf.ni.data_rel_ro = try elf.mf.addOnlyChildNode(gpa, elf.ni.data, .{ + elf.ni.data_rel_ro = try elf.ni.data.addFloatingChild(&elf.mf, gpa, .{ // Must be at least `addr_align` for the `PT_DYNAMIC` node to be placed inside this one // later (if `have_dynamic_section`). Keep in sync with `elf.ni.data` alignment above. .alignment = node_block_align.max(addr_align), @@ -3712,19 +3725,27 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.relro }); - elf.phdrs.items[phndx.relro] = elf.ni.data_rel_ro; + elf.phdrs.items[phndx.relro] = .wrap(elf.ni.data_rel_ro); if (comp.config.any_non_single_threaded) { - elf.ni.tls = try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{ + elf.ni.tls = .wrap(try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align, .moved = true, .bubbles_moved = false, - }); + })); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.tls }); elf.phdrs.items[phndx.tls] = elf.ni.tls; } elf.phdrs.items[phndx.gnu_stack] = .none; + } else { + elf.ni.rodata = elf.ni.elf; + elf.ni.text = elf.ni.elf; + elf.ni.data = elf.ni.elf; + elf.ni.data_rel_ro = elf.ni.elf; + if (comp.config.any_non_single_threaded) { + elf.ni.tls = .wrap(elf.ni.elf); + } } switch (class) { @@ -3736,10 +3757,9 @@ fn initHeaders( .REL => elf.ni.elf, .DYN, .EXEC => elf.ni.rodata, }; - elf.ni.ehdr = try elf.mf.addFirstChildNode(gpa, parent_ni, .{ + elf.ni.ehdr = try parent_ni.addOnlyHeaderChild(&elf.mf, gpa, .{ .size = @sizeOf(ElfN.Ehdr), .alignment = addr_align, - .fixed = true, }); elf.nodes.appendAssumeCapacity(.ehdr); @@ -3785,14 +3805,14 @@ fn initHeaders( ehdr.phentsize = @sizeOf(ElfN.Phdr); ehdr.phnum = @min(phnum, std.elf.PN_XNUM); ehdr.shentsize = @sizeOf(ElfN.Shdr); - ehdr.shnum = 1; // Only the null shdr initially---will be incremented by `addSection` + ehdr.shnum = 1; // Only the SHN_UNDEF shdr initially---will be incremented by `addSection` ehdr.shstrndx = std.elf.SHN_UNDEF; if (elf.targetEndian() != native_endian) std.mem.byteSwapAllFields(ElfN.Ehdr, ehdr); }, } - elf.ni.shdr = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ - .size = 1 * entsize.sh, // as above, only the null shdr initially + elf.ni.shdr = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ + .size = node_block_align.forward(1 * entsize.sh), // as above, only the SHN_UNDEF initially .alignment = addr_align.max(node_block_align), .moved = true, .resized = true, @@ -3916,7 +3936,7 @@ fn initHeaders( }; } - if (comp.config.any_non_single_threaded) { + if (elf.ni.tls.unwrap()) |tls_segment_ni| { const ph_tls = &phdr[phndx.tls]; ph_tls.* = .{ .type = .TLS, @@ -3926,7 +3946,7 @@ fn initHeaders( .filesz = 0, .memsz = 0, .flags = .{ .R = true }, - .@"align" = @intCast(elf.ni.tls.alignment(&elf.mf).toByteUnits()), + .@"align" = @intCast(tls_segment_ni.alignment(&elf.mf).toByteUnits()), }; } @@ -3987,7 +4007,6 @@ fn initHeaders( .entsize = 0, }; if (target_endian != native_endian) std.mem.byteSwapAllFields(ElfN.Shdr, sh_undef); - elf.shdrs.appendAssumeCapacity(.{ .lsi = .null, .ni = .none, .rela = .{ .shndx = .UNDEF } }); elf.symtab.addOneAssumeCapacity().* = .{ .node = .none, @@ -4092,7 +4111,7 @@ fn initHeaders( .node_align = node_block_align, }); } else { - elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt], .{ + elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{ .name = ".plt", .type = .PROGBITS, .flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true }, @@ -4108,14 +4127,14 @@ fn initHeaders( .node_align = node_block_align, }); if (maybe_interp) |interp| { - const interp_ni = try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{ + const interp_ni = try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{ .size = interp.len + 1, .moved = true, .resized = true, .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.interp }); - elf.phdrs.items[phndx.interp] = interp_ni; + elf.phdrs.items[phndx.interp] = .wrap(interp_ni); const sec_interp_shndx = try elf.addSection(interp_ni, .{ .name = ".interp", @@ -4129,13 +4148,13 @@ fn initHeaders( } if (have_dynamic_section) { assert(elf.ni.data_rel_ro.alignment(&elf.mf).compare(.gte, addr_align)); - const dynamic_ni = try elf.mf.addLastChildNode(gpa, elf.ni.data_rel_ro, .{ + const dynamic_ni = try elf.ni.data_rel_ro.addFloatingChild(&elf.mf, gpa, .{ .alignment = addr_align, .moved = true, .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.dynamic }); - elf.phdrs.items[phndx.dynamic] = dynamic_ni; + elf.phdrs.items[phndx.dynamic] = .wrap(dynamic_ni); const dynstr_shndx = try elf.addSection(elf.ni.rodata, .{ .name = ".dynstr", @@ -4207,7 +4226,7 @@ fn initHeaders( .flags = .{ .ALLOC = true, .WRITE = true }, .link = dynstr_shndx.toSection().?, .entsize = @intCast(addr_align.toByteUnits() * 2), - .node_align = addr_align, + .addralign = addr_align, }); switch (elf.targetDynsymHashInfo()) { inline else => |info| { @@ -4230,8 +4249,8 @@ fn initHeaders( if (elf.targetEndian() != std.lang.Endian.native) { std.mem.byteSwapAllFields(info.Header(), header); } - // The initial bucket and chain values are all 0, but `MappedFile` initialized - // the node with zeroes anyway, so no need to memset. + // The initial bucket and chain values are all 0. + @memset(hash_slice[@sizeOf(info.Header())..], 0); }, } @@ -4347,7 +4366,7 @@ fn initHeaders( try elf.ensureUnusedSymbolCapacity(10, .maybe_global); // Despite the name, `__dso_handle` is necessary even in static binaries. _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = Section.Index.text.get(elf).ni, + .node = .wrap(Section.Index.text.get(elf).ni), .name = try .string(elf, "__dso_handle"), .value = Section.Index.text.vaddr(elf), .size = 0, @@ -4359,7 +4378,7 @@ fn initHeaders( error.MultipleDefinitions => unreachable, // no inputs are processed yet }; _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = elf.shndx.plt.get(elf).ni, + .node = .wrap(elf.shndx.plt.get(elf).ni), .name = try .string(elf, "_PROCEDURE_LINKAGE_TABLE_"), .value = elf.shndx.plt.vaddr(elf), .size = 0, @@ -4371,7 +4390,7 @@ fn initHeaders( error.MultipleDefinitions => unreachable, // no inputs are processed yet }; _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = elf.shndx.got.get(elf).ni, + .node = .wrap(elf.shndx.got.get(elf).ni), .name = try .string(elf, "_GLOBAL_OFFSET_TABLE_"), .value = switch (machine) { .AARCH64, @@ -4468,7 +4487,7 @@ fn initHeaders( }; if (have_dynamic_section) { _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = elf.shndx.dynamic.get(elf).ni, + .node = .wrap(elf.shndx.dynamic.get(elf).ni), .name = try .string(elf, "_DYNAMIC"), .value = elf.shndx.dynamic.vaddr(elf), .size = 0, @@ -4484,16 +4503,16 @@ fn initHeaders( assert(maybe_interp == null); assert(!have_dynamic_section); } - if (comp.config.any_non_single_threaded) elf.shndx.tdata = try elf.addSection(elf.ni.tls, .{ + if (elf.ni.tls.unwrap()) |tls_segment_ni| elf.shndx.tdata = try elf.addSection(tls_segment_ni, .{ .name = ".tdata", .flags = .{ .WRITE = true, .ALLOC = true, .TLS = true }, .node_align = node_block_align, }); assert(elf.nodes.len == expected_nodes_len); - assert(elf.shdrs.items.len == shnum); + assert(elf.shdrs.items.len == shnum - 1); // -1 to exclude SHN_UNDEF - for (0..shnum) |shndx_raw| { + for (1..shnum) |shndx_raw| { // start at 1 to exclude SHN_UNDEF const shndx: Section.Index = @fromBackingInt(@intCast(shndx_raw)); elf.section_by_name.putAssumeCapacityNoClobber(shndx.name(elf), {}); } @@ -4520,8 +4539,6 @@ fn initHeaders( break :str try elf.string(.dynstr, slice); }, }; - - try elf.ensureElfNodeSize(); } pub fn startProgress(elf: *Elf, prog_node: std.Progress.Node) void { @@ -4556,6 +4573,7 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index { return switch (elf.getNode(ni)) { .archive, .archive_header, + .archive_elf_footer, .elf, .ehdr, .shdr, @@ -4569,13 +4587,14 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index { .uav, .lazy_code, .lazy_const_data, - => elf.getNode(ni.parent(&elf.mf)).section, + => elf.getNode(ni.parent(&elf.mf).unwrap().?).section, }; } fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { return switch (elf.getNode(ni)) { .archive, .archive_header, + .archive_elf_footer, .elf, .ehdr, .shdr, @@ -4593,8 +4612,8 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { }; } fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { - const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf))) { - .archive, .archive_header => unreachable, + const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) { + .archive, .archive_header, .archive_elf_footer => unreachable, .elf => return 0, .ehdr, .shdr => unreachable, .segment => |phndx| switch (elf.phdrSlice()) { @@ -4620,6 +4639,7 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void { const symbol_relocs: *SymbolReloc.Index, const got_relocs: ?*GotReloc.Index = switch (elf.getNode(ni)) { .archive, .archive_header, + .archive_elf_footer, .elf, .ehdr, .shdr, @@ -4660,7 +4680,7 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void { if (got_relocs) |ptr| { if (ptr.* != .none) { for (elf.got_relocs.items[@backingInt(ptr.*)..]) |*reloc| { - if (reloc.node != ni) break; + if (reloc.node != ni.toOptional()) break; reloc.delete(elf); } } @@ -4691,7 +4711,7 @@ fn flushMovedNodeRelocs( if (first_got_reloc != .none) { for (elf.got_relocs.items[@backingInt(first_got_reloc)..]) |*reloc| { - if (reloc.node != node) break; + if (reloc.node != node.toOptional()) break; reloc.apply(elf); } } @@ -4756,7 +4776,7 @@ fn targetPtrSize(elf: *const Elf) u8 { /// Page alignment for the target platform. /// Usually this returns the maximum page size supported on the /// target to maximize compatibility but there can be exceptions. -fn targetPageAlign(elf: *const Elf) std.mem.Alignment { +fn targetPageAlign(elf: *const Elf) Alignment { return .fromByteUnits(switch (elf.ehdrMachine()) { .AARCH64 => 0x10000, .LOONGARCH => 0x10000, @@ -4810,7 +4830,7 @@ const PltInfo = struct { /// entry, not the `.plt` entry. The `.plt.sec` section has no header entries, and is aligned to /// the same boundary as the `.plt` section. plt_sec: ?struct { entry_size: u8 }, - @"align": std.mem.Alignment, + @"align": Alignment, entry_size: u8, header_entries: u8, @@ -4868,7 +4888,7 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo { // TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`. }; } -fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { +pub fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer; const Child = pointer_ty.child; const alignment = pointer_ty.attrs.@"align" orelse @alignOf(Child); @@ -4941,8 +4961,9 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr { switch (elf.identClass()) { .NONE, _ => unreachable, inline else => |class| { + const shdrs_len = elf.shdrs.items.len + 1; // +1 for SHN_UNDEF const shdr_slice: []class.ElfN().Shdr = @ptrCast(@alignCast( - raw_slice[0 .. elf.shdrs.items.len * @sizeOf(class.ElfN().Shdr)], + raw_slice[0 .. shdrs_len * @sizeOf(class.ElfN().Shdr)], )); const shdr_ptr = &shdr_slice[@backingInt(shndx)]; return @unionInit(ShdrPtr, @tagName(class), shdr_ptr); @@ -4951,7 +4972,7 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr { } fn arHdrPtr(elf: *Elf, ni: MappedFile.Node.Index) *align(2) std.elf.ar_hdr { - assert(elf.ni.elf != MappedFile.Node.Index.root); + assert(elf.ni.elf != .root); const file_offset = ni.fileLocation(&elf.mf, false).offset; return @ptrCast(@alignCast(elf.mf.memory_map.memory[@intCast(switch (elf.getNode(ni)) { else => unreachable, @@ -5049,13 +5070,13 @@ fn mapInputSection(elf: *Elf, opts: struct { const name_shstrtab = try elf.string(.shstrtab, name); const gop = try elf.section_by_name.getOrPut(gpa, name_shstrtab); if (gop.found_existing) { - break :existing @fromBackingInt(@intCast(gop.index)); + break :existing @fromBackingInt(@intCast(gop.index + 1)); // +1 to account for SHN_UDNEF } errdefer assert(elf.section_by_name.pop().?.key == name_shstrtab); const parent_node: MappedFile.Node.Index = parent: { if (!opts.flags.ALLOC) break :parent elf.ni.elf; if (opts.flags.EXECINSTR) break :parent elf.ni.text; - if (opts.flags.TLS) break :parent elf.ni.tls; + if (opts.flags.TLS) break :parent elf.ni.tls.unwrap().?; if (opts.flags.WRITE) break :parent elf.ni.data; break :parent elf.ni.rodata; }; @@ -5148,12 +5169,12 @@ fn navMapIndex(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) Error!Node break :section .data_rel_ro; // TODO: it would be better to use `.rodata` if the NAV value doesn't have relocs } }; - const alignment: InternPool.Alignment = switch (Type.fromInterned(nav.resolved.?.type).zigTypeTag(zcu)) { + const alignment: Alignment = switch (Type.fromInterned(nav.resolved.?.type).zigTypeTag(zcu)) { .@"fn" => a: { const mod = zcu.navFileScope(nav_index).mod.?; const target = &mod.resolved_target.result; const min = target_util.minFunctionAlignment(target); - break :a switch (nav.resolved.?.@"align") { + break :a .fromIp(switch (nav.resolved.?.@"align") { else => |a| a.maxStrict(min), .none => switch (mod.optimize_mode) { .debug, @@ -5162,20 +5183,20 @@ fn navMapIndex(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) Error!Node => target_util.defaultFunctionAlignment(target), .small => min, }.maxStrict(Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu)), - }; + }); }, else => switch (nav.resolved.?.@"align") { - .none => Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu), - else => |a| a, + .none => .fromIp(Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu)), + else => |a| .fromIp(a), }, }; - try shndx.ensureAligned(elf, alignment.toStdMem()); - const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{ - .alignment = alignment.toStdMem(), + try shndx.ensureAligned(elf, alignment); + const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{ + .alignment = alignment, }); nav_gop.value_ptr.* = .{ .lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .name = try elf.string(.strtab, nav.fqn.toSlice(ip)), .value = 0, .size = 0, @@ -5204,19 +5225,19 @@ fn uavMapIndex( try elf.pending_uavs.ensureUnusedCapacity(gpa, 1); const abi_align = Value.fromInterned(uav_val).typeOf(zcu).abiAlignment(zcu); - const resolved_align: InternPool.Alignment = switch (uav_align) { - .none => abi_align, - else => |a| a.minStrict(abi_align), + const resolved_align: Alignment = switch (uav_align) { + .none => .fromIp(abi_align), + else => |a| .fromIp(a.minStrict(abi_align)), }; const uav_gop = elf.uavs.getOrPutAssumeCapacity(uav_val); const umi: Node.UavMapIndex = @fromBackingInt(@intCast(uav_gop.index)); if (!uav_gop.found_existing) { const shndx: Section.Index = .data_rel_ro; // TODO: it would be better to use `.rodata` if the UAV value doesn't have relocs - try shndx.ensureAligned(elf, resolved_align.toStdMem()); - const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{ + try shndx.ensureAligned(elf, resolved_align); + const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{ .moved = true, // see assert at end of `genUav` - .alignment = resolved_align.toStdMem(), + .alignment = resolved_align, }); var name_buf: [32]u8 = undefined; const name = std.fmt.bufPrint( @@ -5226,7 +5247,7 @@ fn uavMapIndex( ) catch unreachable; uav_gop.value_ptr.* = .{ .lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .name = try elf.string(.strtab, name), .value = 0, .size = 0, @@ -5239,11 +5260,11 @@ fn uavMapIndex( elf.const_prog_node.increaseEstimatedTotalItems(1); elf.pending_uavs.appendAssumeCapacity(umi); } else { - const node = uav_gop.value_ptr.lsi.index().ptr(elf).node; - const shndx = elf.getNode(node.parent(&elf.mf)).section; - try shndx.ensureAligned(elf, resolved_align.toStdMem()); - if (resolved_align.toStdMem().order(node.alignment(&elf.mf)).compare(.gt)) { - try node.realign(&elf.mf, gpa, resolved_align.toStdMem(), .{}); + const node = uav_gop.value_ptr.lsi.index().ptr(elf).node.unwrap().?; + const shndx = elf.getNode(node.parent(&elf.mf).unwrap().?).section; + try shndx.ensureAligned(elf, resolved_align); + if (resolved_align.order(node.alignment(&elf.mf)).compare(.gt)) { + try node.realign(&elf.mf, gpa, resolved_align); } } return umi; @@ -5459,10 +5480,11 @@ fn loadObject( .member = if (member) |m| try gpa.dupe(u8, m) else null, .extra = undefined, }; - if (elf.ni.elf != MappedFile.Node.Index.root) { + if (elf.ni.elf != .root) { + const archive_ni: MappedFile.Node.Index = .root; try elf.nodes.ensureUnusedCapacity(gpa, 1); - input.extra = .{ .node = try elf.mf.addLastChildNode(gpa, .root, .{ - .size = fl.size + @sizeOf(std.elf.ar_hdr), + input.extra = .{ .node = try archive_ni.addFloatingChild(&elf.mf, gpa, .{ + .size = Alignment.@"2".forward(fl.size + @sizeOf(std.elf.ar_hdr)), .alignment = .@"2", .next_moved = true, .bubbles_moved = false, @@ -5640,16 +5662,28 @@ fn loadObject( .node_fixed = true, }, }; - const need_align: std.mem.Alignment = .fromByteUnits( + const need_align: Alignment = .fromByteUnits( std.math.ceilPowerOfTwoAssert(usize, @intCast(@max(section.shdr.addralign, 1))), ); try opts.shndx.ensureAligned(elf, need_align); - const ni = try elf.mf.addLastChildNode(gpa, opts.shndx.get(elf).ni, .{ - .size = section.shdr.size, + const add_node_opts: MappedFile.Node.AddOptions = .{ + .size = need_align.forward(section.shdr.size), .alignment = need_align, .moved = true, // see assert at end of `flushInputSection` - .fixed = opts.node_fixed, - }); + }; + const ni = if (opts.node_fixed) ni: { + const shndx_ni = opts.shndx.get(elf).ni; + const after_oni: MappedFile.Node.Index.Optional = after: { + const last_ni = shndx_ni.last(&elf.mf).unwrap() orelse break :after .none; + break :after switch (last_ni.position(&elf.mf)) { + .header => .wrap(last_ni), + .footer, .floating => .none, + }; + }; + break :ni try shndx_ni.addHeaderChildAfter(&elf.mf, gpa, after_oni, add_node_opts); + } else ni: { + break :ni try opts.shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, add_node_opts); + }; elf.nodes.appendAssumeCapacity(.{ .input_section = @fromBackingInt(@intCast(elf.input_sections.items.len)), }); @@ -5754,7 +5788,7 @@ fn loadObject( ), .LOCAL => { const lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = input_section_node, + .node = .wrap(input_section_node), .name = try elf.string(.strtab, name), .value = input_sym.value, .size = input_sym.size, @@ -5765,7 +5799,7 @@ fn loadObject( }, .GLOBAL, .WEAK, .GNU_UNIQUE => |bind| { si.* = elf.addGlobalSymbolAssumeCapacity(.{ - .node = input_section_node, + .node = .wrap(input_section_node), .name = try .string(elf, name), .value = input_sym.value, .size = input_sym.size, @@ -5893,7 +5927,7 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadPars return diags.failParse(path, "bad machine", .{}); if (ehdr.shnum > 0) try fr.seekTo(ehdr.shoff); // We're going to need to know the alignment of every section later. - const section_aligns = try gpa.alloc(std.mem.Alignment, ehdr.shnum); + const section_aligns = try gpa.alloc(Alignment, ehdr.shnum); defer gpa.free(section_aligns); const dynamic_sh: ElfN.Shdr, const dynsym_sh: ElfN.Shdr = sh: { var dynamic_sh: ?ElfN.Shdr = null; @@ -5999,7 +6033,7 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadPars // We need to guess the worst-case alignment of the symbol. Yes, I know this seems // insane---refer to the doc comment on `alignment` in `Elf.dso_globals`. - const sym_align: std.mem.Alignment = switch (sym.value) { + const sym_align: Alignment = switch (sym.value) { 0 => section_aligns[sym.shndx], else => section_aligns[sym.shndx].min(@fromBackingInt(@intCast(@ctz(sym.value)))), }; @@ -6017,8 +6051,8 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadPars // We have a copy relocation for this global, but the amount of space we // reserved for it could be too small or underaligned! try Section.Index.data.ensureAligned(elf, gop.value_ptr.alignment); - try copied_global.node.resize(&elf.mf, gpa, gop.value_ptr.size); - try copied_global.node.realign(&elf.mf, gpa, gop.value_ptr.alignment, .{}); + try copied_global.node.resizeLeaf(&elf.mf, gpa, gop.value_ptr.alignment.forward(gop.value_ptr.size)); + try copied_global.node.realign(&elf.mf, gpa, gop.value_ptr.alignment); const global_ptr = elf.globalByName(name).?; switch (elf.symPtr(global_ptr.symtab_index)) { inline else => |sym_ptr| elf.targetStore(&sym_ptr.size, @intCast(gop.value_ptr.size)), @@ -6158,7 +6192,7 @@ fn createInitFiniArraySection( ) Error!void { assert(shndx.* == .UNDEF); const gpa = elf.base.comp.gpa; - const addr_align: std.mem.Alignment = switch (elf.identClass()) { + const addr_align: Alignment = switch (elf.identClass()) { .NONE, _ => unreachable, .@"32" => .@"4", .@"64" => .@"8", @@ -6178,14 +6212,14 @@ fn createInitFiniArraySection( const start_sym_name = try elf.string(.strtab, "__" ++ name ++ "_start"); const end_sym_name = try elf.string(.strtab, "__" ++ name ++ "_end"); elf.setGlobalSymbolValue(start_sym_name, elf.globals.strong_def.getPtr(start_sym_name).?, .{ - .node = shndx.get(elf).ni, + .node = .wrap(shndx.get(elf).ni), .value = shndx.vaddr(elf), .size = 0, .type = .NOTYPE, .shndx = shndx.*, }); elf.setGlobalSymbolValue(end_sym_name, elf.globals.strong_def.getPtr(end_sym_name).?, .{ - .node = shndx.get(elf).ni, + .node = .wrap(shndx.get(elf).ni), .value = shndx.vaddr(elf), .size = 0, .type = .NOTYPE, @@ -6218,7 +6252,7 @@ fn prelinkInner(elf: *Elf) Error!void { const comp = elf.base.comp; const gpa = comp.gpa; - if (comp.zcu != null and !comp.config.use_llvm and elf.ni.elf == MappedFile.Node.Index.root) { + if (comp.zcu != null and !comp.config.use_llvm and elf.ni.elf == .root) { // We're using self-hosted codegen---add an input representing the Zig "object". try elf.ensureUnusedSymbolCapacity(1, .all_local); try elf.inputs.ensureUnusedCapacity(gpa, 1); @@ -6239,8 +6273,6 @@ fn prelinkInner(elf: *Elf) Error!void { }; elf.input_pending_index += 1; } - - try elf.ensureElfNodeSize(); } fn prepareDynamic(elf: *Elf) Error!void { @@ -6265,7 +6297,7 @@ fn prepareDynamic(elf: *Elf) Error!void { const dynamic_size = dynamic_len * 2 * elf.targetPtrSize(); - try elf.shndx.dynamic.get(elf).ni.resize(&elf.mf, comp.gpa, dynamic_size); + try elf.shndx.dynamic.get(elf).ni.resizeLeaf(&elf.mf, comp.gpa, dynamic_size); switch (elf.shdrPtr(elf.shndx.dynamic)) { inline else => |shdr| elf.targetStore(&shdr.size, @intCast(dynamic_size)), } @@ -6388,10 +6420,9 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct { size: std.elf.Xword = 0, link: std.elf.Word = 0, info: std.elf.Word = 0, - addralign: std.mem.Alignment = .@"1", + addralign: Alignment = .@"1", entsize: std.elf.Word = 0, - node_align: std.mem.Alignment = .@"1", - fixed: bool = false, + node_align: Alignment = .@"1", }) Error!Section.Index { switch (opts.type) { .NULL => assert(opts.size == 0), @@ -6435,19 +6466,20 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct { break :shndx .{ @fromBackingInt(shndx), @as(u64, elf.targetLoad(&ehdr.shentsize)) * @as(u64, shnum) }; }, }; - try elf.ensureNodeSize(elf.ni.shdr, new_shdr_size); - const ni = try elf.mf.addLastChildNode(gpa, switch (elf.ehdrType()) { + try elf.ni.shdr.ensureMinimumSize(&elf.mf, gpa, new_shdr_size); + const parent_ni = switch (elf.ehdrType()) { .REL => elf.ni.elf, .EXEC, .DYN => segment_ni, - }, .{ - .size = opts.size, + }; + assert(opts.addralign.check(opts.size)); + const ni = try parent_ni.addFloatingChild(&elf.mf, gpa, .{ + .size = opts.node_align.forward(opts.size), .alignment = opts.addralign.max(opts.node_align), - .fixed = opts.fixed, .resized = opts.size > 0, }); const addr = elf.computeNodeVAddr(ni); const lsi: Symbol.LocalIndex = if (opts.flags.ALLOC) elf.addLocalSymbolAssumeCapacity(.{ - .node = ni, + .node = .wrap(ni), .name = .empty, .value = addr, .size = 0, @@ -6499,7 +6531,7 @@ fn ensureUnusedRelocCapacity(elf: *Elf, node: MappedFile.Node.Index, len: usize) assert(elf.section_by_name.count() == elf.shdrs.items.len); try elf.section_by_name.ensureUnusedCapacity(gpa, 1); - const rela_shndx = try elf.addSection(.none, .{ + const rela_shndx = try elf.addSection(elf.ni.elf, .{ .name = rela_name, .type = .RELA, .link = @backingInt(Section.Index.symtab), @@ -6528,7 +6560,7 @@ fn ensureUnusedRelocCapacity(elf: *Elf, node: MappedFile.Node.Index, len: usize) .NONE, _ => unreachable, inline else => |ct_class| (elf.got.count() + new_got_entries) * @sizeOf(ct_class.ElfN().Addr), }; - try elf.ensureNodeSize(elf.shndx.got.get(elf).ni, need_got_size); + try elf.shndx.got.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_got_size); if (elf.shndx.dynamic != .UNDEF) { try elf.shndx.rela_dyn.relaEnsureAdditionalCapacity(elf, new_got_entries); @@ -6546,7 +6578,6 @@ fn addRelocAssumeCapacity( addend: i64, @"type": MachineRelocType, ) (Error || error{ UnknownRelocation, NonStaticRelocation, UnimplementedRelocation })!void { - assert(node != .none); switch (elf.ehdrType()) { .REL => { const rela_shndx = elf.getNodeShndx(node).get(elf).rela.shndx; @@ -6894,7 +6925,6 @@ fn addSymbolRelocAssumeCapacity( @"type": SymbolReloc.Type, ) Error!void { assert(elf.ehdrType() != .REL); - assert(node != .none); const rela_index: Section.RelaIndex.Optional = r: { if (elf.shndx.dynamic == .UNDEF) break :r .none; @@ -7042,6 +7072,7 @@ fn addGotRelocAssumeCapacity( switch (elf.getNode(node)) { .archive, .archive_header, + .archive_elf_footer, .elf, .ehdr, .shdr, @@ -7089,7 +7120,7 @@ fn addGotRelocAssumeCapacity( } elf.got_relocs.appendAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .offset = offset, .target = target, .addend = addend, @@ -7111,7 +7142,7 @@ fn updateGotEntry(elf: *Elf, got_index: usize) void { .tpoff => |sym_id| val: { // Only the executable's per-module TLS block is at a known offset from the TLS pointer. if (elf.base.comp.config.output_mode == .Exe and elf.classifySymbolValue(sym_id) != .dynamic) { - const tls_phndx = elf.getNode(elf.ni.tls).segment; + const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment; const tls_size: u64 = switch (elf.phdrSlice()) { inline else => |phdr| tls_size: { assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS); @@ -7284,8 +7315,8 @@ fn maybeAddCopyRelocation(elf: *Elf, global_name: String(.strtab)) Error!bool { try Section.Index.data.ensureAligned(elf, dso_global.alignment); try elf.nodes.ensureUnusedCapacity(gpa, 1); - const node = try elf.mf.addLastChildNode(gpa, Section.Index.data.get(elf).ni, .{ - .size = dso_global.size, + const node = try Section.Index.data.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{ + .size = dso_global.alignment.forward(dso_global.size), .alignment = dso_global.alignment, }); errdefer comptime unreachable; @@ -7336,7 +7367,7 @@ fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) if (!Type.fromInterned(nav.resolved.?.type).hasRuntimeBits(zcu)) return; const nmi = try elf.navMapIndex(zcu, nav_index); - const ni = nmi.symbol(elf).index().ptr(elf).node; + const ni = nmi.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); // Ensure the NAV is marked as moved so that once we're done, `flushMoved` will eventually be @@ -7392,7 +7423,7 @@ fn updateFuncInner( const nmi = try elf.navMapIndex(zcu, func.owner_nav); log.debug("updateFunc({f}) = {d}", .{ nav.fqn.fmt(ip), nmi.symbol(elf) }); - const ni = nmi.symbol(elf).index().ptr(elf).node; + const ni = nmi.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); // Ensure the NAV is marked as moved so that once we're done, `flushMoved` will eventually be @@ -7471,7 +7502,6 @@ fn flushInner( try elf.prepareDynamic(); - try elf.ensureElfNodeSize(); while (try elf.idle(tid)) {} // We've done the final `idle` loop, so everything is at its final place in the file. We have a @@ -7677,7 +7707,7 @@ fn idleProgNode( break :name std.fmt.bufPrint(&name, "{f}{f} {s}", .{ ii.path(elf).fmtEscapeString(), fmtMemberString(ii.member(elf)), - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), }) catch &name; }, .nav => |nmi| { @@ -7724,8 +7754,6 @@ fn genPending(elf: *Elf, pt: Zcu.PerThread) Error!void { }; break; } - - try elf.ensureElfNodeSize(); } fn genUav( @@ -7737,7 +7765,7 @@ fn genUav( const gpa = comp.gpa; const uav_val = umi.uavValue(elf); - const ni = umi.symbol(elf).index().ptr(elf).node; + const ni = umi.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); var nw: MappedFile.Node.Writer = undefined; @@ -7766,7 +7794,7 @@ fn genLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void { const gpa = zcu.gpa; const lazy = lmr.lazySymbol(elf); - const ni = lmr.symbol(elf).index().ptr(elf).node; + const ni = lmr.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); // Ensure the lazy node is marked as moved so that once we're done, `flushMoved` will eventually @@ -7842,7 +7870,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void { fr.seekTo(file_loc.offset) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| return diags.fail("failed to read input section '{s}' from \"{f}{f}\": {t}", .{ - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), path.fmtEscapeString(), fmtMemberString(ii.member(elf)), e, @@ -7853,7 +7881,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void { defer nw.deinit(); const n_bytes = nw.interface.sendFileAll(&fr, .limited(@intCast(file_loc.size))) catch |err| switch (err) { error.ReadFailed => return diags.fail("failed to read input section '{s}' from \"{f}{f}\": {t}", .{ - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), path.fmtEscapeString(), fmtMemberString(ii.member(elf)), fr.err orelse (fr.seek_err orelse fr.size_err.?), @@ -7861,7 +7889,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void { error.WriteFailed => return nw.err.?, }; if (n_bytes != file_loc.size) return diags.fail("failed to read input section '{s}' from \"{f}{f}\": unexpected eof", .{ - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), path.fmtEscapeString(), fmtMemberString(ii.member(elf)), }); @@ -7888,8 +7916,10 @@ fn flushElfOffset(elf: *Elf, ni: MappedFile.Node.Index) void { } }, } - var child_it = ni.children(&elf.mf); - while (child_it.next()) |child_ni| elf.flushElfOffset(child_ni); + var child_oni = ni.first(&elf.mf); + while (child_oni.unwrap()) |child_ni| : (child_oni = child_ni.next(&elf.mf)) { + elf.flushElfOffset(child_ni); + } }, .section => |shndx| switch (elf.shdrPtr(shndx)) { inline else => |shdr| elf.targetStore(&shdr.offset, @intCast(elf_offset)), @@ -7906,7 +7936,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void switch (elf.getNode(ni)) { .archive, .archive_header => unreachable, - .elf => {}, + .archive_elf_footer, .elf => {}, .ehdr, .shdr => elf.flushElfOffset(ni), .segment => |phndx| { elf.flushElfOffset(ni); @@ -7994,7 +8024,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void const ii = isi.input(elf); var lsi, const end_lsi = ii.localSymbolRange(elf); while (lsi != end_lsi) : (lsi = @fromBackingInt(@backingInt(lsi) + 1)) { - if (lsi.index().ptr(elf).node != ni) continue; + if (lsi.index().ptr(elf).node != ni.toOptional()) continue; const visibility: std.elf.STV = switch (elf.symPtr(lsi.index())) { inline else => |sym| elf.targetLoad(&sym.other).visibility, }; @@ -8079,7 +8109,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void /// moving or resizing of a segment could reorder them and thereby affect how we handle *future* /// changes to segments. fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Error!void { - const segment_ni = elf.phdrs.items[orig_phndx]; + const segment_ni = elf.phdrs.items[orig_phndx].unwrap().?; assert(elf.getNode(segment_ni).segment == orig_phndx); const page_align = elf.targetPageAlign(); const node_align = segment_ni.alignment(&elf.mf); @@ -8165,7 +8195,7 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro const next_ni = elf.phdrs.items[next_phndx]; elf.phdrs.items[phndx] = next_ni; elf.nodes.items(.data)[@backingInt(next_ni)] = .{ .segment = phndx }; - elf.phdrs.items[next_phndx] = segment_ni; + elf.phdrs.items[next_phndx] = .wrap(segment_ni); elf.nodes.items(.data)[@backingInt(segment_ni)] = .{ .segment = @intCast(next_phndx) }; phndx = @intCast(next_phndx); } @@ -8189,9 +8219,10 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo _, const size = ni.location(&elf.mf).resolve(&elf.mf); switch (elf.getNode(ni)) { .archive => { - var child_it = ni.reverseChildren(&elf.mf); - if (child_it.next()) |last_ni| { - if (child_it.next()) |prev_ni| if (prev_ni.hasNextMoved(&elf.mf)) return; + if (ni.last(&elf.mf).unwrap()) |last_ni| { + if (last_ni.prev(&elf.mf).unwrap()) |prev_ni| { + if (prev_ni.hasNextMoved(&elf.mf)) return; + } const offset, _ = last_ni.location(&elf.mf).resolve(&elf.mf); _ = std.mem.print(&elf.arHdrPtr(last_ni).ar_size, "{d:<10}", .{ size - offset, @@ -8199,11 +8230,11 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo } }, .archive_header, .elf => {}, - .ehdr => unreachable, + .ehdr, .archive_elf_footer => unreachable, .shdr => {}, .segment => |phndx| switch (elf.phdrSlice()) { inline else => |phdr| { - assert(elf.phdrs.items[phndx] == ni); + assert(elf.phdrs.items[phndx].unwrap().? == ni); const ph = &phdr[phndx]; elf.targetStore(&ph.filesz, @intCast(size)); switch (elf.targetLoad(&ph.type)) { @@ -8284,6 +8315,7 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error! switch (elf.getNode(ni)) { .archive, + .archive_elf_footer, .ehdr, .shdr, .segment, @@ -8301,51 +8333,45 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error! break :member_offset switch (tag) { else => unreachable, .archive_header => .{ offset + std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr), true }, - .elf, .input_member => .{ offset, switch (ni.prev(&elf.mf)) { - .none => unreachable, - else => |prev_ni| !prev_ni.hasNextMoved(&elf.mf), - } }, + .elf, .input_member => .{ offset, !ni.prev(&elf.mf).unwrap().?.hasNextMoved(&elf.mf) }, }; }; - const member_size = member_end: switch (ni.next(&elf.mf)) { - else => |next_ni| { - const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf); - const next_member_size = next_member_end: switch (next_ni.next(&elf.mf)) { - else => |next_next_ni| { - const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf); - break :next_member_end next_next_offset - @sizeOf(std.elf.ar_hdr); - }, - .none => { - _, const parent_size = - ni.parent(&elf.mf).location(&elf.mf).resolve(&elf.mf); - break :next_member_end parent_size; - }, - } - next_offset; - const ar_hdr = elf.arHdrPtr(next_ni); - var name_buf: [16]u8 = undefined; - _ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{ - switch (elf.getNode(next_ni)) { - else => unreachable, - .elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}), - .input_member => |ii| std.mem.print(&name_buf, "{s}/", .{ - std.fs.path.basename(ii.path(elf).sub_path), - }), - } catch @panic("TODO: long archive member names"), - }) catch @panic("TODO: long archive member names"); - ar_hdr.ar_date = "0 ".*; - ar_hdr.ar_uid = "0 ".*; - ar_hdr.ar_gid = "0 ".*; - ar_hdr.ar_mode = "644 ".*; - _ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch - @panic("archive member too large"); - ar_hdr.ar_fmag = std.elf.ARFMAG.*; - break :member_end next_offset - @sizeOf(std.elf.ar_hdr); - }, - .none => { - _, const parent_size = ni.parent(&elf.mf).location(&elf.mf).resolve(&elf.mf); - break :member_end parent_size; - }, - } - member_offset; + const member_size = if (ni.next(&elf.mf).unwrap()) |next_ni| member_size: { + const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf); + const next_member_size = if (next_ni.next(&elf.mf).unwrap()) |next_next_ni| next_member_size: { + const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf); + const next_member_end = next_next_offset - @sizeOf(std.elf.ar_hdr); + break :next_member_size next_member_end - next_offset; + } else next_member_size: { + _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf); + const next_member_end = parent_size; + break :next_member_size next_member_end - next_offset; + }; + const ar_hdr = elf.arHdrPtr(next_ni); + var name_buf: [16]u8 = undefined; + _ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{ + switch (elf.getNode(next_ni)) { + else => unreachable, + .elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}), + .input_member => |ii| std.mem.print(&name_buf, "{s}/", .{ + std.fs.path.basename(ii.path(elf).sub_path), + }), + } catch @panic("TODO: long archive member names"), + }) catch @panic("TODO: long archive member names"); + ar_hdr.ar_date = "0 ".*; + ar_hdr.ar_uid = "0 ".*; + ar_hdr.ar_gid = "0 ".*; + ar_hdr.ar_mode = "644 ".*; + _ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch + @panic("archive member too large"); + ar_hdr.ar_fmag = std.elf.ARFMAG.*; + const member_end = next_offset - @sizeOf(std.elf.ar_hdr); + break :member_size member_end - member_offset; + } else member_size: { + _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf); + const member_end = parent_size; + break :member_size member_end - member_offset; + }; if (update_size) _ = std.mem.print(&elf.arHdrPtr(ni).ar_size, "{d:<10}", .{ member_size, }) catch @panic("archive member too large"); @@ -8735,8 +8761,6 @@ fn updateExportInner( .uav => |uav| (try elf.uavMapIndex(uav, .none)).symbol(elf), }; - try elf.ensureElfNodeSize(); - // Initialize the global symbol with the same values that the local one currently has. If the // NAV/UAV is updated, then `updateNavInner` or `genUav` will update the global symbol sizes, // and `flushMoved` will update their values. @@ -8775,12 +8799,13 @@ fn updateExportInner( // only emitting this error if the symbol we're conflicting with comes from an input // section (as opposed to the ZCU). const conflicting_global = elf.globalByName(try elf.string(.strtab, name)).?; - const conflicting_node = conflicting_global.symtab_index.ptr(elf).node; - if (elf.getNode(conflicting_node) == .input_section) { - return elf.base.comp.link_diags.fail( - "multiple definitions of '{s}'", - .{name}, - ); + if (conflicting_global.symtab_index.ptr(elf).node.unwrap()) |conflicting_node| { + if (elf.getNode(conflicting_node) == .input_section) { + return elf.base.comp.link_diags.fail( + "multiple definitions of '{s}'", + .{name}, + ); + } } }, }; @@ -8842,7 +8867,7 @@ pub fn printNode( try w.print("({f}{f}, {s})", .{ ii.path(elf).fmtEscapeString(), fmtMemberString(ii.member(elf)), - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), }); }, .copied_global => |name| try w.print("(copy:{s})", .{name.slice(elf)}), @@ -8873,25 +8898,27 @@ pub fn printNode( { const mf_node = &elf.mf.nodes.items[@backingInt(ni)]; const off, const size = mf_node.location().resolve(&elf.mf); - try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x}{s}{s}{s}{s}{s}\n", .{ + try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x} {t}{s}{s}{s}{s}\n", .{ @backingInt(ni), off, size, mf_node.flags.alignment.toByteUnits(), - if (mf_node.flags.fixed) " fixed" else "", + mf_node.flags.position, if (mf_node.flags.moved) " moved" else "", if (mf_node.flags.next_moved) " next_moved" else "", if (mf_node.flags.resized) " resized" else "", if (mf_node.flags.has_content) " has_content" else "", }); } - var leaf = true; - var child_it = ni.children(&elf.mf); - while (child_it.next()) |child_ni| { - leaf = false; - try elf.printNode(tid, w, child_ni, indent + 1); + if (ni.first(&elf.mf).unwrap()) |first_ni| { + // non-leaf, just print children + var child_ni = first_ni; + while (true) { + try elf.printNode(tid, w, child_ni, indent + 1); + child_ni = child_ni.next(&elf.mf).unwrap() orelse break; + } + return; } - if (!leaf) return; const file_loc = ni.fileLocation(&elf.mf, false); var address = file_loc.offset; if (file_loc.size == 0) { @@ -8916,16 +8943,16 @@ pub fn printNode( } } -fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignment) Error!void { +fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: Alignment) Error!void { const gpa = elf.base.comp.gpa; // We need to loop through parent nodes because segments may be nested (e.g. a PT_TLS segment // inside a PT_LOAD segment). var phndx = start_phndx; while (true) { // Align the actual node - const seg_ni = elf.phdrs.items[phndx]; + const seg_ni = elf.phdrs.items[phndx].unwrap().?; if (min_align.compare(.gt, seg_ni.alignment(&elf.mf))) { - try seg_ni.realign(&elf.mf, gpa, min_align, .{}); + try seg_ni.realign(&elf.mf, gpa, min_align); } // Update the phdr `@"align"` field if necessary switch (elf.phdrSlice()) { @@ -8948,7 +8975,7 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignmen }, } // Continue on to the parent segment, if any - switch (elf.getNode(seg_ni.parent(&elf.mf))) { + switch (elf.getNode(seg_ni.parent(&elf.mf).unwrap().?)) { .segment => |parent_phndx| phndx = parent_phndx, .elf => return, else => unreachable, @@ -8956,26 +8983,6 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignmen } } -/// Must be called deterministically after any call to `MappedFile.Node.Index.resize` -/// (of `elf.ni.elf` or one of its children) before any possible calls to `idle`. -fn ensureElfNodeSize(elf: *Elf) MappedFile.Error!void { - if (elf.ni.elf == MappedFile.Node.Index.root) return; - var child_it = elf.ni.elf.reverseChildren(&elf.mf); - const last_end = if (child_it.next()) |last_ni| last_end: { - const last_offset, const last_size = last_ni.location(&elf.mf).resolve(&elf.mf); - break :last_end last_offset + last_size; - } else 0; - try elf.ensureNodeSize(elf.ni.elf, last_end + @sizeOf(std.elf.ar_hdr)); -} - -fn ensureNodeSize(elf: *Elf, node: MappedFile.Node.Index, need_size: u64) MappedFile.Error!void { - _, const node_size = node.location(&elf.mf).resolve(&elf.mf); - if (need_size <= node_size) return; - const gpa = elf.base.comp.gpa; - const new_size = need_size + need_size / MappedFile.growth_factor; - try node.resize(&elf.mf, gpa, new_size); -} - /// If `sym` has a PLT entry, returns the address of that entry (specifically, the address which a /// branch to the PLT should target). If `sym` does not have a PLT entry, returns `null`. fn pltEntryTargetAddr(elf: *Elf, sym: Symbol.Id) ?u64 { diff --git a/src/link/MappedFile.zig b/src/link/MappedFile.zig index a6b86e2fc7ac35353027705400fc95e63e229587..a790ecbd3b303fbe97ea860481e41913d2aa3806 100644 --- a/src/link/MappedFile.zig +++ b/src/link/MappedFile.zig @@ -13,14 +13,14 @@ const windows = std.os.windows; io: Io, flags: packed struct { - block_size: std.mem.Alignment, + block_size: Alignment, copy_file_range_unsupported: bool, fallocate_punch_hole_unsupported: bool, fallocate_insert_range_unsupported: bool, }, memory_map: Io.File.MemoryMap, nodes: std.ArrayList(Node), -free_ni: Node.Index, +free_ni: Node.Index.Optional, large: std.ArrayList(u64), updates: std.ArrayList(Node.Index), /// This progress node's estimated total items is increased once for each node appended to `updates`. @@ -62,6 +62,94 @@ pub const Error = Allocator.Error || Io.Cancelable || error{ MappedFileIo, }; +/// This separate `Alignment` type exists because neither of the other options is really suitable: +/// +/// * `std.mem.Alignment` is based on `usize`, which---while technically okay since the file is +/// memory-mapped---is in practice very annoying to work with in linker implementations +/// +/// * `InternPool.Alignment` is based on `u64`, which is better, but it has the value `.none`, which +/// is also really annoying to handle, because no alignment is ever nullable in this API +/// +/// At some point we should probably just change `InternPool.Alignment` to be non-optional, and add +/// a new `InternPool.Alignment.Optional` type for the case where it can actually be `.none`. At +/// that point we can transition this code to using `InternPool.Alignment` (although it should +/// probably be namespaced elsewhere, it has nothing to do with the `InternPool`!). +pub const Alignment = enum(u6) { + @"1" = 0, + @"2" = 1, + @"4" = 2, + @"8" = 3, + @"16" = 4, + @"32" = 5, + @"64" = 6, + _, + + pub fn fromIp(a: @import("../InternPool.zig").Alignment) Alignment { + assert(a != .none); + return @bitCast(a); + } + + pub fn toLog2Units(a: Alignment) u6 { + return @backingInt(a); + } + + pub fn fromLog2Units(a: u6) Alignment { + return @fromBackingInt(a); + } + + pub fn toByteUnits(a: Alignment) u64 { + return @as(u64, 1) << @backingInt(a); + } + + pub fn fromByteUnits(n: u64) Alignment { + assert(std.math.isPowerOfTwo(n)); + return @fromBackingInt(@intCast(@ctz(n))); + } + + pub fn order(lhs: Alignment, rhs: Alignment) std.math.Order { + return std.math.order(@backingInt(lhs), @backingInt(rhs)); + } + + pub fn compare(lhs: Alignment, op: std.math.CompareOperator, rhs: Alignment) bool { + return std.math.compare(@backingInt(lhs), op, @backingInt(rhs)); + } + + pub fn max(lhs: Alignment, rhs: Alignment) Alignment { + return @fromBackingInt(@max(@backingInt(lhs), @backingInt(rhs))); + } + + pub fn min(lhs: Alignment, rhs: Alignment) Alignment { + return @fromBackingInt(@min(@backingInt(lhs), @backingInt(rhs))); + } + + pub inline fn of(comptime T: type) Alignment { + return comptime .fromByteUnits(@alignOf(T)); + } + + /// Given that a base address is known to be aligned to `a`, computes the known alignment of + /// that base address plus `off`. + pub fn offset(a: Alignment, off: u64) Alignment { + return .fromLog2Units(@min(a.toLog2Units(), @ctz(off))); + } + + /// Align an address forwards to this alignment. + pub fn forward(a: Alignment, addr: u64) u64 { + const x = (@as(u64, 1) << @backingInt(a)) - 1; + return (addr + x) & ~x; + } + + /// Align an address backwards to this alignment. + pub fn backward(a: Alignment, addr: u64) u64 { + const x = (@as(u64, 1) << @backingInt(a)) - 1; + return addr & ~x; + } + + /// Check if an address is aligned to this amount. + pub fn check(a: Alignment, addr: u64) bool { + return @ctz(addr) >= @backingInt(a); + } +}; + pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error || Io.Cancelable || IoError)!MappedFile { var mf: MappedFile = .{ .io = io, @@ -95,14 +183,42 @@ pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error || Io.Cancel .fallocate_insert_range_unsupported = false, .fallocate_punch_hole_unsupported = false, }; - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const root_ni = try mf.addNode(gpa, .{ .add_node = .{ - .size = size, - .alignment = mf.flags.block_size, - .fixed = true, - } }); - assert(root_ni == Node.Index.root); - try mf.ensureTotalCapacityInner(@intCast(size)); + + const root_location: Node.Location = l: { + if (std.math.cast(u32, size)) |small_size| { + break :l .{ .small = .{ .offset = 0, .size = small_size } }; + } + try mf.large.appendSlice(gpa, &.{ 0, size }); + break :l .{ .large = .{ .index = 0 } }; + }; + try mf.nodes.append(gpa, .{ + .parent = .none, + .prev = .none, + .next = .none, + .first = .none, + .last = .none, + .flags = .{ + .alignment = mf.flags.block_size, + .position = .floating, + .bubbles_moved = true, + .enable_next_moved = false, + .location_tag = root_location, + .moved = false, + .resized = false, + .next_moved = false, + .has_content = false, + }, + .location_payload = switch (root_location) { + .small => |small| .{ .small = small }, + .large => |large| .{ .large = large }, + }, + }); + + mf.ensureTotalCapacity(@intCast(size)) catch |err| switch (err) { + error.MappedFileIo => return mf.io_err.?, + else => |e| return e, + }; + return mf; } @@ -117,32 +233,61 @@ pub fn deinit(mf: *MappedFile, gpa: Allocator) void { } pub const Node = extern struct { - parent: Node.Index, - prev: Node.Index, - next: Node.Index, - first: Node.Index, - last: Node.Index, + parent: Node.Index.Optional, + prev: Node.Index.Optional, + next: Node.Index.Optional, + first: Node.Index.Optional, + last: Node.Index.Optional, flags: Flags, location_payload: Location.Payload, + /// Any non-leaf node may designate its first N children as "header" nodes. This means that its + /// first N children must be densely packed together and positioned at the start of the parent. + /// The implementation guarantees that it will never re-order these nodes, nor will it introduce + /// padding between them. + /// + /// Likewise, any non-leaf node may designate its *last* M children as "footer" nodes, which are + /// like header nodes except they are positioned at the *end* of the parent rather than the + /// start. + /// + /// Nodes which are neither headers nor footers are called "floating". The implementation is + /// always free to re-order floating nodes relative to one another, and to add or remove padding + /// between them. + pub const Position = enum(u2) { + header, + footer, + floating, + }; + pub const Flags = packed struct(u32) { + /// While the number of header and footer nodes within a parent node is logically a part of + /// that parent, we actually store this information on the child nodes for efficiency: this + /// field indicates whether each child is a header node, a footer node, or a floating node. + /// + /// This value is meaningless for the root node, so is arbitrarily set to `.floating`. + position: Position, + /// For floating nodes, this node's offset into its parent will always be aligned to this + /// boundary. (This is not the case for header and footer nodes due to the requirement that + /// they be densely packed against the start/end of the parent node.) + /// + /// This node's size will also always be aligned to this boundary. (This applies regardless + /// of whether this is a floating node, a header node, or a footer node.) + alignment: Alignment, + /// Whether `moved` events on this node bubble down to children. + bubbles_moved: bool, + /// Whether `next_moved` events are reported in `updates`. + enable_next_moved: bool, + location_tag: Location.Tag, - alignment: std.mem.Alignment, - /// Whether this node can be moved. - fixed: bool, /// Whether this node has been moved. moved: bool, /// Whether this node has been resized. resized: bool, /// Whether the next sibling has moved or is a different node. next_moved: bool, - /// Whether this node might contain non-zero bytes. + /// Whether this node might contain initialized bytes. has_content: bool, - /// Whether `moved` events on this node bubble down to children. - bubbles_moved: bool, - /// Whether `next_moved` events are reported in `updates`. - enable_next_moved: bool, - unused: @Int(.unsigned, 32 - @bitSizeOf(std.mem.Alignment) - 8) = 0, + unused: u17 = 0, }; pub const Location = union(enum(u1)) { @@ -179,74 +324,183 @@ pub const Node = extern struct { } }; + pub const AddOptions = struct { + /// Must be aligned to the given `alignment`. + size: u64 = 0, + alignment: Alignment = .@"1", + bubbles_moved: bool = true, + enable_next_moved: bool = false, + + moved: bool = false, + resized: bool = false, + next_moved: bool = false, + }; + pub const Index = enum(u32) { - none, + root, _, - pub const root: Node.Index = .none; + pub const Optional = enum(u32) { + none = std.math.maxInt(u32), + _, + + pub fn unwrap(oi: Optional) ?Index { + return switch (oi) { + _ => @fromBackingInt(@backingInt(oi)), + .none => null, + }; + } + pub fn wrap(i: Index) Optional { + const oi: Optional = @bitCast(i); + assert(oi != .none); + return oi; + } + }; fn get(ni: Node.Index, mf: *const MappedFile) *Node { return &mf.nodes.items[@backingInt(ni)]; } - pub fn parent(ni: Node.Index, mf: *const MappedFile) Node.Index { + /// Adds a floating child node to `parent_ni`. Returns the index of the new child. + pub fn addFloatingChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { + return mf.addNode(gpa, .{ + .add_options = opts, + .position = .floating, + .parent = parent_ni, + .prev = parent_ni.lastHeader(mf), + }); + } + /// Adds a header child node to `parent_ni`. Returns the index of the new child. + /// + /// Asserts that `parent_ni` has no existing header children. + pub fn addOnlyHeaderChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { + if (parent_ni.first(mf).unwrap()) |first_ni| { + assert(first_ni.position(mf) != .header); // `parent_ni` already has a header child + } + return parent_ni.addHeaderChildAfter(mf, gpa, .none, opts); + } + /// Adds a header child node to `parent_ni`. Returns the index of the new child. + /// + /// If `prev_oni` is `.none`, the new child is placed at the very start of the parent, + /// before any existing header nodes. + /// + /// Otherwise, asserts that `prev_oni` is a header node and a child of `parent_ni`, and + /// places the new child node immediately after `prev_oni`. + pub fn addHeaderChildAfter(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, prev_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index { + return mf.addNode(gpa, .{ + .add_options = opts, + .position = .header, + .parent = parent_ni, + .prev = prev_oni, + }); + } + /// Adds a footer child node to `parent_ni`. Returns the index of the new child. + /// + /// Asserts that `parent_ni` has no existing footer children. + pub fn addOnlyFooterChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { + if (parent_ni.last(mf).unwrap()) |last_ni| { + assert(last_ni.position(mf) != .footer); // `parent_ni` already has a footer child + } + return parent_ni.addFooterChildBefore(mf, gpa, .none, opts); + } + /// Adds a footer child node to `parent_ni`. Returns the index of the new child. + /// + /// If `next_oni` is `.none`, the new child is placed at the very end of the parent, after + /// any existing footer nodes. + /// + /// Otherwise, asserts that `next_oni` is a footer node and a child of `parent_ni`, and + /// places the new child node immediately before `next_oni`. + pub fn addFooterChildBefore(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, next_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index { + const prev_oni: Node.Index.Optional = prev: { + const next_ni = next_oni.unwrap() orelse { + break :prev parent_ni.last(mf); + }; + break :prev next_ni.prev(mf); + }; + return mf.addNode(gpa, .{ + .add_options = opts, + .position = .footer, + .parent = parent_ni, + .prev = prev_oni, + }); + } + + /// Alias for `Optional.wrap`, provided for convenience when a result type is not available. + pub const toOptional = Optional.wrap; + + pub fn parent(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { return ni.get(mf).parent; } - pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index { + pub fn first(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + return ni.get(mf).first; + } + + pub fn last(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + return ni.get(mf).last; + } + + fn lastHeader(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + var header_ni = ni.first(mf).unwrap() orelse return .none; + if (header_ni.position(mf) != .header) return .none; + while (true) { + const next_ni = header_ni.next(mf).unwrap() orelse break; + if (next_ni.position(mf) != .header) break; + header_ni = next_ni; + } + return .wrap(header_ni); + } + fn firstFooter(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + var footer_ni = ni.last(mf).unwrap() orelse return .none; + if (footer_ni.position(mf) != .footer) return .none; + while (true) { + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } + return .wrap(footer_ni); + } + + /// Asserts that `ni` is not `.root`, because `Position` is meaningless for the root node. + pub fn position(ni: Node.Index, mf: *const MappedFile) Node.Position { + assert(ni != .root); + return ni.get(mf).flags.position; + } + + pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { return ni.get(mf).next; } fn setNext( - prev_ni: Node.Index, + ni: Node.Index, gpa: Allocator, - next_ni: Node.Index, + next_ni: Node.Index.Optional, mf: *MappedFile, ) Allocator.Error!void { - assert(prev_ni != .none); - const prev_next = &prev_ni.get(mf).next; - if (prev_next.* == next_ni) return; - prev_next.* = next_ni; - try prev_ni.nextMoved(gpa, mf); + const next_ptr = &ni.get(mf).next; + if (next_ptr.* == next_ni) return; + next_ptr.* = next_ni; + try ni.nextMoved(gpa, mf); } - pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index { + pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { return ni.get(mf).prev; } - pub fn ChildIterator(comptime direction: enum { prev, next }) type { - return struct { - mf: *const MappedFile, - ni: Node.Index, - pub fn next(it: *@This()) ?Node.Index { - const ni = it.ni; - if (ni == .none) return null; - it.ni = @field(ni.get(it.mf), @tagName(direction)); - return ni; - } - }; - } - pub fn children(ni: Node.Index, mf: *const MappedFile) ChildIterator(.next) { - return .{ .mf = mf, .ni = ni.get(mf).first }; - } - pub fn reverseChildren(ni: Node.Index, mf: *const MappedFile) ChildIterator(.prev) { - return .{ .mf = mf, .ni = ni.get(mf).last }; - } - pub fn childrenMoved(ni: Node.Index, gpa: Allocator, mf: *MappedFile) Allocator.Error!void { - var child_ni = ni.get(mf).last; - while (child_ni != .none) { + var child_oni = ni.get(mf).last; + while (child_oni.unwrap()) |child_ni| { try child_ni.moved(gpa, mf); - child_ni = child_ni.get(mf).prev; + child_oni = child_ni.get(mf).prev; } } pub fn hasMoved(ni: Node.Index, mf: *const MappedFile) bool { var parent_ni = ni; - while (parent_ni != Node.Index.root) { + while (parent_ni != .root) { const parent_node = parent_ni.get(mf); if (!parent_node.flags.bubbles_moved) break; if (parent_node.flags.moved) return true; - parent_ni = parent_node.parent; + parent_ni = parent_node.parent.unwrap().?; } return false; } @@ -263,9 +517,8 @@ pub const Node = extern struct { if (ni.hasMoved(mf)) return; const node = ni.get(mf); node.flags.moved = true; - switch (node.prev) { - .none => {}, - else => |prev_ni| prev_ni.nextMovedAssumeCapacity(mf), + if (node.prev.unwrap()) |prev_ni| { + prev_ni.nextMovedAssumeCapacity(mf); } if (node.flags.resized or node.flags.next_moved) return; mf.updates.appendAssumeCapacity(ni); @@ -314,12 +567,18 @@ pub const Node = extern struct { mf.update_prog_node.increaseEstimatedTotalItems(1); } - pub fn alignment(ni: Node.Index, mf: *const MappedFile) std.mem.Alignment { + pub fn alignment(ni: Node.Index, mf: *const MappedFile) Alignment { return ni.get(mf).flags.alignment; } - fn setLocationAssumeCapacity(ni: Node.Index, mf: *MappedFile, offset: u64, size: u64) void { + fn setLocation(ni: Node.Index, mf: *MappedFile, gpa: Allocator, offset: u64, size: u64) Allocator.Error!void { + try mf.large.ensureUnusedCapacity(gpa, 2); + try mf.updates.ensureUnusedCapacity(gpa, 2); const node = ni.get(mf); + if (node.flags.position == .floating) { + assert(node.flags.alignment.check(offset)); + } + assert(node.flags.alignment.check(size)); if (size == 0) node.flags.has_content = false; switch (node.location()) { .small => |small| { @@ -361,8 +620,11 @@ pub const Node = extern struct { while (true) { const parent_node = parent_ni.get(mf); if (set_has_content) parent_node.flags.has_content = true; - if (parent_ni == .none) break; - parent_ni = parent_node.parent; + if (parent_ni == .root) { + assert(parent_node.parent == .none); + break; + } + parent_ni = parent_node.parent.unwrap().?; const parent_offset, _ = parent_ni.location(mf).resolve(mf); offset += parent_offset; } @@ -379,62 +641,46 @@ pub const Node = extern struct { return mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)]; } - pub fn resize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void { - mf.resizeNode(gpa, ni, size) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; - var writers_it = mf.writers.first; - while (writers_it) |writer_node| : (writers_it = writer_node.next) { - const w: *Node.Writer = @fieldParentPtr("writer_node", writer_node); - w.interface.buffer = w.ni.slice(mf); + /// Ensures that the size of `ni` is at least `min_size`. Valid for any node. + /// + /// Applies `growth_factor` if necessary (so the caller should *not* apply `growth_factor`). + pub fn ensureMinimumSize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, min_size: u64) Error!void { + _, const current_size = ni.location(mf).resolve(mf); + if (current_size >= min_size) return; + const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor); + try mf.growNode(gpa, ni, new_size, .minimum); + mf.updateWriters(); + } + + /// Sets the size of `ni` to exactly `size`. + /// + /// Asserts that `ni` is a leaf node, i.e. has no children. + /// + /// Asserts that `size` is aligned to `ni.alignment(mf)`. + pub fn resizeLeaf(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void { + assert(ni.first(mf) == .none); + // The alignment of `size` is asserted by `shrinkLeafNode` and `growNode`. + _, const old_size = ni.location(mf).resolve(mf); + switch (std.math.order(size, old_size)) { + .lt => try mf.shrinkLeafNode(gpa, ni, size), + .eq => {}, // `old_size` must be well-aligned, so `size` is too + .gt => try mf.growNode(gpa, ni, size, .exact), } + mf.updateWriters(); } - pub const RealignNodeOptions = struct { - /// Shift the node backwards if possible - try_backwards: bool = false, - }; - - /// Moves and expands a node such that its offset and size are aligned to `new_alignment`. - /// Asserts that `ni` is not `Node.Index.root`. + /// Updates a node's alignment to exactly `new_alignment`. Valid for any node. + /// + /// If the node's current offset or size is not sufficiently aligned, it will be moved + /// and/or resized to match the new alignment. The node's size may be increased by any + /// amount, as if `ensureMinimumSize` were used. pub fn realign( ni: Node.Index, mf: *MappedFile, gpa: Allocator, - new_alignment: std.mem.Alignment, - opts: RealignNodeOptions, + new_alignment: Alignment, ) Error!void { - mf.realignNode(gpa, ni, new_alignment, opts) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; - mf.updateWriters(); - } - - /// Shrink a node to `size`, exactly. - /// Asserts that the new size can contain all the children. - /// If `shift_next` is set, then the following node is shifted backwards into - /// the free space as much as alignment allows. - /// Asserts that `size` is >= the end of the last child node. - pub fn shrink( - ni: Node.Index, - mf: *MappedFile, - gpa: Allocator, - size: u64, - shift_next: bool, - ) Error!void { - try mf.shrinkNode(gpa, ni, size, shift_next); + try mf.realignNode(gpa, ni, new_alignment); mf.updateWriters(); } @@ -538,16 +784,9 @@ pub const Node = extern struct { file_reader.pos, w.ni.fileLocation(w.mf, true).offset + interface.end, limit.minInt(interface.unusedCapacityLen()), - ) catch |err| switch (err) { - error.Canceled => |e| { - w.err = e; - return error.WriteFailed; - }, - else => |e| { - w.mf.io_err = e; - w.err = error.MappedFileIo; - return error.WriteFailed; - }, + ) catch |err| { + w.err = err; + return error.WriteFailed; }); if (n == 0) return error.Unimplemented; file_reader.pos += n; @@ -574,10 +813,8 @@ pub const Node = extern struct { unused_capacity: usize, ) Io.Writer.Error!void { _ = preserve; - const total_capacity = interface.end + unused_capacity; - if (interface.buffer.len >= total_capacity) return; const w: *Writer = @fieldParentPtr("interface", interface); - w.ni.resize(w.mf, w.gpa, total_capacity +| total_capacity / growth_factor) catch |err| { + w.ni.ensureMinimumSize(w.mf, w.gpa, interface.end + unused_capacity) catch |err| { w.err = err; return error.WriteFailed; }; @@ -585,617 +822,1265 @@ pub const Node = extern struct { }; comptime { - if (!std.debug.runtime_safety) std.debug.assert(@sizeOf(Node) == 32); + if (!std.debug.runtime_safety) assert(@sizeOf(Node) == 32); } }; +/// Asserts that `opts.position` is compatible with `opts.prev` (i.e. that this addition will not +/// violate the requirement that header nodes come before floating nodes come before footer nodes). fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { - parent: Node.Index = .none, - prev: Node.Index = .none, - next: Node.Index = .none, - offset: u64 = 0, - add_node: AddNodeOptions, -}) (Allocator.Error || Io.Cancelable || IoError)!Node.Index { + add_options: Node.AddOptions, + position: Node.Position, + parent: Node.Index, + /// If `position == .floating`, this is just used as an initial value, and may be immediately + /// replaced when finding a location for this node. In this case, it is still necessary that + /// `prev` be compatible with `position` (so `prev` must be either a floating node or the last + /// header node in `parent`). + prev: Node.Index.Optional, +}) Error!Node.Index { mf.nodes_lock.assertUnlocked(); - const location_tag: Node.Location.Tag, const location_payload: Node.Location.Payload = location: { - if (std.math.cast(u32, opts.offset)) |small_offset| break :location .{ .small, .{ - .small = .{ .offset = small_offset, .size = 0 }, - } }; - try mf.large.ensureUnusedCapacity(gpa, 2); - defer mf.large.appendSliceAssumeCapacity(&.{ opts.offset, 0 }); - break :location .{ .large, .{ .large = .{ .index = mf.large.items.len } } }; + + try mf.nodes.ensureUnusedCapacity(gpa, 1); + try mf.large.ensureUnusedCapacity(gpa, 2); + + const new_ni: Node.Index = new: { + if (mf.free_ni.unwrap()) |free_ni| { + mf.free_ni = free_ni.get(mf).next; + break :new free_ni; + } + const new_ni: Node.Index = @fromBackingInt(@intCast(mf.nodes.items.len)); + _ = mf.nodes.addOneAssumeCapacity(); + break :new new_ni; }; - const free_ni: Node.Index, const free_node = free: switch (mf.free_ni) { - .none => .{ @fromBackingInt(@intCast(mf.nodes.items.len)), mf.nodes.addOneAssumeCapacity() }, - else => |free_ni| { - const free_node = free_ni.get(mf); - mf.free_ni = free_node.next; - break :free .{ free_ni, free_node }; + + const next_oni: Node.Index.Optional = if (opts.prev.unwrap()) |prev_ni| next: { + assert(prev_ni.parent(mf) == opts.parent.toOptional()); // `prev` is not a child of `parent` + break :next prev_ni.get(mf).next; + } else opts.parent.first(mf); + + // Validate node ordering + switch (opts.position) { + .floating => { + if (opts.prev.unwrap()) |prev_ni| { + assert(prev_ni.position(mf) != .footer); // tried to add floating node after footer node + } + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.position(mf) != .header); // tried to add floating node before header node + } + }, + .header => if (opts.prev.unwrap()) |prev_ni| { + switch (prev_ni.position(mf)) { + .header => {}, + .floating => unreachable, // tried to add header node after floating node + .footer => unreachable, // tried to add header node after footer node + } + }, + .footer => if (next_oni.unwrap()) |next_ni| { + switch (next_ni.position(mf)) { + .header => unreachable, // tried to add footer node before header node + .floating => unreachable, // tried to add footer node before floating node + .footer => {}, + } }, - }; - switch (opts.prev) { - .none => opts.parent.get(mf).first = free_ni, - else => |prev_ni| try prev_ni.setNext(gpa, free_ni, mf), - } - switch (opts.next) { - .none => opts.parent.get(mf).last = free_ni, - else => |next_ni| next_ni.get(mf).prev = free_ni, } - free_node.* = .{ - .parent = opts.parent, - .prev = opts.prev, - .next = opts.next, + + // Initialize the node as empty with alignment 1 + const location: Node.Location = loc: { + const offset: u64 = switch (opts.position) { + .header, .floating => offset: { + const prev_ni = opts.prev.unwrap() orelse break :offset 0; + const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); + break :offset prev_offset + prev_size; + }, + .footer => offset: { + const next_ni = next_oni.unwrap() orelse { + _, const parent_size = opts.parent.location(mf).resolve(mf); + break :offset parent_size; + }; + const next_offset, _ = next_ni.location(mf).resolve(mf); + break :offset next_offset; + }, + }; + if (std.math.cast(u32, offset)) |small_offset| { + break :loc .{ .small = .{ .offset = small_offset, .size = 0 } }; + } + const large_index = mf.large.items.len; + mf.large.appendSliceAssumeCapacity(&.{ offset, 0 }); + break :loc .{ .large = .{ .index = large_index } }; + }; + new_ni.get(mf).* = .{ + .parent = .wrap(opts.parent), + .prev = .none, + .next = .none, .first = .none, .last = .none, .flags = .{ - .location_tag = location_tag, + .position = opts.position, .alignment = .@"1", - .fixed = opts.add_node.fixed, - .moved = true, - .resized = true, - .next_moved = true, + .bubbles_moved = opts.add_options.bubbles_moved, + .enable_next_moved = opts.add_options.enable_next_moved, + .location_tag = location, + .moved = false, + .resized = false, + .next_moved = false, .has_content = false, - .bubbles_moved = opts.add_node.bubbles_moved, - .enable_next_moved = opts.add_node.enable_next_moved, }, - .location_payload = location_payload, + .location_payload = switch (location) { + .small => |small| .{ .small = small }, + .large => |large| .{ .large = large }, + }, }; - { - defer { - free_node.flags.moved = false; - free_node.flags.resized = false; - free_node.flags.next_moved = false; - } - try mf.realignNode(gpa, free_ni, opts.add_node.alignment, .{}); - try mf.resizeNode(gpa, free_ni, opts.add_node.size); + try mf.addNodesToChildListBefore(gpa, next_oni, new_ni, new_ni); + + try mf.realignNode(gpa, new_ni, opts.add_options.alignment); + if (opts.add_options.size > 0) { + try mf.growNode(gpa, new_ni, opts.add_options.size, .exact); } mf.updateWriters(); - if (opts.add_node.moved) try free_ni.moved(gpa, mf); - if (opts.add_node.resized) try free_ni.resized(gpa, mf); - if (opts.add_node.next_moved) try free_ni.nextMoved(gpa, mf); - return free_ni; -} -pub const AddNodeOptions = struct { - size: u64 = 0, - alignment: std.mem.Alignment = .@"1", - fixed: bool = false, - moved: bool = false, - resized: bool = false, - next_moved: bool = false, - bubbles_moved: bool = true, - enable_next_moved: bool = false, -}; + new_ni.get(mf).flags.moved = false; + new_ni.get(mf).flags.resized = false; + new_ni.get(mf).flags.next_moved = false; -pub fn addOnlyChildNode( - mf: *MappedFile, - gpa: Allocator, - parent_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const parent = parent_ni.get(mf); - assert(parent.first == .none and parent.last == .none); - return mf.addNode(gpa, .{ - .parent = parent_ni, - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; -} + if (opts.add_options.moved) try new_ni.moved(gpa, mf); + if (opts.add_options.resized) try new_ni.resized(gpa, mf); + if (opts.add_options.next_moved) try new_ni.nextMoved(gpa, mf); -pub fn addFirstChildNode( - mf: *MappedFile, - gpa: Allocator, - parent_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const parent = parent_ni.get(mf); - return mf.addNode(gpa, .{ - .parent = parent_ni, - .next = parent.first, - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; + return new_ni; } -pub fn addLastChildNode( +fn shrinkLeafNode( mf: *MappedFile, gpa: Allocator, - parent_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const parent = parent_ni.get(mf); - return mf.addNode(gpa, .{ - .parent = parent_ni, - .prev = parent.last, - .offset = offset: switch (parent.last) { - .none => 0, - else => |last_ni| { - const last_offset, const last_size = last_ni.location(mf).resolve(mf); - break :offset last_offset + last_size; + ni: Node.Index, + new_size: u64, +) Error!void { + mf.nodes_lock.assertUnlocked(); + + const old_offset, const old_size = ni.location(mf).resolve(mf); + + assert(new_size < old_size); + assert(ni.alignment(mf).check(new_size)); + assert(ni.first(mf) == .none); // `ni` must be a leaf node + + const parent_ni = ni.parent(mf).unwrap() orelse { + assert(ni == .root); + mf.memory_map.write(mf.io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; + }; + mf.memory_map.file.setLength(mf.io, new_size) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; }, - }, - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, + }; + try mf.ensureTotalCapacityPrecise(@intCast(new_size)); + try ni.setLocation(mf, gpa, old_offset, new_size); + return; }; -} -pub fn addNodeAfter( - mf: *MappedFile, - gpa: Allocator, - prev_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - assert(prev_ni != .none); - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const prev = prev_ni.get(mf); - const prev_offset, const prev_size = prev.location().resolve(mf); - return mf.addNode(gpa, .{ - .parent = prev.parent, - .prev = prev_ni, - .next = prev.next, - .offset = prev_offset + prev_size, - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; + switch (ni.position(mf)) { + .header => { + const shift = old_size - new_size; + + try ni.setLocation(mf, gpa, old_offset, new_size); + + // We need to shift backwards all header nodes following us. + const next_header_ni = ni.next(mf).unwrap() orelse return; + if (next_header_ni.position(mf) != .header) return; + + var header_ni = next_header_ni; + while (true) { + const old_header_off, const old_header_size = header_ni.location(mf).resolve(mf); + try header_ni.setLocation(mf, gpa, old_header_off - shift, old_header_size); + + const next_ni = header_ni.next(mf).unwrap() orelse break; + if (next_ni.position(mf) != .header) break; + header_ni = next_ni; + } + + // Now we must shift the actual header bytes of those nodes backwards. + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + const move_src_off = old_offset + old_size; + const move_dest_off = old_offset + new_size; + assert(next_header_ni.location(mf).resolve(mf)[0] == move_dest_off); // `move_dest_off` because we already updated the location + const move_size = size: { + // `header_ni` is the last header in the parent. + const last_off, const last_size = header_ni.location(mf).resolve(mf); + const move_end = last_off + last_size; + break :size move_end - move_dest_off; // `move_dest_off` because we already updated the location + }; + try mf.moveRange( + parent_file_off + move_src_off, + parent_file_off + move_dest_off, + move_size, + ); }, - }; -} + .floating => { + try ni.setLocation(mf, gpa, old_offset, new_size); + }, + .footer => { + const shift = old_size - new_size; -fn shrinkNode( - mf: *MappedFile, - gpa: Allocator, - ni: Node.Index, - size: u64, - shift_next: bool, -) !void { - mf.nodes_lock.assertUnlocked(); - const node = ni.get(mf); - const old_offset, _ = node.location().resolve(mf); + const new_offset = old_offset + shift; + try ni.setLocation(mf, gpa, new_offset, new_size); - // This would require unmapping first - assert(ni != Node.Index.root); + const prev_footers_size = prev_footers_size: { + // We need to shift forwards all footer nodes preceding us. + const prev_footer_ni = ni.prev(mf).unwrap() orelse { + break :prev_footers_size 0; + }; + if (prev_footer_ni.position(mf) != .footer) { + break :prev_footers_size 0; + } - if (node.last != .none) { - const last = node.last.get(mf); - const last_offset, const last_size = last.location().resolve(mf); - assert(last_offset + last_size > size); - } + var footer_ni = prev_footer_ni; + while (true) { + const old_footer_off, const old_footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_off + shift, old_footer_size); - try mf.large.ensureUnusedCapacity(gpa, 4); - try mf.updates.ensureUnusedCapacity(gpa, 4); - - ni.setLocationAssumeCapacity(mf, old_offset, size); - if (!shift_next or node.next == .none) return; - - const next = node.next.get(mf); - const old_next_offset, const next_size = next.location().resolve(mf); - const padding = old_next_offset - (old_offset + size); - const new_next_offset = next.flags.alignment.forward(@intCast(old_next_offset - padding)); - - if (next.flags.has_content and new_next_offset < old_next_offset) { - const old_file_offset = node.next.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - old_next_offset) + new_next_offset; - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(next_size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(next_size)], - ); - @memset(mf.memory_map.memory[@intCast(new_file_offset + next_size)..@intCast(old_file_offset + next_size)], 0); - } + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } + + // `footer_ni` is the first footer in the parent. This expression gets its *new* + // offset because we already did the `setLocation` calls. + const first_footer_new_offset = footer_ni.location(mf).resolve(mf)[0]; - node.next.setLocationAssumeCapacity(mf, new_next_offset, next_size); + break :prev_footers_size new_offset - first_footer_new_offset; + }; + + // Now we must shift the actual footer bytes forwards, including our own. + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + old_offset - prev_footers_size, + parent_file_offset + new_offset - prev_footers_size, + prev_footers_size + new_size, + ); + }, + } } -fn resizeNode( +const GrowMode = enum { exact, minimum }; + +/// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`. +/// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`. +/// +/// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!). +/// +/// Asserts that `new_size` is greater than the current size of `ni`. +fn growNode( mf: *MappedFile, gpa: Allocator, ni: Node.Index, - requested_size: u64, -) (Allocator.Error || Io.Cancelable || IoError)!void { + new_size: u64, + grow_mode: GrowMode, +) Error!void { mf.nodes_lock.assertUnlocked(); - const io = mf.io; + const node = ni.get(mf); + const old_offset, const old_size = node.location().resolve(mf); - const new_size = node.flags.alignment.forward(@intCast(requested_size)); - - // Resize the entire file - if (ni == Node.Index.root) { - try mf.ensureCapacityForSetLocation(gpa); - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + + assert(node.flags.alignment.check(old_size)); + assert(node.flags.alignment.check(new_size)); + assert(new_size > old_size); + + const parent_ni = node.parent.unwrap() orelse { + assert(ni == .root); + + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + return; + } + + mf.memory_map.write(mf.io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; + }; + mf.memory_map.file.setLength(mf.io, new_size) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, }; - try mf.memory_map.file.setLength(io, new_size); - try mf.ensureTotalCapacityInner(@intCast(new_size)); - ni.setLocationAssumeCapacity(mf, old_offset, new_size); + try mf.ensureTotalCapacityPrecise(@intCast(new_size)); + try ni.setLocation(mf, gpa, old_offset, new_size); + // We need to move any footers to be at the *new* end of the file. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = old_size - old_footers_offset; + try mf.moveRange( + old_footers_offset, + old_footers_offset + (new_size - old_size), + footers_size, + ); + // Also update the footers' locations. + var cur_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + } return; + }; + + switch (node.flags.position) { + .header => { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + return; + } + + try mf.ensureAdditionalHeaderCapacity(gpa, parent_ni, new_size - old_size); + + // `old_offset` is still valid because header nodes don't move when the parent resizes. + + const last_header_ni: Node.Index = last_header: { + var header_ni = ni; + while (true) { + const next_ni = header_ni.next(mf).unwrap() orelse break; + if (next_ni.position(mf) != .header) break; + header_ni = next_ni; + } + break :last_header header_ni; + }; + const last_header_offset, const last_header_size = last_header_ni.location(mf).resolve(mf); + const old_headers_size = last_header_offset + last_header_size; + + // This is the first footer *inside* of `ni`. + const first_sub_footer_oni = ni.firstFooter(mf); + const sub_footers_size = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + break :size old_size - first_sub_footer_offset; + }; + + // We need to shift two things forwards; any header nodes which follow us, and any + // footer nodes *within* us (since they need to be at the end of our new size). + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + old_offset + old_size - sub_footers_size, + parent_file_offset + old_offset + new_size - sub_footers_size, + old_headers_size - (old_offset + old_size - sub_footers_size), + ); + + // Any footers inside of us have had their offsets changed due to us growing: + if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { + var cur_ni = first_sub_footer_ni; + while (true) { + const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation( + mf, + gpa, + old_sub_footer_offset + (new_size - old_size), + sub_footer_size, + ); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + } + + // Update the offsets of all header nodes following us: + { + var moved_header_ni = last_header_ni; + while (moved_header_ni != ni) { + assert(moved_header_ni.position(mf) == .header); + const moved_header_offset, const moved_header_size = moved_header_ni.location(mf).resolve(mf); + try moved_header_ni.setLocation( + mf, + gpa, + moved_header_offset - old_size + new_size, + moved_header_size, + ); + moved_header_ni = moved_header_ni.prev(mf).unwrap().?; + } + } + + // Finally, update our own size: + try ni.setLocation(mf, gpa, old_offset, new_size); + return; + }, + .floating => { + try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode); + }, + .footer => { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + return; + } + + try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size); + + const first_footer_ni: Node.Index = first_footer: { + var footer_ni = ni; + while (true) { + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } + break :first_footer footer_ni; + }; + + // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself + // a footer within its parent). + const first_sub_footer_oni = ni.firstFooter(mf); + const sub_footers_size = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + break :size old_size - first_sub_footer_offset; + }; + + _, const parent_size = parent_ni.location(mf).resolve(mf); + + const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0]; + const new_footers_size = old_footers_size - old_size + new_size; + + // Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node + // itself needs to shift its contents, because our offset was shifted backwards by + // `new_size - old_size`, and the added bytes should go at the end of this footer node. + // However, if we *contain* any footer nodes, they need to stay at the end of `ni`, so + // we *shouldn't* shift *that* data. + const old_footers_start = parent_size - old_footers_size; + const new_footers_start = parent_size - new_footers_size; + const end_offset = node.location().resolve(mf)[0] + old_size; + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + old_footers_start, + parent_file_offset + new_footers_start, + end_offset - old_footers_start - sub_footers_size, + ); + + // Update our own offset and size: + try ni.setLocation(mf, gpa, end_offset - new_size, new_size); + + // Any footers inside of us have had their offsets changed due to us growing: + if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { + var cur_ni = first_sub_footer_ni; + while (true) { + const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation( + mf, + gpa, + old_sub_footer_offset + (new_size - old_size), + sub_footer_size, + ); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + } + + // Finally, update the offsets of every footer before us: + if (node.prev.unwrap()) |prev_ni| { + var maybe_footer_ni = prev_ni; + while (true) { + switch (maybe_footer_ni.position(mf)) { + .header, .floating => break, + .footer => {}, + } + const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf); + try maybe_footer_ni.setLocation( + mf, + gpa, + moved_footer_offset + old_size - new_size, + moved_footer_size, + ); + maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break; + } + } + + return; + }, } - const parent = node.parent.get(mf); - _, var old_parent_size = parent.location().resolve(mf); - const trailing_end = trailing_end: switch (node.next) { - .none => old_parent_size, - else => |next_ni| { +} + +/// Moves a floating node to an unused region with the given size, which may be greater than the +/// current size. If `new_alignment` is not `null`, then the offset and size of the new region will +/// have that alignment instead of `ni.alignment(mf)`. +/// +/// Asserts that `ni` is a floating node (and not `.root`). +/// +/// Asserts that `new_size` is aligned to `new_alignment orelse ni.alignment(mf)`. +/// +/// Asserts that `new_size` is greater than or equal to the current size of `ni`. +fn growFloatingNodeWithAlignment( + mf: *MappedFile, + gpa: Allocator, + ni: Node.Index, + new_alignment: ?Alignment, + new_size: u64, + grow_mode: GrowMode, +) Error!void { + mf.nodes_lock.assertUnlocked(); + + const parent_ni = ni.parent(mf).unwrap().?; // `ni` cannot be `.root` + const old_offset, const old_size = ni.location(mf).resolve(mf); + + const alignment = new_alignment orelse ni.alignment(mf); + + assert(new_size >= old_size); + assert(ni.position(mf) == .floating); + assert(alignment.check(new_size)); + + grow_in_place: { + if (!alignment.check(old_offset)) { + break :grow_in_place; + } + const limit: u64 = limit: { + const next_ni = ni.next(mf).unwrap() orelse break :limit parent_ni.location(mf).resolve(mf)[1]; const next_offset, _ = next_ni.location(mf).resolve(mf); - break :trailing_end next_offset; - }, - }; - assert(old_offset + old_size <= trailing_end); - if (old_offset + new_size <= trailing_end) { - // Expand the node into trailing free space - try mf.ensureCapacityForSetLocation(gpa); - ni.setLocationAssumeCapacity(mf, old_offset, new_size); + break :limit next_offset; + }; + if (old_offset + new_size > limit) { + break :grow_in_place; // the parent is not big enough + } + // Great, we can grow this node without changing its offset or moving any siblings. + try ni.setLocation(mf, gpa, old_offset, new_size); + // If we have any footers, we need to move them to the end of our new size, and update their + // offsets accordingly. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var cur_ni = first_footer_ni; + var footers_have_content = false; + while (true) { + footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + if (footers_have_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + // This gets the *new* offset because we already updated the offsets above. + const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = new_size - new_footers_offset; + try mf.moveRange( + parent_file_off + old_offset + old_size - footers_size, + parent_file_off + old_offset + new_size - footers_size, + footers_size, + ); + } + } return; } - insert_range: { - if (!is_linux) break :insert_range; - if (mf.flags.fallocate_insert_range_unsupported) break :insert_range; - // We need the node to be aligned to `mf.flags.block_size` in the file in order to use this - // fast path. It is not sufficient to check `node.flags.alignment`, because that doesn't - // necessarily mean that all *parent* nodes are equally aligned; instead we must compute the - // actual file offset. - const range_file_offset = ni.fileLocation(mf, false).offset + old_size; - const range_size = node.flags.alignment.forward( - @intCast(requested_size +| requested_size / growth_factor), - ) - old_size; - if (!mf.flags.block_size.check(@intCast(range_file_offset))) break :insert_range; - if (!mf.flags.block_size.check(@intCast(range_size))) break :insert_range; + const new_loc: struct { + offset: u64, + prev: Node.Index.Optional, + } = new_loc: { + _, const parent_size = parent_ni.location(mf).resolve(mf); - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, - }; - // Ask the filesystem driver to insert extents into the file without copying any data - const last_offset, const last_size = parent.last.location(mf).resolve(mf); - const last_end = last_offset + last_size; - assert(last_end <= old_parent_size); - _, const file_size = Node.Index.root.location(mf).resolve(mf); - while (true) switch (linux.errno(switch (std.math.order(range_file_offset, file_size)) { - .lt => linux.fallocate( - mf.memory_map.file.handle, - linux.FALLOC.FL_INSERT_RANGE, - @intCast(range_file_offset), - @intCast(range_size), - ), - .eq => linux.ftruncate(mf.memory_map.file.handle, @intCast(range_file_offset + range_size)), - .gt => unreachable, - })) { - .SUCCESS => { - var enclosing_ni = ni; - while (true) { - try mf.ensureCapacityForSetLocation(gpa); - const enclosing = enclosing_ni.get(mf); - const enclosing_offset, const old_enclosing_size = - enclosing.location().resolve(mf); - const new_enclosing_size = old_enclosing_size + range_size; - enclosing_ni.setLocationAssumeCapacity(mf, enclosing_offset, new_enclosing_size); - if (enclosing_ni == Node.Index.root) { - assert(enclosing_offset == 0); - try mf.ensureTotalCapacityInner(@intCast(new_enclosing_size)); - break; + { + // See if there's space at the start of the parent. + const last_header_oni = parent_ni.lastHeader(mf); + const headers_end: u64 = if (last_header_oni.unwrap()) |last_header_ni| headers_end: { + const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf); + break :headers_end last_header_off + last_header_size; + } else 0; + const limit: u64 = limit: { + const after_header_oni: Node.Index.Optional = after_header: { + if (last_header_oni.unwrap()) |last_header_ni| { + break :after_header last_header_ni.next(mf); } - var after_ni = enclosing.next; - while (after_ni != .none) { - try mf.ensureCapacityForSetLocation(gpa); - const after = after_ni.get(mf); - const after_offset, const after_size = after.location().resolve(mf); - after_ni.setLocationAssumeCapacity( - mf, - range_size + after_offset, - after_size, - ); - after_ni = after.next; - } - enclosing_ni = enclosing.parent; + break :after_header parent_ni.first(mf); + }; + if (after_header_oni.unwrap()) |after_header_ni| { + break :limit after_header_ni.location(mf).resolve(mf)[0]; + } else { + break :limit parent_size; } + }; + if (alignment.forward(headers_end) + new_size <= limit) { + // There's space here! + break :new_loc .{ + // Put ourselves at the *end* of this range, so that the free space remains at the start of the parent. + .offset = alignment.backward(limit - new_size), + .prev = last_header_oni, + }; + } + } + + // Otherwise, use space at the end of the parent, or make space there if necessary. + + const first_footer_oni = parent_ni.firstFooter(mf); + + // We know there is a node before the footer[s], because `ni` itself is such a node. + const prev_ni: Node.Index = if (first_footer_oni.unwrap()) |first_footer_ni| prev: { + break :prev first_footer_ni.prev(mf).unwrap().?; + } else prev: { + break :prev parent_ni.last(mf).unwrap().?; + }; + + const result_offset: u64 = result_offset: { + if (prev_ni == ni and alignment.check(old_offset)) { + // We're already at the end of the parent, and our offset is already well-aligned. + // The only reason we didn't simply grow in place earlier is that the parent wasn't + // big enough---but now we're resizing the parent anyway, so growing in-place stops + // us from unnecessarily moving! + break :result_offset old_offset; + } + // Otherwise, just move after the last node. + const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); + break :result_offset alignment.forward(prev_offset + prev_size); + }; + + const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| footers_size: { + const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); + break :footers_size parent_size - first_footer_offset; + } else 0; + + const min_parent_size = result_offset + new_size + footers_size; + if (parent_size < min_parent_size) { + // Okay, at this point we're planning to expand the parent---so before we actually do + // that, let's first try the Linux "insert range" fast path. We didn't try it before now + // because it would have been more efficient to just move ourselves into existing space. + // + // If we were given a custom alignment, we cannot pass `grow_mode` directly into the + // "insert range" path, because that function is unaware of `new_alignment`. + const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact; + if (alignment.check(old_offset) and + try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode)) + { + // The Linux fast path did our job for us! return; + } + + // Grow the parent and move to the end of the parent. + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, + ); + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + } + + break :new_loc .{ + .offset = result_offset, + .prev = .wrap(prev_ni), + }; + }; + + // We've found our new location in `parent_ni`, now to actually move ourselves there. + + // Footers need to move to a different place than the rest of our content. + const footers_size: u64, const footers_have_content: bool = footers: { + const first_footer_ni = ni.firstFooter(mf).unwrap() orelse { + break :footers .{ 0, false }; + }; + + var cur_ni = first_footer_ni; + var footers_have_content = false; + while (true) { + footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + // Our footers' offsets must change to be at the end of our new size. + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + + // This is the *new* offset because we already updated the offsets above. + const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = new_size - new_footers_offset; + + break :footers .{ footers_size, footers_have_content }; + }; + + if (ni.get(mf).flags.has_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_off + old_offset, + parent_file_off + new_loc.offset, + old_size - footers_size, + ); + if (footers_have_content) try mf.moveRange( + parent_file_off + old_offset + old_size - footers_size, + parent_file_off + new_loc.offset + new_size - footers_size, + footers_size, + ); + } else { + assert(!footers_have_content); + } + + try ni.setLocation(mf, gpa, new_loc.offset, new_size); + + if (new_loc.prev != ni.toOptional()) { + // We're potentially in a different place in `parent_ni`'s child list, so remove and re-add ourselves. + try mf.removeNodesFromChildList(gpa, ni, ni); + try mf.addNodesToChildListAfter(gpa, new_loc.prev, ni, ni); + } +} + +/// Attempts to grow `ni` to `new_size` using `FALLOCATE_FL_INSERT_RANGE` on Linux. This strategy +/// has the advantage that it does not require manually moving any bytes in the file, but has the +/// disadvantages that it may increase the file size more than necessary, and that it changes the +/// offsets of all following nodes, recursively. +/// +/// If this strategy is inapplicable or unsuitable for this operation, this function returns `false` +/// without changing any nodes' locations or invalidating any slices. +/// +/// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node +/// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`, +/// the actual new size of `ni` may be greater than `new_size`. +fn growNodeViaInsertRange( + mf: *MappedFile, + gpa: Allocator, + ni: Node.Index, + new_size: u64, + grow_mode: GrowMode, +) Error!bool { + if (!is_linux or mf.flags.fallocate_insert_range_unsupported) { + return false; + } + + _, const old_size = ni.location(mf).resolve(mf); + + // We don't compute the size of the range yet, because depending on `grow_mode` we might want to + // bump it based on our sibling and parent nodes' alignments. However, we can do an early check + // for cases where we should obviously exit. + const requested_range_size = new_size - old_size; + if (!mf.flags.block_size.check(requested_range_size)) { + // The requested size isn't exactly aligned. + switch (grow_mode) { + .exact => return false, + .minimum => { + // We can still choose to allow it by increasing the size a bit, but we shouldn't do + // that if it would *significantly* increase the requested size. + const block_size = mf.flags.block_size.toByteUnits(); + if (requested_range_size < block_size * 2) { + // Bumping this size up to the next block boundary would be a quite significant + // increase; let's not do it. + return false; + } }, + } + } + // If `grow_mode` is exact, we will use exactly this size, but if it is `.minimum`, we may bump + // the size a little more. + const min_range_size: u64 = s: { + const exact_size = new_size - old_size; + if (mf.flags.block_size.check(exact_size)) { + break :s exact_size; + } + switch (grow_mode) { + .exact => return false, + .minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) { + // We're growing by at least a few blocks, so allow ourselves to bump the size + // slightly to give it the needed alignment. + break :s mf.flags.block_size.forward(exact_size); + } else { + return false; + }, + } + }; + assert(min_range_size > 0); + assert(mf.flags.block_size.check(min_range_size)); + + const range_file_offset: u64 = range_file_offset: { + const node_file_offset = ni.fileLocation(mf, false).offset; + const last_ni = ni.last(mf).unwrap() orelse { + // If `ni` has no children (i.e. is a leaf node), we need to insert exactly at its end. + const range_file_offset = node_file_offset + old_size; + if (!mf.flags.block_size.check(range_file_offset)) { + return false; + } + break :range_file_offset range_file_offset; + }; + const first_footer_oni = ni.firstFooter(mf); + const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| size: { + const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); + break :size old_size - first_footer_offset; + } else 0; + const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) |first_footer_ni| pre_footer: { + break :pre_footer first_footer_ni.prev(mf); + } else .wrap(last_ni); + const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: { + const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf); + break :end pre_footer_off + pre_footer_size; + } else 0; + + const min_file_offset = node_file_offset + pre_footer_end; + const max_file_offset = node_file_offset + old_size - footers_size; + // We can go anywhere between `min_file_offset` and `max_file_offset`. + const candidate_file_offset = mf.flags.block_size.forward(min_file_offset); + if (candidate_file_offset > max_file_offset) { + return false; + } + break :range_file_offset candidate_file_offset; + }; + assert(mf.flags.block_size.check(range_file_offset)); + + const range_size: u64 = range_size: { + // For this strategy to be valid, the number of bytes we insert needs to be compatible with + // the alignments of all nodes following us (and following our parents, their parents, etc). + // We also probably don't want to trigger too many "node moved" events, since doing that + // repeatedly could result in a lot of extra work. Therefore, while we traverse parents and + // siblings to check their alignment requirements, we will also set an arbitrary limit on + // the number of nodes we can move, and give up if we walk more than that. + const max_moved_nodes = 32; + var num_moved: u32 = 0; + var cur_ni = ni; + // Alignment required for `range_size`: initially the block size (required for the syscall), + // then updated as we traverse based on how the operation would affect surrounding nodes. + var need_range_align: Alignment = mf.flags.block_size.max(ni.alignment(mf)); + while (true) { + // `cur_ni` will grow as a result of the range insertion. Its size must be well-aligned. + need_range_align = need_range_align.max(cur_ni.alignment(mf)); + + // Siblings following `cur_ni` don't get bigger, but their offsets change. + while (cur_ni.next(mf).unwrap()) |next_ni| { + // Only floating children need well-aligned offsets. + if (next_ni.position(mf) == .floating) { + need_range_align = need_range_align.max(next_ni.alignment(mf)); + } + num_moved += 1; + if (num_moved > max_moved_nodes) return false; + cur_ni = next_ni; + } + + // Move up to the parent. + cur_ni = cur_ni.parent(mf).unwrap() orelse break; + } + // Traversal done. We didn't hit `max_moved_nodes`, so now we can use the computed alignment + // requirement to figure out whether we're actually going to insert a range. + if (need_range_align.check(requested_range_size)) { + break :range_size requested_range_size; + } + // Perhaps we're allowed to grow by more than `requested_range_size`? + switch (grow_mode) { + .exact => return false, + .minimum => { + const candidate_range_size = need_range_align.forward(min_range_size); + // Allow growing by up to 50% more than was requested. + if (candidate_range_size <= requested_range_size +| requested_range_size / 2) { + break :range_size candidate_range_size; + } else { + return false; + } + }, + } + }; + + // This `range_size` is compatible with everyone's alignment requirements, and we won't move too + // many nodes, so let's do it! + + mf.memory_map.write(mf.io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; + }; + + // If we happen to be inserting at the very end of the file, we need to resize the file instead + // of using `FALLOCATE_FL_INSERT_RANGE`. + if (range_file_offset == Node.Index.root.location(mf).resolve(mf)[1]) { + mf.memory_map.file.setLength(mf.io, range_file_offset + range_size) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, + }; + } else { + while (true) switch (linux.errno(linux.fallocate( + mf.memory_map.file.handle, + linux.FALLOC.FL_INSERT_RANGE, + @intCast(range_file_offset), + @intCast(range_size), + ))) { + .SUCCESS => break, .INTR => continue, - .BADF, .FBIG, .INVAL => unreachable, - .IO => return error.InputOutput, - .NODEV => return error.NotFile, - .NOSPC => return error.NoSpaceLeft, .NOSYS, .OPNOTSUPP => { + // After all that setup work, it turns out the operation is actually unsupported! mf.flags.fallocate_insert_range_unsupported = true; - break :insert_range; + return false; + }, + else => |e| { + mf.io_err = switch (e) { + .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above + .BADF => unreachable, + .FBIG => unreachable, + .INVAL => unreachable, + .IO => error.InputOutput, + .NODEV => error.NotFile, + .NOSPC => error.NoSpaceLeft, + .PERM => error.PermissionDenied, + .SPIPE => error.Unseekable, + .TXTBSY => error.FileBusy, + else => std.posix.unexpectedErrno(e), + }; + return error.MappedFileIo; }, - .PERM => return error.PermissionDenied, - .SPIPE => return error.Unseekable, - .TXTBSY => return error.FileBusy, - else => |e| return std.posix.unexpectedErrno(e), }; } - if (node.next == .none) { - // As this is the last node, we simply need more space in the parent - const new_parent_size = old_offset + new_size; - try mf.resizeNode(gpa, node.parent, new_parent_size +| new_parent_size / growth_factor); - try mf.ensureCapacityForSetLocation(gpa); - ni.setLocationAssumeCapacity(mf, old_offset, new_size); - return; - } - if (!node.flags.fixed) { - // Make space at the end of the parent for this floating node - const last = parent.last.get(mf); - const last_offset, const last_size = last.location().resolve(mf); - const new_offset = node.flags.alignment.forward(@intCast(last_offset + last_size)); - const new_parent_size = new_offset + new_size; - if (new_parent_size > old_parent_size) - try mf.resizeNode(gpa, node.parent, new_parent_size +| new_parent_size / growth_factor); - try mf.ensureCapacityForSetLocation(gpa); - const next_ni = node.next; - next_ni.get(mf).prev = node.prev; - switch (node.prev) { - .none => parent.first = next_ni, - else => |prev_ni| try prev_ni.setNext(gpa, next_ni, mf), + + // We did it! Now to update all the sizes and offsets. This loop is exactly the same shape as + // above, except we're updating locations instead of checking alignments. + var cur_ni = ni; + while (true) { + const this_offset, const this_old_size = cur_ni.location(mf).resolve(mf); + if (cur_ni == .root) { + try mf.ensureTotalCapacityPrecise(@intCast(this_old_size + range_size)); } - try parent.last.setNext(gpa, ni, mf); - node.prev = parent.last; - try ni.setNext(gpa, .none, mf); - parent.last = ni; - if (node.flags.has_content) { - const parent_file_offset = node.parent.fileLocation(mf, false).offset; - try mf.moveRange( - parent_file_offset + old_offset, - parent_file_offset + new_offset, - old_size, - ); + try cur_ni.setLocation(mf, gpa, this_offset, this_old_size + range_size); + + while (cur_ni.next(mf).unwrap()) |next_ni| { + const next_old_offset, const next_size = next_ni.location(mf).resolve(mf); + try next_ni.setLocation(mf, gpa, next_old_offset + range_size, next_size); + cur_ni = next_ni; } - ni.setLocationAssumeCapacity(mf, new_offset, new_size); - return; + + cur_ni = cur_ni.parent(mf).unwrap() orelse break; } - // Search for the first floating node following this fixed node - var last_fixed_ni = ni; - var first_floating_ni = node.next; - var shift = new_size - old_size; - var max_shift_align: std.mem.Alignment = .@"1"; - var direction: enum { forward, reverse } = .forward; - while (true) { - assert(last_fixed_ni != .none); - const last_fixed = last_fixed_ni.get(mf); - assert(last_fixed.flags.fixed); - const old_last_fixed_offset, const last_fixed_size = last_fixed.location().resolve(mf); - const new_last_fixed_offset = old_last_fixed_offset + shift; - make_space: switch (first_floating_ni) { - else => { - const first_floating = first_floating_ni.get(mf); - const old_first_floating_offset, const first_floating_size = - first_floating.location().resolve(mf); - assert(old_last_fixed_offset + last_fixed_size <= old_first_floating_offset); - if (new_last_fixed_offset + last_fixed_size <= old_first_floating_offset) - break :make_space; - assert(direction == .forward); - max_shift_align = max_shift_align.max(first_floating.flags.alignment.max(last_fixed.flags.alignment)); - if (first_floating.flags.fixed) { - shift = max_shift_align.forward(@intCast( - @max(shift, first_floating_size), - )); - - // Not enough space, try the next node - last_fixed_ni = first_floating_ni; - first_floating_ni = first_floating.next; - continue; - } - // Move the found floating node to make space for preceding fixed nodes - const last = parent.last.get(mf); - const last_offset, const last_size = last.location().resolve(mf); - const new_first_floating_offset = max_shift_align.forward( - @intCast(@max(new_last_fixed_offset + last_fixed_size, last_offset + last_size)), - ); - const new_parent_size = new_first_floating_offset + first_floating_size; - if (new_parent_size > old_parent_size) { - try mf.resizeNode( - gpa, - node.parent, - new_parent_size +| new_parent_size / growth_factor, - ); - _, old_parent_size = parent.location().resolve(mf); - } - try mf.ensureCapacityForSetLocation(gpa); - if (parent.last != first_floating_ni) { - const old_last = parent.last; - first_floating.prev = old_last; - parent.last = first_floating_ni; - try old_last.setNext(gpa, first_floating_ni, mf); - try last_fixed_ni.setNext(gpa, first_floating.next, mf); - switch (first_floating.next) { - .none => {}, - else => |next_ni| next_ni.get(mf).prev = last_fixed_ni, - } - try first_floating_ni.setNext(gpa, .none, mf); - } - if (first_floating.flags.has_content) { - const parent_file_offset = - node.parent.fileLocation(mf, false).offset; - try mf.moveRange( - parent_file_offset + old_first_floating_offset, - parent_file_offset + new_first_floating_offset, - first_floating_size, - ); - } - first_floating_ni.setLocationAssumeCapacity( - mf, - new_first_floating_offset, - first_floating_size, - ); - // Continue the search after the just-moved floating node - first_floating_ni = last_fixed.next; - continue; - }, - .none => { - assert(direction == .forward); - const new_parent_size = new_last_fixed_offset + last_fixed_size; - if (new_parent_size > old_parent_size) { - try mf.resizeNode( - gpa, - node.parent, - new_parent_size +| new_parent_size / growth_factor, - ); - _, old_parent_size = parent.location().resolve(mf); - } - }, + + // The only thing left is to update the offsets of any footers inside of `ni`. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var footer_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size); + footer_ni = footer_ni.next(mf).unwrap() orelse break; } - try mf.ensureCapacityForSetLocation(gpa); - if (last_fixed_ni == ni) { - // The original fixed node now has enough space - last_fixed_ni.setLocationAssumeCapacity( - mf, - old_last_fixed_offset, - new_size, + } + + return true; +} + +/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes following its current +/// headers, so that the headers can grow into that space. +fn ensureAdditionalHeaderCapacity( + mf: *MappedFile, + gpa: Allocator, + parent_ni: Node.Index, + extra_capacity: u64, +) Error!void { + _, const parent_size = parent_ni.location(mf).resolve(mf); + + const last_header_oni = parent_ni.lastHeader(mf); + const first_footer_oni = parent_ni.firstFooter(mf); + + const headers_size: u64 = headers_size: { + const last_header_ni = last_header_oni.unwrap() orelse break :headers_size 0; + const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf); + break :headers_size last_header_off + last_header_size; + }; + + const footers_size: u64 = footers_size: { + const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; + const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); + break :footers_size parent_size - first_footer_off; + }; + + const first_floating_oni: Node.Index.Optional = if (last_header_oni.unwrap()) |last_header_ni| first_floating: { + const after_header_ni = last_header_ni.next(mf).unwrap() orelse break :first_floating .none; + break :first_floating switch (after_header_ni.position(mf)) { + .header => unreachable, + .floating => .wrap(after_header_ni), + .footer => .none, + }; + } else first_floating: { + const first_ni = parent_ni.first(mf).unwrap() orelse break :first_floating .none; + break :first_floating switch (first_ni.position(mf)) { + .header => unreachable, + .floating => .wrap(first_ni), + .footer => .none, + }; + }; + const first_floating_ni = first_floating_oni.unwrap() orelse { + // This node has only headers and footers. + const min_parent_size = headers_size + extra_capacity + footers_size; + if (parent_size < min_parent_size) { + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, ); - return; + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); } - // Move a fixed node into trailing free space - if (last_fixed.flags.has_content) { - const parent_file_offset = node.parent.fileLocation(mf, false).offset; - try mf.moveRange( - parent_file_offset + old_last_fixed_offset, - parent_file_offset + new_last_fixed_offset, - last_fixed_size, - ); + return; + }; + + const last_floating_ni = if (first_footer_oni.unwrap()) |first_footer_ni| last_floating: { + break :last_floating first_footer_ni.prev(mf).unwrap().?; + } else last_floating: { + break :last_floating parent_ni.last(mf).unwrap().?; + }; + assert(last_floating_ni.position(mf) == .floating); // we know `parent_ni` contains at least `first_floating_ni` + + // Find the first floating child, if any, which does not overlap the new header space. + const first_good_floating_oni: Node.Index.Optional = first_good_floating: { + var floating_ni = first_floating_ni; + while (true) { + const floating_offset, _ = floating_ni.location(mf).resolve(mf); + if (floating_offset >= headers_size + extra_capacity) { + break :first_good_floating .wrap(floating_ni); + } + const next_ni = floating_ni.next(mf).unwrap() orelse { + break :first_good_floating .none; + }; + switch (next_ni.position(mf)) { + .header => unreachable, // after the last header + .floating => floating_ni = next_ni, + .footer => break :first_good_floating .none, + } } - last_fixed_ni.setLocationAssumeCapacity(mf, new_last_fixed_offset, last_fixed_size); - // Retry the previous nodes now that there is enough space - first_floating_ni = last_fixed_ni; - last_fixed_ni = last_fixed.prev; - direction = .reverse; + }; + + if (first_good_floating_oni == first_floating_ni.toOptional()) { + // None of the floating children are in our way! That means there's already enough space. + return; + } + + const last_moving_ni = if (first_good_floating_oni.unwrap()) |first_good_floating_ni| last_moving: { + break :last_moving first_good_floating_ni.prev(mf).unwrap().?; + } else last_moving: { + break :last_moving last_floating_ni; + }; + + // We are going to move all nodes between `first_floating_ni` and `last_moving_ni` to the end of + // the parent. We'll move all the node data in one big block. + + const moving_offset: u64 = first_floating_ni.location(mf).resolve(mf)[0]; + const moving_size: u64 = size: { + const last_moving_off, const last_moving_size = last_moving_ni.location(mf).resolve(mf); + break :size last_moving_off + last_moving_size - moving_offset; + }; + + var moving_alignment: Alignment = .@"1"; + var moving_has_content = false; // optimization: no need to move data if it's all uninitialized + { + var cur_ni = first_floating_ni; + while (true) { + moving_alignment = moving_alignment.max(cur_ni.alignment(mf)); + moving_has_content = moving_has_content or cur_ni.get(mf).flags.has_content; + if (cur_ni == last_moving_ni) break; + cur_ni = cur_ni.next(mf).unwrap().?; + } + } + + const first_free_offset = free_offset: { + const last_floating_off, const last_floating_size = last_floating_ni.location(mf).resolve(mf); + break :free_offset @max(last_floating_off + last_floating_size, headers_size + extra_capacity); + }; + // Alignment is a little tricky here. We don't necessarily want the new offset to be aligned to + // `moving_alignment` exactly, because if (e.g.) the first floating node is align(2) and the + // second is align(4), then the overall range we're moving may not be 4-byte aligned even though + // one of the nodes is. Instead, the old and new offsets must be congruent modulo the alignment. + const aligned_dest_offset = moving_alignment.forward(first_free_offset); + const dest_offset = aligned_dest_offset + (moving_offset - moving_alignment.backward(moving_offset)); + assert(dest_offset % moving_alignment.toByteUnits() == moving_offset % moving_alignment.toByteUnits()); + + // This expression is correct because `dest_offset` is after all floating nodes (except the ones + // we're moving there of course). + const min_parent_size = dest_offset + moving_size + footers_size; + if (parent_size < min_parent_size) { + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, + ); + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + } + + if (moving_has_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_off + moving_offset, + parent_file_off + dest_offset, + moving_size, + ); + } + + // Remove everything between `first_floating_ni` and `last_moving_ni` from the linked list, then + // re-insert them in their new position. + try mf.removeNodesFromChildList(gpa, first_floating_ni, last_moving_ni); + try mf.addNodesToChildListBefore(gpa, first_footer_oni, first_floating_ni, last_moving_ni); + + // Finally, we need to update the locations of all of those nodes. + var cur_ni = first_floating_ni; + while (true) { + assert(cur_ni.position(mf) == .floating); + const old_offset, const old_size = cur_ni.location(mf).resolve(mf); + const new_offset = old_offset - moving_offset + dest_offset; + assert(cur_ni.alignment(mf).check(new_offset)); + try cur_ni.setLocation(mf, gpa, new_offset, old_size); + if (cur_ni == last_moving_ni) break; + cur_ni = cur_ni.next(mf).unwrap().?; + } +} + +/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current +/// footers, so that the footers can grow into that space. +fn ensureAdditionalFooterCapacity( + mf: *MappedFile, + gpa: Allocator, + parent_ni: Node.Index, + extra_capacity: u64, +) Error!void { + // This is way easier than the header case, because we don't need to actually move anything; we + // just need to expand the parent if there isn't space, and that will add padding after the + // parent's floating children, which is exactly where we want it. + + const first_footer_oni = parent_ni.firstFooter(mf); + + _, const parent_size = parent_ni.location(mf).resolve(mf); + + const footers_size: u64 = footers_size: { + const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; + const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); + break :footers_size parent_size - first_footer_off; + }; + + const header_and_floating_end: u64 = end: { + const before_footers_oni = if (first_footer_oni.unwrap()) |first_footer_ni| before_footers: { + break :before_footers first_footer_ni.prev(mf); + } else before_footers: { + break :before_footers parent_ni.last(mf); + }; + const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0; + const offset, const size = before_footers_ni.location(mf).resolve(mf); + break :end offset + size; + }; + + assert(header_and_floating_end + footers_size <= parent_size); + + const min_parent_size = header_and_floating_end + footers_size + extra_capacity; + if (parent_size < min_parent_size) { + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, + ); + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + } +} + +fn removeNodesFromChildList( + mf: *MappedFile, + gpa: Allocator, + first_remove_ni: Node.Index, + last_remove_ni: Node.Index, +) Allocator.Error!void { + const parent_ni = first_remove_ni.parent(mf).unwrap().?; + assert(last_remove_ni.parent(mf).unwrap().? == parent_ni); + + const prev_oni = first_remove_ni.prev(mf); + const next_oni = last_remove_ni.next(mf); + + if (prev_oni.unwrap()) |prev_ni| { + assert(prev_ni.next(mf).unwrap().? == first_remove_ni); + try prev_ni.setNext(gpa, next_oni, mf); + } else { + assert(parent_ni.first(mf).unwrap().? == first_remove_ni); + parent_ni.get(mf).first = next_oni; + } + + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.prev(mf).unwrap().? == last_remove_ni); + next_ni.get(mf).prev = prev_oni; + } else { + assert(parent_ni.last(mf).unwrap().? == last_remove_ni); + parent_ni.get(mf).last = prev_oni; + } +} +/// Assumes `first_add_ni` and `last_add_ni` are connected, and that all nodes in between them +/// already have their `parent` field correctly populated. +/// +/// To add a single node, set `first_add_ni` equal to `last_add_ni`. +fn addNodesToChildListBefore( + mf: *MappedFile, + gpa: Allocator, + /// `null` means to add at the end of the parent. + next_oni: Node.Index.Optional, + first_add_ni: Node.Index, + last_add_ni: Node.Index, +) Allocator.Error!void { + const parent_ni = first_add_ni.parent(mf).unwrap().?; + assert(last_add_ni.parent(mf).unwrap().? == parent_ni); + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.parent(mf).unwrap().? == parent_ni); + } + + const prev_oni: Node.Index.Optional = if (next_oni.unwrap()) |next_ni| prev: { + break :prev next_ni.prev(mf); + } else prev: { + break :prev parent_ni.last(mf); + }; + + first_add_ni.get(mf).prev = prev_oni; + try last_add_ni.setNext(gpa, next_oni, mf); + + if (prev_oni.unwrap()) |prev_ni| { + assert(prev_ni.next(mf) == next_oni); + try prev_ni.setNext(gpa, .wrap(first_add_ni), mf); + } else { + assert(parent_ni.first(mf) == next_oni); + parent_ni.get(mf).first = .wrap(first_add_ni); + } + + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.prev(mf) == prev_oni); + next_ni.get(mf).prev = .wrap(last_add_ni); + } else { + assert(parent_ni.last(mf) == prev_oni); + parent_ni.get(mf).last = .wrap(last_add_ni); } } +fn addNodesToChildListAfter( + mf: *MappedFile, + gpa: Allocator, + /// `null` means to add at the start of the parent. + prev_oni: Node.Index.Optional, + first_add_ni: Node.Index, + last_add_ni: Node.Index, +) Allocator.Error!void { + const next_oni: Node.Index.Optional = next: { + if (prev_oni.unwrap()) |prev_ni| break :next prev_ni.next(mf); + const parent_ni = first_add_ni.parent(mf).unwrap().?; + break :next parent_ni.first(mf); + }; + return mf.addNodesToChildListBefore(gpa, next_oni, first_add_ni, last_add_ni); +} fn realignNode( mf: *MappedFile, gpa: Allocator, ni: Node.Index, - new_alignment: std.mem.Alignment, - opts: Node.Index.RealignNodeOptions, -) (Allocator.Error || Io.Cancelable || IoError)!void { + new_alignment: Alignment, +) Error!void { mf.nodes_lock.assertUnlocked(); - const node = ni.get(mf); - { - const prev_alignment = node.flags.alignment; - node.flags.alignment = new_alignment; - if (new_alignment.compare(.lte, prev_alignment)) return; - } - - const old_offset, const size = node.location().resolve(mf); - if (ni == Node.Index.root) return mf.resizeNode(gpa, ni, size); - - const new_size = new_alignment.forward(@intCast(size)); - if (new_alignment.check(@intCast(old_offset))) return mf.resizeNode(gpa, ni, new_size); - - _, const parent_size = node.parent.location(mf).resolve(mf); - const trailing_end = trailing_end: switch (node.next) { - .none => parent_size, - else => |next_ni| { - const next_offset, _ = next_ni.location(mf).resolve(mf); - break :trailing_end next_offset; - }, - }; - - if (opts.try_backwards) { - const backward_offset = new_alignment.backward(@intCast(old_offset)); - const prev_end = if (node.prev == .none) 0 else prev: { - const prev_offset, const prev_size = node.prev.location(mf).resolve(mf); - break :prev prev_offset + prev_size; - }; - - if (backward_offset >= prev_end) { - try mf.ensureCapacityForSetLocation(gpa); - - if (node.flags.has_content) { - const old_file_offset = ni.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - old_offset) + backward_offset; - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], - ); - @memset(mf.memory_map.memory[@intCast(new_file_offset + size)..@intCast(old_file_offset + size)], 0); - } - - if (backward_offset + new_size <= trailing_end) { - ni.setLocationAssumeCapacity(mf, backward_offset, new_size); - } else { - ni.setLocationAssumeCapacity(mf, backward_offset, size); - try mf.resizeNode(gpa, ni, new_size); - } - - return; + const old_offset, const old_size = ni.location(mf).resolve(mf); + + if (ni == .root or ni.position(mf) != .floating) { + // Only this node's size is aligned, not its offset. + if (!new_alignment.check(old_size)) { + assert(new_alignment.compare(.gt, ni.alignment(mf))); + try mf.growNode( + gpa, + ni, + new_alignment.forward(old_size), + .exact, // because `growNode` is not aware that the size needs to match `new_alignment` + ); } - } - - const forward_offset = new_alignment.forward(@intCast(old_offset)); - if (forward_offset + new_size <= trailing_end) { - // Shift into the free space if possible - try mf.ensureCapacityForSetLocation(gpa); - if (node.flags.has_content) { - const old_file_offset = ni.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - old_offset) + forward_offset; - if (new_file_offset < old_file_offset + size) { - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], - ); - } else try mf.moveRange(old_file_offset, new_file_offset, size); - @memset(mf.memory_map.memory[@intCast(new_file_offset + size)..][0..@intCast(new_size - size)], 0); - } - - ni.setLocationAssumeCapacity(mf, forward_offset, new_size); } else { - const temp_size = new_alignment.forward(@intCast(new_size + 1)); - try mf.resizeNode(gpa, ni, temp_size); - const new_offset, _ = ni.location(mf).resolve(mf); - - try mf.ensureCapacityForSetLocation(gpa); - - // Non-fixed nodes may now be aligned if the resize moved them - const new_forward_offset = new_alignment.forward(@intCast(new_offset)); - const final_offset = if (new_forward_offset != new_offset) final_offset: { - if (node.flags.has_content) { - const old_file_offset = ni.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - new_offset) + new_forward_offset; - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], - ); - @memset(mf.memory_map.memory[@intCast(old_file_offset)..@intCast(new_file_offset)], 0); - } - - break :final_offset new_forward_offset; - } else new_offset; - - ni.setLocationAssumeCapacity(mf, final_offset, new_size); + // This is a floating node, so its size and offset are both aligned. + if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) { + assert(new_alignment.compare(.gt, ni.alignment(mf))); + try mf.growFloatingNodeWithAlignment( + gpa, + ni, + new_alignment, + new_alignment.forward(old_size), + .minimum, + ); + } } + + ni.get(mf).flags.alignment = new_alignment; } fn updateWriters(mf: *MappedFile) void { @@ -1206,10 +2091,47 @@ fn updateWriters(mf: *MappedFile) void { } } -fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable || IoError)!void { - // make a copy of this node at the new location - try mf.copyRange(old_file_offset, new_file_offset, size); - // delete the copy of this node at the old location +fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) Error!void { + if (old_file_offset == new_file_offset) return; + + if (old_file_offset >= new_file_offset + size or + new_file_offset >= old_file_offset + size) + { + const n = try mf.copyFileRange( + mf.memory_map.file, + old_file_offset, + new_file_offset, + size, + ); + @memcpy( + mf.memory_map.memory[@intCast(new_file_offset + n)..][0..@intCast(size - n)], + mf.memory_map.memory[@intCast(old_file_offset + n)..][0..@intCast(size - n)], + ); + + try mf.zeroRange(old_file_offset, size); + + return; + } + + // TODO: if the non-overlapping region is greater than or equal to a filesystem block, is it + // ever worth doing multiple `copyFileRange` calls instead of a big `@memmove`? + + @memmove( + mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], + mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], + ); + + if (new_file_offset > old_file_offset) { + const clear_size = new_file_offset - old_file_offset; + assert(clear_size < size); + try mf.zeroRange(old_file_offset, clear_size); + } else { + const clear_size = old_file_offset - new_file_offset; + assert(clear_size < size); + try mf.zeroRange(new_file_offset + size, clear_size); + } +} +fn zeroRange(mf: *MappedFile, file_offset: u64, size: u64) Error!void { if (is_linux and !mf.flags.fallocate_punch_hole_unsupported and size >= mf.flags.block_size.toByteUnits() * 2 - 1) @@ -1217,147 +2139,149 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: while (true) switch (linux.errno(linux.fallocate( mf.memory_map.file.handle, linux.FALLOC.FL_PUNCH_HOLE | linux.FALLOC.FL_KEEP_SIZE, - @intCast(old_file_offset), + @intCast(file_offset), @intCast(size), ))) { .SUCCESS => return, .INTR => continue, - .BADF, .FBIG, .INVAL => unreachable, - .IO => return error.InputOutput, - .NODEV => return error.NotFile, - .NOSPC => return error.NoSpaceLeft, .NOSYS, .OPNOTSUPP => { mf.flags.fallocate_punch_hole_unsupported = true; break; // fall back to slow path }, - .PERM => return error.PermissionDenied, - .SPIPE => return error.Unseekable, - .TXTBSY => return error.FileBusy, - else => |e| return std.posix.unexpectedErrno(e), + else => |e| { + mf.io_err = switch (e) { + .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above + .BADF => unreachable, + .FBIG => unreachable, + .INVAL => unreachable, + .IO => error.InputOutput, + .NODEV => error.NotFile, + .NOSPC => error.NoSpaceLeft, + .PERM => error.PermissionDenied, + .SPIPE => error.Unseekable, + .TXTBSY => error.FileBusy, + else => std.posix.unexpectedErrno(e), + }; + return error.MappedFileIo; + }, }; } - @memset(mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], 0); + @memset(mf.memory_map.memory[@intCast(file_offset)..][0..@intCast(size)], 0); } - -fn copyRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable || IoError)!void { - const copy_size = try mf.copyFileRange(mf.memory_map.file, old_file_offset, new_file_offset, size); - if (copy_size < size) @memcpy( - mf.memory_map.memory[@intCast(new_file_offset + copy_size)..][0..@intCast(size - copy_size)], - mf.memory_map.memory[@intCast(old_file_offset + copy_size)..][0..@intCast(size - copy_size)], - ); -} - fn copyFileRange( mf: *MappedFile, old_file: Io.File, old_file_offset: u64, new_file_offset: u64, size: u64, -) (Io.Cancelable || IoError)!u64 { +) Error!u64 { + if (!is_linux or mf.flags.copy_file_range_unsupported) { + return 0; + } + + const min_size = mf.flags.block_size.toByteUnits() * 2 - 1; + if (size < min_size) return 0; + const io = mf.io; - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + mf.memory_map.write(io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; }; var remaining_size = size; - if (is_linux and !mf.flags.copy_file_range_unsupported) { - var old_file_offset_mut: i64 = @intCast(old_file_offset); - var new_file_offset_mut: i64 = @intCast(new_file_offset); - while (remaining_size >= mf.flags.block_size.toByteUnits() * 2 - 1) { - const copy_len = linux.copy_file_range( - old_file.handle, - &old_file_offset_mut, - mf.memory_map.file.handle, - &new_file_offset_mut, - @intCast(remaining_size), - 0, - ); - switch (linux.errno(copy_len)) { - .SUCCESS => { - if (copy_len == 0) break; - remaining_size -= copy_len; - if (remaining_size == 0) break; - }, - .INTR => continue, - .BADF, .FBIG, .INVAL, .OVERFLOW => unreachable, - .IO => return error.InputOutput, - .ISDIR => return error.IsDir, - .NOMEM => return error.SystemResources, - .NOSPC => return error.NoSpaceLeft, - .NOSYS, .OPNOTSUPP, .XDEV => { - mf.flags.copy_file_range_unsupported = true; - break; - }, - .PERM => return error.PermissionDenied, - .TXTBSY => return error.FileBusy, - else => |e| return std.posix.unexpectedErrno(e), - } + var old_file_offset_mut: i64 = @intCast(old_file_offset); + var new_file_offset_mut: i64 = @intCast(new_file_offset); + while (remaining_size >= min_size) { + const copy_len = linux.copy_file_range( + old_file.handle, + &old_file_offset_mut, + mf.memory_map.file.handle, + &new_file_offset_mut, + @intCast(remaining_size), + 0, + ); + switch (linux.errno(copy_len)) { + .SUCCESS => { + if (copy_len == 0) break; + remaining_size -= copy_len; + if (remaining_size == 0) break; + }, + .INTR => continue, + .NOSYS, .OPNOTSUPP, .XDEV => { + mf.flags.copy_file_range_unsupported = true; + break; + }, + else => |e| { + mf.io_err = switch (e) { + .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP, .XDEV => unreachable, // handled above + .BADF => unreachable, + .FBIG => unreachable, + .INVAL => unreachable, + .OVERFLOW => unreachable, + .IO => error.InputOutput, + .ISDIR => error.IsDir, + .NOMEM => error.SystemResources, + .NOSPC => error.NoSpaceLeft, + .PERM => error.PermissionDenied, + .TXTBSY => error.FileBusy, + else => std.posix.unexpectedErrno(e), + }; + return error.MappedFileIo; + }, } } return size - remaining_size; } -fn ensureCapacityForSetLocation(mf: *MappedFile, gpa: Allocator) Allocator.Error!void { - try mf.large.ensureUnusedCapacity(gpa, 2); - try mf.updates.ensureUnusedCapacity(gpa, 2); -} - pub fn ensureTotalCapacity(mf: *MappedFile, new_capacity: usize) Error!void { - mf.ensureTotalCapacityInner(new_capacity) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; -} -fn ensureTotalCapacityInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error || Io.Cancelable || IoError)!void { if (mf.memory_map.memory.len >= new_capacity) return; - try mf.ensureTotalCapacityPreciseInner(new_capacity +| new_capacity / growth_factor); + try mf.ensureTotalCapacityPrecise(new_capacity +| new_capacity / growth_factor); } pub fn ensureTotalCapacityPrecise(mf: *MappedFile, new_capacity: usize) Error!void { - mf.ensureTotalCapacityPreciseInner(new_capacity) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; -} -fn ensureTotalCapacityPreciseInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error || Io.Cancelable || IoError)!void { if (mf.memory_map.memory.len >= new_capacity) return; const io = mf.io; - const aligned_capacity = mf.flags.block_size.forward(new_capacity); + const aligned_capacity: usize = @intCast( + mf.flags.block_size.forward(new_capacity), + ); if (mf.memory_map.memory.len > 0) { if (mf.memory_map.setLength(io, aligned_capacity)) |_| { return; } else |err| switch (err) { error.OperationUnsupported => {}, - else => |e| return e, + error.OutOfMemory, error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, } - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + mf.memory_map.write(io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; }; unmap(mf); } const file = mf.memory_map.file; - mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForReading => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch |err| { + mf.io_err = switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForReading => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; }; } @@ -1376,7 +2300,7 @@ pub fn flush(mf: *MappedFile) (Io.Cancelable || error{MappedFileIo})!void { error.WouldBlock, // file was not opened as non-blocking error.NotOpenForWriting, // we definitely opened the file for writing - error.ReadOnlyFileSystem, + error.ReadOnlyFileSystem, // again, we opened the file for writing => { mf.io_err = error.Unexpected; return error.MappedFileIo; @@ -1399,213 +2323,278 @@ fn verify(mf: *MappedFile) void { assert(root.parent == .none); assert(root.prev == .none); assert(root.next == .none); - mf.verifyNode(Node.Index.root); + mf.verifyNode(.root); } - fn verifyNode(mf: *MappedFile, parent_ni: Node.Index) void { const parent = parent_ni.get(mf); - const parent_offset, const parent_size = parent.location().resolve(mf); - var prev_ni: Node.Index = .none; + _, const parent_size = parent.location().resolve(mf); + + var prev_oni: Node.Index.Optional = .none; var prev_end: u64 = 0; - var ni = parent.first; - while (true) { - if (ni == .none) { - assert(parent.last == prev_ni); - return; - } + var prev_pos: Node.Position = .header; + var oni = parent.first; + while (oni.unwrap()) |ni| { const node = ni.get(mf); - assert(node.parent == parent_ni); + assert(node.parent == parent_ni.toOptional()); + assert(node.prev == prev_oni); + const offset, const size = node.location().resolve(mf); - assert(node.flags.alignment.check(@intCast(offset))); - assert(node.flags.alignment.check(@intCast(size))); const end = offset + size; - assert(end <= parent_offset + parent_size); + + assert(node.flags.alignment.check(size)); assert(offset >= prev_end); - assert(node.prev == prev_ni); + assert(end <= parent_size); + + switch (node.flags.position) { + .header => { + assert(prev_pos == .header); + assert(offset == prev_end); + }, + .floating => { + assert(prev_pos != .footer); + assert(node.flags.alignment.check(offset)); + }, + .footer => { + if (prev_pos == .footer) assert(offset == prev_end); + }, + } + mf.verifyNode(ni); - prev_ni = ni; + + prev_oni = .wrap(ni); prev_end = end; - ni = node.next; + prev_pos = ni.position(mf); + + oni = node.next; + } + assert(parent.last == prev_oni); + if (prev_pos == .footer) { + assert(prev_end == parent_size); } } -const testing = std.testing; -fn testVerifyContent(mf: *@This(), ni: Node.Index, value: u8, init_len: usize) !void { - // Not using std.mem.allEqual, so we can get useful output - const slice = ni.slice(mf); - var buf: [256]u8 = undefined; - @memset(buf[0..init_len], value); - @memset(buf[init_len..], 0); - try testing.expectEqualSlices(u8, buf[0..slice.len], slice); +test "fuzz node operations" { + try std.testing.fuzz({}, fuzzOneNodeOperations, .{}); } +fn fuzzOneNodeOperations(_: void, smith: *std.testing.Smith) anyerror!void { + const gpa = std.testing.allocator; + const io = std.testing.io; -test { - const gpa = testing.allocator; - - var tmp_dir = testing.tmpDir(.{}); + var tmp_dir = std.testing.tmpDir(.{}); defer tmp_dir.cleanup(); - var file = try tmp_dir.dir.createFile(testing.io, "test.mf", .{ .read = true }); - defer file.close(testing.io); + var tmp_file = try tmp_dir.dir.createFile(io, "test.mf", .{ .read = true }); + defer tmp_file.close(io); - var mf = try init(file, gpa, testing.io); + var mf: MappedFile = try .init(tmp_file, gpa, io); defer mf.deinit(gpa); - const a = try mf.addFirstChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" }); - const c = try mf.addLastChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" }); - const b = try mf.addNodeAfter(gpa, a, .{ .fixed = true, .alignment = .@"16" }); - const d = try mf.addNodeAfter(gpa, b, .{ .alignment = .@"4" }); - - const a_init_size = 8; - const b_init_size = 16; - const c_init_size = 24; - const d_init_size = 28; - - // Resize without content - { - // Verify size is aligned forward - try d.resize(&mf, gpa, d_init_size - 1); - try a.resize(&mf, gpa, a_init_size - 2); - try c.resize(&mf, gpa, c_init_size); - try b.resize(&mf, gpa, b_init_size); - mf.verify(); - - const a_loc, const a_size = a.location(&mf).resolve(&mf); - const b_loc, const b_size = b.location(&mf).resolve(&mf); - const c_loc, const c_size = c.location(&mf).resolve(&mf); - _, const d_size = d.location(&mf).resolve(&mf); - try testing.expect(a_size >= a_init_size); - try testing.expect(b_size >= b_init_size); - try testing.expect(c_size >= c_init_size); - try testing.expect(d_size >= d_init_size); - try testing.expect(b_loc >= a_loc + a_size); - try testing.expect(c_loc >= b_loc + b_size); - } - - const a_exp_size = 24; - const b_exp_size = 28; - const c_exp_size = 48; - const d_exp_size = 32; - - // Resize with content - { - @memset(a.slice(&mf)[0..a_init_size], 0xaa); - @memset(b.slice(&mf)[0..b_init_size], 0xbb); - @memset(c.slice(&mf)[0..c_init_size], 0xcc); - @memset(d.slice(&mf)[0..d_init_size], 0xdd); - - try a.resize(&mf, gpa, a_exp_size); - try b.resize(&mf, gpa, b_exp_size); - try c.resize(&mf, gpa, c_exp_size); - try d.resize(&mf, gpa, d_exp_size); - mf.verify(); - - const a_loc, const a_size = a.location(&mf).resolve(&mf); - const b_loc, const b_size = b.location(&mf).resolve(&mf); - const c_loc, const c_size = c.location(&mf).resolve(&mf); - _, const d_size = d.location(&mf).resolve(&mf); - try testing.expect(a_size >= a_exp_size); - try testing.expect(b_size >= b_exp_size); - try testing.expect(c_size >= c_exp_size); - try testing.expect(d_size >= d_exp_size); - try testing.expect(b_loc >= a_loc + a_size); - try testing.expect(c_loc >= b_loc + b_size); - - try testVerifyContent(&mf, a, 0xaa, a_init_size); - try testVerifyContent(&mf, b, 0xbb, b_init_size); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - try testVerifyContent(&mf, d, 0xdd, d_init_size); - } - - const child_init: []const struct { std.mem.Alignment, usize } = &.{ - .{ .@"16", 16 }, - .{ .@"1", 1 }, - .{ .@"1", 19 }, - .{ .@"1", 3 }, - .{ .@"8", 30 }, - .{ .@"2", 5 }, - .{ .@"1", 60 }, - .{ .@"2", 2 }, - .{ .@"16", 32 }, + var nodes: std.array_hash_map.Auto(MappedFile.Node.Index, struct { + parent: MappedFile.Node.Index.Optional, + position: MappedFile.Node.Position, + num_headers: u32, + num_footers: u32, + /// For leaf nodes, this value is whether we have initialized the contents of the node or + /// not. For non-leaf nodes, this value is unspecified and should be ignored. + initialized: bool, + }) = .empty; + defer nodes.deinit(gpa); + + // When initializing a leaf node, we will place its 4-byte node index at the start of its range, + // and the bitwise NOT of its node index at the end of its range (both little-endian). This is + // just a simple way to put distinct values we can validate at all node boundaries. + + try nodes.putNoClobber(gpa, .root, .{ + .parent = .none, + .position = .floating, + .num_headers = 0, + .num_footers = 0, + .initialized = false, + }); + + // Allow a range of alignments, with most nodes having a small alignment of 1--32 bytes (most + // commonly 1 byte), but with a small chance for some large alignments too. + const alignment_weights: []const std.testing.Smith.Weight = comptime &.{ + .value(Alignment, .@"1", 20), + .value(Alignment, .@"2", 5), + .value(Alignment, .@"4", 5), + .value(Alignment, .@"8", 5), + .value(Alignment, .@"16", 5), + .value(Alignment, .@"32", 5), + .value(Alignment, .fromByteUnits(0x200), 1), + .value(Alignment, .fromByteUnits(0x400), 1), + .value(Alignment, .fromByteUnits(0x800), 1), + .value(Alignment, .fromByteUnits(0x1000), 1), + .value(Alignment, .fromByteUnits(0x2000), 1), + .value(Alignment, .fromByteUnits(0x4000), 1), + .value(Alignment, .fromByteUnits(0x8000), 1), }; - var children: [child_init.len]Node.Index = undefined; + const min_nonzero_size = 2 * @sizeOf(MappedFile.Node.Index); + const max_size = 0x10_000; + const initial_size_weights: []const std.testing.Smith.Weight = comptime &.{ + // initially, make nodes just as likely to be empty as non-empty + .value(u64, 0, max_size - min_nonzero_size + 1), + .rangeAtMost(u64, min_nonzero_size, max_size, 1), + }; + + while (!smith.eos()) switch (smith.value(enum { add, resize, realign })) { + .add => { + const parent_ni = nodes.keys()[smith.index(nodes.count())]; + + const alignment = smith.valueWeighted(Alignment, alignment_weights); + const size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); - // Differently-aligned fixed sibling nodes - { - for (children[0 .. children.len - 1], child_init[0 .. children.len - 1], 0..) |*ni, opts, i| { - ni.* = try mf.addLastChildNode(gpa, b, .{ - .alignment = opts.@"0", - .size = opts.@"1", - .fixed = true, + const position = smith.valueWeighted(Node.Position, comptime &.{ + // make floating nodes more common than header and footer nodes + .value(Node.Position, .header, 1), + .value(Node.Position, .footer, 1), + .value(Node.Position, .floating, 4), }); + const new_ni: Node.Index = switch (position) { + .header => new_ni: { + const parent_info = nodes.getPtr(parent_ni).?; + const prev_oni: Node.Index.Optional = prev_oni: { + const n = smith.valueRangeAtMost(u32, 0, parent_info.num_headers); + if (n == 0) break :prev_oni .none; + var cur_ni = parent_ni.first(&mf).unwrap().?; + for (1..n) |_| cur_ni = cur_ni.next(&mf).unwrap().?; + break :prev_oni .wrap(cur_ni); + }; + const new_ni = try parent_ni.addHeaderChildAfter(&mf, gpa, prev_oni, .{ + .size = size, + .alignment = alignment, + }); + parent_info.num_headers += 1; + break :new_ni new_ni; + }, + + .floating => try parent_ni.addFloatingChild(&mf, gpa, .{ + .size = size, + .alignment = alignment, + }), + + .footer => new_ni: { + const parent_info = nodes.getPtr(parent_ni).?; + const next_oni: Node.Index.Optional = next_oni: { + const n = smith.valueRangeAtMost(u32, 0, parent_info.num_footers); + if (n == 0) break :next_oni .none; + var cur_ni = parent_ni.last(&mf).unwrap().?; + for (1..n) |_| cur_ni = cur_ni.prev(&mf).unwrap().?; + break :next_oni .wrap(cur_ni); + }; + const new_ni = try parent_ni.addFooterChildBefore(&mf, gpa, next_oni, .{ + .size = size, + .alignment = alignment, + }); + parent_info.num_footers += 1; + break :new_ni new_ni; + }, + }; + + const initialize = size > 0 and smith.value(bool); + if (initialize) { + const slice = new_ni.slice(&mf); + std.mem.writeInt(u32, slice[0..4], @backingInt(new_ni), .little); + std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(new_ni), .little); + } + + try nodes.putNoClobber(gpa, new_ni, .{ + .parent = .wrap(parent_ni), + .position = position, + .num_headers = 0, + .num_footers = 0, + .initialized = initialize, + }); + }, + + .resize => { + const ni = nodes.keys()[smith.index(nodes.count())]; + const node_info = nodes.getPtr(ni).?; + + const alignment = ni.alignment(&mf); + + if (ni.first(&mf) == .none and smith.value(bool)) { + // Since this is a leaf node, we can use `resizeLeaf`. + const new_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); + try ni.resizeLeaf(&mf, gpa, new_size); + if (new_size == 0) { + node_info.initialized = false; + } + } else { + const min_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); + try ni.ensureMinimumSize(&mf, gpa, min_size); + } + + if (ni.first(&mf) == .none) { + // This is a leaf node, so it can contain data. + if (node_info.initialized) { + // It's already initialized, so we'll write the expected footer at the new end. + const slice = ni.slice(&mf); + std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little); + } else if (ni.location(&mf).resolve(&mf)[1] > 0) { + // It was uninitialized, but it has a non-zero size, so maybe we'd like to + // initialize it now? + if (smith.value(bool)) { + node_info.initialized = true; + const slice = ni.slice(&mf); + std.mem.writeInt(u32, slice[0..4], @backingInt(ni), .little); + std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little); + } + } + } + }, + .realign => { + const ni = nodes.keys()[smith.index(nodes.count())]; + const new_alignment = smith.valueWeighted(Alignment, alignment_weights); + if (new_alignment.compare(.gt, ni.alignment(&mf))) { + _, const old_size = ni.location(&mf).resolve(&mf); + try ni.realign(&mf, gpa, new_alignment); + if (ni.first(&mf) == .none and nodes.get(ni).?.initialized) { + const slice = ni.slice(&mf); + @memmove(slice[slice.len - 4 ..][0..4], slice[old_size - 4 ..][0..4]); + } + } + }, + }; - @memset(ni.slice(&mf)[0..opts.@"1"], @intCast(i + 1)); + mf.verify(); + + for (nodes.keys(), nodes.values()) |ni, expected| { + try std.testing.expectEqual(expected.parent, ni.parent(&mf)); + if (ni != .root) { + try std.testing.expectEqual(expected.position, ni.position(&mf)); } - // Shift differently-aligned nodes by inserting a node - children[children.len - 1] = try mf.addNodeAfter(gpa, children[3], .{ - .alignment = child_init[children.len - 1].@"0", - .size = child_init[children.len - 1].@"1", - .fixed = true, - }); - @memset(children[children.len - 1].slice(&mf), @intCast(children.len)); - - mf.verify(); - for (children, child_init, 0..) |ni, opts, i| { - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); + + { + var num_headers: u32 = 0; + var header_oni = ni.lastHeader(&mf); + while (header_oni.unwrap()) |header_ni| { + num_headers += 1; + header_oni = header_ni.prev(&mf); + } + try std.testing.expectEqual(expected.num_headers, num_headers); } - } - - // Shifting child nodes forward due via resize of parent.prev - { - try testing.expect(a.location(&mf).resolve(&mf)[1] < 64); - try a.resize(&mf, gpa, 64); - - try testVerifyContent(&mf, a, 0xaa, a_init_size); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - try testVerifyContent(&mf, d, 0xdd, d_init_size); - for (children, child_init, 0..) |ni, opts, i| { - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); + + { + var num_footers: u32 = 0; + var footer_oni = ni.firstFooter(&mf); + while (footer_oni.unwrap()) |footer_ni| { + num_footers += 1; + footer_oni = footer_ni.next(&mf); + } + try std.testing.expectEqual(expected.num_footers, num_footers); } - } - - // Re-align last node into trailing free space within parent - { - try b.resize(&mf, gpa, b.location(&mf).resolve(&mf)[1] + 64); - - const last = children[children.len - 2]; - try last.realign(&mf, gpa, .@"4", true); - mf.verify(); - - for (children, child_init, 0..) |ni, opts, i| - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - } - - // Re-align, shifting sibling nodes - { - try children[1].realign(&mf, gpa, .@"8", true); - mf.verify(); - - for (children, child_init, 0..) |ni, opts, i| - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - } - - // Shrink and shift start of trailing node into free space - { - try mf.shrinkNode(gpa, a, 16, true); - mf.verify(); - - const a_loc, const a_size = a.location(&mf).resolve(&mf); - const b_loc, _ = b.location(&mf).resolve(&mf); - try testing.expectEqual(b_loc, a_loc + a_size); - - try testVerifyContent(&mf, a, 0xaa, a_init_size); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - try testVerifyContent(&mf, d, 0xdd, d_init_size); - for (children, child_init, 0..) |ni, opts, i| { - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); + + if (ni.first(&mf) == .none and expected.initialized) { + const slice = ni.sliceConst(&mf); + if (slice.len > 0) { + try std.testing.expect(slice.len >= min_nonzero_size); + const header = std.mem.readInt(u32, slice[0..4], .little); + const footer = std.mem.readInt(u32, slice[slice.len - 4 ..][0..4], .little); + try std.testing.expectEqual(@backingInt(ni), header); + try std.testing.expectEqual(~@backingInt(ni), footer); + } } } } diff --git a/src/main.zig b/src/main.zig index 7e33a7fdf2d15ec9fa8e1417c0533fb702d22dfd..4584c950abd7855d781492de48e25204a8de10c1 100644 --- a/src/main.zig +++ b/src/main.zig @@ -36,6 +36,7 @@ const Module = @import("Module.zig"); test { _ = @import("codegen.zig"); + _ = @import("link/MappedFile.zig"); } const thread_stack_size = 60 << 20;