Commit 8a6f4ae1827a5b6b28b07f57852008ea001bf02e | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2026-08-30 23:12:25+02:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2026-08-30 23:12:25+02:00
log	8a6f4ae1827a5b6b28b07f57852008ea001bf02e
tree	adf8e4ba8463c0f7e18d1402d8c50248d17408bf
parent	107eafe87ebc24b2e644857f445188ffdd7a4799
parent	a6ab86acd92b81dcdfa3f8192690de7a26fa13d7

Merge pull request 'Elf2 and MappedFile enhancements' (#36677) from mlugg/elf2-yet-again into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36677

2 files changed, 874 insertions(+), 448 deletions(-)

src
- link
  - Elf2.zig
  - MappedFile.zig

src/link/Elf2.zig+469-212

...	...	@@ -24,6 +24,7 @@ base: link.File,
24	24	options: link.File.OpenOptions,
25	25	mf: MappedFile,
26	26	ni: Node.Known,
	27	archive: ?Archive,
27	28	nodes: std.MultiArrayList(Node),
28	29	/// Does not contain an item for `SHN_UNDEF`.
29	30	shdrs: std.ArrayList(Section),
...	...	@@ -200,19 +201,39 @@ input_prog_node: std.Progress.Node,
200	201	const Error = link.Error \|\| error{MappedFileIo};
201	202
202	203	const Node = union(enum) {
	204	/// Only used when emitting a static library.
	205	///
	206	/// Contains a header node which is an `.archive_header`.
	207	///
	208	/// Contains the following footer nodes:
	209	/// * One `.archive_input_member` for each external input in the archive
	210	/// * One `.archive_elf_member_header` containing the `ar_hdr` for the ZCU
	211	/// * One `.elf` containing the ZCU's actual ELF object
	212	///
	213	/// Padding between the headers and footers is absorbed into the "//" member (whose actual
	214	/// content is in the `.archive_header` node).
203	215	archive,
204		/// This includes the archive magic and long file member.
	216	/// Only used when emitting a static library.
	217	///
	218	/// Contains the archive magic (`ARMAG`), as well as the `ar_hdr` and content for the long file
	219	/// name string table member ("//").
205	220	archive_header,
206		/// This is a footer of the `.elf` node, and contains the next archive entry's file header.
207		archive_elf_footer,
	221	/// Only used when emitting a static library.
	222	///
	223	/// Contains the `ar_hdr` and content for one non-ZCU archive member (external link input). Also
	224	/// includes the single byte '\n' padding at the end of this archive member, if necessary.
	225	archive_input_member: InputIndex,
	226	/// Only used when emitting a static library.
	227	///
	228	/// Contains the `ar_hdr` for the `.elf` node.
	229	archive_elf_member_header,
	230
208	231	elf,
209	232	ehdr,
210	233	shdr,
211	234	segment: u32,
212	235	/// The section '.plt' may contain relocations via `elf.plt_first_symbol_reloc`.
213	236	section: Section.Index,
214		/// Only valid for static libraries, represents one non-zcu archive member.
215		input_member: InputIndex,
216	237	/// May contain relocations.
217	238	input_section: InputSection.Index,
218	239	/// Value is the name of a global which has an entry in `elf.copied_globals`, so, a global for
...	...	@@ -404,6 +425,15 @@ const InputSection = struct {
404	425	};
405	426	};
406	427
	428	const Archive = struct {
	429	ni: MappedFile.Node.Index,
	430	header_ni: MappedFile.Node.Index,
	431	elf_member_header_ni: MappedFile.Node.Index,
	432
	433	elf_member_too_big: bool,
	434	strtab_member_too_big: bool,
	435	};
	436
407	437	const Section = struct {
408	438	/// The node corresponding to this section.
409	439	ni: MappedFile.Node.Index,
...	...	@@ -2954,13 +2984,13 @@ pub fn symbolForAtom(elf: *Elf, atom: link.File.AtomId) link.File.SymbolId {
2954	2984	const lsi: Symbol.LocalIndex = switch (elf.getNode(Node.fromAtom(atom))) {
2955	2985	.archive,
2956	2986	.archive_header,
2957		.archive_elf_footer,
	2987	.archive_input_member,
	2988	.archive_elf_member_header,
2958	2989	.elf,
2959	2990	.ehdr,
2960	2991	.shdr,
2961	2992	.segment,
2962	2993	.section,
2963		.input_member,
2964	2994	.input_section,
2965	2995	.copied_global,
2966	2996	=> unreachable,
...	...	@@ -3364,6 +3394,7 @@ fn create(
3364	3394	.data_rel_ro = undefined,
3365	3395	.tls = .none,
3366	3396	},
	3397	.archive = null,
3367	3398	.nodes = .empty,
3368	3399	.shdrs = .empty,
3369	3400	.phdrs = .empty,
...	...	@@ -3558,51 +3589,58 @@ fn initHeaders(
3558	3589	.EXEC, .DYN => {},
3559	3590	}
3560	3591	var phnum: u32 = 0;
3561		break :ph .{ .{
3562		.phdr = phndx: {
3563		defer phnum += 1;
3564		break :phndx phnum;
3565		},
3566		.interp = if (maybe_interp) \|_\| phndx: {
3567		defer phnum += 1;
3568		break :phndx phnum;
3569		} else undefined,
3570		.rodata = phndx: {
3571		defer phnum += 1;
3572		break :phndx phnum;
3573		},
3574		.text = phndx: {
3575		defer phnum += 1;
3576		break :phndx phnum;
3577		},
3578		.data = phndx: {
3579		defer phnum += 1;
3580		break :phndx phnum;
3581		},
3582		.plt = if (plt.got_plt == null) phndx: {
3583		defer phnum += 1;
3584		break :phndx phnum;
3585		} else undefined,
3586		.tls = if (comp.config.any_non_single_threaded) phndx: {
3587		defer phnum += 1;
3588		break :phndx phnum;
3589		} else undefined,
3590		.dynamic = if (have_dynamic_section) phndx: {
3591		defer phnum += 1;
3592		break :phndx phnum;
3593		} else undefined,
3594		.relro = phndx: {
3595		defer phnum += 1;
3596		break :phndx phnum;
3597		},
3598		.gnu_stack = phndx: {
3599		defer phnum += 1;
3600		break :phndx phnum;
	3592	break :ph .{
	3593	.{
	3594	.phdr = phndx: {
	3595	defer phnum += 1;
	3596	break :phndx phnum;
	3597	},
	3598	.interp = if (maybe_interp) \|_\| phndx: {
	3599	defer phnum += 1;
	3600	break :phndx phnum;
	3601	} else undefined,
	3602	.rodata = phndx: {
	3603	defer phnum += 1;
	3604	break :phndx phnum;
	3605	},
	3606	.text = phndx: {
	3607	defer phnum += 1;
	3608	break :phndx phnum;
	3609	},
	3610	.plt = if (plt.got_plt == null) phndx: {
	3611	defer phnum += 1;
	3612	break :phndx phnum;
	3613	} else undefined,
	3614	// `data` must be assigned after all other loadable segments so that it has the greatest
	3615	// phndx of any loadable segment. This is so that `targetSegmentLoadAddressRestrictions`
	3616	// can be obeyed (specifically, the `.data_last` restriction, needed on SPARC).
	3617	.data = phndx: {
	3618	defer phnum += 1;
	3619	break :phndx phnum;
	3620	},
	3621	.tls = if (comp.config.any_non_single_threaded) phndx: {
	3622	defer phnum += 1;
	3623	break :phndx phnum;
	3624	} else undefined,
	3625	.dynamic = if (have_dynamic_section) phndx: {
	3626	defer phnum += 1;
	3627	break :phndx phnum;
	3628	} else undefined,
	3629	.relro = phndx: {
	3630	defer phnum += 1;
	3631	break :phndx phnum;
	3632	},
	3633	.gnu_stack = phndx: {
	3634	defer phnum += 1;
	3635	break :phndx phnum;
	3636	},
3601	3637	},
3602		}, phnum };
	3638	// (I don't actually want the trailing comma below, but a `zig fmt` bug forces it.)
	3639	phnum,
	3640	};
3603	3641	};
3604	3642
3605		const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_footer
	3643	const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_member_header
3606	3644	3 + // `.elf`, `.ehdr`, and `.shdr` nodes
3607	3645	(shnum - 1) + // -1 because the SHN_UNDEF shdr does not have a `.section` node
3608	3646	(phnum -\| 1); // -1 because the GNU_STACK phdr does not have a `.segment` node
...	...	@@ -3619,11 +3657,14 @@ fn initHeaders(
3619	3657	const archive_ni: MappedFile.Node.Index = .root;
3620	3658
3621	3659	const archive_header_ni = try archive_ni.addOnlyHeaderChild(&elf.mf, gpa, .{
3622		.size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr) * 2,
3623		.alignment = .@"2",
3624		.next_moved = true,
3625		.bubbles_moved = false,
	3660	// We intentionally do not set `.alignment = .@"2"` here, because the string table data
	3661	// in this node does not need to have an aligned length. (This node's offset is aligned
	3662	// regardless by virtue of it being a header.)
	3663	.size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr),
	3664	// The archive header uses 'next_moved' events to resize the "//" member, so that it
	3665	// absorbs all padding between `archive_header_ni` and the actual object file members.
3626	3666	.enable_next_moved = true,
	3667	.next_moved = true,
3627	3668	});
3628	3669	elf.nodes.appendAssumeCapacity(.archive_header);
3629	3670	const archive_header_slice = archive_header_ni.slice(&elf.mf);
...	...	@@ -3635,23 +3676,47 @@ fn initHeaders(
3635	3676	.ar_uid = @splat(' '),
3636	3677	.ar_gid = @splat(' '),
3637	3678	.ar_mode = @splat(' '),
3638		.ar_size = @splat(' '),
	3679	.ar_size = undefined, // populated by `flushNextMoved` for `archive_header_ni`
3639	3680	.ar_fmag = std.elf.ARFMAG.*,
3640	3681	};
3641	3682
3642		elf.ni.elf = try archive_ni.addFloatingChild(&elf.mf, gpa, .{
	3683	elf.ni.elf = try archive_ni.addOnlyFooterChild(&elf.mf, gpa, .{
3643	3684	.alignment = node_block_align.max(.@"2"),
3644		.next_moved = true,
3645	3685	.bubbles_moved = false,
3646		.enable_next_moved = true,
	3686	.resized = true, // ensure that this node's `ar_hdr.ar_size` is updated at least once
3647	3687	});
3648	3688	elf.nodes.appendAssumeCapacity(.elf);
3649	3689
3650		_ = try elf.ni.elf.addOnlyFooterChild(&elf.mf, gpa, .{
	3690	const elf_ar_hdr_ni = try archive_ni.addFooterChildBefore(&elf.mf, gpa, .wrap(elf.ni.elf), .{
3651	3691	.alignment = .@"2",
3652	3692	.size = @sizeOf(std.elf.ar_hdr),
3653	3693	});
3654		elf.nodes.appendAssumeCapacity(.archive_elf_footer);
	3694	elf.nodes.appendAssumeCapacity(.archive_elf_member_header);
	3695
	3696	// Must be populated before we call `populateArchiveMemberName` below.
	3697	elf.archive = .{
	3698	.ni = archive_ni,
	3699	.header_ni = archive_header_ni,
	3700	.elf_member_header_ni = elf_ar_hdr_ni,
	3701
	3702	.elf_member_too_big = false,
	3703	.strtab_member_too_big = false,
	3704	};
	3705
	3706	const elf_ar_hdr: *std.elf.ar_hdr = @ptrCast(elf_ar_hdr_ni.slice(&elf.mf));
	3707	elf_ar_hdr.* = .{
	3708	.ar_name = undefined, // populated below
	3709	.ar_date = "0 ".*,
	3710	.ar_uid = "0 ".*,
	3711	.ar_gid = "0 ".*,
	3712	.ar_mode = "644 ".*,
	3713	.ar_size = undefined, // populated by `flushResized` for the `.elf` node
	3714	.ar_fmag = std.elf.ARFMAG.*,
	3715	};
	3716	const zcu_member_name = try std.fmt.allocPrint(gpa, "{s}_zcu.o", .{comp.root_name});
	3717	defer gpa.free(zcu_member_name);
	3718	// After this call returns, `elf_ar_hdr` is invalidated.
	3719	try elf.populateArchiveMemberName(elf_ar_hdr, zcu_member_name);
3655	3720	} else {
3656	3721	elf.ni.elf = .root;
3657	3722	elf.nodes.appendAssumeCapacity(.elf);
...	...	@@ -4092,33 +4157,44 @@ fn initHeaders(
4092	4157	.addralign = addr_align,
4093	4158	.entsize = @intCast(addr_align.toByteUnits()),
4094	4159	});
4095		if (plt.got_plt) \|got_plt\| {
4096		const got_plt_segment_ni = if (elf.options.z_now) elf.ni.data_rel_ro else elf.ni.data;
4097		elf.shndx.got_plt = try elf.addSection(got_plt_segment_ni, .{
4098		.name = ".got.plt",
4099		.type = .PROGBITS,
4100		.flags = .{ .WRITE = true, .ALLOC = true },
4101		.size = got_plt.header_entries * elf.targetPtrSize(),
4102		.addralign = addr_align,
4103		.entsize = @intCast(addr_align.toByteUnits()),
4104		});
4105		elf.shndx.plt = try elf.addSection(elf.ni.text, .{
4106		.name = ".plt",
4107		.type = .PROGBITS,
4108		.flags = .{ .ALLOC = true, .EXECINSTR = true },
4109		.size = plt.entry_size * plt.header_entries,
4110		.addralign = plt.@"align",
4111		.node_align = node_block_align,
4112		});
4113		} else {
4114		elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{
4115		.name = ".plt",
4116		.type = .PROGBITS,
4117		.flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true },
4118		.size = plt.entry_size * plt.header_entries,
4119		.addralign = plt.@"align",
4120		.node_align = node_block_align,
4121		});
	4160	{
	4161	const init_plt_size = plt.entry_size * plt.header_entries;
	4162	if (plt.got_plt) \|got_plt\| {
	4163	const got_plt_segment_ni = if (elf.options.z_now) elf.ni.data_rel_ro else elf.ni.data;
	4164	elf.shndx.got_plt = try elf.addSection(got_plt_segment_ni, .{
	4165	.name = ".got.plt",
	4166	.type = .PROGBITS,
	4167	.flags = .{ .WRITE = true, .ALLOC = true },
	4168	.size = got_plt.header_entries * elf.targetPtrSize(),
	4169	.addralign = addr_align,
	4170	.entsize = @intCast(addr_align.toByteUnits()),
	4171	});
	4172	elf.shndx.plt = try elf.addSection(elf.ni.text, .{
	4173	.name = ".plt",
	4174	.type = .PROGBITS,
	4175	.flags = .{ .ALLOC = true, .EXECINSTR = true },
	4176	.size = plt.@"align".forward(init_plt_size),
	4177	.addralign = plt.@"align",
	4178	.node_align = node_block_align,
	4179	});
	4180	} else {
	4181	elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{
	4182	.name = ".plt",
	4183	.type = .PROGBITS,
	4184	.flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true },
	4185	.size = plt.@"align".forward(init_plt_size),
	4186	.addralign = plt.@"align",
	4187	.node_align = node_block_align,
	4188	});
	4189	}
	4190	// And the award for most annoying PLT requirement goes to SPARC, which decided that the
	4191	// whole table should have a greater alignment than the size of the individual entries,
	4192	// hence this bullshit:
	4193	if (plt.@"align".forward(init_plt_size) != init_plt_size) {
	4194	switch (elf.shdrPtr(elf.shndx.plt)) {
	4195	inline else => \|shdr\| elf.targetStore(&shdr.size, init_plt_size),
	4196	}
	4197	}
4122	4198	}
4123	4199	if (plt.plt_sec != null) elf.shndx.plt_sec = try elf.addSection(elf.ni.text, .{
4124	4200	.name = ".plt.sec",
...	...	@@ -4358,6 +4434,12 @@ fn initHeaders(
4358	4434	assert(elf.targetLoad(&shdr.size) == elf.got.count() * @sizeOf(Addr));
4359	4435	},
4360	4436	}
	4437	if (elf.shndx.dynamic != .UNDEF) {
	4438	try elf.shndx.rela_dyn.relaEnsureAdditionalCapacity(elf, elf.got.count());
	4439	}
	4440	for (0..elf.got.count()) \|got_index\| {
	4441	elf.updateGotEntry(got_index);
	4442	}
4361	4443
4362	4444	// Create any always-provided linker-defined symbols. The symbols marking the `INIT_ARRAY`/
4363	4445	// `FINI_ARRAY`/`PREINIT_ARRAY` sections are instead created by `createInitFiniArraySection`
...	...	@@ -4539,6 +4621,22 @@ fn initHeaders(
4539	4621	break :str try elf.string(.dynstr, slice);
4540	4622	},
4541	4623	};
	4624
	4625	if (@"type" != .REL) switch (elf.targetSegmentLoadAddressRestrictions()) {
	4626	.none => {},
	4627	.data_last => switch (elf.phdrSlice()) {
	4628	inline else => \|phdr\| {
	4629	// Ensure that the segment after `.data` (if any) is not a loadable segment.
	4630	const next_phndx = phndx.data + 1;
	4631	if (next_phndx < phdr.len) {
	4632	switch (elf.targetLoad(&phdr[next_phndx].type)) {
	4633	.NULL, .LOAD => unreachable, // data segment should be the last loadable segment
	4634	else => {},
	4635	}
	4636	}
	4637	},
	4638	},
	4639	};
4542	4640	}
4543	4641
4544	4642	pub fn startProgress(elf: *Elf, prog_node: std.Progress.Node) void {
...	...	@@ -4573,12 +4671,12 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index {
4573	4671	return switch (elf.getNode(ni)) {
4574	4672	.archive,
4575	4673	.archive_header,
4576		.archive_elf_footer,
	4674	.archive_input_member,
	4675	.archive_elf_member_header,
4577	4676	.elf,
4578	4677	.ehdr,
4579	4678	.shdr,
4580	4679	.segment,
4581		.input_member,
4582	4680	=> unreachable,
4583	4681	.section => \|shndx\| shndx,
4584	4682	.input_section,
...	...	@@ -4594,12 +4692,12 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {
4594	4692	return switch (elf.getNode(ni)) {
4595	4693	.archive,
4596	4694	.archive_header,
4597		.archive_elf_footer,
	4695	.archive_input_member,
	4696	.archive_elf_member_header,
4598	4697	.elf,
4599	4698	.ehdr,
4600	4699	.shdr,
4601	4700	.segment,
4602		.input_member,
4603	4701	.copied_global,
4604	4702	=> unreachable,
4605	4703	.section => \|shndx\| shndx.vaddr(elf),
...	...	@@ -4613,14 +4711,18 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {
4613	4711	}
4614	4712	fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {
4615	4713	const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) {
4616		.archive, .archive_header, .archive_elf_footer => unreachable,
	4714	.archive,
	4715	.archive_header,
	4716	.archive_input_member,
	4717	.archive_elf_member_header,
	4718	=> unreachable,
4617	4719	.elf => return 0,
4618	4720	.ehdr, .shdr => unreachable,
4619	4721	.segment => \|phndx\| switch (elf.phdrSlice()) {
4620	4722	inline else => \|phdr\| elf.targetLoad(&phdr[phndx].vaddr),
4621	4723	},
4622	4724	.section => \|shndx\| if (shndx == elf.shndx.tdata) 0 else shndx.vaddr(elf),
4623		.input_member, .input_section, .copied_global => unreachable,
	4725	.input_section, .copied_global => unreachable,
4624	4726	inline .nav, .uav, .lazy_code, .lazy_const_data => \|i\| Symbol.Id.local(i.symbol(elf)).value(elf),
4625	4727	};
4626	4728	const offset, _ = ni.location(&elf.mf).resolve(&elf.mf);
...	...	@@ -4639,12 +4741,12 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void {
4639	4741	const symbol_relocs: SymbolReloc.Index, const got_relocs: ?GotReloc.Index = switch (elf.getNode(ni)) {
4640	4742	.archive,
4641	4743	.archive_header,
4642		.archive_elf_footer,
	4744	.archive_input_member,
	4745	.archive_elf_member_header,
4643	4746	.elf,
4644	4747	.ehdr,
4645	4748	.shdr,
4646	4749	.segment,
4647		.input_member,
4648	4750	.copied_global,
4649	4751	=> unreachable, // cannot contain relocs
4650	4752	.section => unreachable, // cannot contain relocs (.plt and .dynamic unsupported)
...	...	@@ -4705,7 +4807,12 @@ fn flushMovedNodeRelocs(
4705	4807	// changed, so update the `offset` field of the `ElfN.Rela` entry.
4706	4808	reloc.relaSection(elf).relaSetOffset(elf, rela_index, node_vaddr + reloc.offset);
4707	4809	}
4708		reloc.apply(elf);
	4810	// This is not just the inverse of the above condition, because if `reloc` is relative
	4811	// to the base of this DSO, then `rela_index` is an `R_*_RELATIVE` relocation, but we
	4812	// still need to call `SymbolReloc.apply` to update that relocation's addend.
	4813	if (elf.ehdrType() != .REL) {
	4814	reloc.apply(elf);
	4815	}
4709	4816	}
4710	4817	}
4711	4818
...	...	@@ -4888,7 +4995,31 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo {
4888	4995	// TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`.
4889	4996	};
4890	4997	}
4891		pub fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child {
	4998	/// Specifies any restrictions the current target has regarding how segments are ordered in the
	4999	/// virtual address space. Most targets do not have any such restrictions.
	5000	fn targetSegmentLoadAddressRestrictions(elf: *const Elf) enum {
	5001	none,
	5002	/// The "mutable data" segment must be the last loadable segment in the virtual address space.
	5003	data_last,
	5004	} {
	5005	return switch (elf.ehdrMachine()) {
	5006	.AARCH64,
	5007	.PPC64,
	5008	.RISCV,
	5009	.X86_64,
	5010	.LOONGARCH,
	5011	=> .none,
	5012
	5013	// SPARC uses `R_SPARC_PC{10,22}` relocations to construct pointers to the GOT, but these
	5014	// relocations write an unsigned PC-relative offset. This cannot even be worked around by
	5015	// using a larger code model, because the crt `_start` assembly always uses these specific
	5016	// relocations. Therefore, to avoid relocation errors, all code must appear before the GOT
	5017	// in the virtual address space. The easiest way for us to do that is to ensure that the
	5018	// "mutable data" segment, containing the GOT, is the last segment in the address space.
	5019	.SPARCV9 => .data_last,
	5020	};
	5021	}
	5022	fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child {
4892	5023	const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer;
4893	5024	const Child = pointer_ty.child;
4894	5025	const alignment = pointer_ty.attrs.@"align" orelse @alignOf(Child);
...	...	@@ -4971,16 +5102,6 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr {
4971	5102	}
4972	5103	}
4973	5104
4974		fn arHdrPtr(elf: Elf, ni: MappedFile.Node.Index) align(2) std.elf.ar_hdr {
4975		assert(elf.ni.elf != .root);
4976		const file_offset = ni.fileLocation(&elf.mf, false).offset;
4977		return @ptrCast(@alignCast(elf.mf.memory_map.memory[@intCast(switch (elf.getNode(ni)) {
4978		else => unreachable,
4979		.archive_header => file_offset + std.elf.ARMAG.len,
4980		.elf, .input_member => file_offset - @sizeOf(std.elf.ar_hdr),
4981		})..][0..@sizeOf(std.elf.ar_hdr)]));
4982		}
4983
4984	5105	const SymPtr = union(std.elf.CLASS) {
4985	5106	NONE: noreturn,
4986	5107	@"32": *std.elf.Elf32.Sym,
...	...	@@ -5480,19 +5601,61 @@ fn loadObject(
5480	5601	.member = if (member) \|m\| try gpa.dupe(u8, m) else null,
5481	5602	.extra = undefined,
5482	5603	};
5483		if (elf.ni.elf != .root) {
5484		const archive_ni: MappedFile.Node.Index = .root;
	5604	if (elf.archive) \|*archive\| {
	5605	// We're creating a static library, so just add this input as an archive member.
	5606	assert(member == null); // don't try to put static library members into other static libraries
	5607
	5608	const first_member_oni = archive.header_ni.next(&elf.mf);
	5609
	5610	if (first_member_oni.unwrap()) \|first_member_ni\| switch (elf.getNode(first_member_ni)) {
	5611	.archive_input_member, .archive_elf_member_header => {},
	5612	.elf => unreachable, // always preceded by `.archive_elf_member_header`
	5613	else => unreachable, // never a child of `.archive`
	5614	};
	5615
5485	5616	try elf.nodes.ensureUnusedCapacity(gpa, 1);
5486		input.extra = .{ .node = try archive_ni.addFloatingChild(&elf.mf, gpa, .{
5487		.size = Alignment.@"2".forward(fl.size + @sizeOf(std.elf.ar_hdr)),
	5617	const new_member_ni = try archive.ni.addFooterChildBefore(&elf.mf, gpa, first_member_oni, .{
	5618	.size = Alignment.@"2".forward(@sizeOf(std.elf.ar_hdr) + fl.size),
5488	5619	.alignment = .@"2",
5489		.next_moved = true,
5490		.bubbles_moved = false,
5491		.enable_next_moved = true,
5492		}) };
5493		elf.nodes.appendAssumeCapacity(.{ .input_member = input_index });
	5620	});
	5621	elf.nodes.appendAssumeCapacity(.{ .archive_input_member = input_index });
	5622	input.extra = .{ .node = new_member_ni };
5494	5623	elf.input_prog_node.increaseEstimatedTotalItems(1);
5495	5624
	5625	// The contents of the input will be written to the file by an idle task (`flushInput`), but
	5626	// we do need to write the input's archive member header (`ar_hdr`) now, for two reasons:
	5627	//
	5628	// * If the input file has a long name, we need to add it to the archive member name string
	5629	// table, which must happen deterministically (i.e. not in an idle task).
	5630	//
	5631	// * `flushInput` needs to know the actual file size (before padding to the alignment).
	5632	const member_ar_hdr: *std.elf.ar_hdr = @ptrCast(
	5633	new_member_ni.slice(&elf.mf)[0..@sizeOf(std.elf.ar_hdr)],
	5634	);
	5635	member_ar_hdr.* = .{
	5636	.ar_name = undefined, // populated below
	5637	.ar_date = "0 ".*,
	5638	.ar_uid = "0 ".*,
	5639	.ar_gid = "0 ".*,
	5640	.ar_mode = "644 ".*,
	5641	.ar_size = undefined, // populated below
	5642	.ar_fmag = std.elf.ARFMAG.*,
	5643	};
	5644
	5645	if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{fl.size})) \|size_str\| {
	5646	@memset(member_ar_hdr.ar_size[size_str.len..], ' ');
	5647	} else \|err\| switch (err) {
	5648	error.NoSpaceLeft => return diags.failParse(
	5649	path,
	5650	"file size of {Bi} exceeds maximum size of archive member",
	5651	.{fl.size},
	5652	),
	5653	}
	5654
	5655	const member_name = std.fs.path.basename(path.sub_path);
	5656	// After this call returns, `member_ar_hdr` is invalidated.
	5657	try elf.populateArchiveMemberName(member_ar_hdr, member_name);
	5658
5496	5659	// Since we are not emitting the archive symbol table (yet?) we do not need to parse
5497	5660	// the symbols in this input.
5498	5661	return;
...	...	@@ -5905,6 +6068,46 @@ fn loadObject(
5905	6068	},
5906	6069	}
5907	6070	}
	6071	/// This function may resize the archive header, so therefore invalidates `member_ar_hdr`.
	6072	fn populateArchiveMemberName(elf: Elf, member_ar_hdr: std.elf.ar_hdr, member_name: []const u8) Error!void {
	6073	if (std.mem.print(&member_ar_hdr.ar_name, "{s}/", .{member_name})) \|name_str\| {
	6074	@memset(member_ar_hdr.ar_name[name_str.len..], ' ');
	6075	return;
	6076	} else \|err\| switch (err) {
	6077	error.NoSpaceLeft => {}, // handled below
	6078	}
	6079
	6080	const gpa = elf.base.comp.gpa;
	6081	const archive_header_ni = elf.archive.?.header_ni;
	6082
	6083	// The member's name is too big to put directly in the `ar_name` field, so it needs to go in the
	6084	// "long name" string table instead (in the special member named "//").
	6085
	6086	_, const old_archive_header_size = archive_header_ni.location(&elf.mf).resolve(&elf.mf);
	6087
	6088	// We're going to add a new string at the end of the table. Update `member_ar_hdr` first,
	6089	// because resizing the string table will invalidate it.
	6090	const string_table_offset = old_archive_header_size - (std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr));
	6091	if (std.mem.print(&member_ar_hdr.ar_name, "/{d}", .{string_table_offset})) \|name_str\| {
	6092	@memset(member_ar_hdr.ar_name[name_str.len..], ' ');
	6093	} else \|inner_err\| switch (inner_err) {
	6094	error.NoSpaceLeft => {
	6095	// The string table offset is itself too big to represent. This means the string table's
	6096	// size is definitely too big to represent (we only get 10 bytes for that whereas we
	6097	// get 16 here!), so as long as we still add the string, we're guaranteed to get a link
	6098	// error for that reason. Therefore, we can just ignore this error and carry on.
	6099	},
	6100	}
	6101
	6102	// We set the size of the archive header node exactly, because we want padding bytes to go into
	6103	// the root `.archive` node. That way, those bytes could still be used to grow the string table
	6104	// if necessary, but they could also be used for new archive members.
	6105	try archive_header_ni.resizeLeaf(&elf.mf, gpa, old_archive_header_size + member_name.len + 2);
	6106
	6107	const dest_slice = archive_header_ni.slice(&elf.mf)[@intCast(old_archive_header_size)..];
	6108	@memcpy(dest_slice[0 .. dest_slice.len - 2], member_name);
	6109	@memcpy(dest_slice[dest_slice.len - 2 ..], "/\n"); // yes, the terminator is weird
	6110	}
5908	6111	fn loadDso(elf: Elf, path: std.Build.Cache.Path, fr: Io.File.Reader) (LoadParseInputError \|\| error{BadMagic})!void {
5909	6112	const comp = elf.base.comp;
5910	6113	const gpa = comp.gpa;
...	...	@@ -7072,12 +7275,12 @@ fn addGotRelocAssumeCapacity(
7072	7275	switch (elf.getNode(node)) {
7073	7276	.archive,
7074	7277	.archive_header,
7075		.archive_elf_footer,
	7278	.archive_input_member,
	7279	.archive_elf_member_header,
7076	7280	.elf,
7077	7281	.ehdr,
7078	7282	.shdr,
7079	7283	.segment,
7080		.input_member,
7081	7284	.copied_global,
7082	7285	=> unreachable, // cannot contain relocs,
7083	7286	.section,
...	...	@@ -7129,6 +7332,7 @@ fn addGotRelocAssumeCapacity(
7129	7332	});
7130	7333	}
7131	7334	fn updateGotEntry(elf: *Elf, got_index: usize) void {
	7335	assert(elf.ehdrType() != .REL);
7132	7336	const entry_value: union(enum) {
7133	7337	unsigned: u64,
7134	7338	signed: i64,
...	...	@@ -7514,6 +7718,17 @@ fn flushInner(
7514	7718	diags.addError("failed to apply {d} relocations: misaligned value", .{elf.misaligned_reloc_count});
7515	7719	}
7516	7720
	7721	if (elf.archive) \|*archive\| {
	7722	if (archive.elf_member_too_big) diags.addError(
	7723	"file size of {Bi} exceeds maximum size of archive member",
	7724	.{elf.ni.elf.location(&elf.mf).resolve(&elf.mf)[1]},
	7725	);
	7726	if (archive.strtab_member_too_big) diags.addError(
	7727	"archive file name string table exceeds maximum size",
	7728	.{},
	7729	);
	7730	}
	7731
7517	7732	elf.flushDynamic();
7518	7733
7519	7734	const entry_addr: u64 = entry: {
...	...	@@ -7545,6 +7760,9 @@ pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) link.Error!bool {
7545	7760	const comp = elf.base.comp;
7546	7761	const diags = &comp.link_diags;
7547	7762
	7763	elf.mf.nodes_lock.lock();
	7764	defer elf.mf.nodes_lock.unlock();
	7765
7548	7766	assert(elf.pending_uavs.items.len == 0);
7549	7767	for (&elf.lazy.values) \|*lazy\| {
7550	7768	assert(lazy.pending_index == lazy.map.count());
...	...	@@ -7698,7 +7916,7 @@ fn idleProgNode(
7698	7916	return prog_node.start(name: switch (node) {
7699	7917	else => \|tag\| @tagName(tag),
7700	7918	.section => \|shndx\| shndx.name(elf).slice(elf),
7701		.input_member => \|ii\| std.fmt.bufPrint(&name, "{f}{f}", .{
	7919	.archive_input_member => \|ii\| std.fmt.bufPrint(&name, "{f}{f}", .{
7702	7920	ii.path(elf).fmtEscapeString(),
7703	7921	fmtMemberString(ii.member(elf)),
7704	7922	}) catch &name,
...	...	@@ -7825,7 +8043,6 @@ fn genLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void {
7825	8043	fn flushInput(elf: *Elf, ii: Node.InputIndex) Error!void {
7826	8044	const comp = elf.base.comp;
7827	8045	const io = comp.io;
7828		const gpa = comp.gpa;
7829	8046	const diags = &comp.link_diags;
7830	8047	const path = ii.path(elf);
7831	8048	const file = path.root_dir.handle.openFile(io, path.sub_path, .{}) catch \|err\| switch (err) {
...	...	@@ -7833,23 +8050,40 @@ fn flushInput(elf: *Elf, ii: Node.InputIndex) Error!void {
7833	8050	else => \|e\| return diags.fail("failed to open input file \"{f}\": {t}", .{ path.fmtEscapeString(), e }),
7834	8051	};
7835	8052	defer file.close(io);
	8053
	8054	const slice = ii.node(elf).slice(&elf.mf);
	8055
	8056	const member_ar_hdr: *const std.elf.ar_hdr = @ptrCast(slice[0..@sizeOf(std.elf.ar_hdr)]);
	8057	const input_size: u32 = member_ar_hdr.size() catch \|err\| switch (err) {
	8058	// We wrote the `ar_hdr` ourselves (in `loadObject`), so it is definitely valid.
	8059	error.Overflow, error.InvalidCharacter => unreachable,
	8060	};
	8061
	8062	switch (slice.len - @sizeOf(std.elf.ar_hdr) - input_size) {
	8063	0 => {},
	8064	1 => {
	8065	// Alignment added one padding byte, which the format requires to have value '\n'.
	8066	slice[slice.len - 1] = '\n';
	8067	},
	8068	else => unreachable, // node size should agree with the value we wrote into `ar_hdr.ar_size`
	8069	}
	8070
7836	8071	var fr = file.reader(io, &.{});
7837		var nw: MappedFile.Node.Writer = undefined;
7838		ii.node(elf).writer(&elf.mf, gpa, &nw);
7839		defer nw.deinit();
7840		const size = nw.interface.buffer.len - @sizeOf(std.elf.ar_hdr);
7841		const n_bytes = nw.interface.sendFileAll(&fr, .limited(size)) catch \|err\| switch (err) {
	8072	var w: Io.Writer = .fixed(slice[@sizeOf(std.elf.ar_hdr)..]);
	8073	const n_bytes_read = w.sendFileAll(&fr, .limited(input_size)) catch \|err\| switch (err) {
7842	8074	error.ReadFailed => return diags.fail("failed to read input \"{f}{f}\": {t}", .{
7843	8075	path.fmtEscapeString(),
7844	8076	fmtMemberString(ii.member(elf)),
7845	8077	fr.err orelse (fr.seek_err orelse fr.size_err.?),
7846	8078	}),
7847		error.WriteFailed => return nw.err.?,
	8079	error.WriteFailed => unreachable, // `.limited(input_size)` prevents us writing too many bytes
7848	8080	};
7849		if (n_bytes + 1 < size) return diags.fail("failed to read input \"{f}{f}\": unexpected eof", .{
7850		path.fmtEscapeString(),
7851		fmtMemberString(ii.member(elf)),
7852		});
	8081	if (n_bytes_read != input_size) {
	8082	return diags.fail("failed to load input \"{f}{f}\": file truncated during compilation", .{
	8083	path.fmtEscapeString(),
	8084	fmtMemberString(ii.member(elf)),
	8085	});
	8086	}
7853	8087	}
7854	8088
7855	8089	fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void {
...	...	@@ -7931,12 +8165,18 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void
7931	8165	const trace = tracy.trace(@src());
7932	8166	defer trace.end();
7933	8167
7934		elf.mf.nodes_lock.lock();
7935		defer elf.mf.nodes_lock.unlock();
7936
7937	8168	switch (elf.getNode(ni)) {
7938		.archive, .archive_header => unreachable,
7939		.archive_elf_footer, .elf => {},
	8169	.archive => unreachable,
	8170	.archive_header => unreachable,
	8171
	8172	.archive_input_member,
	8173	.archive_elf_member_header,
	8174	.elf,
	8175	=> {
	8176	assert(elf.archive != null);
	8177	return;
	8178	},
	8179
7940	8180	.ehdr, .shdr => elf.flushElfOffset(ni),
7941	8181	.segment => \|phndx\| {
7942	8182	elf.flushElfOffset(ni);
...	...	@@ -8014,7 +8254,6 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void
8014	8254	elf.flushMovedPltSection(.plt_sec, old_addr, addr);
8015	8255	}
8016	8256	},
8017		.input_member => {},
8018	8257	.input_section => \|isi\| {
8019	8258	const old_section_addr = isi.ptr(elf).vaddr;
8020	8259	const new_section_addr = elf.computeNodeVAddr(ni);
...	...	@@ -8114,6 +8353,18 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro
8114	8353	const page_align = elf.targetPageAlign();
8115	8354	const node_align = segment_ni.alignment(&elf.mf);
8116	8355	const ph_align = page_align.max(node_align);
	8356
	8357	// If we determine that the segment's virtual address needs to move, then it's a good idea to
	8358	// make it less likely that it needs to move again in the future, because it is expensive to
	8359	// change a segment's load address (a lot of re-flushing is necessary). To do that, we reserve
	8360	// more virtual address space than we need (multiplying the actual size by this value). That
	8361	// way, there will usually be padding between segments which they can grow into.
	8362	//
	8363	// TODO: we might want to decrease this multiplier, or even omit it entirely, in cases where
	8364	// virtual address space is constrained. For instance, 32-bit targets, or targets where short
	8365	// PC-relative relocations between segments are common.
	8366	const reserve_size_multiplier = 4;
	8367
8117	8368	switch (elf.phdrSlice()) {
8118	8369	inline else => \|phdr\| {
8119	8370	const offset = elf.targetLoad(&phdr[orig_phndx].offset);
...	...	@@ -8172,15 +8423,46 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro
8172	8423	// backwards to the start of the page.
8173	8424	const next_page_vaddr = std.mem.alignBackward(u64, next_vaddr, page_align.toByteUnits());
8174	8425
8175		// If we're at the same vaddr we started at, then all we're worried about is the
8176		// segment fitting here. However, if we've already changed our virtual address, then
8177		// we might as well try to reserve a bit more virtual address space while we're at
8178		// it, because changing virtual address is quite disruptive (we need to re-flush a
8179		// lot of stuff!) and giving ourselves more space will make it less likely to happen
8180		// again.
8181		const target_size = if (vaddr == orig_vaddr) size else size * 4;
8182		if (vaddr + target_size <= next_page_vaddr) {
8183		break; // hooray, we fit here!
	8426	// Check if the segment fits here. We apply `reserve_size_multiplier`, but only if
	8427	// the segment is already known to be moving---making it easier to grow in-place is
	8428	// the whole point of the multiplier!
	8429	{
	8430	const target_size = if (vaddr == orig_vaddr) size else size * reserve_size_multiplier;
	8431	if (vaddr + target_size <= next_page_vaddr) {
	8432	break; // hooray, we fit here!
	8433	}
	8434	}
	8435
	8436	const next_ni = elf.phdrs.items[next_phndx].unwrap().?;
	8437
	8438	// This segment don't fit here, but before deciding how to proceed, we need to
	8439	// consider any target-specific restrictions we are subject to.
	8440	switch (elf.targetSegmentLoadAddressRestrictions()) {
	8441	.none => {},
	8442	.data_last => if (next_ni == elf.ni.data) {
	8443	// We can't leapfrog over the data segment. Instead, that segment just needs
	8444	// to be shifted forwards to make space for us, and we'll then `break` with
	8445	// our current vaddr.
	8446
	8447	if (next_phndx + 1 < phdr.len) switch (elf.targetLoad(&phdr[next_phndx + 1].type)) {
	8448	.NULL, .LOAD => unreachable, // data segment should be the last loadable segment
	8449	else => {},
	8450	};
	8451
	8452	const free_vaddr = vaddr + size * reserve_size_multiplier;
	8453
	8454	const next_align = page_align.max(next_ni.alignment(&elf.mf));
	8455	const next_offset = elf.targetLoad(&next_ph.offset);
	8456	const next_new_vaddr = next_align.forward(free_vaddr) + next_offset % next_align.toByteUnits();
	8457
	8458	// This logic for updating the data segment's vaddr is identical to how we
	8459	// will update the vaddr of `phndx` when we break from the loop.
	8460	elf.targetStore(&next_ph.vaddr, @intCast(next_new_vaddr));
	8461	elf.targetStore(&next_ph.paddr, @intCast(next_new_vaddr));
	8462	try next_ni.childrenMoved(elf.base.comp.gpa, &elf.mf);
	8463
	8464	break;
	8465	},
8184	8466	}
8185	8467
8186	8468	// We don't fit here, so shift ourselves forward (i.e. swap with `next_phndx`). But
...	...	@@ -8192,8 +8474,7 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro
8192	8474
8193	8475	// Now just swap the phdrs and update our `phndx`.
8194	8476	std.mem.swap(@TypeOf(next_ph.*), &phdr[phndx], next_ph);
8195		const next_ni = elf.phdrs.items[next_phndx];
8196		elf.phdrs.items[phndx] = next_ni;
	8477	elf.phdrs.items[phndx] = .wrap(next_ni);
8197	8478	elf.nodes.items(.data)[@backingInt(next_ni)] = .{ .segment = phndx };
8198	8479	elf.phdrs.items[next_phndx] = .wrap(segment_ni);
8199	8480	elf.nodes.items(.data)[@backingInt(segment_ni)] = .{ .segment = @intCast(next_phndx) };
...	...	@@ -8213,24 +8494,23 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo
8213	8494	const trace = tracy.trace(@src());
8214	8495	defer trace.end();
8215	8496
8216		elf.mf.nodes_lock.lock();
8217		defer elf.mf.nodes_lock.unlock();
8218
8219	8497	_, const size = ni.location(&elf.mf).resolve(&elf.mf);
8220	8498	switch (elf.getNode(ni)) {
8221		.archive => {
8222		if (ni.last(&elf.mf).unwrap()) \|last_ni\| {
8223		if (last_ni.prev(&elf.mf).unwrap()) \|prev_ni\| {
8224		if (prev_ni.hasNextMoved(&elf.mf)) return;
8225		}
8226		const offset, _ = last_ni.location(&elf.mf).resolve(&elf.mf);
8227		_ = std.mem.print(&elf.arHdrPtr(last_ni).ar_size, "{d:<10}", .{
8228		size - offset,
8229		}) catch @panic("archive member too large");
	8499	.archive, .archive_header => {},
	8500	.archive_input_member => unreachable,
	8501	.archive_elf_member_header => unreachable,
	8502	.elf => if (elf.archive) \|*archive\| {
	8503	const member_ar_hdr: *std.elf.ar_hdr = @ptrCast(
	8504	archive.elf_member_header_ni.slice(&elf.mf),
	8505	);
	8506	if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{size})) \|size_str\| {
	8507	@memset(member_ar_hdr.ar_size[size_str.len..], ' ');
	8508	archive.elf_member_too_big = false;
	8509	} else \|err\| switch (err) {
	8510	error.NoSpaceLeft => archive.elf_member_too_big = true,
8230	8511	}
8231	8512	},
8232		.archive_header, .elf => {},
8233		.ehdr, .archive_elf_footer => unreachable,
	8513	.ehdr => unreachable,
8234	8514	.shdr => {},
8235	8515	.segment => \|phndx\| switch (elf.phdrSlice()) {
8236	8516	inline else => \|phdr\| {
...	...	@@ -8302,7 +8582,7 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo
8302	8582	}
8303	8583	},
8304	8584	},
8305		.input_member, .input_section, .copied_global, .nav, .uav, .lazy_code, .lazy_const_data => {},
	8585	.input_section, .copied_global, .nav, .uav, .lazy_code, .lazy_const_data => {},
8306	8586	}
8307	8587	}
8308	8588
...	...	@@ -8310,12 +8590,11 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!
8310	8590	const trace = tracy.trace(@src());
8311	8591	defer trace.end();
8312	8592
8313		elf.mf.nodes_lock.lock();
8314		defer elf.mf.nodes_lock.unlock();
8315
8316	8593	switch (elf.getNode(ni)) {
8317	8594	.archive,
8318		.archive_elf_footer,
	8595	.archive_input_member,
	8596	.archive_elf_member_header,
	8597	.elf,
8319	8598	.ehdr,
8320	8599	.shdr,
8321	8600	.segment,
...	...	@@ -8327,54 +8606,32 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!
8327	8606	.lazy_code,
8328	8607	.lazy_const_data,
8329	8608	=> unreachable,
8330		.archive_header, .elf, .input_member => \|_, tag\| {
8331		const member_offset, const update_size = member_offset: {
8332		const offset, _ = ni.location(&elf.mf).resolve(&elf.mf);
8333		break :member_offset switch (tag) {
8334		else => unreachable,
8335		.archive_header => .{ offset + std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr), true },
8336		.elf, .input_member => .{ offset, !ni.prev(&elf.mf).unwrap().?.hasNextMoved(&elf.mf) },
8337		};
8338		};
8339		const member_size = if (ni.next(&elf.mf).unwrap()) \|next_ni\| member_size: {
8340		const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf);
8341		const next_member_size = if (next_ni.next(&elf.mf).unwrap()) \|next_next_ni\| next_member_size: {
8342		const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf);
8343		const next_member_end = next_next_offset - @sizeOf(std.elf.ar_hdr);
8344		break :next_member_size next_member_end - next_offset;
8345		} else next_member_size: {
8346		_, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf);
8347		const next_member_end = parent_size;
8348		break :next_member_size next_member_end - next_offset;
8349		};
8350		const ar_hdr = elf.arHdrPtr(next_ni);
8351		var name_buf: [16]u8 = undefined;
8352		_ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{
8353		switch (elf.getNode(next_ni)) {
8354		else => unreachable,
8355		.elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}),
8356		.input_member => \|ii\| std.mem.print(&name_buf, "{s}/", .{
8357		std.fs.path.basename(ii.path(elf).sub_path),
8358		}),
8359		} catch @panic("TODO: long archive member names"),
8360		}) catch @panic("TODO: long archive member names");
8361		ar_hdr.ar_date = "0 ".*;
8362		ar_hdr.ar_uid = "0 ".*;
8363		ar_hdr.ar_gid = "0 ".*;
8364		ar_hdr.ar_mode = "644 ".*;
8365		_ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch
8366		@panic("archive member too large");
8367		ar_hdr.ar_fmag = std.elf.ARFMAG.*;
8368		const member_end = next_offset - @sizeOf(std.elf.ar_hdr);
8369		break :member_size member_end - member_offset;
8370		} else member_size: {
8371		_, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf);
8372		const member_end = parent_size;
8373		break :member_size member_end - member_offset;
	8609
	8610	.archive_header => {
	8611	const archive = &elf.archive.?;
	8612
	8613	// Because we can't just throw padding bytes in the middle of an archive file, we need
	8614	// the member name string table (the "//" member) to absorb all the padding bytes
	8615	// between it (in the `.archive_header` node) and the first actual member.
	8616	const next_member_ni = ni.next(&elf.mf).unwrap() orelse {
	8617	// I guess there are no link inputs yet? But there will be eventually!
	8618	return;
8374	8619	};
8375		if (update_size) _ = std.mem.print(&elf.arHdrPtr(ni).ar_size, "{d:<10}", .{
8376		member_size,
8377		}) catch @panic("archive member too large");
	8620	const next_member_offset: u64, _ = next_member_ni.location(&elf.mf).resolve(&elf.mf);
	8621	const strtab_member_offset = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr);
	8622	assert(Alignment.@"2".check(next_member_offset));
	8623	assert(Alignment.@"2".check(strtab_member_offset));
	8624	const strtab_size = next_member_offset - strtab_member_offset;
	8625
	8626	const member_ar_hdr: *std.elf.ar_hdr = @ptrCast(
	8627	archive.header_ni.slice(&elf.mf)[std.elf.ARMAG.len..][0..@sizeOf(std.elf.ar_hdr)],
	8628	);
	8629	if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{strtab_size})) \|size_str\| {
	8630	@memset(member_ar_hdr.ar_size[size_str.len..], ' ');
	8631	archive.strtab_member_too_big = false;
	8632	} else \|err\| switch (err) {
	8633	error.NoSpaceLeft => archive.strtab_member_too_big = true,
	8634	}
8378	8635	},
8379	8636	}
8380	8637	}

src/link/MappedFile.zig+405-236

...	...	@@ -648,7 +648,10 @@ pub const Node = extern struct {
648	648	_, const current_size = ni.location(mf).resolve(mf);
649	649	if (current_size >= min_size) return;
650	650	const new_size = ni.alignment(mf).forward(min_size +\| min_size / growth_factor);
651		try mf.growNode(gpa, ni, new_size, .minimum);
	651	try mf.growNode(gpa, ni, new_size, .{
	652	.exact_size = false,
	653	.move_footers = true,
	654	});
652	655	mf.updateWriters();
653	656	}
654	657
...	...	@@ -664,7 +667,10 @@ pub const Node = extern struct {
664	667	switch (std.math.order(size, old_size)) {
665	668	.lt => try mf.shrinkLeafNode(gpa, ni, size),
666	669	.eq => {}, // `old_size` must be well-aligned, so `size` is too
667		.gt => try mf.growNode(gpa, ni, size, .exact),
	670	.gt => try mf.growNode(gpa, ni, size, .{
	671	.exact_size = true,
	672	.move_footers = false, // irrelevant, since we have no footers
	673	}),
668	674	}
669	675	mf.updateWriters();
670	676	}
...	...	@@ -935,7 +941,10 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct {
935	941
936	942	try mf.realignNode(gpa, new_ni, opts.add_options.alignment);
937	943	if (opts.add_options.size > 0) {
938		try mf.growNode(gpa, new_ni, opts.add_options.size, .exact);
	944	try mf.growNode(gpa, new_ni, opts.add_options.size, .{
	945	.exact_size = true,
	946	.move_footers = false, // irrelevant, since we have no footers
	947	});
939	948	}
940	949	mf.updateWriters();
941	950
...	...	@@ -1070,12 +1079,21 @@ fn shrinkLeafNode(
1070	1079	}
1071	1080	}
1072	1081
1073		const GrowMode = enum { exact, minimum };
	1082	const GrowOptions = struct {
	1083	/// If `true`, the node size must be set to exactly the given size.
	1084	///
	1085	/// If `false`, the given size is a minimum, and the actual new node size may be larger.
	1086	exact_size: bool,
	1087	/// If `true`, footers within the resized node will be moved forwards to its new end.
	1088	///
	1089	/// If `false`, footers will all remain at their current offsets (so the nodes are in a
	1090	/// temporarily invalid state), and moving them is the responsibility of the caller.
	1091	move_footers: bool,
	1092	};
1074	1093
1075		/// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`.
1076		/// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`.
	1094	/// Increases the size of a node.
1077	1095	///
1078		/// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!).
	1096	/// Asserts that `new_size` is aligned to `ni.alignment(mf)`, even if `!grow_options.exact_size`.
1079	1097	///
1080	1098	/// Asserts that `new_size` is greater than the current size of `ni`.
1081	1099	fn growNode(
...	...	@@ -1083,7 +1101,7 @@ fn growNode(
1083	1101	gpa: Allocator,
1084	1102	ni: Node.Index,
1085	1103	new_size: u64,
1086		grow_mode: GrowMode,
	1104	grow_options: GrowOptions,
1087	1105	) Error!void {
1088	1106	mf.nodes_lock.assertUnlocked();
1089	1107
...	...	@@ -1098,7 +1116,7 @@ fn growNode(
1098	1116	const parent_ni = node.parent.unwrap() orelse {
1099	1117	assert(ni == .root);
1100	1118
1101		if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {
	1119	if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) {
1102	1120	return;
1103	1121	}
1104	1122
...	...	@@ -1120,21 +1138,23 @@ fn growNode(
1120	1138	};
1121	1139	try mf.ensureTotalCapacityPrecise(@intCast(new_size));
1122	1140	try ni.setLocation(mf, gpa, old_offset, new_size);
1123		// We need to move any footers to be at the new end of the file.
1124		if (ni.firstFooter(mf).unwrap()) \|first_footer_ni\| {
1125		const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);
1126		const footers_size = old_size - old_footers_offset;
1127		try mf.moveRange(
1128		old_footers_offset,
1129		old_footers_offset + (new_size - old_size),
1130		footers_size,
1131		);
1132		// Also update the footers' locations.
1133		var cur_ni = first_footer_ni;
1134		while (true) {
1135		const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);
1136		try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);
1137		cur_ni = cur_ni.next(mf).unwrap() orelse break;
	1141	if (grow_options.move_footers) {
	1142	// We need to move any footers to be at the new end of the file.
	1143	if (ni.firstFooter(mf).unwrap()) \|first_footer_ni\| {
	1144	const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);
	1145	const footers_size = old_size - old_footers_offset;
	1146	try mf.moveRange(
	1147	old_footers_offset,
	1148	old_footers_offset + (new_size - old_size),
	1149	footers_size,
	1150	);
	1151	// Also update the footers' locations.
	1152	var cur_ni = first_footer_ni;
	1153	while (true) {
	1154	const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);
	1155	try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);
	1156	cur_ni = cur_ni.next(mf).unwrap() orelse break;
	1157	}
1138	1158	}
1139	1159	}
1140	1160	return;
...	...	@@ -1142,7 +1162,7 @@ fn growNode(
1142	1162
1143	1163	switch (node.flags.position) {
1144	1164	.header => {
1145		if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {
	1165	if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) {
1146	1166	return;
1147	1167	}
1148	1168
...	...	@@ -1163,7 +1183,13 @@ fn growNode(
1163	1183	const old_headers_size = last_header_offset + last_header_size;
1164	1184
1165	1185	// This is the first footer inside of `ni`.
1166		const first_sub_footer_oni = ni.firstFooter(mf);
	1186	const first_sub_footer_oni: Node.Index.Optional = footer: {
	1187	if (!grow_options.move_footers) {
	1188	// Pretend there are no footers so as to not move them.
	1189	break :footer .none;
	1190	}
	1191	break :footer ni.firstFooter(mf);
	1192	};
1167	1193	const sub_footers_size = size: {
1168	1194	const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
1169	1195	const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
...	...	@@ -1215,92 +1241,264 @@ fn growNode(
1215	1241	return;
1216	1242	},
1217	1243	.floating => {
1218		try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode);
	1244	try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_options);
1219	1245	},
1220	1246	.footer => {
1221		if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {
	1247	if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) {
1222	1248	return;
1223	1249	}
1224	1250
1225		try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size);
1226
1227		const first_footer_ni: Node.Index = first_footer: {
1228		var footer_ni = ni;
1229		while (true) {
1230		const prev_ni = footer_ni.prev(mf).unwrap() orelse break;
1231		if (prev_ni.position(mf) != .footer) break;
1232		footer_ni = prev_ni;
	1251	// This is the first footer inside of `ni` (unrelated to the fact that `ni` is itself
	1252	// a footer within its parent). We'll need this later in any case, so just find it now.
	1253	const first_sub_footer_oni: Node.Index.Optional = footer: {
	1254	if (!grow_options.move_footers) {
	1255	// Pretend there are no nested footers so as to not move them.
	1256	break :footer .none;
1233	1257	}
1234		break :first_footer footer_ni;
	1258	break :footer ni.firstFooter(mf);
1235	1259	};
1236	1260
1237		// This is the first footer inside of `ni` (unrelated to the fact that `ni` is itself
1238		// a footer within its parent).
1239		const first_sub_footer_oni = ni.firstFooter(mf);
1240		const sub_footers_size = size: {
1241		const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
1242		const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
1243		break :size old_size - first_sub_footer_offset;
	1261	// We have two different strategies for growing a footer node, with different advantages
	1262	// and disadvantages; so first we must decide which to use.
	1263	const strat: union(enum) {
	1264	/// Expand into pre-footer padding space in the parent node (growing the parent if
	1265	/// necessary). This strategy has the benefit that it can reclaim padding bytes in
	1266	/// the parent, but it has the disadvantage that it requires moving this node's
	1267	/// existing content backwards in the file, which may be expensive (particularly
	1268	/// since the src and dest ranges are likely to overlap).
	1269	grow_backwards,
	1270
	1271	/// Grow the parent node with `GrowOptions.move_footers` set to `false`, and
	1272	/// implicitly grow ourselves into the newly available space. This usually requires
	1273	/// a lot less moving of bytes, but never reclaims unused space before the parent's
	1274	/// footers, and is sometimes straight-up impossible.
	1275	grow_parent_at_end: struct {
	1276	add_size: u64,
	1277	exact_size: bool,
	1278	},
	1279	} = strat: {
	1280	// If this node is small, the move overhead is trivial, so prefer `.grow_backwards`
	1281	// to avoid unnecessary growth of the parent node.
	1282	if (old_size <= mf.flags.block_size.toByteUnits() * 2) {
	1283	break :strat .grow_backwards;
	1284	}
	1285
	1286	// It may also be worth doing `.grow_backwards` if the parent has a lot of space
	1287	// we could grow into. More specifically, if "free space we can grow into" makes up
	1288	// a significant proportion of the parent's total size, then that implies the parent
	1289	// has quite poor utilization of space, and that we can significantly improve that
	1290	// statistic by growing into that space.
	1291	if (old_size + mf.availableFooterCapacity(parent_ni) >= new_size) {
	1292	break :strat .grow_backwards;
	1293	}
	1294
	1295	if (grow_options.exact_size) {
	1296	const add_size = new_size - old_size;
	1297	if (parent_ni.alignment(mf).check(add_size)) {
	1298	break :strat .{ .grow_parent_at_end = .{
	1299	.add_size = add_size,
	1300	.exact_size = true,
	1301	} };
	1302	} else {
	1303	// We can't ask the parent to grow by this much, so we have no choice.
	1304	break :strat .grow_backwards;
	1305	}
	1306	}
	1307
	1308	if (parent_ni.alignment(mf).compare(.lt, node.flags.alignment)) {
	1309	// Because the parent's alignment is less than our own, if we gave them the
	1310	// freedom to pick a size, they might choose one which results in us having a
	1311	// size incompatible with our alignment. Therefore, to prevent that, we need to
	1312	// request an exact size from the parent in this case.
	1313	break :strat .{ .grow_parent_at_end = .{
	1314	.add_size = new_size - old_size,
	1315	.exact_size = true,
	1316	} };
	1317	}
	1318
	1319	// The parent's alignment is greater than or equal to our own, so we only need to
	1320	// give the parent a minimum size (although we need to ensure it matches their
	1321	// alignment since it could be greater than our own).
	1322	break :strat .{ .grow_parent_at_end = .{
	1323	.add_size = parent_ni.alignment(mf).forward(new_size - old_size),
	1324	.exact_size = false,
	1325	} };
1244	1326	};
1245	1327
1246		_, const parent_size = parent_ni.location(mf).resolve(mf);
	1328	switch (strat) {
	1329	.grow_backwards => {
	1330	// First, we might need to grow the parent to make enough space.
	1331	{
	1332	const available_size = mf.availableFooterCapacity(parent_ni);
	1333	if (old_size + available_size < new_size) {
	1334	_, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf);
	1335	const min_parent_size = old_parent_size + (new_size - old_size - available_size);
	1336	const new_parent_size = parent_ni.alignment(mf).forward(
	1337	min_parent_size +\| min_parent_size / growth_factor,
	1338	);
	1339	try mf.growNode(gpa, parent_ni, new_parent_size, .{
	1340	.exact_size = false,
	1341	.move_footers = true,
	1342	});
	1343	assert(old_size + mf.availableFooterCapacity(parent_ni) >= new_size);
	1344	}
	1345	}
1247	1346
1248		const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0];
1249		const new_footers_size = old_footers_size - old_size + new_size;
1250
1251		// Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node
1252		// itself needs to shift its contents, because our offset was shifted backwards by
1253		// `new_size - old_size`, and the added bytes should go at the end of this footer node.
1254		// However, if we contain any footer nodes, they need to stay at the end of `ni`, so
1255		// we shouldn't shift that data.
1256		const old_footers_start = parent_size - old_footers_size;
1257		const new_footers_start = parent_size - new_footers_size;
1258		const end_offset = node.location().resolve(mf)[0] + old_size;
1259		const parent_file_offset = parent_ni.fileLocation(mf, false).offset;
1260		try mf.moveRange(
1261		parent_file_offset + old_footers_start,
1262		parent_file_offset + new_footers_start,
1263		end_offset - old_footers_start - sub_footers_size,
1264		);
	1347	// Now we need to grow! To do that, we must move `ni` itself, and every footer
	1348	// before it in `parent_ni`, backwards. Unlike header nodes, `ni` is included in
	1349	// the shift, because the bytes we're adding need to go at the end of `ni`
	1350	// rather than its start.
	1351
	1352	// This is the same as `parent_ni.firstFooter(mf)`, it's just more efficient to
	1353	// start at `ni` than to start at `parent_ni.last(mf)`.
	1354	const first_parent_footer_ni: Node.Index = first_footer: {
	1355	var footer_ni = ni;
	1356	while (true) {
	1357	const prev_ni = footer_ni.prev(mf).unwrap() orelse break;
	1358	if (prev_ni.position(mf) != .footer) break;
	1359	footer_ni = prev_ni;
	1360	}
	1361	break :first_footer footer_ni;
	1362	};
1265	1363
1266		// Update our own offset and size:
1267		try ni.setLocation(mf, gpa, end_offset - new_size, new_size);
	1364	const shift = new_size - old_size;
1268	1365
1269		// Any footers inside of us have had their offsets changed due to us growing:
1270		if (first_sub_footer_oni.unwrap()) \|first_sub_footer_ni\| {
1271		var cur_ni = first_sub_footer_ni;
1272		while (true) {
1273		const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf);
1274		try cur_ni.setLocation(
	1366	// Update our own offset and size:
	1367	try ni.setLocation(
1275	1368	mf,
1276	1369	gpa,
1277		old_sub_footer_offset + (new_size - old_size),
1278		sub_footer_size,
	1370	node.location().resolve(mf)[0] - shift,
	1371	new_size,
1279	1372	);
1280		cur_ni = cur_ni.next(mf).unwrap() orelse break;
1281		}
1282		}
1283	1373
1284		// Finally, update the offsets of every footer before us:
1285		if (node.prev.unwrap()) \|prev_ni\| {
1286		var maybe_footer_ni = prev_ni;
1287		while (true) {
1288		switch (maybe_footer_ni.position(mf)) {
1289		.header, .floating => break,
1290		.footer => {},
	1374	// Any footers inside of `ni` have had their offsets changed, because they are
	1375	// now positioned at the new end of `ni`:
	1376	{
	1377	var footer_oni = first_sub_footer_oni;
	1378	while (footer_oni.unwrap()) \|footer_ni\| : (footer_oni = footer_ni.next(mf)) {
	1379	const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);
	1380	try footer_ni.setLocation(mf, gpa, old_footer_offset + shift, footer_size);
	1381	}
	1382	}
	1383
	1384	// Any footers before `ni` (in `parent_ni`) have been shifted backwards. We'll
	1385	// also be moving their actual bytes in a moment, so track whether they have
	1386	// content (if nothing does then we'll be able to skip the `moveRange`). That
	1387	// flag is initially whether `ni` has content because we're shifting our own
	1388	// bytes backwards too.
	1389	var moved_has_content: bool = node.flags.has_content;
	1390	{
	1391	var footer_ni = first_parent_footer_ni;
	1392	while (footer_ni != ni) : (footer_ni = footer_ni.next(mf).unwrap().?) {
	1393	moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content;
	1394	const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);
	1395	try footer_ni.setLocation(mf, gpa, old_footer_offset - shift, footer_size);
	1396	}
1291	1397	}
1292		const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf);
1293		try maybe_footer_ni.setLocation(
	1398
	1399	if (moved_has_content) {
	1400	// We moved at least one thing containing initialized bytes, so we need to
	1401	// move the actual data. However, we should not move the bytes of any
	1402	// nested footers inside of `ni`, because they've been "moved" to the end
	1403	// of our new size, which is the same file location as before.
	1404	const sub_footers_size = size: {
	1405	const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
	1406	const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
	1407	break :size new_size - first_sub_footer_offset;
	1408	};
	1409	const new_offset: u64, _ = node.location().resolve(mf);
	1410	const new_footers_offset: u64, _ = first_parent_footer_ni.location(mf).resolve(mf);
	1411	const parent_file_offset = parent_ni.fileLocation(mf, false).offset;
	1412	try mf.moveRange(
	1413	parent_file_offset + new_footers_offset + shift,
	1414	parent_file_offset + new_footers_offset,
	1415	(new_offset - new_footers_offset) + // accounts for every footer before `ni`
	1416	(old_size - sub_footers_size), // accounts for `ni` itself, excluding nested footers
	1417	);
	1418	}
	1419	},
	1420	.grow_parent_at_end => \|grow_parent\| {
	1421	_, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf);
	1422	try mf.growNode(gpa, parent_ni, old_parent_size + grow_parent.add_size, .{
	1423	.exact_size = grow_parent.exact_size,
	1424	.move_footers = false,
	1425	});
	1426	_, const new_parent_size: u64 = parent_ni.location(mf).resolve(mf);
	1427	const shift = new_parent_size - old_parent_size;
	1428
	1429	// Here's what we have left to do:
	1430	//
	1431	// * Increase our own size by `shift` to absorb the added space.
	1432	//
	1433	// * If there are any footers inside `ni`, increase their offsets by `shift`.
	1434	//
	1435	// * If there are any footers after `ni` (inside `parent_ni`), increase their
	1436	// offsets by `shift`.
	1437	//
	1438	// * Do a `moveRange` corresponding to those offset changes. This is a single
	1439	// range which starts at the footers inside `ni`.
	1440
	1441	const actual_new_size = old_size + shift;
	1442	if (grow_options.exact_size) {
	1443	assert(actual_new_size == new_size);
	1444	}
	1445
	1446	try ni.setLocation(
1294	1447	mf,
1295	1448	gpa,
1296		moved_footer_offset + old_size - new_size,
1297		moved_footer_size,
	1449	node.location().resolve(mf)[0],
	1450	actual_new_size,
1298	1451	);
1299		maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break;
1300		}
1301		}
1302	1452
1303		return;
	1453	// This will track whether any node with a changed offset actually contains
	1454	// initialized bytes. If not, there'll be no need to call `moveRange`.
	1455	var moved_has_content: bool = false;
	1456
	1457	// Set any nested footers' offsets (and include them in `moved_has_content`).
	1458	{
	1459	var footer_oni = first_sub_footer_oni;
	1460	while (footer_oni.unwrap()) \|footer_ni\| : (footer_oni = footer_ni.next(mf)) {
	1461	assert(footer_ni.position(mf) == .footer);
	1462	moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content;
	1463	const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf);
	1464	try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size);
	1465	}
	1466	}
	1467
	1468	// Now set offsets for footers after `ni` inside of `parent_ni`.
	1469	{
	1470	var footer_oni = ni.next(mf);
	1471	while (footer_oni.unwrap()) \|footer_ni\| : (footer_oni = footer_ni.next(mf)) {
	1472	assert(footer_ni.position(mf) == .footer);
	1473	moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content;
	1474	const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf);
	1475	try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size);
	1476	}
	1477	}
	1478
	1479	if (moved_has_content) {
	1480	// We moved at least one footer containing initialized bytes, so we need to
	1481	// move the actual data. Compute how big the footers inside `ni` are...
	1482	const sub_footers_size: u64 = size: {
	1483	const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
	1484	const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
	1485	// `actual_new_size` is used here since we already updated the nested footers' offsets above.
	1486	break :size actual_new_size - first_sub_footer_offset;
	1487	};
	1488	// ...and how big the footers after `ni`, inside `parent_ni`, are...
	1489	const post_footers_size: u64 = old_parent_size - (old_offset + old_size);
	1490	// ...and move them both.
	1491	const parent_file_off = parent_ni.fileLocation(mf, false).offset;
	1492	const total_move_size = sub_footers_size + post_footers_size;
	1493	assert(total_move_size != 0);
	1494	try mf.moveRange(
	1495	parent_file_off + old_parent_size - total_move_size,
	1496	parent_file_off + new_parent_size - total_move_size,
	1497	total_move_size,
	1498	);
	1499	}
	1500	},
	1501	}
1304	1502	},
1305	1503	}
1306	1504	}
...	...	@@ -1320,7 +1518,7 @@ fn growFloatingNodeWithAlignment(
1320	1518	ni: Node.Index,
1321	1519	new_alignment: ?Alignment,
1322	1520	new_size: u64,
1323		grow_mode: GrowMode,
	1521	grow_options: GrowOptions,
1324	1522	) Error!void {
1325	1523	mf.nodes_lock.assertUnlocked();
1326	1524
...	...	@@ -1347,27 +1545,29 @@ fn growFloatingNodeWithAlignment(
1347	1545	}
1348	1546	// Great, we can grow this node without changing its offset or moving any siblings.
1349	1547	try ni.setLocation(mf, gpa, old_offset, new_size);
1350		// If we have any footers, we need to move them to the end of our new size, and update their
1351		// offsets accordingly.
1352		if (ni.firstFooter(mf).unwrap()) \|first_footer_ni\| {
1353		var cur_ni = first_footer_ni;
1354		var footers_have_content = false;
1355		while (true) {
1356		footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content;
1357		const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);
1358		try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);
1359		cur_ni = cur_ni.next(mf).unwrap() orelse break;
1360		}
1361		if (footers_have_content) {
1362		const parent_file_off = parent_ni.fileLocation(mf, false).offset;
1363		// This gets the new offset because we already updated the offsets above.
1364		const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);
1365		const footers_size = new_size - new_footers_offset;
1366		try mf.moveRange(
1367		parent_file_off + old_offset + old_size - footers_size,
1368		parent_file_off + old_offset + new_size - footers_size,
1369		footers_size,
1370		);
	1548	if (grow_options.move_footers) {
	1549	// If we have any footers, we need to move them to the end of our new size, and update
	1550	// their offsets accordingly.
	1551	if (ni.firstFooter(mf).unwrap()) \|first_footer_ni\| {
	1552	var cur_ni = first_footer_ni;
	1553	var footers_have_content = false;
	1554	while (true) {
	1555	footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content;
	1556	const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);
	1557	try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);
	1558	cur_ni = cur_ni.next(mf).unwrap() orelse break;
	1559	}
	1560	if (footers_have_content) {
	1561	const parent_file_off = parent_ni.fileLocation(mf, false).offset;
	1562	// This gets the new offset because we already updated the offsets above.
	1563	const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);
	1564	const footers_size = new_size - new_footers_offset;
	1565	try mf.moveRange(
	1566	parent_file_off + old_offset + old_size - footers_size,
	1567	parent_file_off + old_offset + new_size - footers_size,
	1568	footers_size,
	1569	);
	1570	}
1371	1571	}
1372	1572	}
1373	1573	return;
...	...	@@ -1444,11 +1644,14 @@ fn growFloatingNodeWithAlignment(
1444	1644	// that, let's first try the Linux "insert range" fast path. We didn't try it before now
1445	1645	// because it would have been more efficient to just move ourselves into existing space.
1446	1646	//
1447		// If we were given a custom alignment, we cannot pass `grow_mode` directly into the
	1647	// If we were given a custom alignment, we need to set `GrowOptions.exact_size` for the
1448	1648	// "insert range" path, because that function is unaware of `new_alignment`.
1449		const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact;
	1649	const insert_range_grow_options: GrowOptions = .{
	1650	.exact_size = grow_options.exact_size or new_alignment != null,
	1651	.move_footers = grow_options.move_footers,
	1652	};
1450	1653	if (alignment.check(old_offset) and
1451		try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode))
	1654	try mf.growNodeViaInsertRange(gpa, ni, new_size, insert_range_grow_options))
1452	1655	{
1453	1656	// The Linux fast path did our job for us!
1454	1657	return;
...	...	@@ -1458,7 +1661,10 @@ fn growFloatingNodeWithAlignment(
1458	1661	const new_parent_size = parent_ni.alignment(mf).forward(
1459	1662	min_parent_size +\| min_parent_size / growth_factor,
1460	1663	);
1461		try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);
	1664	try mf.growNode(gpa, parent_ni, new_parent_size, .{
	1665	.exact_size = false,
	1666	.move_footers = true,
	1667	});
1462	1668	}
1463	1669
1464	1670	break :new_loc .{
...	...	@@ -1471,6 +1677,10 @@ fn growFloatingNodeWithAlignment(
1471	1677
1472	1678	// Footers need to move to a different place than the rest of our content.
1473	1679	const footers_size: u64, const footers_have_content: bool = footers: {
	1680	if (!grow_options.move_footers) {
	1681	// Pretend there are no footers so as to not move them.
	1682	break :footers .{ 0, false };
	1683	}
1474	1684	const first_footer_ni = ni.firstFooter(mf).unwrap() orelse {
1475	1685	break :footers .{ 0, false };
1476	1686	};
...	...	@@ -1525,15 +1735,14 @@ fn growFloatingNodeWithAlignment(
1525	1735	/// If this strategy is inapplicable or unsuitable for this operation, this function returns `false`
1526	1736	/// without changing any nodes' locations or invalidating any slices.
1527	1737	///
1528		/// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node
1529		/// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`,
1530		/// the actual new size of `ni` may be greater than `new_size`.
	1738	/// Otherwise, this function grows `ni` to `new_size` (maybe larger if `!grow_options.exact_size`),
	1739	/// updates the location of `ni` and every node whose offset has changed, and returns `true`.
1531	1740	fn growNodeViaInsertRange(
1532	1741	mf: *MappedFile,
1533	1742	gpa: Allocator,
1534	1743	ni: Node.Index,
1535	1744	new_size: u64,
1536		grow_mode: GrowMode,
	1745	grow_options: GrowOptions,
1537	1746	) Error!bool {
1538	1747	if (!is_linux or mf.flags.fallocate_insert_range_unsupported) {
1539	1748	return false;
...	...	@@ -1541,43 +1750,22 @@ fn growNodeViaInsertRange(
1541	1750
1542	1751	_, const old_size = ni.location(mf).resolve(mf);
1543	1752
1544		// We don't compute the size of the range yet, because depending on `grow_mode` we might want to
1545		// bump it based on our sibling and parent nodes' alignments. However, we can do an early check
1546		// for cases where we should obviously exit.
1547		const requested_range_size = new_size - old_size;
1548		if (!mf.flags.block_size.check(requested_range_size)) {
1549		// The requested size isn't exactly aligned.
1550		switch (grow_mode) {
1551		.exact => return false,
1552		.minimum => {
1553		// We can still choose to allow it by increasing the size a bit, but we shouldn't do
1554		// that if it would significantly increase the requested size.
1555		const block_size = mf.flags.block_size.toByteUnits();
1556		if (requested_range_size < block_size * 2) {
1557		// Bumping this size up to the next block boundary would be a quite significant
1558		// increase; let's not do it.
1559		return false;
1560		}
1561		},
1562		}
1563		}
1564		// If `grow_mode` is exact, we will use exactly this size, but if it is `.minimum`, we may bump
1565		// the size a little more.
	1753	// We don't compute the size of the range yet, because depending on `grow_options` we might want
	1754	// to bump it based on our sibling and parent nodes' alignments. However, we can do an early
	1755	// check for cases where we should obviously exit.
1566	1756	const min_range_size: u64 = s: {
1567		const exact_size = new_size - old_size;
1568		if (mf.flags.block_size.check(exact_size)) {
1569		break :s exact_size;
1570		}
1571		switch (grow_mode) {
1572		.exact => return false,
1573		.minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) {
1574		// We're growing by at least a few blocks, so allow ourselves to bump the size
1575		// slightly to give it the needed alignment.
1576		break :s mf.flags.block_size.forward(exact_size);
1577		} else {
1578		return false;
1579		},
	1757	const requested_size = new_size - old_size;
	1758	if (mf.flags.block_size.check(requested_size)) {
	1759	break :s requested_size;
1580	1760	}
	1761	if (!grow_options.exact_size and
	1762	requested_size >= mf.flags.block_size.toByteUnits() * 2)
	1763	{
	1764	// We're growing by at least a few blocks, so allow ourselves to bump the size
	1765	// slightly to give it the needed alignment.
	1766	break :s mf.flags.block_size.forward(requested_size);
	1767	}
	1768	return false;
1581	1769	};
1582	1770	assert(min_range_size > 0);
1583	1771	assert(mf.flags.block_size.check(min_range_size));
...	...	@@ -1592,14 +1780,17 @@ fn growNodeViaInsertRange(
1592	1780	}
1593	1781	break :range_file_offset range_file_offset;
1594	1782	};
1595		const first_footer_oni = ni.firstFooter(mf);
1596		const footers_size: u64 = if (first_footer_oni.unwrap()) \|first_footer_ni\| size: {
	1783	const pre_footer_oni: Node.Index.Optional, const footers_size: u64 = footers: {
	1784	if (!grow_options.move_footers) {
	1785	// Pretend there are no footers so as to not move them.
	1786	break :footers .{ .wrap(last_ni), 0 };
	1787	}
	1788	const first_footer_ni = ni.firstFooter(mf).unwrap() orelse {
	1789	break :footers .{ .wrap(last_ni), 0 };
	1790	};
1597	1791	const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf);
1598		break :size old_size - first_footer_offset;
1599		} else 0;
1600		const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) \|first_footer_ni\| pre_footer: {
1601		break :pre_footer first_footer_ni.prev(mf);
1602		} else .wrap(last_ni);
	1792	break :footers .{ first_footer_ni.prev(mf), old_size - first_footer_offset };
	1793	};
1603	1794	const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) \|pre_footer_ni\| end: {
1604	1795	const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf);
1605	1796	break :end pre_footer_off + pre_footer_size;
...	...	@@ -1649,22 +1840,18 @@ fn growNodeViaInsertRange(
1649	1840	}
1650	1841	// Traversal done. We didn't hit `max_moved_nodes`, so now we can use the computed alignment
1651	1842	// requirement to figure out whether we're actually going to insert a range.
1652		if (need_range_align.check(requested_range_size)) {
1653		break :range_size requested_range_size;
1654		}
1655		// Perhaps we're allowed to grow by more than `requested_range_size`?
1656		switch (grow_mode) {
1657		.exact => return false,
1658		.minimum => {
1659		const candidate_range_size = need_range_align.forward(min_range_size);
1660		// Allow growing by up to 50% more than was requested.
1661		if (candidate_range_size <= requested_range_size +\| requested_range_size / 2) {
1662		break :range_size candidate_range_size;
1663		} else {
1664		return false;
1665		}
1666		},
	1843	if (need_range_align.check(min_range_size)) {
	1844	break :range_size min_range_size;
	1845	}
	1846	// Perhaps we're allowed to grow by more than `min_range_size`?
	1847	const candidate_range_size = need_range_align.forward(min_range_size);
	1848	if (!grow_options.exact_size and
	1849	// Allow growing by up to 50% more than was requested.
	1850	candidate_range_size <= min_range_size +\| min_range_size / 2)
	1851	{
	1852	break :range_size candidate_range_size;
1667	1853	}
	1854	return false;
1668	1855	};
1669	1856
1670	1857	// This `range_size` is compatible with everyone's alignment requirements, and we won't move too
...	...	@@ -1742,13 +1929,15 @@ fn growNodeViaInsertRange(
1742	1929	cur_ni = cur_ni.parent(mf).unwrap() orelse break;
1743	1930	}
1744	1931
1745		// The only thing left is to update the offsets of any footers inside of `ni`.
1746		if (ni.firstFooter(mf).unwrap()) \|first_footer_ni\| {
1747		var footer_ni = first_footer_ni;
1748		while (true) {
1749		const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);
1750		try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size);
1751		footer_ni = footer_ni.next(mf).unwrap() orelse break;
	1932	if (grow_options.move_footers) {
	1933	// The only thing left is to update the offsets of any footers inside of `ni`.
	1934	if (ni.firstFooter(mf).unwrap()) \|first_footer_ni\| {
	1935	var footer_ni = first_footer_ni;
	1936	while (true) {
	1937	const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);
	1938	try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size);
	1939	footer_ni = footer_ni.next(mf).unwrap() orelse break;
	1940	}
1752	1941	}
1753	1942	}
1754	1943
...	...	@@ -1802,7 +1991,10 @@ fn ensureAdditionalHeaderCapacity(
1802	1991	const new_parent_size = parent_ni.alignment(mf).forward(
1803	1992	min_parent_size +\| min_parent_size / growth_factor,
1804	1993	);
1805		try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);
	1994	try mf.growNode(gpa, parent_ni, new_parent_size, .{
	1995	.exact_size = false,
	1996	.move_footers = true,
	1997	});
1806	1998	}
1807	1999	return;
1808	2000	};
...	...	@@ -1884,7 +2076,10 @@ fn ensureAdditionalHeaderCapacity(
1884	2076	const new_parent_size = parent_ni.alignment(mf).forward(
1885	2077	min_parent_size +\| min_parent_size / growth_factor,
1886	2078	);
1887		try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);
	2079	try mf.growNode(gpa, parent_ni, new_parent_size, .{
	2080	.exact_size = false,
	2081	.move_footers = true,
	2082	});
1888	2083	}
1889	2084
1890	2085	if (moving_has_content) {
...	...	@@ -1914,48 +2109,27 @@ fn ensureAdditionalHeaderCapacity(
1914	2109	}
1915	2110	}
1916	2111
1917		/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current
1918		/// footers, so that the footers can grow into that space.
1919		fn ensureAdditionalFooterCapacity(
1920		mf: *MappedFile,
1921		gpa: Allocator,
1922		parent_ni: Node.Index,
1923		extra_capacity: u64,
1924		) Error!void {
1925		// This is way easier than the header case, because we don't need to actually move anything; we
1926		// just need to expand the parent if there isn't space, and that will add padding after the
1927		// parent's floating children, which is exactly where we want it.
1928
	2112	/// Returns how many padding bytes `parent_ni` currently has directly preceding its footers, which
	2113	/// footers can therefore grow into.
	2114	fn availableFooterCapacity(mf: *const MappedFile, parent_ni: Node.Index) u64 {
1929	2115	const first_footer_oni = parent_ni.firstFooter(mf);
1930	2116
1931		_, const parent_size = parent_ni.location(mf).resolve(mf);
1932
1933		const footers_size: u64 = footers_size: {
1934		const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0;
	2117	const before_footers_oni: Node.Index.Optional, const footers_off: u64 = footers: {
	2118	const first_footer_ni = first_footer_oni.unwrap() orelse {
	2119	_, const parent_size = parent_ni.location(mf).resolve(mf);
	2120	break :footers .{ parent_ni.last(mf), parent_size };
	2121	};
1935	2122	const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf);
1936		break :footers_size parent_size - first_footer_off;
	2123	break :footers .{ first_footer_ni.prev(mf), first_footer_off };
1937	2124	};
1938	2125
1939	2126	const header_and_floating_end: u64 = end: {
1940		const before_footers_oni = if (first_footer_oni.unwrap()) \|first_footer_ni\| before_footers: {
1941		break :before_footers first_footer_ni.prev(mf);
1942		} else before_footers: {
1943		break :before_footers parent_ni.last(mf);
1944		};
1945	2127	const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0;
1946	2128	const offset, const size = before_footers_ni.location(mf).resolve(mf);
1947	2129	break :end offset + size;
1948	2130	};
1949	2131
1950		assert(header_and_floating_end + footers_size <= parent_size);
1951
1952		const min_parent_size = header_and_floating_end + footers_size + extra_capacity;
1953		if (parent_size < min_parent_size) {
1954		const new_parent_size = parent_ni.alignment(mf).forward(
1955		min_parent_size +\| min_parent_size / growth_factor,
1956		);
1957		try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);
1958		}
	2132	return footers_off - header_and_floating_end;
1959	2133	}
1960	2134
1961	2135	fn removeNodesFromChildList(
...	...	@@ -2049,7 +2223,7 @@ fn realignNode(
2049	2223	mf: *MappedFile,
2050	2224	gpa: Allocator,
2051	2225	ni: Node.Index,
2052		new_alignment: Alignment,
	2226	new_align: Alignment,
2053	2227	) Error!void {
2054	2228	mf.nodes_lock.assertUnlocked();
2055	2229
...	...	@@ -2057,30 +2231,25 @@ fn realignNode(
2057	2231
2058	2232	if (ni == .root or ni.position(mf) != .floating) {
2059	2233	// Only this node's size is aligned, not its offset.
2060		if (!new_alignment.check(old_size)) {
2061		assert(new_alignment.compare(.gt, ni.alignment(mf)));
2062		try mf.growNode(
2063		gpa,
2064		ni,
2065		new_alignment.forward(old_size),
2066		.exact, // because `growNode` is not aware that the size needs to match `new_alignment`
2067		);
	2234	if (!new_align.check(old_size)) {
	2235	assert(new_align.compare(.gt, ni.alignment(mf)));
	2236	try mf.growNode(gpa, ni, new_align.forward(old_size), .{
	2237	.exact_size = true, // because `growNode` is not aware that the size needs to match `new_align`
	2238	.move_footers = true,
	2239	});
2068	2240	}
2069	2241	} else {
2070	2242	// This is a floating node, so its size and offset are both aligned.
2071		if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) {
2072		assert(new_alignment.compare(.gt, ni.alignment(mf)));
2073		try mf.growFloatingNodeWithAlignment(
2074		gpa,
2075		ni,
2076		new_alignment,
2077		new_alignment.forward(old_size),
2078		.minimum,
2079		);
	2243	if (!new_align.check(old_offset) or !new_align.check(old_size)) {
	2244	assert(new_align.compare(.gt, ni.alignment(mf)));
	2245	try mf.growFloatingNodeWithAlignment(gpa, ni, new_align, new_align.forward(old_size), .{
	2246	.exact_size = false,
	2247	.move_footers = true,
	2248	});
2080	2249	}
2081	2250	}
2082	2251
2083		ni.get(mf).flags.alignment = new_alignment;
	2252	ni.get(mf).flags.alignment = new_align;
2084	2253	}
2085	2254
2086	2255	fn updateWriters(mf: *MappedFile) void {