| ... | @@ -5376,9 +5376,10 @@ fn initHeaders( | ... | @@ -5376,9 +5376,10 @@ fn initHeaders( |
| 5376 | defer phnum += 1; | 5376 | defer phnum += 1; |
| 5377 | break :phndx phnum; | 5377 | break :phndx phnum; |
| 5378 | } else undefined, | 5378 | } else undefined, |
| 5379 | // `data` must be assigned after all other loadable segments so that it has the greatest | 5379 | // `data` must be assigned after all other loadable segments so that it has the last |
| 5380 | // phndx of any loadable segment. This is so that `targetSegmentLoadAddressRestrictions` | 5380 | // phndx of the loadable segments. This causes `updateSegmentLoadAddresses` to put |
| 5381 | // can be obeyed (specifically, the `.data_last` restriction, needed on SPARC). | 5381 | // that segment's virtual address after the others, which is a de-facto requirement |
| | 5382 | // on SPARC. More details are in a comment in `updateSegmentLoadAddresses`. |
| 5382 | .data = phndx: { | 5383 | .data = phndx: { |
| 5383 | defer phnum += 1; | 5384 | defer phnum += 1; |
| 5384 | break :phndx phnum; | 5385 | break :phndx phnum; |
| ... | @@ -5654,25 +5655,10 @@ fn initHeaders( | ... | @@ -5654,25 +5655,10 @@ fn initHeaders( |
| 5654 | const ElfN = ct_class.ElfN(); | 5655 | const ElfN = ct_class.ElfN(); |
| 5655 | const target_endian = elf.targetEndian(); | 5656 | const target_endian = elf.targetEndian(); |
| 5656 | | 5657 | |
| 5657 | populate_phdrs: { | 5658 | if (@"type" != .REL) { |
| 5658 | // Initially we will give every `PT_LOAD` segment this address. When we re-allocate | 5659 | // Initially we will give every `PT_LOAD` segment the base address. They will be |
| 5659 | // segments in the virtual address space in `flushMoved` and `flushResized`, we will | 5660 | // moved to non-overlapping addresses by `updateSegmentLoadAddresses`. |
| 5660 | // move some segments to higher addresses to prevent overlap. This address therefore | 5661 | const base_vaddr: u64 = elf.baseLoadableSegmentVAddr(); |
| 5661 | // becomes the image's "base address"; i.e. the first `PT_LOAD` segment will start | | |
| 5662 | // at this address. The base address could eventually end up higher than this due to | | |
| 5663 | // how we re-allocate the address space, but never lower. | | |
| 5664 | const base_vaddr: u64 = switch (@"type") { | | |
| 5665 | .REL => break :populate_phdrs, | | |
| 5666 | .DYN => 0, | | |
| 5667 | .EXEC => switch (machine) { | | |
| 5668 | .AARCH64 => 0x200000, | | |
| 5669 | .LOONGARCH => 0x10000, | | |
| 5670 | .PPC64 => 0x10000000, | | |
| 5671 | .RISCV => 0x10000, | | |
| 5672 | .SPARCV9 => 0x100000, | | |
| 5673 | .X86_64 => 0x200000, | | |
| 5674 | }, | | |
| 5675 | }; | | |
| 5676 | | 5662 | |
| 5677 | // All `PT_LOAD` segments are given this `.@"align"`. However, to avoid bloating the | 5663 | // All `PT_LOAD` segments are given this `.@"align"`. However, to avoid bloating the |
| 5678 | // binary, their *nodes* are not aligned to this boundary---ELF only requires that | 5664 | // binary, their *nodes* are not aligned to this boundary---ELF only requires that |
| ... | @@ -6621,21 +6607,7 @@ fn initHeaders( | ... | @@ -6621,21 +6607,7 @@ fn initHeaders( |
| 6621 | }, | 6607 | }, |
| 6622 | }; | 6608 | }; |
| 6623 | | 6609 | |
| 6624 | if (@"type" != .REL) switch (elf.targetSegmentLoadAddressRestrictions()) { | 6610 | try elf.updateSegmentLoadAddresses(); |
| 6625 | .none => {}, | | |
| 6626 | .data_last => switch (elf.phdrSlice()) { | | |
| 6627 | inline else => |phdr| { | | |
| 6628 | // Ensure that the segment after `.data` (if any) is not a loadable segment. | | |
| 6629 | const next_phndx = phndx.data + 1; | | |
| 6630 | if (next_phndx < phdr.len) { | | |
| 6631 | switch (elf.targetLoad(&phdr[next_phndx].type)) { | | |
| 6632 | .NULL, .LOAD => unreachable, // data segment should be the last loadable segment | | |
| 6633 | else => {}, | | |
| 6634 | } | | |
| 6635 | } | | |
| 6636 | }, | | |
| 6637 | }, | | |
| 6638 | }; | | |
| 6639 | } | 6611 | } |
| 6640 | | 6612 | |
| 6641 | pub fn startProgress(elf: *Elf, prog_node: std.Progress.Node) void { | 6613 | pub fn startProgress(elf: *Elf, prog_node: std.Progress.Node) void { |
| ... | @@ -7035,6 +7007,21 @@ fn ehdrType(elf: *const Elf) EhdrType { | ... | @@ -7035,6 +7007,21 @@ fn ehdrType(elf: *const Elf) EhdrType { |
| 7035 | } | 7007 | } |
| 7036 | } | 7008 | } |
| 7037 | | 7009 | |
| | 7010 | fn baseLoadableSegmentVAddr(elf: *const Elf) u64 { |
| | 7011 | return switch (elf.ehdrType()) { |
| | 7012 | .REL => unreachable, |
| | 7013 | .DYN => 0, |
| | 7014 | .EXEC => switch (elf.ehdrMachine()) { |
| | 7015 | .AARCH64 => 0x200000, |
| | 7016 | .LOONGARCH => 0x10000, |
| | 7017 | .PPC64 => 0x10000000, |
| | 7018 | .RISCV => 0x10000, |
| | 7019 | .SPARCV9 => 0x100000, |
| | 7020 | .X86_64 => 0x200000, |
| | 7021 | }, |
| | 7022 | }; |
| | 7023 | } |
| | 7024 | |
| 7038 | fn targetPtrSize(elf: *const Elf) u8 { | 7025 | fn targetPtrSize(elf: *const Elf) u8 { |
| 7039 | return elf.identClass().size(); | 7026 | return elf.identClass().size(); |
| 7040 | } | 7027 | } |
| ... | @@ -7153,30 +7140,6 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo { | ... | @@ -7153,30 +7140,6 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo { |
| 7153 | // TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`. | 7140 | // TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`. |
| 7154 | }; | 7141 | }; |
| 7155 | } | 7142 | } |
| 7156 | /// Specifies any restrictions the current target has regarding how segments are ordered in the | | |
| 7157 | /// virtual address space. Most targets do not have any such restrictions. | | |
| 7158 | fn targetSegmentLoadAddressRestrictions(elf: *const Elf) enum { | | |
| 7159 | none, | | |
| 7160 | /// The "mutable data" segment must be the last loadable segment in the virtual address space. | | |
| 7161 | data_last, | | |
| 7162 | } { | | |
| 7163 | return switch (elf.ehdrMachine()) { | | |
| 7164 | .AARCH64, | | |
| 7165 | .PPC64, | | |
| 7166 | .RISCV, | | |
| 7167 | .X86_64, | | |
| 7168 | .LOONGARCH, | | |
| 7169 | => .none, | | |
| 7170 | | | |
| 7171 | // SPARC uses `R_SPARC_PC{10,22}` relocations to construct pointers to the GOT, but these | | |
| 7172 | // relocations write an *unsigned* PC-relative offset. This cannot even be worked around by | | |
| 7173 | // using a larger code model, because the crt `_start` assembly always uses these specific | | |
| 7174 | // relocations. Therefore, to avoid relocation errors, all code must appear before the GOT | | |
| 7175 | // in the virtual address space. The easiest way for us to do that is to ensure that the | | |
| 7176 | // "mutable data" segment, containing the GOT, is the last segment in the address space. | | |
| 7177 | .SPARCV9 => .data_last, | | |
| 7178 | }; | | |
| 7179 | } | | |
| 7180 | fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { | 7143 | fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { |
| 7181 | const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer; | 7144 | const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer; |
| 7182 | const Child = pointer_ty.child; | 7145 | const Child = pointer_ty.child; |
| ... | @@ -7606,6 +7569,7 @@ pub fn loadInput(elf: *Elf, input: link.Input) (link.Error || error{BadMagic})!v | ... | @@ -7606,6 +7569,7 @@ pub fn loadInput(elf: *Elf, input: link.Input) (link.Error || error{BadMagic})!v |
| 7606 | .{elf.mf.io_err.?}, | 7569 | .{elf.mf.io_err.?}, |
| 7607 | ), | 7570 | ), |
| 7608 | }; | 7571 | }; |
| | 7572 | try elf.updateSegmentLoadAddresses(); |
| 7609 | } | 7573 | } |
| 7610 | fn loadInputInner(elf: *Elf, input: link.Input) (Error || error{BadMagic})!void { | 7574 | fn loadInputInner(elf: *Elf, input: link.Input) (Error || error{BadMagic})!void { |
| 7611 | const comp = elf.base.comp; | 7575 | const comp = elf.base.comp; |
| ... | @@ -8885,6 +8849,7 @@ pub fn prelink(elf: *Elf, prog_node: std.Progress.Node) link.Error!void { | ... | @@ -8885,6 +8849,7 @@ pub fn prelink(elf: *Elf, prog_node: std.Progress.Node) link.Error!void { |
| 8885 | error.MappedFileIo => return diags.fail("failed to write output file: {t}", .{elf.mf.io_err.?}), | 8849 | error.MappedFileIo => return diags.fail("failed to write output file: {t}", .{elf.mf.io_err.?}), |
| 8886 | else => |e| return e, | 8850 | else => |e| return e, |
| 8887 | }; | 8851 | }; |
| | 8852 | try elf.updateSegmentLoadAddresses(); |
| 8888 | } | 8853 | } |
| 8889 | fn prelinkInner(elf: *Elf) Error!void { | 8854 | fn prelinkInner(elf: *Elf) Error!void { |
| 8890 | const comp = elf.base.comp; | 8855 | const comp = elf.base.comp; |
| ... | @@ -9038,6 +9003,7 @@ pub fn zcuFilesReady(elf: *Elf, zcu: *Zcu) link.Error!void { | ... | @@ -9038,6 +9003,7 @@ pub fn zcuFilesReady(elf: *Elf, zcu: *Zcu) link.Error!void { |
| 9038 | .{elf.mf.io_err.?}, | 9003 | .{elf.mf.io_err.?}, |
| 9039 | ), | 9004 | ), |
| 9040 | }; | 9005 | }; |
| | 9006 | try elf.updateSegmentLoadAddresses(); |
| 9041 | } | 9007 | } |
| 9042 | fn zcuFilesReadyInner(elf: *Elf, zcu: *Zcu) Error!void { | 9008 | fn zcuFilesReadyInner(elf: *Elf, zcu: *Zcu) Error!void { |
| 9043 | const gpa = zcu.gpa; | 9009 | const gpa = zcu.gpa; |
| ... | @@ -10327,6 +10293,7 @@ pub fn updateNav(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) | ... | @@ -10327,6 +10293,7 @@ pub fn updateNav(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) |
| 10327 | .{elf.mf.io_err.?}, | 10293 | .{elf.mf.io_err.?}, |
| 10328 | ), | 10294 | ), |
| 10329 | }; | 10295 | }; |
| | 10296 | try elf.updateSegmentLoadAddresses(); |
| 10330 | } | 10297 | } |
| 10331 | fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) Error!void { | 10298 | fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) Error!void { |
| 10332 | const comp = elf.base.comp; | 10299 | const comp = elf.base.comp; |
| ... | @@ -10416,6 +10383,7 @@ pub fn updateContainerType( | ... | @@ -10416,6 +10383,7 @@ pub fn updateContainerType( |
| 10416 | .{elf.mf.io_err.?}, | 10383 | .{elf.mf.io_err.?}, |
| 10417 | ), | 10384 | ), |
| 10418 | }; | 10385 | }; |
| | 10386 | try elf.updateSegmentLoadAddresses(); |
| 10419 | } | 10387 | } |
| 10420 | fn updateContainerTypeInner( | 10388 | fn updateContainerTypeInner( |
| 10421 | elf: *Elf, | 10389 | elf: *Elf, |
| ... | @@ -10557,6 +10525,7 @@ pub fn updateFunc( | ... | @@ -10557,6 +10525,7 @@ pub fn updateFunc( |
| 10557 | .{elf.mf.io_err.?}, | 10525 | .{elf.mf.io_err.?}, |
| 10558 | ), | 10526 | ), |
| 10559 | }; | 10527 | }; |
| | 10528 | try elf.updateSegmentLoadAddresses(); |
| 10560 | } | 10529 | } |
| 10561 | fn updateFuncInner( | 10530 | fn updateFuncInner( |
| 10562 | elf: *Elf, | 10531 | elf: *Elf, |
| ... | @@ -10804,6 +10773,7 @@ pub fn lostTracking( | ... | @@ -10804,6 +10773,7 @@ pub fn lostTracking( |
| 10804 | elf.mf.io_err.?, | 10773 | elf.mf.io_err.?, |
| 10805 | }), | 10774 | }), |
| 10806 | }; | 10775 | }; |
| | 10776 | try elf.updateSegmentLoadAddresses(); |
| 10807 | } | 10777 | } |
| 10808 | | 10778 | |
| 10809 | pub fn updateErrorData(elf: *Elf, pt: Zcu.PerThread) link.Error!void { | 10779 | pub fn updateErrorData(elf: *Elf, pt: Zcu.PerThread) link.Error!void { |
| ... | @@ -10813,6 +10783,7 @@ pub fn updateErrorData(elf: *Elf, pt: Zcu.PerThread) link.Error!void { | ... | @@ -10813,6 +10783,7 @@ pub fn updateErrorData(elf: *Elf, pt: Zcu.PerThread) link.Error!void { |
| 10813 | }); | 10783 | }); |
| 10814 | if (elf.dwarf.const_pool.getIfExists(.anyerror_type)) |cpi| | 10784 | if (elf.dwarf.const_pool.getIfExists(.anyerror_type)) |cpi| |
| 10815 | try elf.updateConstInner(pt, cpi, .anyerror_type, .complete); | 10785 | try elf.updateConstInner(pt, cpi, .anyerror_type, .complete); |
| | 10786 | try elf.updateSegmentLoadAddresses(); |
| 10816 | } | 10787 | } |
| 10817 | | 10788 | |
| 10818 | pub fn flush( | 10789 | pub fn flush( |
| ... | @@ -10860,6 +10831,8 @@ fn flushInner( | ... | @@ -10860,6 +10831,8 @@ fn flushInner( |
| 10860 | | 10831 | |
| 10861 | try elf.prepareDynamic(); | 10832 | try elf.prepareDynamic(); |
| 10862 | | 10833 | |
| | 10834 | try elf.updateSegmentLoadAddresses(); |
| | 10835 | |
| 10863 | while (try elf.idle()) {} | 10836 | while (try elf.idle()) {} |
| 10864 | | 10837 | |
| 10865 | assert(elf.input_pending_index == elf.inputs.items.len); | 10838 | assert(elf.input_pending_index == elf.inputs.items.len); |
| ... | @@ -11566,9 +11539,10 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void | ... | @@ -11566,9 +11539,10 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void |
| 11566 | else => unreachable, | 11539 | else => unreachable, |
| 11567 | | 11540 | |
| 11568 | .NULL, .LOAD => { | 11541 | .NULL, .LOAD => { |
| 11569 | try elf.allocateSegmentLoadAddress(phndx); | 11542 | // Our address changes are handled by `updateSegmentLoadAddresses`, so |
| 11570 | // Do not bubble moved---our child nodes only care if our virtual | 11543 | // there's nothing we need to do here. Don't bubble moved, because our |
| 11571 | // address changes, which is handled by `updateSegmentLoadAddress`. | 11544 | // child nodes only care if our virtual address changes, which is |
| | 11545 | // handled by that other function. |
| 11572 | return; | 11546 | return; |
| 11573 | }, | 11547 | }, |
| 11574 | | 11548 | |
| ... | @@ -11987,166 +11961,156 @@ fn flushMovedNobitsSymbols( | ... | @@ -11987,166 +11961,156 @@ fn flushMovedNobitsSymbols( |
| 11987 | } | 11961 | } |
| 11988 | } | 11962 | } |
| 11989 | | 11963 | |
| 11990 | /// Updates `vaddr` and `paddr` of the segment header, and marks everything in the segment as moved. | 11964 | /// If any loadable segments have invalid virtual addresses, assigns them new valid addresses and |
| 11991 | fn updateSegmentLoadAddress( | 11965 | /// queues the necessary idle tasks (e.g. re-applying relocations targeting those segments). |
| 11992 | elf: *Elf, | | |
| 11993 | segment_ni: MappedFile.Node.Index, | | |
| 11994 | new_vaddr: u64, | | |
| 11995 | ) std.mem.Allocator.Error!void { | | |
| 11996 | const phndx = elf.getNode(segment_ni).segment; | | |
| 11997 | | | |
| 11998 | switch (elf.phdrSlice()) { | | |
| 11999 | inline else => |phdr| { | | |
| 12000 | assert(new_vaddr != elf.targetLoad(&phdr[phndx].vaddr)); | | |
| 12001 | elf.targetStore(&phdr[phndx].vaddr, @intCast(new_vaddr)); | | |
| 12002 | elf.targetStore(&phdr[phndx].paddr, @intCast(new_vaddr)); | | |
| 12003 | }, | | |
| 12004 | } | | |
| 12005 | | | |
| 12006 | try segment_ni.childrenMoved(elf.base.comp.gpa, &elf.mf); | | |
| 12007 | if (segment_ni == elf.ni.data) elf.bss_moved = true; | | |
| 12008 | } | | |
| 12009 | | | |
| 12010 | /// Given the index of a `PT_LOAD`/`PT_NULL` segment, assumes that the phdr's `offset` and `filesz` | | |
| 12011 | /// have been updated as needed by the caller, and updates the `@"align"`, `vaddr`, `paddr`, and | | |
| 12012 | /// `memsz` fields of the segment, in order to place it at a valid virtual address. | | |
| 12013 | /// | 11966 | /// |
| 12014 | /// TODO: this function is currently a source of non-determinism in the linker, because handling the | 11967 | /// This function must be called at the end of every link task in case tghe task invalidated segment |
| 12015 | /// moving or resizing of a segment could reorder them and thereby affect how we handle *future* | 11968 | /// addresses (which can happen in response to any node movement/resize, or any section resize). It |
| 12016 | /// changes to segments. | 11969 | /// cannot be called from `idle` as this function does not have the required idempotency guarantee |
| 12017 | fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Error!void { | 11970 | /// so calling it non-deterministically it would lead to unreproducible output files. |
| 12018 | const segment_ni = elf.phdrs.items[orig_phndx].unwrap().?; | 11971 | fn updateSegmentLoadAddresses(elf: *Elf) std.mem.Allocator.Error!void { |
| 12019 | assert(elf.getNode(segment_ni).segment == orig_phndx); | 11972 | if (elf.ehdrType() == .REL) return; |
| | 11973 | |
| 12020 | const page_align = elf.targetPageAlign(); | 11974 | const page_align = elf.targetPageAlign(); |
| 12021 | const node_align = segment_ni.alignment(&elf.mf); | | |
| 12022 | const ph_align = page_align.max(node_align); | | |
| 12023 | | | |
| 12024 | // If we determine that the segment's virtual address needs to move, then it's a good idea to | | |
| 12025 | // make it less likely that it needs to move *again* in the future, because it is expensive to | | |
| 12026 | // change a segment's load address (a lot of re-flushing is necessary). To do that, we reserve | | |
| 12027 | // more virtual address space than we need (multiplying the actual size by this value). That | | |
| 12028 | // way, there will usually be padding between segments which they can grow into. | | |
| 12029 | // | | |
| 12030 | // TODO: we might want to decrease this multiplier, or even omit it entirely, in cases where | | |
| 12031 | // virtual address space is constrained. For instance, 32-bit targets, or targets where short | | |
| 12032 | // PC-relative relocations between segments are common. | | |
| 12033 | const reserve_size_multiplier = 4; | | |
| 12034 | | 11975 | |
| 12035 | switch (elf.phdrSlice()) { | 11976 | switch (elf.phdrSlice()) { |
| 12036 | inline else => |phdr| { | 11977 | inline else => |phdr| { |
| 12037 | const offset = elf.targetLoad(&phdr[orig_phndx].offset); | 11978 | // Somewhere in the phdrs is a contiguous sequence of all "loadable" segments (segments |
| 12038 | const filesz = elf.targetLoad(&phdr[orig_phndx].filesz); | 11979 | // with a `type` of `PT_NULL` or `PT_LOAD`); find the start of that sequence. In theory |
| 12039 | | 11980 | // we could memoize this, but there's no real need since the number of phdrs is tiny. |
| 12040 | const orig_vaddr = elf.targetLoad(&phdr[orig_phndx].vaddr); | 11981 | const load_phdrs_start = for (phdr, 0..) |*ph, phndx| { |
| 12041 | assert(elf.targetLoad(&phdr[orig_phndx].paddr) == orig_vaddr); | 11982 | switch (elf.targetLoad(&ph.type)) { |
| 12042 | | 11983 | .NULL, .LOAD => break phndx, |
| 12043 | const bss_size = if (segment_ni == elf.ni.data) | 11984 | else => continue, |
| 12044 | elf.shndx.bss.size(elf) | | |
| 12045 | else | | |
| 12046 | 0; | | |
| 12047 | const memsz = filesz + bss_size; | | |
| 12048 | | | |
| 12049 | var vaddr: u64 = orig_vaddr; | | |
| 12050 | | | |
| 12051 | elf.targetStore(&phdr[orig_phndx].@"align", @intCast(ph_align.toByteUnits())); | | |
| 12052 | elf.targetStore(&phdr[orig_phndx].memsz, @intCast(memsz)); | | |
| 12053 | elf.targetStore(&phdr[orig_phndx].type, if (memsz > 0) .LOAD else .NULL); | | |
| 12054 | | | |
| 12055 | // First, we will shift the virtual address as needed in order to maintain the required | | |
| 12056 | // property that vaddr is congruent to offset modulo the phdr alignment. | | |
| 12057 | { | | |
| 12058 | // Compute the candidate address by undoing the current offset and then re-offsetting | | |
| 12059 | vaddr = std.mem.alignBackward(u64, vaddr, ph_align.toByteUnits()) + offset % ph_align.toByteUnits(); | | |
| 12060 | // If `node_align` is greater than `page_align`, the address we just set might be in | | |
| 12061 | // the previous segment. The first page we "own" is the one in which the old vaddr | | |
| 12062 | // resides, so check against that. | | |
| 12063 | const first_good_vaddr = std.mem.alignBackward(u64, orig_vaddr, page_align.toByteUnits()); | | |
| 12064 | if (vaddr < first_good_vaddr) { | | |
| 12065 | // Yep, we crossed into the previous segment's pages, so correct for that by | | |
| 12066 | // offsetting our address by another `ph_align`. | | |
| 12067 | vaddr += ph_align.toByteUnits(); | | |
| 12068 | assert(vaddr >= first_good_vaddr); | | |
| 12069 | } | 11985 | } |
| 12070 | } | 11986 | } else unreachable; // we always create loadable (`PT_NULL`/`PT_LOAD`) segments |
| 12071 | | 11987 | |
| 12072 | // If our size has changed, or if the address shift above caused our "end" address to | 11988 | // Update these fields for each loadable segment: |
| 12073 | // cross a page boundary, then we might be overlapping with the next segment's pages. In | 11989 | // |
| 12074 | // that case, we will jump past that segment and give ourselves a new address after it. | 11990 | // * `memsz` |
| 12075 | // We'll need to repeat this for every loadable phdr after us, until we're no longer | 11991 | // * `type` (`PT_NULL` vs `PT_LOAD`) |
| 12076 | // overlapping anything. | 11992 | // * `align` |
| 12077 | var phndx = orig_phndx; | 11993 | // * `offset` (in theory this should happen in `updateElfOffset`, but this logic needs |
| 12078 | for (phdr[orig_phndx + 1 ..], orig_phndx + 1..) |*next_ph, next_phndx| { | 11994 | // to track that value, so it's easier to just populate it here instead) |
| 12079 | switch (elf.targetLoad(&next_ph.type)) { | 11995 | // |
| | 11996 | // ...and check if the current `vaddr` values for the loadable segments are valid (i.e. |
| | 11997 | // compatible with the segments' file offsets and address ranges do not overlap). |
| | 11998 | var vaddrs_valid: bool = true; |
| | 11999 | var first_free_vaddr: u64 = elf.baseLoadableSegmentVAddr(); |
| | 12000 | const load_phdrs_end: usize = for ( |
| | 12001 | phdr[load_phdrs_start..], |
| | 12002 | elf.phdrs.items[load_phdrs_start..], |
| | 12003 | load_phdrs_start.., |
| | 12004 | ) |*ph, segment_oni, phndx| { |
| | 12005 | switch (elf.targetLoad(&ph.type)) { |
| 12080 | .NULL, .LOAD => {}, | 12006 | .NULL, .LOAD => {}, |
| 12081 | else => { | 12007 | else => break phndx, // reached end of loadable segments |
| 12082 | // All loadable segments have contiguous indices, so this indicates we have | | |
| 12083 | // become the last loadable segment, meaning we definitely don't overlap any | | |
| 12084 | // other loadable segment. | | |
| 12085 | break; | | |
| 12086 | }, | | |
| 12087 | } | 12008 | } |
| 12088 | | 12009 | const segment_ni = segment_oni.unwrap().?; // all loadable segments have nodes |
| 12089 | const next_vaddr = elf.targetLoad(&next_ph.vaddr); | 12010 | |
| 12090 | // Find the first virtual address which the next phdr "owns" by aligning its vaddr | 12011 | // We need to know the segment's file offset here anyway, so we may as well just |
| 12091 | // backwards to the start of the page. | 12012 | // populate the phdr `offset` field now instead of in `flushElfOffset`. |
| 12092 | const next_page_vaddr = std.mem.alignBackward(u64, next_vaddr, page_align.toByteUnits()); | 12013 | const ph_offset = elf.computeNodeElfOffset(segment_ni); |
| 12093 | | 12014 | elf.targetStore(&ph.offset, @intCast(ph_offset)); |
| 12094 | // Check if the segment fits here. We apply `reserve_size_multiplier`, but only if | 12015 | |
| 12095 | // the segment is already known to be moving---making it easier to grow in-place is | 12016 | // The alignment of the segment affects which virtual addresses are valid. |
| 12096 | // the whole point of the multiplier! | 12017 | const ph_align = segment_ni.alignment(&elf.mf).max(page_align); |
| 12097 | { | 12018 | elf.targetStore(&ph.@"align", @intCast(ph_align.toByteUnits())); |
| 12098 | const target_size = if (vaddr == orig_vaddr) memsz else memsz * reserve_size_multiplier; | 12019 | |
| 12099 | if (vaddr + target_size <= next_page_vaddr) { | 12020 | // The size of the segment in memory is equal to the size of the segment in the file |
| 12100 | break; // hooray, we fit here! | 12021 | // plus the size of any `SHT_NOBITS` section(s) at the end of the segment. |
| | 12022 | const ph_memsz: u64 = memsz: { |
| | 12023 | _, const file_size: u64 = segment_ni.location(&elf.mf).resolve(&elf.mf); |
| | 12024 | if (segment_ni == elf.ni.data) { |
| | 12025 | break :memsz file_size + elf.shndx.bss.size(elf); |
| | 12026 | } else { |
| | 12027 | break :memsz file_size; |
| 12101 | } | 12028 | } |
| | 12029 | }; |
| | 12030 | elf.targetStore(&ph.memsz, @intCast(ph_memsz)); |
| | 12031 | elf.targetStore(&ph.type, if (ph_memsz == 0) .NULL else .LOAD); |
| | 12032 | |
| | 12033 | // Check if the existing virtual address of the segment is valid. |
| | 12034 | const ph_vaddr = elf.targetLoad(&ph.vaddr); |
| | 12035 | assert(elf.targetLoad(&ph.paddr) == ph_vaddr); |
| | 12036 | if (ph_vaddr < first_free_vaddr or |
| | 12037 | ph_offset % ph_align.toByteUnits() != ph_vaddr % ph_align.toByteUnits()) |
| | 12038 | { |
| | 12039 | @branchHint(.unlikely); |
| | 12040 | vaddrs_valid = false; |
| 12102 | } | 12041 | } |
| 12103 | | 12042 | |
| 12104 | const next_ni = elf.phdrs.items[next_phndx].unwrap().?; | 12043 | // The next segment's pages must not overlap our own. |
| | 12044 | first_free_vaddr = page_align.forward(ph_vaddr + ph_memsz); |
| | 12045 | } else phdr.len; |
| 12105 | | 12046 | |
| 12106 | // This segment don't fit here, but before deciding how to proceed, we need to | 12047 | // There should be no more loadable segments after the contiguous range. |
| 12107 | // consider any target-specific restrictions we are subject to. | 12048 | for (phdr[load_phdrs_end..]) |*ph| switch (elf.targetLoad(&ph.type)) { |
| 12108 | switch (elf.targetSegmentLoadAddressRestrictions()) { | 12049 | .NULL, .LOAD => unreachable, // non-contiguous loadable segment |
| 12109 | .none => {}, | 12050 | else => {}, |
| 12110 | .data_last => if (next_ni == elf.ni.data) { | 12051 | }; |
| 12111 | // We can't leapfrog over the data segment. Instead, that segment just needs | | |
| 12112 | // to be shifted forwards to make space for us, and we'll then `break` with | | |
| 12113 | // our current vaddr. | | |
| 12114 | | | |
| 12115 | if (next_phndx + 1 < phdr.len) switch (elf.targetLoad(&phdr[next_phndx + 1].type)) { | | |
| 12116 | .NULL, .LOAD => unreachable, // data segment should be the last loadable segment | | |
| 12117 | else => {}, | | |
| 12118 | }; | | |
| 12119 | | 12052 | |
| 12120 | const free_vaddr = vaddr + memsz * reserve_size_multiplier; | 12053 | if (vaddrs_valid) { |
| | 12054 | @branchHint(.likely); |
| | 12055 | return; |
| | 12056 | } |
| 12121 | | 12057 | |
| 12122 | const next_align = page_align.max(next_ni.alignment(&elf.mf)); | 12058 | // It's time to actually allocate virtual addresses for all segments. Our current |
| 12123 | const next_offset = elf.targetLoad(&next_ph.offset); | 12059 | // strategy for this preserves the order of segments in memory, but this isn't a hard |
| 12124 | const next_new_vaddr = next_align.forward(free_vaddr) + next_offset % next_align.toByteUnits(); | 12060 | // requirement in general. |
| | 12061 | // |
| | 12062 | // However, a warning! If this logic *is* ever changed to re-order segments, it may need |
| | 12063 | // to abide by target-specific restrictions. For instance, on SPARC, because the GOT is |
| | 12064 | // usually accessed with the unsigned relocations `R_SPARC_PC{10,22}`, it is effectively |
| | 12065 | // a requirement for all code to have a lower virtual address than the GOT, which means |
| | 12066 | // that the "text" segment must be ordered before the "data" segment on that target. |
| 12125 | | 12067 | |
| 12126 | try elf.updateSegmentLoadAddress(next_ni, next_new_vaddr); | 12068 | const gpa = elf.base.comp.gpa; |
| 12127 | | 12069 | |
| 12128 | break; | 12070 | // If we determine that a segment's virtual address needs to move, then it's a good idea |
| 12129 | }, | 12071 | // to space it out a bit from the surrounding segments, making it less likely that more |
| | 12072 | // segment moves are necessary in the future (because every segment movement invalidates |
| | 12073 | // a lot of idle work). We do that by multiplying each segment's size by this multiplier |
| | 12074 | // and treating that amount of space as "reserved" for the segment to grow into. |
| | 12075 | // |
| | 12076 | // TODO: we might want to decrease this multiplier, or even omit it entirely, in cases |
| | 12077 | // where the virtual address space is constrained; for instance, 32-bit targets, or |
| | 12078 | // targets where short PC-relative relocations between segments are common. |
| | 12079 | const reserve_size_multiplier = 4; |
| | 12080 | |
| | 12081 | first_free_vaddr = elf.baseLoadableSegmentVAddr(); |
| | 12082 | var preferred_vaddr = first_free_vaddr; |
| | 12083 | |
| | 12084 | for (load_phdrs_start..load_phdrs_end) |phndx| { |
| | 12085 | const ph_align: Alignment = .fromByteUnits(elf.targetLoad(&phdr[phndx].@"align")); |
| | 12086 | assert(ph_align.compare(.gte, page_align)); |
| | 12087 | const ph_offset = elf.targetLoad(&phdr[phndx].offset); |
| | 12088 | const ph_memsz = elf.targetLoad(&phdr[phndx].memsz); |
| | 12089 | |
| | 12090 | // If the existing address is valid, then don't do anything. |
| | 12091 | const ph_vaddr: u64 = elf.targetLoad(&phdr[phndx].vaddr); |
| | 12092 | if (ph_vaddr >= first_free_vaddr and |
| | 12093 | ph_vaddr % ph_align.toByteUnits() == ph_offset % ph_align.toByteUnits()) |
| | 12094 | { |
| | 12095 | // The current virtual address is already valid, so just stay put. Perhaps the |
| | 12096 | // next segment will have to move, but the only alternative was *us* moving! |
| | 12097 | first_free_vaddr = page_align.forward(ph_vaddr + ph_memsz); |
| | 12098 | preferred_vaddr = page_align.forward(ph_vaddr + ph_memsz * reserve_size_multiplier); |
| | 12099 | continue; |
| 12130 | } | 12100 | } |
| 12131 | | 12101 | |
| 12132 | // We don't fit here, so shift ourselves forward (i.e. swap with `next_phndx`). But | 12102 | // Otherwise, put ourselves at a new address just after the previous segment. |
| 12133 | // first we need to adjust `vaddr` to come after it. | 12103 | const new_vaddr = ph_align.forward(preferred_vaddr) + ph_offset % ph_align.toByteUnits(); |
| 12134 | const next_size = elf.targetLoad(&next_ph.memsz); | 12104 | assert(new_vaddr != ph_vaddr); // `ph_vaddr` was not valid |
| 12135 | // Instead of putting ourselves right after `next_ph`, we'll go a bit later in the | | |
| 12136 | // address space so that `next_ph` has address space to grow into (like above). | | |
| 12137 | vaddr = ph_align.forward(@intCast(next_vaddr + next_size * 4)) + offset % ph_align.toByteUnits(); | | |
| 12138 | | 12105 | |
| 12139 | // Now just swap the phdrs and update our `phndx`. | 12106 | elf.targetStore(&phdr[phndx].vaddr, @intCast(new_vaddr)); |
| 12140 | std.mem.swap(@TypeOf(next_ph.*), &phdr[phndx], next_ph); | 12107 | elf.targetStore(&phdr[phndx].paddr, @intCast(new_vaddr)); |
| 12141 | elf.phdrs.items[phndx] = .wrap(next_ni); | 12108 | const segment_ni = elf.phdrs.items[phndx].unwrap().?; |
| 12142 | elf.nodes.items(.data)[@backingInt(next_ni)] = .{ .segment = phndx }; | 12109 | try segment_ni.childrenMoved(gpa, &elf.mf); |
| 12143 | elf.phdrs.items[next_phndx] = .wrap(segment_ni); | 12110 | if (segment_ni == elf.ni.data) elf.bss_moved = true; |
| 12144 | elf.nodes.items(.data)[@backingInt(segment_ni)] = .{ .segment = @intCast(next_phndx) }; | | |
| 12145 | phndx = @intCast(next_phndx); | | |
| 12146 | } | | |
| 12147 | | 12111 | |
| 12148 | if (vaddr != orig_vaddr) { | 12112 | first_free_vaddr = page_align.forward(new_vaddr + ph_memsz); |
| 12149 | try elf.updateSegmentLoadAddress(segment_ni, vaddr); | 12113 | preferred_vaddr = page_align.forward(new_vaddr + ph_memsz * reserve_size_multiplier); |
| 12150 | } | 12114 | } |
| 12151 | }, | 12115 | }, |
| 12152 | } | 12116 | } |
| ... | @@ -12184,7 +12148,6 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo | ... | @@ -12184,7 +12148,6 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo |
| 12184 | else => unreachable, | 12148 | else => unreachable, |
| 12185 | .NULL, .LOAD => { | 12149 | .NULL, .LOAD => { |
| 12186 | elf.targetStore(&ph.filesz, @intCast(size)); | 12150 | elf.targetStore(&ph.filesz, @intCast(size)); |
| 12187 | try elf.allocateSegmentLoadAddress(phndx); | | |
| 12188 | if (ni == elf.ni.data) elf.bss_moved = true; | 12151 | if (ni == elf.ni.data) elf.bss_moved = true; |
| 12189 | }, | 12152 | }, |
| 12190 | .DYNAMIC, .INTERP, .PHDR, .GNU_EH_FRAME, .GNU_RELRO => { | 12153 | .DYNAMIC, .INTERP, .PHDR, .GNU_EH_FRAME, .GNU_RELRO => { |
| ... | @@ -12197,7 +12160,11 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo | ... | @@ -12197,7 +12160,11 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo |
| 12197 | | 12160 | |
| 12198 | elf.targetStore(&ph.filesz, @intCast(size)); | 12161 | elf.targetStore(&ph.filesz, @intCast(size)); |
| 12199 | | 12162 | |
| 12200 | // Our memsz must include the `.tbss` section. | 12163 | // Our memsz must include the `.tbss` section. After adding that in, we will |
| | 12164 | // realign to the TLS segment alignment. This is not strictly required, but |
| | 12165 | // the TLS runtime will pad the module's TLS block to the segment alignment |
| | 12166 | // anyway, so it's simpler for us to just make the memsz aligned so that our |
| | 12167 | // memsz matches the size of the module's TLS block. |
| 12201 | const tls_align: Alignment = .fromByteUnits(elf.targetLoad(&ph.@"align")); | 12168 | const tls_align: Alignment = .fromByteUnits(elf.targetLoad(&ph.@"align")); |
| 12202 | elf.targetStore(&ph.memsz, @intCast( | 12169 | elf.targetStore(&ph.memsz, @intCast( |
| 12203 | tls_align.forward(size + elf.shndx.tbss.size(elf)), | 12170 | tls_align.forward(size + elf.shndx.tbss.size(elf)), |
| ... | @@ -12948,6 +12915,7 @@ pub fn updateExports( | ... | @@ -12948,6 +12915,7 @@ pub fn updateExports( |
| 12948 | }; | 12915 | }; |
| 12949 | } | 12916 | } |
| 12950 | try elf.genPending(pt); | 12917 | try elf.genPending(pt); |
| | 12918 | try elf.updateSegmentLoadAddresses(); |
| 12951 | } | 12919 | } |
| 12952 | fn updateExportInner(elf: *Elf, export_index: Zcu.Export.Index) Error!void { | 12920 | fn updateExportInner(elf: *Elf, export_index: Zcu.Export.Index) Error!void { |
| 12953 | const zcu = elf.base.comp.zcu.?; | 12921 | const zcu = elf.base.comp.zcu.?; |
| ... | @@ -13122,15 +13090,8 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: Alignment) Error | ... | @@ -13122,15 +13090,8 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: Alignment) Error |
| 13122 | switch (elf.phdrSlice()) { | 13090 | switch (elf.phdrSlice()) { |
| 13123 | inline else => |phdr| switch (elf.targetLoad(&phdr[phndx].type)) { | 13091 | inline else => |phdr| switch (elf.targetLoad(&phdr[phndx].type)) { |
| 13124 | .NULL, .LOAD => { | 13092 | .NULL, .LOAD => { |
| 13125 | // The `@"align"` field is managed by `allocateSegmentLoadAddress`. | 13093 | // The `@"align"` field is managed by `updateSegmentLoadAddresses`, which will |
| 13126 | // | 13094 | // be called at the end of the current link task. |
| 13127 | // It's very likely that the node was moved and/or resized when we realigned it | | |
| 13128 | // just above, but it is possible that it was not moved *but* still has an | | |
| 13129 | // unaligned virtual address. In that case, we need to ensure the segment's | | |
| 13130 | // virtual address range will be recomputed. | | |
| 13131 | if (!min_align.check(@intCast(elf.targetLoad(&phdr[phndx].vaddr)))) { | | |
| 13132 | try seg_ni.moved(gpa, &elf.mf); | | |
| 13133 | } | | |
| 13134 | }, | 13095 | }, |
| 13135 | else => elf.targetStore(&phdr[phndx].@"align", @intCast(@max( | 13096 | else => elf.targetStore(&phdr[phndx].@"align", @intCast(@max( |
| 13136 | elf.targetLoad(&phdr[phndx].@"align"), | 13097 | elf.targetLoad(&phdr[phndx].@"align"), |
Matthew Lugg <mlugg@mlugg.co.uk> 2026-10-18 13:57:23+01:00