| ... | ... | @@ -5376,9 +5376,10 @@ fn initHeaders( |
| 5376 | 5376 | defer phnum += 1; |
| 5377 | 5377 | break :phndx phnum; |
| 5378 | 5378 | } else undefined, |
| 5379 | | // `data` must be assigned after all other loadable segments so that it has the greatest |
| 5380 | | // phndx of any loadable segment. This is so that `targetSegmentLoadAddressRestrictions` |
| 5381 | | // can be obeyed (specifically, the `.data_last` restriction, needed on SPARC). |
| 5379 | // `data` must be assigned after all other loadable segments so that it has the last |
| 5380 | // phndx of the loadable segments. This causes `updateSegmentLoadAddresses` to put |
| 5381 | // that segment's virtual address after the others, which is a de-facto requirement |
| 5382 | // on SPARC. More details are in a comment in `updateSegmentLoadAddresses`. |
| 5382 | 5383 | .data = phndx: { |
| 5383 | 5384 | defer phnum += 1; |
| 5384 | 5385 | break :phndx phnum; |
| ... | ... | @@ -5654,25 +5655,10 @@ fn initHeaders( |
| 5654 | 5655 | const ElfN = ct_class.ElfN(); |
| 5655 | 5656 | const target_endian = elf.targetEndian(); |
| 5656 | 5657 | |
| 5657 | | populate_phdrs: { |
| 5658 | | // Initially we will give every `PT_LOAD` segment this address. When we re-allocate |
| 5659 | | // segments in the virtual address space in `flushMoved` and `flushResized`, we will |
| 5660 | | // move some segments to higher addresses to prevent overlap. This address therefore |
| 5661 | | // becomes the image's "base address"; i.e. the first `PT_LOAD` segment will start |
| 5662 | | // at this address. The base address could eventually end up higher than this due to |
| 5663 | | // how we re-allocate the address space, but never lower. |
| 5664 | | const base_vaddr: u64 = switch (@"type") { |
| 5665 | | .REL => break :populate_phdrs, |
| 5666 | | .DYN => 0, |
| 5667 | | .EXEC => switch (machine) { |
| 5668 | | .AARCH64 => 0x200000, |
| 5669 | | .LOONGARCH => 0x10000, |
| 5670 | | .PPC64 => 0x10000000, |
| 5671 | | .RISCV => 0x10000, |
| 5672 | | .SPARCV9 => 0x100000, |
| 5673 | | .X86_64 => 0x200000, |
| 5674 | | }, |
| 5675 | | }; |
| 5658 | if (@"type" != .REL) { |
| 5659 | // Initially we will give every `PT_LOAD` segment the base address. They will be |
| 5660 | // moved to non-overlapping addresses by `updateSegmentLoadAddresses`. |
| 5661 | const base_vaddr: u64 = elf.baseLoadableSegmentVAddr(); |
| 5676 | 5662 | |
| 5677 | 5663 | // All `PT_LOAD` segments are given this `.@"align"`. However, to avoid bloating the |
| 5678 | 5664 | // binary, their *nodes* are not aligned to this boundary---ELF only requires that |
| ... | ... | @@ -6621,21 +6607,7 @@ fn initHeaders( |
| 6621 | 6607 | }, |
| 6622 | 6608 | }; |
| 6623 | 6609 | |
| 6624 | | if (@"type" != .REL) switch (elf.targetSegmentLoadAddressRestrictions()) { |
| 6625 | | .none => {}, |
| 6626 | | .data_last => switch (elf.phdrSlice()) { |
| 6627 | | inline else => |phdr| { |
| 6628 | | // Ensure that the segment after `.data` (if any) is not a loadable segment. |
| 6629 | | const next_phndx = phndx.data + 1; |
| 6630 | | if (next_phndx < phdr.len) { |
| 6631 | | switch (elf.targetLoad(&phdr[next_phndx].type)) { |
| 6632 | | .NULL, .LOAD => unreachable, // data segment should be the last loadable segment |
| 6633 | | else => {}, |
| 6634 | | } |
| 6635 | | } |
| 6636 | | }, |
| 6637 | | }, |
| 6638 | | }; |
| 6610 | try elf.updateSegmentLoadAddresses(); |
| 6639 | 6611 | } |
| 6640 | 6612 | |
| 6641 | 6613 | pub fn startProgress(elf: *Elf, prog_node: std.Progress.Node) void { |
| ... | ... | @@ -7035,6 +7007,21 @@ fn ehdrType(elf: *const Elf) EhdrType { |
| 7035 | 7007 | } |
| 7036 | 7008 | } |
| 7037 | 7009 | |
| 7010 | fn baseLoadableSegmentVAddr(elf: *const Elf) u64 { |
| 7011 | return switch (elf.ehdrType()) { |
| 7012 | .REL => unreachable, |
| 7013 | .DYN => 0, |
| 7014 | .EXEC => switch (elf.ehdrMachine()) { |
| 7015 | .AARCH64 => 0x200000, |
| 7016 | .LOONGARCH => 0x10000, |
| 7017 | .PPC64 => 0x10000000, |
| 7018 | .RISCV => 0x10000, |
| 7019 | .SPARCV9 => 0x100000, |
| 7020 | .X86_64 => 0x200000, |
| 7021 | }, |
| 7022 | }; |
| 7023 | } |
| 7024 | |
| 7038 | 7025 | fn targetPtrSize(elf: *const Elf) u8 { |
| 7039 | 7026 | return elf.identClass().size(); |
| 7040 | 7027 | } |
| ... | ... | @@ -7153,30 +7140,6 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo { |
| 7153 | 7140 | // TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`. |
| 7154 | 7141 | }; |
| 7155 | 7142 | } |
| 7156 | | /// Specifies any restrictions the current target has regarding how segments are ordered in the |
| 7157 | | /// virtual address space. Most targets do not have any such restrictions. |
| 7158 | | fn targetSegmentLoadAddressRestrictions(elf: *const Elf) enum { |
| 7159 | | none, |
| 7160 | | /// The "mutable data" segment must be the last loadable segment in the virtual address space. |
| 7161 | | data_last, |
| 7162 | | } { |
| 7163 | | return switch (elf.ehdrMachine()) { |
| 7164 | | .AARCH64, |
| 7165 | | .PPC64, |
| 7166 | | .RISCV, |
| 7167 | | .X86_64, |
| 7168 | | .LOONGARCH, |
| 7169 | | => .none, |
| 7170 | | |
| 7171 | | // SPARC uses `R_SPARC_PC{10,22}` relocations to construct pointers to the GOT, but these |
| 7172 | | // relocations write an *unsigned* PC-relative offset. This cannot even be worked around by |
| 7173 | | // using a larger code model, because the crt `_start` assembly always uses these specific |
| 7174 | | // relocations. Therefore, to avoid relocation errors, all code must appear before the GOT |
| 7175 | | // in the virtual address space. The easiest way for us to do that is to ensure that the |
| 7176 | | // "mutable data" segment, containing the GOT, is the last segment in the address space. |
| 7177 | | .SPARCV9 => .data_last, |
| 7178 | | }; |
| 7179 | | } |
| 7180 | 7143 | fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { |
| 7181 | 7144 | const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer; |
| 7182 | 7145 | const Child = pointer_ty.child; |
| ... | ... | @@ -7606,6 +7569,7 @@ pub fn loadInput(elf: *Elf, input: link.Input) (link.Error || error{BadMagic})!v |
| 7606 | 7569 | .{elf.mf.io_err.?}, |
| 7607 | 7570 | ), |
| 7608 | 7571 | }; |
| 7572 | try elf.updateSegmentLoadAddresses(); |
| 7609 | 7573 | } |
| 7610 | 7574 | fn loadInputInner(elf: *Elf, input: link.Input) (Error || error{BadMagic})!void { |
| 7611 | 7575 | const comp = elf.base.comp; |
| ... | ... | @@ -8885,6 +8849,7 @@ pub fn prelink(elf: *Elf, prog_node: std.Progress.Node) link.Error!void { |
| 8885 | 8849 | error.MappedFileIo => return diags.fail("failed to write output file: {t}", .{elf.mf.io_err.?}), |
| 8886 | 8850 | else => |e| return e, |
| 8887 | 8851 | }; |
| 8852 | try elf.updateSegmentLoadAddresses(); |
| 8888 | 8853 | } |
| 8889 | 8854 | fn prelinkInner(elf: *Elf) Error!void { |
| 8890 | 8855 | const comp = elf.base.comp; |
| ... | ... | @@ -9038,6 +9003,7 @@ pub fn zcuFilesReady(elf: *Elf, zcu: *Zcu) link.Error!void { |
| 9038 | 9003 | .{elf.mf.io_err.?}, |
| 9039 | 9004 | ), |
| 9040 | 9005 | }; |
| 9006 | try elf.updateSegmentLoadAddresses(); |
| 9041 | 9007 | } |
| 9042 | 9008 | fn zcuFilesReadyInner(elf: *Elf, zcu: *Zcu) Error!void { |
| 9043 | 9009 | const gpa = zcu.gpa; |
| ... | ... | @@ -10327,6 +10293,7 @@ pub fn updateNav(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) |
| 10327 | 10293 | .{elf.mf.io_err.?}, |
| 10328 | 10294 | ), |
| 10329 | 10295 | }; |
| 10296 | try elf.updateSegmentLoadAddresses(); |
| 10330 | 10297 | } |
| 10331 | 10298 | fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) Error!void { |
| 10332 | 10299 | const comp = elf.base.comp; |
| ... | ... | @@ -10416,6 +10383,7 @@ pub fn updateContainerType( |
| 10416 | 10383 | .{elf.mf.io_err.?}, |
| 10417 | 10384 | ), |
| 10418 | 10385 | }; |
| 10386 | try elf.updateSegmentLoadAddresses(); |
| 10419 | 10387 | } |
| 10420 | 10388 | fn updateContainerTypeInner( |
| 10421 | 10389 | elf: *Elf, |
| ... | ... | @@ -10557,6 +10525,7 @@ pub fn updateFunc( |
| 10557 | 10525 | .{elf.mf.io_err.?}, |
| 10558 | 10526 | ), |
| 10559 | 10527 | }; |
| 10528 | try elf.updateSegmentLoadAddresses(); |
| 10560 | 10529 | } |
| 10561 | 10530 | fn updateFuncInner( |
| 10562 | 10531 | elf: *Elf, |
| ... | ... | @@ -10804,6 +10773,7 @@ pub fn lostTracking( |
| 10804 | 10773 | elf.mf.io_err.?, |
| 10805 | 10774 | }), |
| 10806 | 10775 | }; |
| 10776 | try elf.updateSegmentLoadAddresses(); |
| 10807 | 10777 | } |
| 10808 | 10778 | |
| 10809 | 10779 | pub fn updateErrorData(elf: *Elf, pt: Zcu.PerThread) link.Error!void { |
| ... | ... | @@ -10813,6 +10783,7 @@ pub fn updateErrorData(elf: *Elf, pt: Zcu.PerThread) link.Error!void { |
| 10813 | 10783 | }); |
| 10814 | 10784 | if (elf.dwarf.const_pool.getIfExists(.anyerror_type)) |cpi| |
| 10815 | 10785 | try elf.updateConstInner(pt, cpi, .anyerror_type, .complete); |
| 10786 | try elf.updateSegmentLoadAddresses(); |
| 10816 | 10787 | } |
| 10817 | 10788 | |
| 10818 | 10789 | pub fn flush( |
| ... | ... | @@ -10860,6 +10831,8 @@ fn flushInner( |
| 10860 | 10831 | |
| 10861 | 10832 | try elf.prepareDynamic(); |
| 10862 | 10833 | |
| 10834 | try elf.updateSegmentLoadAddresses(); |
| 10835 | |
| 10863 | 10836 | while (try elf.idle()) {} |
| 10864 | 10837 | |
| 10865 | 10838 | assert(elf.input_pending_index == elf.inputs.items.len); |
| ... | ... | @@ -11566,9 +11539,10 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void |
| 11566 | 11539 | else => unreachable, |
| 11567 | 11540 | |
| 11568 | 11541 | .NULL, .LOAD => { |
| 11569 | | try elf.allocateSegmentLoadAddress(phndx); |
| 11570 | | // Do not bubble moved---our child nodes only care if our virtual |
| 11571 | | // address changes, which is handled by `updateSegmentLoadAddress`. |
| 11542 | // Our address changes are handled by `updateSegmentLoadAddresses`, so |
| 11543 | // there's nothing we need to do here. Don't bubble moved, because our |
| 11544 | // child nodes only care if our virtual address changes, which is |
| 11545 | // handled by that other function. |
| 11572 | 11546 | return; |
| 11573 | 11547 | }, |
| 11574 | 11548 | |
| ... | ... | @@ -11987,166 +11961,156 @@ fn flushMovedNobitsSymbols( |
| 11987 | 11961 | } |
| 11988 | 11962 | } |
| 11989 | 11963 | |
| 11990 | | /// Updates `vaddr` and `paddr` of the segment header, and marks everything in the segment as moved. |
| 11991 | | fn updateSegmentLoadAddress( |
| 11992 | | elf: *Elf, |
| 11993 | | segment_ni: MappedFile.Node.Index, |
| 11994 | | new_vaddr: u64, |
| 11995 | | ) std.mem.Allocator.Error!void { |
| 11996 | | const phndx = elf.getNode(segment_ni).segment; |
| 11997 | | |
| 11998 | | switch (elf.phdrSlice()) { |
| 11999 | | inline else => |phdr| { |
| 12000 | | assert(new_vaddr != elf.targetLoad(&phdr[phndx].vaddr)); |
| 12001 | | elf.targetStore(&phdr[phndx].vaddr, @intCast(new_vaddr)); |
| 12002 | | elf.targetStore(&phdr[phndx].paddr, @intCast(new_vaddr)); |
| 12003 | | }, |
| 12004 | | } |
| 12005 | | |
| 12006 | | try segment_ni.childrenMoved(elf.base.comp.gpa, &elf.mf); |
| 12007 | | if (segment_ni == elf.ni.data) elf.bss_moved = true; |
| 12008 | | } |
| 12009 | | |
| 12010 | | /// Given the index of a `PT_LOAD`/`PT_NULL` segment, assumes that the phdr's `offset` and `filesz` |
| 12011 | | /// have been updated as needed by the caller, and updates the `@"align"`, `vaddr`, `paddr`, and |
| 12012 | | /// `memsz` fields of the segment, in order to place it at a valid virtual address. |
| 11964 | /// If any loadable segments have invalid virtual addresses, assigns them new valid addresses and |
| 11965 | /// queues the necessary idle tasks (e.g. re-applying relocations targeting those segments). |
| 12013 | 11966 | /// |
| 12014 | | /// TODO: this function is currently a source of non-determinism in the linker, because handling the |
| 12015 | | /// moving or resizing of a segment could reorder them and thereby affect how we handle *future* |
| 12016 | | /// changes to segments. |
| 12017 | | fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Error!void { |
| 12018 | | const segment_ni = elf.phdrs.items[orig_phndx].unwrap().?; |
| 12019 | | assert(elf.getNode(segment_ni).segment == orig_phndx); |
| 11967 | /// This function must be called at the end of every link task in case tghe task invalidated segment |
| 11968 | /// addresses (which can happen in response to any node movement/resize, or any section resize). It |
| 11969 | /// cannot be called from `idle` as this function does not have the required idempotency guarantee |
| 11970 | /// so calling it non-deterministically it would lead to unreproducible output files. |
| 11971 | fn updateSegmentLoadAddresses(elf: *Elf) std.mem.Allocator.Error!void { |
| 11972 | if (elf.ehdrType() == .REL) return; |
| 11973 | |
| 12020 | 11974 | const page_align = elf.targetPageAlign(); |
| 12021 | | const node_align = segment_ni.alignment(&elf.mf); |
| 12022 | | const ph_align = page_align.max(node_align); |
| 12023 | | |
| 12024 | | // If we determine that the segment's virtual address needs to move, then it's a good idea to |
| 12025 | | // make it less likely that it needs to move *again* in the future, because it is expensive to |
| 12026 | | // change a segment's load address (a lot of re-flushing is necessary). To do that, we reserve |
| 12027 | | // more virtual address space than we need (multiplying the actual size by this value). That |
| 12028 | | // way, there will usually be padding between segments which they can grow into. |
| 12029 | | // |
| 12030 | | // TODO: we might want to decrease this multiplier, or even omit it entirely, in cases where |
| 12031 | | // virtual address space is constrained. For instance, 32-bit targets, or targets where short |
| 12032 | | // PC-relative relocations between segments are common. |
| 12033 | | const reserve_size_multiplier = 4; |
| 12034 | 11975 | |
| 12035 | 11976 | switch (elf.phdrSlice()) { |
| 12036 | 11977 | inline else => |phdr| { |
| 12037 | | const offset = elf.targetLoad(&phdr[orig_phndx].offset); |
| 12038 | | const filesz = elf.targetLoad(&phdr[orig_phndx].filesz); |
| 12039 | | |
| 12040 | | const orig_vaddr = elf.targetLoad(&phdr[orig_phndx].vaddr); |
| 12041 | | assert(elf.targetLoad(&phdr[orig_phndx].paddr) == orig_vaddr); |
| 12042 | | |
| 12043 | | const bss_size = if (segment_ni == elf.ni.data) |
| 12044 | | elf.shndx.bss.size(elf) |
| 12045 | | else |
| 12046 | | 0; |
| 12047 | | const memsz = filesz + bss_size; |
| 12048 | | |
| 12049 | | var vaddr: u64 = orig_vaddr; |
| 12050 | | |
| 12051 | | elf.targetStore(&phdr[orig_phndx].@"align", @intCast(ph_align.toByteUnits())); |
| 12052 | | elf.targetStore(&phdr[orig_phndx].memsz, @intCast(memsz)); |
| 12053 | | elf.targetStore(&phdr[orig_phndx].type, if (memsz > 0) .LOAD else .NULL); |
| 12054 | | |
| 12055 | | // First, we will shift the virtual address as needed in order to maintain the required |
| 12056 | | // property that vaddr is congruent to offset modulo the phdr alignment. |
| 12057 | | { |
| 12058 | | // Compute the candidate address by undoing the current offset and then re-offsetting |
| 12059 | | vaddr = std.mem.alignBackward(u64, vaddr, ph_align.toByteUnits()) + offset % ph_align.toByteUnits(); |
| 12060 | | // If `node_align` is greater than `page_align`, the address we just set might be in |
| 12061 | | // the previous segment. The first page we "own" is the one in which the old vaddr |
| 12062 | | // resides, so check against that. |
| 12063 | | const first_good_vaddr = std.mem.alignBackward(u64, orig_vaddr, page_align.toByteUnits()); |
| 12064 | | if (vaddr < first_good_vaddr) { |
| 12065 | | // Yep, we crossed into the previous segment's pages, so correct for that by |
| 12066 | | // offsetting our address by another `ph_align`. |
| 12067 | | vaddr += ph_align.toByteUnits(); |
| 12068 | | assert(vaddr >= first_good_vaddr); |
| 11978 | // Somewhere in the phdrs is a contiguous sequence of all "loadable" segments (segments |
| 11979 | // with a `type` of `PT_NULL` or `PT_LOAD`); find the start of that sequence. In theory |
| 11980 | // we could memoize this, but there's no real need since the number of phdrs is tiny. |
| 11981 | const load_phdrs_start = for (phdr, 0..) |*ph, phndx| { |
| 11982 | switch (elf.targetLoad(&ph.type)) { |
| 11983 | .NULL, .LOAD => break phndx, |
| 11984 | else => continue, |
| 12069 | 11985 | } |
| 12070 | | } |
| 12071 | | |
| 12072 | | // If our size has changed, or if the address shift above caused our "end" address to |
| 12073 | | // cross a page boundary, then we might be overlapping with the next segment's pages. In |
| 12074 | | // that case, we will jump past that segment and give ourselves a new address after it. |
| 12075 | | // We'll need to repeat this for every loadable phdr after us, until we're no longer |
| 12076 | | // overlapping anything. |
| 12077 | | var phndx = orig_phndx; |
| 12078 | | for (phdr[orig_phndx + 1 ..], orig_phndx + 1..) |*next_ph, next_phndx| { |
| 12079 | | switch (elf.targetLoad(&next_ph.type)) { |
| 11986 | } else unreachable; // we always create loadable (`PT_NULL`/`PT_LOAD`) segments |
| 11987 | |
| 11988 | // Update these fields for each loadable segment: |
| 11989 | // |
| 11990 | // * `memsz` |
| 11991 | // * `type` (`PT_NULL` vs `PT_LOAD`) |
| 11992 | // * `align` |
| 11993 | // * `offset` (in theory this should happen in `updateElfOffset`, but this logic needs |
| 11994 | // to track that value, so it's easier to just populate it here instead) |
| 11995 | // |
| 11996 | // ...and check if the current `vaddr` values for the loadable segments are valid (i.e. |
| 11997 | // compatible with the segments' file offsets and address ranges do not overlap). |
| 11998 | var vaddrs_valid: bool = true; |
| 11999 | var first_free_vaddr: u64 = elf.baseLoadableSegmentVAddr(); |
| 12000 | const load_phdrs_end: usize = for ( |
| 12001 | phdr[load_phdrs_start..], |
| 12002 | elf.phdrs.items[load_phdrs_start..], |
| 12003 | load_phdrs_start.., |
| 12004 | ) |*ph, segment_oni, phndx| { |
| 12005 | switch (elf.targetLoad(&ph.type)) { |
| 12080 | 12006 | .NULL, .LOAD => {}, |
| 12081 | | else => { |
| 12082 | | // All loadable segments have contiguous indices, so this indicates we have |
| 12083 | | // become the last loadable segment, meaning we definitely don't overlap any |
| 12084 | | // other loadable segment. |
| 12085 | | break; |
| 12086 | | }, |
| 12007 | else => break phndx, // reached end of loadable segments |
| 12087 | 12008 | } |
| 12088 | | |
| 12089 | | const next_vaddr = elf.targetLoad(&next_ph.vaddr); |
| 12090 | | // Find the first virtual address which the next phdr "owns" by aligning its vaddr |
| 12091 | | // backwards to the start of the page. |
| 12092 | | const next_page_vaddr = std.mem.alignBackward(u64, next_vaddr, page_align.toByteUnits()); |
| 12093 | | |
| 12094 | | // Check if the segment fits here. We apply `reserve_size_multiplier`, but only if |
| 12095 | | // the segment is already known to be moving---making it easier to grow in-place is |
| 12096 | | // the whole point of the multiplier! |
| 12097 | | { |
| 12098 | | const target_size = if (vaddr == orig_vaddr) memsz else memsz * reserve_size_multiplier; |
| 12099 | | if (vaddr + target_size <= next_page_vaddr) { |
| 12100 | | break; // hooray, we fit here! |
| 12009 | const segment_ni = segment_oni.unwrap().?; // all loadable segments have nodes |
| 12010 | |
| 12011 | // We need to know the segment's file offset here anyway, so we may as well just |
| 12012 | // populate the phdr `offset` field now instead of in `flushElfOffset`. |
| 12013 | const ph_offset = elf.computeNodeElfOffset(segment_ni); |
| 12014 | elf.targetStore(&ph.offset, @intCast(ph_offset)); |
| 12015 | |
| 12016 | // The alignment of the segment affects which virtual addresses are valid. |
| 12017 | const ph_align = segment_ni.alignment(&elf.mf).max(page_align); |
| 12018 | elf.targetStore(&ph.@"align", @intCast(ph_align.toByteUnits())); |
| 12019 | |
| 12020 | // The size of the segment in memory is equal to the size of the segment in the file |
| 12021 | // plus the size of any `SHT_NOBITS` section(s) at the end of the segment. |
| 12022 | const ph_memsz: u64 = memsz: { |
| 12023 | _, const file_size: u64 = segment_ni.location(&elf.mf).resolve(&elf.mf); |
| 12024 | if (segment_ni == elf.ni.data) { |
| 12025 | break :memsz file_size + elf.shndx.bss.size(elf); |
| 12026 | } else { |
| 12027 | break :memsz file_size; |
| 12101 | 12028 | } |
| 12029 | }; |
| 12030 | elf.targetStore(&ph.memsz, @intCast(ph_memsz)); |
| 12031 | elf.targetStore(&ph.type, if (ph_memsz == 0) .NULL else .LOAD); |
| 12032 | |
| 12033 | // Check if the existing virtual address of the segment is valid. |
| 12034 | const ph_vaddr = elf.targetLoad(&ph.vaddr); |
| 12035 | assert(elf.targetLoad(&ph.paddr) == ph_vaddr); |
| 12036 | if (ph_vaddr < first_free_vaddr or |
| 12037 | ph_offset % ph_align.toByteUnits() != ph_vaddr % ph_align.toByteUnits()) |
| 12038 | { |
| 12039 | @branchHint(.unlikely); |
| 12040 | vaddrs_valid = false; |
| 12102 | 12041 | } |
| 12103 | 12042 | |
| 12104 | | const next_ni = elf.phdrs.items[next_phndx].unwrap().?; |
| 12043 | // The next segment's pages must not overlap our own. |
| 12044 | first_free_vaddr = page_align.forward(ph_vaddr + ph_memsz); |
| 12045 | } else phdr.len; |
| 12105 | 12046 | |
| 12106 | | // This segment don't fit here, but before deciding how to proceed, we need to |
| 12107 | | // consider any target-specific restrictions we are subject to. |
| 12108 | | switch (elf.targetSegmentLoadAddressRestrictions()) { |
| 12109 | | .none => {}, |
| 12110 | | .data_last => if (next_ni == elf.ni.data) { |
| 12111 | | // We can't leapfrog over the data segment. Instead, that segment just needs |
| 12112 | | // to be shifted forwards to make space for us, and we'll then `break` with |
| 12113 | | // our current vaddr. |
| 12114 | | |
| 12115 | | if (next_phndx + 1 < phdr.len) switch (elf.targetLoad(&phdr[next_phndx + 1].type)) { |
| 12116 | | .NULL, .LOAD => unreachable, // data segment should be the last loadable segment |
| 12117 | | else => {}, |
| 12118 | | }; |
| 12047 | // There should be no more loadable segments after the contiguous range. |
| 12048 | for (phdr[load_phdrs_end..]) |*ph| switch (elf.targetLoad(&ph.type)) { |
| 12049 | .NULL, .LOAD => unreachable, // non-contiguous loadable segment |
| 12050 | else => {}, |
| 12051 | }; |
| 12119 | 12052 | |
| 12120 | | const free_vaddr = vaddr + memsz * reserve_size_multiplier; |
| 12053 | if (vaddrs_valid) { |
| 12054 | @branchHint(.likely); |
| 12055 | return; |
| 12056 | } |
| 12121 | 12057 | |
| 12122 | | const next_align = page_align.max(next_ni.alignment(&elf.mf)); |
| 12123 | | const next_offset = elf.targetLoad(&next_ph.offset); |
| 12124 | | const next_new_vaddr = next_align.forward(free_vaddr) + next_offset % next_align.toByteUnits(); |
| 12058 | // It's time to actually allocate virtual addresses for all segments. Our current |
| 12059 | // strategy for this preserves the order of segments in memory, but this isn't a hard |
| 12060 | // requirement in general. |
| 12061 | // |
| 12062 | // However, a warning! If this logic *is* ever changed to re-order segments, it may need |
| 12063 | // to abide by target-specific restrictions. For instance, on SPARC, because the GOT is |
| 12064 | // usually accessed with the unsigned relocations `R_SPARC_PC{10,22}`, it is effectively |
| 12065 | // a requirement for all code to have a lower virtual address than the GOT, which means |
| 12066 | // that the "text" segment must be ordered before the "data" segment on that target. |
| 12125 | 12067 | |
| 12126 | | try elf.updateSegmentLoadAddress(next_ni, next_new_vaddr); |
| 12068 | const gpa = elf.base.comp.gpa; |
| 12127 | 12069 | |
| 12128 | | break; |
| 12129 | | }, |
| 12070 | // If we determine that a segment's virtual address needs to move, then it's a good idea |
| 12071 | // to space it out a bit from the surrounding segments, making it less likely that more |
| 12072 | // segment moves are necessary in the future (because every segment movement invalidates |
| 12073 | // a lot of idle work). We do that by multiplying each segment's size by this multiplier |
| 12074 | // and treating that amount of space as "reserved" for the segment to grow into. |
| 12075 | // |
| 12076 | // TODO: we might want to decrease this multiplier, or even omit it entirely, in cases |
| 12077 | // where the virtual address space is constrained; for instance, 32-bit targets, or |
| 12078 | // targets where short PC-relative relocations between segments are common. |
| 12079 | const reserve_size_multiplier = 4; |
| 12080 | |
| 12081 | first_free_vaddr = elf.baseLoadableSegmentVAddr(); |
| 12082 | var preferred_vaddr = first_free_vaddr; |
| 12083 | |
| 12084 | for (load_phdrs_start..load_phdrs_end) |phndx| { |
| 12085 | const ph_align: Alignment = .fromByteUnits(elf.targetLoad(&phdr[phndx].@"align")); |
| 12086 | assert(ph_align.compare(.gte, page_align)); |
| 12087 | const ph_offset = elf.targetLoad(&phdr[phndx].offset); |
| 12088 | const ph_memsz = elf.targetLoad(&phdr[phndx].memsz); |
| 12089 | |
| 12090 | // If the existing address is valid, then don't do anything. |
| 12091 | const ph_vaddr: u64 = elf.targetLoad(&phdr[phndx].vaddr); |
| 12092 | if (ph_vaddr >= first_free_vaddr and |
| 12093 | ph_vaddr % ph_align.toByteUnits() == ph_offset % ph_align.toByteUnits()) |
| 12094 | { |
| 12095 | // The current virtual address is already valid, so just stay put. Perhaps the |
| 12096 | // next segment will have to move, but the only alternative was *us* moving! |
| 12097 | first_free_vaddr = page_align.forward(ph_vaddr + ph_memsz); |
| 12098 | preferred_vaddr = page_align.forward(ph_vaddr + ph_memsz * reserve_size_multiplier); |
| 12099 | continue; |
| 12130 | 12100 | } |
| 12131 | 12101 | |
| 12132 | | // We don't fit here, so shift ourselves forward (i.e. swap with `next_phndx`). But |
| 12133 | | // first we need to adjust `vaddr` to come after it. |
| 12134 | | const next_size = elf.targetLoad(&next_ph.memsz); |
| 12135 | | // Instead of putting ourselves right after `next_ph`, we'll go a bit later in the |
| 12136 | | // address space so that `next_ph` has address space to grow into (like above). |
| 12137 | | vaddr = ph_align.forward(@intCast(next_vaddr + next_size * 4)) + offset % ph_align.toByteUnits(); |
| 12102 | // Otherwise, put ourselves at a new address just after the previous segment. |
| 12103 | const new_vaddr = ph_align.forward(preferred_vaddr) + ph_offset % ph_align.toByteUnits(); |
| 12104 | assert(new_vaddr != ph_vaddr); // `ph_vaddr` was not valid |
| 12138 | 12105 | |
| 12139 | | // Now just swap the phdrs and update our `phndx`. |
| 12140 | | std.mem.swap(@TypeOf(next_ph.*), &phdr[phndx], next_ph); |
| 12141 | | elf.phdrs.items[phndx] = .wrap(next_ni); |
| 12142 | | elf.nodes.items(.data)[@backingInt(next_ni)] = .{ .segment = phndx }; |
| 12143 | | elf.phdrs.items[next_phndx] = .wrap(segment_ni); |
| 12144 | | elf.nodes.items(.data)[@backingInt(segment_ni)] = .{ .segment = @intCast(next_phndx) }; |
| 12145 | | phndx = @intCast(next_phndx); |
| 12146 | | } |
| 12106 | elf.targetStore(&phdr[phndx].vaddr, @intCast(new_vaddr)); |
| 12107 | elf.targetStore(&phdr[phndx].paddr, @intCast(new_vaddr)); |
| 12108 | const segment_ni = elf.phdrs.items[phndx].unwrap().?; |
| 12109 | try segment_ni.childrenMoved(gpa, &elf.mf); |
| 12110 | if (segment_ni == elf.ni.data) elf.bss_moved = true; |
| 12147 | 12111 | |
| 12148 | | if (vaddr != orig_vaddr) { |
| 12149 | | try elf.updateSegmentLoadAddress(segment_ni, vaddr); |
| 12112 | first_free_vaddr = page_align.forward(new_vaddr + ph_memsz); |
| 12113 | preferred_vaddr = page_align.forward(new_vaddr + ph_memsz * reserve_size_multiplier); |
| 12150 | 12114 | } |
| 12151 | 12115 | }, |
| 12152 | 12116 | } |
| ... | ... | @@ -12184,7 +12148,6 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo |
| 12184 | 12148 | else => unreachable, |
| 12185 | 12149 | .NULL, .LOAD => { |
| 12186 | 12150 | elf.targetStore(&ph.filesz, @intCast(size)); |
| 12187 | | try elf.allocateSegmentLoadAddress(phndx); |
| 12188 | 12151 | if (ni == elf.ni.data) elf.bss_moved = true; |
| 12189 | 12152 | }, |
| 12190 | 12153 | .DYNAMIC, .INTERP, .PHDR, .GNU_EH_FRAME, .GNU_RELRO => { |
| ... | ... | @@ -12197,7 +12160,11 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo |
| 12197 | 12160 | |
| 12198 | 12161 | elf.targetStore(&ph.filesz, @intCast(size)); |
| 12199 | 12162 | |
| 12200 | | // Our memsz must include the `.tbss` section. |
| 12163 | // Our memsz must include the `.tbss` section. After adding that in, we will |
| 12164 | // realign to the TLS segment alignment. This is not strictly required, but |
| 12165 | // the TLS runtime will pad the module's TLS block to the segment alignment |
| 12166 | // anyway, so it's simpler for us to just make the memsz aligned so that our |
| 12167 | // memsz matches the size of the module's TLS block. |
| 12201 | 12168 | const tls_align: Alignment = .fromByteUnits(elf.targetLoad(&ph.@"align")); |
| 12202 | 12169 | elf.targetStore(&ph.memsz, @intCast( |
| 12203 | 12170 | tls_align.forward(size + elf.shndx.tbss.size(elf)), |
| ... | ... | @@ -12948,6 +12915,7 @@ pub fn updateExports( |
| 12948 | 12915 | }; |
| 12949 | 12916 | } |
| 12950 | 12917 | try elf.genPending(pt); |
| 12918 | try elf.updateSegmentLoadAddresses(); |
| 12951 | 12919 | } |
| 12952 | 12920 | fn updateExportInner(elf: *Elf, export_index: Zcu.Export.Index) Error!void { |
| 12953 | 12921 | const zcu = elf.base.comp.zcu.?; |
| ... | ... | @@ -13122,15 +13090,8 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: Alignment) Error |
| 13122 | 13090 | switch (elf.phdrSlice()) { |
| 13123 | 13091 | inline else => |phdr| switch (elf.targetLoad(&phdr[phndx].type)) { |
| 13124 | 13092 | .NULL, .LOAD => { |
| 13125 | | // The `@"align"` field is managed by `allocateSegmentLoadAddress`. |
| 13126 | | // |
| 13127 | | // It's very likely that the node was moved and/or resized when we realigned it |
| 13128 | | // just above, but it is possible that it was not moved *but* still has an |
| 13129 | | // unaligned virtual address. In that case, we need to ensure the segment's |
| 13130 | | // virtual address range will be recomputed. |
| 13131 | | if (!min_align.check(@intCast(elf.targetLoad(&phdr[phndx].vaddr)))) { |
| 13132 | | try seg_ni.moved(gpa, &elf.mf); |
| 13133 | | } |
| 13093 | // The `@"align"` field is managed by `updateSegmentLoadAddresses`, which will |
| 13094 | // be called at the end of the current link task. |
| 13134 | 13095 | }, |
| 13135 | 13096 | else => elf.targetStore(&phdr[phndx].@"align", @intCast(@max( |
| 13136 | 13097 | elf.targetLoad(&phdr[phndx].@"align"), |
Matthew Lugg <mlugg@mlugg.co.uk> 2026-10-18 13:57:23+01:00