| ... | @@ -154,6 +154,9 @@ tls_size_symbol_relocs: std.array_hash_map.Auto(SymbolReloc.Index, void), | ... | @@ -154,6 +154,9 @@ tls_size_symbol_relocs: std.array_hash_map.Auto(SymbolReloc.Index, void), |
| 154 | section_by_name: std.array_hash_map.Auto(String(.shstrtab), void), | 154 | section_by_name: std.array_hash_map.Auto(String(.shstrtab), void), |
| 155 | /// Key is the name of a global symbol which has been moved to a new symtab index. Any relocation | 155 | /// Key is the name of a global symbol which has been moved to a new symtab index. Any relocation |
| 156 | /// entries which target that symbol must be updated to reference the correct symbol index. | 156 | /// entries which target that symbol must be updated to reference the correct symbol index. |
| | 157 | /// |
| | 158 | /// When emitting a relocatable (`ET_REL`), this refers to the index in `.symtab`. Otherwise, it |
| | 159 | /// refers to the index in `.dynsym`. |
| 157 | changed_symtab_index: std.array_hash_map.Auto(String(.strtab), void), | 160 | changed_symtab_index: std.array_hash_map.Auto(String(.strtab), void), |
| 158 | /// Counts how many relocations are currently in `.rela.dyn` which would require a `DT_TEXTREL` | 161 | /// Counts how many relocations are currently in `.rela.dyn` which would require a `DT_TEXTREL` |
| 159 | /// entry in the `.dynamic` section. This allows adding `DT_TEXTREL` to the output `.dynamic` | 162 | /// entry in the `.dynamic` section. This allows adding `DT_TEXTREL` to the output `.dynamic` |
| ... | @@ -1331,12 +1334,9 @@ fn ensureUnusedSymbolCapacity(elf: *Elf, len: u32, kind: enum { all_local, maybe | ... | @@ -1331,12 +1334,9 @@ fn ensureUnusedSymbolCapacity(elf: *Elf, len: u32, kind: enum { all_local, maybe |
| 1331 | try elf.symtab.ensureUnusedCapacity(gpa, len); | 1334 | try elf.symtab.ensureUnusedCapacity(gpa, len); |
| 1332 | | 1335 | |
| 1333 | // If adding locals, we may need to move one global out of the way for each local. If adding | 1336 | // If adding locals, we may need to move one global out of the way for each local. If adding |
| 1334 | // globals, they could all get demoted to STB_LOCAL, which would mean we move those N globals | 1337 | // globals, they could all get demoted to STB_LOCAL, meaning we have to move N other globals |
| 1335 | // *and* we move up to N other globals out of their way. | 1338 | // around to keep `.dynsym` compact. Either way, the maximum is N. |
| 1336 | try elf.changed_symtab_index.ensureUnusedCapacity(gpa, switch (kind) { | 1339 | try elf.changed_symtab_index.ensureUnusedCapacity(gpa, len); |
| 1337 | .all_local => len, | | |
| 1338 | .maybe_global => len * 2, | | |
| 1339 | }); | | |
| 1340 | | 1340 | |
| 1341 | { | 1341 | { |
| 1342 | // Ensure the symtab section's node is big enough | 1342 | // Ensure the symtab section's node is big enough |
| ... | @@ -1468,7 +1468,7 @@ fn addLocalSymbolAssumeCapacity(elf: *Elf, opts: AddLocalSymbolOptions) Symbol.L | ... | @@ -1468,7 +1468,7 @@ fn addLocalSymbolAssumeCapacity(elf: *Elf, opts: AddLocalSymbolOptions) Symbol.L |
| 1468 | const global_name: String(.strtab) = @enumFromInt(elf.targetLoad(&new_sym.name)); | 1468 | const global_name: String(.strtab) = @enumFromInt(elf.targetLoad(&new_sym.name)); |
| 1469 | elf.globalByName(global_name).?.symtab_index = new_index; | 1469 | elf.globalByName(global_name).?.symtab_index = new_index; |
| 1470 | | 1470 | |
| 1471 | if (target_index.ptr(elf).first_target_reloc != .none) { | 1471 | if (elf.ehdrField(.type) == .REL and target_index.ptr(elf).first_target_reloc != .none) { |
| 1472 | // This symbol's index is changing, so queue an update of relocs targeting it. | 1472 | // This symbol's index is changing, so queue an update of relocs targeting it. |
| 1473 | elf.changed_symtab_index.putAssumeCapacity(global_name, {}); | 1473 | elf.changed_symtab_index.putAssumeCapacity(global_name, {}); |
| 1474 | } | 1474 | } |
| ... | @@ -1950,61 +1950,68 @@ fn moveDemotedGlobal(elf: *Elf, global_ptr: *Symbol.Global) void { | ... | @@ -1950,61 +1950,68 @@ fn moveDemotedGlobal(elf: *Elf, global_ptr: *Symbol.Global) void { |
| 1950 | | 1950 | |
| 1951 | elf.targetStore(&shdr.info, @intFromEnum(dest_index) + 1); | 1951 | elf.targetStore(&shdr.info, @intFromEnum(dest_index) + 1); |
| 1952 | | 1952 | |
| 1953 | if (src_index == dest_index) { | 1953 | if (src_index != dest_index) { |
| 1954 | // The demoted global was already the first global, so we don't need to do any swap. | 1954 | // The demoted global was not the first global in the symtab, so we need to swap it |
| 1955 | return; | 1955 | // to its new location. |
| 1956 | } | | |
| 1957 | | 1956 | |
| 1958 | const src_sym_ptr = @field(elf.symPtr(src_index), @tagName(class)); | 1957 | const src_sym_ptr = @field(elf.symPtr(src_index), @tagName(class)); |
| 1959 | const dest_sym_ptr = @field(elf.symPtr(dest_index), @tagName(class)); | 1958 | const dest_sym_ptr = @field(elf.symPtr(dest_index), @tagName(class)); |
| 1960 | | 1959 | |
| 1961 | const this_name: String(.strtab) = @enumFromInt(elf.targetLoad(&src_sym_ptr.name)); | 1960 | const this_name: String(.strtab) = @enumFromInt(elf.targetLoad(&src_sym_ptr.name)); |
| 1962 | assert(elf.globalByName(this_name).? == global_ptr); | 1961 | assert(elf.globalByName(this_name).? == global_ptr); |
| 1963 | if (global_ptr.symtab_index.ptr(elf).first_target_reloc != .none) { | 1962 | |
| 1964 | // This symbol's index is changing, so queue an update of relocs targeting it. | 1963 | const other_name: String(.strtab) = @enumFromInt(elf.targetLoad(&dest_sym_ptr.name)); |
| 1965 | elf.changed_symtab_index.putAssumeCapacity(this_name, {}); | 1964 | const other_global_ptr = elf.globalByName(other_name).?; |
| 1966 | } | 1965 | assert(other_global_ptr.symtab_index == dest_index); |
| 1967 | | 1966 | |
| 1968 | const other_name: String(.strtab) = @enumFromInt(elf.targetLoad(&dest_sym_ptr.name)); | 1967 | // First swap the symtab entries... |
| 1969 | const other_global_ptr = elf.globalByName(other_name).?; | 1968 | std.mem.swap(class.ElfN().Sym, src_sym_ptr, dest_sym_ptr); |
| 1970 | assert(other_global_ptr.symtab_index == dest_index); | 1969 | // ...then the `elf.symtab` metadata... |
| 1971 | if (other_global_ptr.symtab_index.ptr(elf).first_target_reloc != .none) { | 1970 | std.mem.swap(Symbol, src_index.ptr(elf), dest_index.ptr(elf)); |
| 1972 | // This other symbol's index is changing, so queue an update of relocs targeting it. | 1971 | // ...then update the `elf.globals` tracking. |
| 1973 | elf.changed_symtab_index.putAssumeCapacity(other_name, {}); | 1972 | global_ptr.symtab_index = dest_index; |
| | 1973 | other_global_ptr.symtab_index = src_index; |
| 1974 | } | 1974 | } |
| 1975 | | 1975 | |
| 1976 | // First swap the symtab entries... | 1976 | // We also need to get rid of the dynsym entry if there is one. To keep dynsym compact, |
| 1977 | std.mem.swap(class.ElfN().Sym, src_sym_ptr, dest_sym_ptr); | 1977 | // we'll move another symbol into its place just like we did above. |
| 1978 | // ...then the `elf.symtab` metadata... | | |
| 1979 | std.mem.swap(Symbol, src_index.ptr(elf), dest_index.ptr(elf)); | | |
| 1980 | // ...then update the `elf.globals` tracking. | | |
| 1981 | global_ptr.symtab_index = dest_index; | | |
| 1982 | other_global_ptr.symtab_index = src_index; | | |
| 1983 | | | |
| 1984 | // We also need to get rid of the dynsym entry if there is one. For simplicity, just | | |
| 1985 | // replace it with a dummy entry which will never be used and will not cause problems. | | |
| 1986 | // TODO: we should have a free-list of dynsym slots so that other symbols can go here. | | |
| 1987 | // TODO: it would also be best to just avoid having gaps in the dynsym altogether. | | |
| 1988 | if (global_ptr.dynsym_index != 0) { | 1978 | if (global_ptr.dynsym_index != 0) { |
| 1989 | const dynsym = @field(elf.dynsymPtr(global_ptr.dynsym_index), @tagName(class)); | 1979 | const dynsym_shdr = @field(elf.shdrPtr(elf.shndx.dynsym), @tagName(class)); |
| 1990 | dynsym.* = .{ | 1980 | |
| 1991 | .name = @intFromEnum(String(.dynstr).empty), | 1981 | const ent_size = @sizeOf(class.ElfN().Sym); |
| 1992 | .value = 0, | 1982 | assert(elf.targetLoad(&dynsym_shdr.entsize) == ent_size); |
| 1993 | .size = 0, | 1983 | |
| 1994 | .info = .{ | 1984 | // We're going to decrease the size of `.dynsym`, thereby removing its last index. |
| 1995 | .type = .NOTYPE, | 1985 | const old_size = elf.targetLoad(&dynsym_shdr.size); |
| 1996 | // STB_WEAK is important: we mustn't cause a dynamic linker error if the | 1986 | const new_size = old_size - ent_size; |
| 1997 | // symbol can't be resolved. | 1987 | const remove_dynsym_index: u32 = @intCast(@divExact(new_size, ent_size)); |
| 1998 | .bind = .WEAK, | 1988 | |
| 1999 | }, | 1989 | const free_dynsym_index = global_ptr.dynsym_index; |
| 2000 | // SHN_UNDEF is important: we mustn't define this symbol for other DSOs. | | |
| 2001 | .shndx = std.elf.SHN_UNDEF, | | |
| 2002 | .other = .{ .visibility = .DEFAULT }, | | |
| 2003 | }; | | |
| 2004 | if (elf.targetEndian() != native_endian) { | | |
| 2005 | std.mem.byteSwapAllFields(class.ElfN().Sym, dynsym); | | |
| 2006 | } | | |
| 2007 | global_ptr.dynsym_index = 0; | 1990 | global_ptr.dynsym_index = 0; |
| | 1991 | |
| | 1992 | if (free_dynsym_index != remove_dynsym_index) { |
| | 1993 | // The demoted global wasn't the last entry, so move whatever entry we just |
| | 1994 | // truncated out of dynsym into its place. |
| | 1995 | |
| | 1996 | const src_dynsym_ptr = @field(elf.dynsymPtr(remove_dynsym_index), @tagName(class)); |
| | 1997 | const dest_dynsym_ptr = @field(elf.dynsymPtr(free_dynsym_index), @tagName(class)); |
| | 1998 | |
| | 1999 | const moved_name_dynstr: String(.dynstr) = @enumFromInt(elf.targetLoad(&src_dynsym_ptr.name)); |
| | 2000 | const moved_name = elf.stringExisting(.strtab, moved_name_dynstr.slice(elf)); |
| | 2001 | const moved_global_ptr = elf.globalByName(moved_name).?; |
| | 2002 | |
| | 2003 | dest_dynsym_ptr.* = src_dynsym_ptr.*; |
| | 2004 | |
| | 2005 | assert(moved_global_ptr.dynsym_index == remove_dynsym_index); |
| | 2006 | moved_global_ptr.dynsym_index = free_dynsym_index; |
| | 2007 | |
| | 2008 | // Since that symbol's dynsym index has changed, we'll have to update any |
| | 2009 | // relocation entries targeting it. |
| | 2010 | elf.changed_symtab_index.putAssumeCapacity(moved_name, {}); |
| | 2011 | } |
| | 2012 | |
| | 2013 | // Now that we've given that symbol a new home, actually decrease the section size. |
| | 2014 | elf.targetStore(&dynsym_shdr.size, new_size); |
| 2008 | } | 2015 | } |
| 2009 | }, | 2016 | }, |
| 2010 | } | 2017 | } |
| ... | @@ -2596,6 +2603,11 @@ fn string(elf: *Elf, comptime section: StringSection, key: []const u8) Error!Str | ... | @@ -2596,6 +2603,11 @@ fn string(elf: *Elf, comptime section: StringSection, key: []const u8) Error!Str |
| 2596 | const st: *StringTable = &@field(elf, @tagName(section)); | 2603 | const st: *StringTable = &@field(elf, @tagName(section)); |
| 2597 | return @enumFromInt(try st.get(elf, section.shndx(elf), key)); | 2604 | return @enumFromInt(try st.get(elf, section.shndx(elf), key)); |
| 2598 | } | 2605 | } |
| | 2606 | /// Like `string`, but asserts that the string is already in `section`. |
| | 2607 | fn stringExisting(elf: *Elf, comptime section: StringSection, key: []const u8) String(section) { |
| | 2608 | const st: *StringTable = &@field(elf, @tagName(section)); |
| | 2609 | return @enumFromInt(st.getExisting(elf, section.shndx(elf), key)); |
| | 2610 | } |
| 2599 | | 2611 | |
| 2600 | const StringTable = struct { | 2612 | const StringTable = struct { |
| 2601 | map: std.HashMapUnmanaged(u32, void, StringTable.Context, std.hash_map.default_max_load_percentage), | 2613 | map: std.HashMapUnmanaged(u32, void, StringTable.Context, std.hash_map.default_max_load_percentage), |
| ... | @@ -2626,7 +2638,14 @@ const StringTable = struct { | ... | @@ -2626,7 +2638,14 @@ const StringTable = struct { |
| 2626 | } | 2638 | } |
| 2627 | }; | 2639 | }; |
| 2628 | | 2640 | |
| 2629 | pub fn get(st: *StringTable, elf: *Elf, shndx: Section.Index, key: []const u8) Error!u32 { | 2641 | fn getExisting(st: *StringTable, elf: *Elf, shndx: Section.Index, key: []const u8) u32 { |
| | 2642 | if (key.len == 0) return 0; |
| | 2643 | const slice_const = shndx.get(elf).ni.sliceConst(&elf.mf); |
| | 2644 | const adapter: StringTable.Adapter = .{ .slice = slice_const }; |
| | 2645 | return st.map.getKeyAdapted(key, adapter).?; |
| | 2646 | } |
| | 2647 | |
| | 2648 | fn get(st: *StringTable, elf: *Elf, shndx: Section.Index, key: []const u8) Error!u32 { |
| 2630 | // If we are in `initHeaders` the strtab might not be initalized yet, so we need to special | 2649 | // If we are in `initHeaders` the strtab might not be initalized yet, so we need to special |
| 2631 | // case the empty string. | 2650 | // case the empty string. |
| 2632 | if (key.len == 0) return 0; | 2651 | if (key.len == 0) return 0; |
| ... | @@ -5201,7 +5220,7 @@ fn updateInitFiniArraySectionSize( | ... | @@ -5201,7 +5220,7 @@ fn updateInitFiniArraySectionSize( |
| 5201 | const end_vaddr: u64 = switch (elf.shdrPtr(shndx)) { | 5220 | const end_vaddr: u64 = switch (elf.shdrPtr(shndx)) { |
| 5202 | inline else => |shdr| shndx.vaddr(elf) + elf.targetLoad(&shdr.size), | 5221 | inline else => |shdr| shndx.vaddr(elf) + elf.targetLoad(&shdr.size), |
| 5203 | }; | 5222 | }; |
| 5204 | const end_sym_name = elf.string(.strtab, "__" ++ name ++ "_end") catch unreachable; // string definitely already exists | 5223 | const end_sym_name = elf.stringExisting(.strtab, "__" ++ name ++ "_end"); |
| 5205 | Symbol.Id.global(end_sym_name).flushMoved(elf, end_vaddr); | 5224 | Symbol.Id.global(end_sym_name).flushMoved(elf, end_vaddr); |
| 5206 | } | 5225 | } |
| 5207 | | 5226 | |
| ... | @@ -6491,26 +6510,72 @@ pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) link.Error!bool { | ... | @@ -6491,26 +6510,72 @@ pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) link.Error!bool { |
| 6491 | }; | 6510 | }; |
| 6492 | break :task; | 6511 | break :task; |
| 6493 | } | 6512 | } |
| 6494 | while (elf.changed_symtab_index.pop()) |kv| { | 6513 | if (elf.changed_symtab_index.pop()) |kv| { |
| 6495 | // We only need to do work in relocatables, because in ELF modules (non-relocatables) | 6514 | const sub_prog_node = elf.mf.update_prog_node.start(kv.key.slice(elf), 0); |
| 6496 | // our `ElfN.Rela` entries use `.dynsym` indices rather than `.symtab` indices, and | 6515 | defer sub_prog_node.end(); |
| 6497 | // `.dynsym` indices are (at the time of writing) always immutable. | 6516 | |
| 6498 | if (elf.ehdrField(.type) == .REL) { | 6517 | const global_name = kv.key; |
| 6499 | const sub_prog_node = elf.mf.update_prog_node.start(kv.key.slice(elf), 0); | 6518 | const global = elf.globalByName(global_name).?; |
| 6500 | defer sub_prog_node.end(); | 6519 | const sym_id: Symbol.Id = .global(global_name); |
| 6501 | const sym = elf.globalByName(kv.key).?.symtab_index.ptr(elf); | 6520 | const sym = global.symtab_index.ptr(elf); |
| 6502 | var ri = sym.first_target_reloc; | 6521 | |
| 6503 | while (ri != .none) { | 6522 | switch (elf.ehdrField(.type)) { |
| 6504 | const reloc = ri.get(elf); | 6523 | .REL => { |
| 6505 | reloc.relaSection(elf).relaUpdateSym( | 6524 | // Index in `.symtab` has changed. Relocatables are easy, we just need to update |
| 6506 | elf, | 6525 | // all of the output relocations. |
| 6507 | reloc.rela_index.unwrap().?, | 6526 | const symtab_index = @intFromEnum(global.symtab_index); |
| 6508 | @intFromEnum(reloc.target.index(elf)), | 6527 | var ri = sym.first_target_reloc; |
| 6509 | ); | 6528 | while (ri != .none) { |
| 6510 | ri = reloc.next; | 6529 | const reloc = ri.get(elf); |
| 6511 | } | 6530 | assert(reloc.target == sym_id); |
| 6512 | break :task; | 6531 | // In relocatables, every symbol relocation has an output relocation. |
| | 6532 | const rela_index = reloc.rela_index.unwrap().?; |
| | 6533 | reloc.relaSection(elf).relaUpdateSym(elf, rela_index, symtab_index); |
| | 6534 | ri = reloc.next; |
| | 6535 | } |
| | 6536 | }, |
| | 6537 | else => { |
| | 6538 | // Index in `.dynsym` has changed. This case is slightly trickier because there |
| | 6539 | // are a few things which may have emitted runtime relocations, including symbol |
| | 6540 | // relocs... |
| | 6541 | const dynsym_index = global.dynsym_index; |
| | 6542 | var ri = sym.first_target_reloc; |
| | 6543 | while (ri != .none) { |
| | 6544 | const reloc = ri.get(elf); |
| | 6545 | assert(reloc.target == sym_id); |
| | 6546 | // There may or may not be a runtime relocation for this symbol reloc. |
| | 6547 | if (reloc.rela_index.unwrap()) |rela_index| { |
| | 6548 | reloc.relaSection(elf).relaUpdateSym(elf, rela_index, dynsym_index); |
| | 6549 | } |
| | 6550 | ri = reloc.next; |
| | 6551 | } |
| | 6552 | |
| | 6553 | // ...a copy relocation... |
| | 6554 | if (elf.copied_globals.get(global_name)) |copied| { |
| | 6555 | elf.shndx.rela_dyn.relaUpdateSym(elf, copied.rela_index, dynsym_index); |
| | 6556 | } |
| | 6557 | |
| | 6558 | // ...a PLT entry... |
| | 6559 | if (elf.plt.getIndex(global_name)) |plt_index| { |
| | 6560 | // PLT indices exactly match `.rela.plt` relocation indices. |
| | 6561 | elf.shndx.rela_plt.relaUpdateSym(elf, @enumFromInt(plt_index), dynsym_index); |
| | 6562 | } |
| | 6563 | |
| | 6564 | // ...and any relevant GOT entries. |
| | 6565 | if (elf.got.getIndex(.{ .symbol = sym_id })) |got_index| { |
| | 6566 | elf.updateGotEntry(got_index); |
| | 6567 | } |
| | 6568 | if (elf.got.getIndex(.{ .tpoff = sym_id })) |got_index| { |
| | 6569 | elf.updateGotEntry(got_index); |
| | 6570 | } |
| | 6571 | if (elf.got.getIndex(.{ .tlsgd0 = sym_id })) |got_index| { |
| | 6572 | elf.updateGotEntry(got_index); |
| | 6573 | elf.updateGotEntry(got_index + 1); // tlsgd1 |
| | 6574 | } |
| | 6575 | }, |
| 6513 | } | 6576 | } |
| | 6577 | |
| | 6578 | break :task; |
| 6514 | } | 6579 | } |
| 6515 | while (elf.mf.updates.pop()) |ni| { | 6580 | while (elf.mf.updates.pop()) |ni| { |
| 6516 | const clean_moved = ni.cleanMoved(&elf.mf); | 6581 | const clean_moved = ni.cleanMoved(&elf.mf); |