authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-04-18 14:04:42+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-04-21 22:44:25+02:00
log8a3ad3f6204747b5621e14cae0564ee7929a7cd8
tree349b9386d0acd3f5314ec58482f5b3d14bf2baee
parent528b66f6ec9cfb140abff3dc0c4735c179520f42

elf: do not reserve a GOT slot for every Atom

8 files changed, 132 insertions(+), 53 deletions(-)

src/arch/aarch64/CodeGen.zig+1
......@@ -4290,6 +4290,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
42904290 if (self.bin_file.cast(link.File.Elf)) |elf_file| {
42914291 const atom_index = try elf_file.getOrCreateAtomForDecl(func.owner_decl);
42924292 const atom = elf_file.getAtom(atom_index);
4293 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
42934294 const got_addr = @intCast(u32, atom.getOffsetTableAddress(elf_file));
42944295 try self.genSetReg(Type.initTag(.usize), .x30, .{ .memory = got_addr });
42954296 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
src/arch/arm/CodeGen.zig+1
......@@ -4270,6 +4270,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
42704270 if (self.bin_file.cast(link.File.Elf)) |elf_file| {
42714271 const atom_index = try elf_file.getOrCreateAtomForDecl(func.owner_decl);
42724272 const atom = elf_file.getAtom(atom_index);
4273 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
42734274 const got_addr = @intCast(u32, atom.getOffsetTableAddress(elf_file));
42744275 try self.genSetReg(Type.initTag(.usize), .lr, .{ .memory = got_addr });
42754276 } else if (self.bin_file.cast(link.File.MachO)) |_| {
src/arch/riscv64/CodeGen.zig+1
......@@ -1734,6 +1734,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
17341734 const func = func_payload.data;
17351735 const atom_index = try elf_file.getOrCreateAtomForDecl(func.owner_decl);
17361736 const atom = elf_file.getAtom(atom_index);
1737 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
17371738 const got_addr = @intCast(u32, atom.getOffsetTableAddress(elf_file));
17381739 try self.genSetReg(Type.initTag(.usize), .ra, .{ .memory = got_addr });
17391740 _ = try self.addInst(.{
src/arch/sparc64/CodeGen.zig+1
......@@ -1254,6 +1254,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
12541254 const got_addr = if (self.bin_file.cast(link.File.Elf)) |elf_file| blk: {
12551255 const atom_index = try elf_file.getOrCreateAtomForDecl(func.owner_decl);
12561256 const atom = elf_file.getAtom(atom_index);
1257 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
12571258 break :blk @intCast(u32, atom.getOffsetTableAddress(elf_file));
12581259 } else unreachable;
12591260
src/arch/x86_64/CodeGen.zig+9-3
......@@ -5624,7 +5624,9 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
56245624
56255625 if (self.bin_file.cast(link.File.Elf)) |elf_file| {
56265626 const atom_index = try elf_file.getOrCreateAtomForDecl(func.owner_decl);
5627 const got_addr = elf_file.getAtom(atom_index).getOffsetTableAddress(elf_file);
5627 const atom = elf_file.getAtom(atom_index);
5628 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
5629 const got_addr = atom.getOffsetTableAddress(elf_file);
56285630 try self.asmMemory(.call, Memory.sib(.qword, .{
56295631 .base = .ds,
56305632 .disp = @intCast(i32, got_addr),
......@@ -5853,7 +5855,9 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
58535855 .{ .kind = .const_data, .ty = Type.anyerror },
58545856 4, // dword alignment
58555857 );
5856 const got_addr = elf_file.getAtom(atom_index).getOffsetTableAddress(elf_file);
5858 const atom = elf_file.getAtom(atom_index);
5859 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
5860 const got_addr = atom.getOffsetTableAddress(elf_file);
58575861 try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
58585862 .base = .ds,
58595863 .disp = @intCast(i32, got_addr),
......@@ -8230,7 +8234,9 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
82308234 .{ .kind = .const_data, .ty = Type.anyerror },
82318235 4, // dword alignment
82328236 );
8233 const got_addr = elf_file.getAtom(atom_index).getOffsetTableAddress(elf_file);
8237 const atom = elf_file.getAtom(atom_index);
8238 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
8239 const got_addr = atom.getOffsetTableAddress(elf_file);
82348240 try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
82358241 .base = .ds,
82368242 .disp = @intCast(i32, got_addr),
src/codegen.zig+1
......@@ -1006,6 +1006,7 @@ fn genDeclRef(
10061006 if (bin_file.cast(link.File.Elf)) |elf_file| {
10071007 const atom_index = try elf_file.getOrCreateAtomForDecl(decl_index);
10081008 const atom = elf_file.getAtom(atom_index);
1009 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
10091010 return GenResult.mcv(.{ .memory = atom.getOffsetTableAddress(elf_file) });
10101011 } else if (bin_file.cast(link.File.MachO)) |macho_file| {
10111012 const atom_index = try macho_file.getOrCreateAtomForDecl(decl_index);
src/link/Elf.zig+105-45
......@@ -63,6 +63,88 @@ const Section = struct {
6363 free_list: std.ArrayListUnmanaged(Atom.Index) = .{},
6464};
6565
66const SectionTable = struct {
67 entries: std.ArrayListUnmanaged(SymIndex) = .{},
68 free_list: std.ArrayListUnmanaged(Index) = .{},
69 lookup: std.AutoHashMapUnmanaged(SymIndex, Index) = .{},
70
71 const SymIndex = u32;
72 const Index = u32;
73
74 pub fn deinit(st: *ST, allocator: Allocator) void {
75 st.entries.deinit(allocator);
76 st.free_list.deinit(allocator);
77 st.lookup.deinit(allocator);
78 }
79
80 pub fn allocateEntry(st: *ST, allocator: Allocator, target: SymIndex) !Index {
81 try st.entries.ensureUnusedCapacity(allocator, 1);
82 const index = blk: {
83 if (st.free_list.popOrNull()) |index| {
84 log.debug(" (reusing entry index {d})", .{index});
85 break :blk index;
86 } else {
87 log.debug(" (allocating entry at index {d})", .{st.entries.items.len});
88 const index = @intCast(u32, st.entries.items.len);
89 _ = st.entries.addOneAssumeCapacity();
90 break :blk index;
91 }
92 };
93 st.entries.items[index] = target;
94 try st.lookup.putNoClobber(allocator, target, index);
95 return index;
96 }
97
98 pub fn freeEntry(st: *ST, allocator: Allocator, target: SymIndex) void {
99 const index = st.lookup.get(target) orelse return;
100 st.free_list.append(allocator, index) catch {};
101 st.entries.items[index] = 0;
102 _ = st.lookup.remove(target);
103 }
104
105 const FormatContext = struct {
106 ctx: *Elf,
107 st: *const ST,
108 };
109
110 fn fmt(
111 ctx: FormatContext,
112 comptime unused_format_string: []const u8,
113 options: std.fmt.FormatOptions,
114 writer: anytype,
115 ) @TypeOf(writer).Error!void {
116 _ = options;
117 comptime assert(unused_format_string.len == 0);
118
119 const base_addr = ctx.ctx.program_headers.items[ctx.ctx.phdr_got_index.?].p_vaddr;
120 const target = ctx.ctx.base.options.target;
121 const ptr_bits = target.cpu.arch.ptrBitWidth();
122 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
123
124 try writer.writeAll("SectionTable:\n");
125 for (ctx.st.entries.items, 0..) |entry, i| {
126 try writer.print(" {d}@{x} => local(%{d})\n", .{ i, base_addr + i * ptr_bytes, entry });
127 }
128 }
129
130 fn format(st: ST, comptime unused_format_string: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void {
131 _ = st;
132 _ = unused_format_string;
133 _ = options;
134 _ = writer;
135 @compileError("do not format SectionTable directly; use st.fmtDebug()");
136 }
137
138 pub fn fmtDebug(st: ST, ctx: *Elf) std.fmt.Formatter(fmt) {
139 return .{ .data = .{
140 .ctx = ctx,
141 .st = st,
142 } };
143 }
144
145 const ST = @This();
146};
147
66148const LazySymbolMetadata = struct {
67149 text_atom: ?Atom.Index = null,
68150 rodata_atom: ?Atom.Index = null,
......@@ -148,17 +230,13 @@ global_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = .{},
148230
149231local_symbol_free_list: std.ArrayListUnmanaged(u32) = .{},
150232global_symbol_free_list: std.ArrayListUnmanaged(u32) = .{},
151offset_table_free_list: std.ArrayListUnmanaged(u32) = .{},
152233
153/// Same order as in the file. The value is the absolute vaddr value.
154/// If the vaddr of the executable program header changes, the entire
155/// offset table needs to be rewritten.
156offset_table: std.ArrayListUnmanaged(u64) = .{},
234got_table: SectionTable = .{},
157235
158236phdr_table_dirty: bool = false,
159237shdr_table_dirty: bool = false,
160238shstrtab_dirty: bool = false,
161offset_table_count_dirty: bool = false,
239got_table_count_dirty: bool = false,
162240
163241debug_strtab_dirty: bool = false,
164242debug_abbrev_section_dirty: bool = false,
......@@ -329,8 +407,7 @@ pub fn deinit(self: *Elf) void {
329407 self.global_symbols.deinit(gpa);
330408 self.global_symbol_free_list.deinit(gpa);
331409 self.local_symbol_free_list.deinit(gpa);
332 self.offset_table_free_list.deinit(gpa);
333 self.offset_table.deinit(gpa);
410 self.got_table.deinit(gpa);
334411
335412 {
336413 var it = self.decls.iterator();
......@@ -1289,6 +1366,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
12891366 assert(!self.shdr_table_dirty);
12901367 assert(!self.shstrtab_dirty);
12911368 assert(!self.debug_strtab_dirty);
1369 assert(!self.got_table_count_dirty);
12921370}
12931371
12941372fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !void {
......@@ -2168,7 +2246,7 @@ fn freeAtom(self: *Elf, atom_index: Atom.Index) void {
21682246 _ = self.atom_by_index_table.remove(local_sym_index);
21692247 self.getAtomPtr(atom_index).local_sym_index = 0;
21702248
2171 self.offset_table_free_list.append(self.base.allocator, atom.offset_table_index) catch {};
2249 self.got_table.freeEntry(gpa, local_sym_index);
21722250}
21732251
21742252fn shrinkAtom(self: *Elf, atom_index: Atom.Index, new_block_size: u64) void {
......@@ -2191,11 +2269,9 @@ pub fn createAtom(self: *Elf) !Atom.Index {
21912269 const atom_index = @intCast(Atom.Index, self.atoms.items.len);
21922270 const atom = try self.atoms.addOne(gpa);
21932271 const local_sym_index = try self.allocateLocalSymbol();
2194 const offset_table_index = try self.allocateGotOffset();
21952272 try self.atom_by_index_table.putNoClobber(gpa, local_sym_index, atom_index);
21962273 atom.* = .{
21972274 .local_sym_index = local_sym_index,
2198 .offset_table_index = offset_table_index,
21992275 .prev_index = null,
22002276 .next_index = null,
22012277 };
......@@ -2352,26 +2428,6 @@ pub fn allocateLocalSymbol(self: *Elf) !u32 {
23522428 return index;
23532429}
23542430
2355pub fn allocateGotOffset(self: *Elf) !u32 {
2356 try self.offset_table.ensureUnusedCapacity(self.base.allocator, 1);
2357
2358 const index = blk: {
2359 if (self.offset_table_free_list.popOrNull()) |index| {
2360 log.debug(" (reusing GOT offset at index {d})", .{index});
2361 break :blk index;
2362 } else {
2363 log.debug(" (allocating GOT offset at index {d})", .{self.offset_table.items.len});
2364 const index = @intCast(u32, self.offset_table.items.len);
2365 _ = self.offset_table.addOneAssumeCapacity();
2366 self.offset_table_count_dirty = true;
2367 break :blk index;
2368 }
2369 };
2370
2371 self.offset_table.items[index] = 0;
2372 return index;
2373}
2374
23752431fn freeUnnamedConsts(self: *Elf, decl_index: Module.Decl.Index) void {
23762432 const unnamed_consts = self.unnamed_const_atoms.getPtr(decl_index) orelse return;
23772433 for (unnamed_consts.items) |atom| {
......@@ -2465,6 +2521,7 @@ fn updateDeclCode(self: *Elf, decl_index: Module.Decl.Index, code: []const u8, s
24652521 const decl_metadata = self.decls.get(decl_index).?;
24662522 const atom_index = decl_metadata.atom;
24672523 const atom = self.getAtom(atom_index);
2524 const local_sym_index = atom.getSymbolIndex().?;
24682525
24692526 const shdr_index = decl_metadata.shdr;
24702527 if (atom.getSymbol(self).st_size != 0 and self.base.child_pid == null) {
......@@ -2485,8 +2542,9 @@ fn updateDeclCode(self: *Elf, decl_index: Module.Decl.Index, code: []const u8, s
24852542 local_sym.st_value = vaddr;
24862543
24872544 log.debug(" (writing new offset table entry)", .{});
2488 self.offset_table.items[atom.offset_table_index] = vaddr;
2489 try self.writeOffsetTableEntry(atom.offset_table_index);
2545 const got_entry_index = self.got_table.lookup.get(local_sym_index).?;
2546 self.got_table.entries.items[got_entry_index] = local_sym_index;
2547 try self.writeOffsetTableEntry(got_entry_index);
24902548 }
24912549 } else if (code.len < local_sym.st_size) {
24922550 self.shrinkAtom(atom_index, code.len);
......@@ -2494,7 +2552,7 @@ fn updateDeclCode(self: *Elf, decl_index: Module.Decl.Index, code: []const u8, s
24942552 local_sym.st_size = code.len;
24952553
24962554 // TODO this write could be avoided if no fields of the symbol were changed.
2497 try self.writeSymbol(atom.getSymbolIndex().?);
2555 try self.writeSymbol(local_sym_index);
24982556 } else {
24992557 const local_sym = atom.getSymbolPtr(self);
25002558 local_sym.* = .{
......@@ -2509,12 +2567,12 @@ fn updateDeclCode(self: *Elf, decl_index: Module.Decl.Index, code: []const u8, s
25092567 errdefer self.freeAtom(atom_index);
25102568 log.debug("allocated text block for {s} at 0x{x}", .{ decl_name, vaddr });
25112569
2512 self.offset_table.items[atom.offset_table_index] = vaddr;
25132570 local_sym.st_value = vaddr;
25142571 local_sym.st_size = code.len;
25152572
2516 try self.writeSymbol(atom.getSymbolIndex().?);
2517 try self.writeOffsetTableEntry(atom.offset_table_index);
2573 try self.writeSymbol(local_sym_index);
2574 const got_entry_index = try atom.getOrCreateOffsetTableEntry(self);
2575 try self.writeOffsetTableEntry(got_entry_index);
25182576 }
25192577
25202578 const local_sym = atom.getSymbolPtr(self);
......@@ -2755,12 +2813,12 @@ fn updateLazySymbolAtom(
27552813 errdefer self.freeAtom(atom_index);
27562814 log.debug("allocated text block for {s} at 0x{x}", .{ name, vaddr });
27572815
2758 self.offset_table.items[atom.offset_table_index] = vaddr;
27592816 local_sym.st_value = vaddr;
27602817 local_sym.st_size = code.len;
27612818
27622819 try self.writeSymbol(local_sym_index);
2763 try self.writeOffsetTableEntry(atom.offset_table_index);
2820 const got_entry_index = try atom.getOrCreateOffsetTableEntry(self);
2821 try self.writeOffsetTableEntry(got_entry_index);
27642822
27652823 const section_offset = vaddr - self.program_headers.items[phdr_index].p_vaddr;
27662824 const file_offset = self.sections.items(.shdr)[shdr_index].sh_offset + section_offset;
......@@ -2991,30 +3049,32 @@ fn writeSectHeader(self: *Elf, index: usize) !void {
29913049
29923050fn writeOffsetTableEntry(self: *Elf, index: usize) !void {
29933051 const entry_size: u16 = self.archPtrWidthBytes();
2994 if (self.offset_table_count_dirty) {
2995 const needed_size = self.offset_table.items.len * entry_size;
3052 if (self.got_table_count_dirty) {
3053 const needed_size = self.got_table.entries.items.len * entry_size;
29963054 try self.growAllocSection(self.got_section_index.?, needed_size);
2997 self.offset_table_count_dirty = false;
3055 self.got_table_count_dirty = false;
29983056 }
29993057 const endian = self.base.options.target.cpu.arch.endian();
30003058 const shdr = &self.sections.items(.shdr)[self.got_section_index.?];
30013059 const off = shdr.sh_offset + @as(u64, entry_size) * index;
30023060 const phdr = &self.program_headers.items[self.phdr_got_index.?];
30033061 const vaddr = phdr.p_vaddr + @as(u64, entry_size) * index;
3062 const got_entry = self.got_table.entries.items[index];
3063 const got_value = self.getSymbol(got_entry).st_value;
30043064 switch (entry_size) {
30053065 2 => {
30063066 var buf: [2]u8 = undefined;
3007 mem.writeInt(u16, &buf, @intCast(u16, self.offset_table.items[index]), endian);
3067 mem.writeInt(u16, &buf, @intCast(u16, got_value), endian);
30083068 try self.base.file.?.pwriteAll(&buf, off);
30093069 },
30103070 4 => {
30113071 var buf: [4]u8 = undefined;
3012 mem.writeInt(u32, &buf, @intCast(u32, self.offset_table.items[index]), endian);
3072 mem.writeInt(u32, &buf, @intCast(u32, got_value), endian);
30133073 try self.base.file.?.pwriteAll(&buf, off);
30143074 },
30153075 8 => {
30163076 var buf: [8]u8 = undefined;
3017 mem.writeInt(u64, &buf, self.offset_table.items[index], endian);
3077 mem.writeInt(u64, &buf, got_value, endian);
30183078 try self.base.file.?.pwriteAll(&buf, off);
30193079
30203080 if (self.base.child_pid) |pid| {
src/link/Elf/Atom.zig+13-5
......@@ -14,9 +14,6 @@ const Elf = @import("../Elf.zig");
1414/// offset table entry.
1515local_sym_index: u32,
1616
17/// This field is undefined for symbols with size = 0.
18offset_table_index: u32,
19
2017/// Points to the previous and next neighbors, based on the `text_offset`.
2118/// This can be used to find, for example, the capacity of this `TextBlock`.
2219prev_index: ?Index,
......@@ -48,13 +45,24 @@ pub fn getName(self: Atom, elf_file: *const Elf) []const u8 {
4845 return elf_file.getSymbolName(self.getSymbolIndex().?);
4946}
5047
48/// If entry already exists, returns index to it.
49/// Otherwise, creates a new entry in the Global Offset Table for this Atom.
50pub fn getOrCreateOffsetTableEntry(self: Atom, elf_file: *Elf) !u32 {
51 const sym_index = self.getSymbolIndex().?;
52 if (elf_file.got_table.lookup.get(sym_index)) |index| return index;
53 const index = try elf_file.got_table.allocateEntry(elf_file.base.allocator, sym_index);
54 elf_file.got_table_count_dirty = true;
55 return index;
56}
57
5158pub fn getOffsetTableAddress(self: Atom, elf_file: *Elf) u64 {
52 assert(self.getSymbolIndex() != null);
59 const sym_index = self.getSymbolIndex().?;
60 const got_entry_index = elf_file.got_table.lookup.get(sym_index).?;
5361 const target = elf_file.base.options.target;
5462 const ptr_bits = target.cpu.arch.ptrBitWidth();
5563 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
5664 const got = elf_file.program_headers.items[elf_file.phdr_got_index.?];
57 return got.p_vaddr + self.offset_table_index * ptr_bytes;
65 return got.p_vaddr + got_entry_index * ptr_bytes;
5866}
5967
6068/// Returns how much room there is to grow in virtual address space.