From 27e1e636710ca72fb4508e2ea731dcb39cedbc95 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Aug 2024 07:45:05 +0200 Subject: [PATCH 01/39] elf: introduce OffsetTable in ZigObject for funcs only --- src/link/Elf.zig | 6 +++- src/link/Elf/ZigObject.zig | 58 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 103c69202bad983b3486a090cda6fff46f323c45..41c4f202d7fb1400ef8b9934baf70675f2c063d0 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -5657,10 +5657,14 @@ fn fmtDumpState( if (self.zigObjectPtr()) |zig_object| { try writer.print("zig_object({d}) : {s}\n", .{ zig_object.index, zig_object.path }); - try writer.print("{}{}\n", .{ + try writer.print("{}{}", .{ zig_object.fmtAtoms(self), zig_object.fmtSymtab(self), }); + if (zig_object.func_offset_table) |ot| { + try writer.print("{}", .{ot.fmt(zig_object, self)}); + } + try writer.writeByte('\n'); } for (self.objects.items) |index| { diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index ef3e2ed77c7828fc9b4491072a2041e73e50dc17..671dc285e35983267d0436b0706b2b456bef1814 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -55,6 +55,8 @@ debug_str_section_zig_size: u64 = 0, debug_aranges_section_zig_size: u64 = 0, debug_line_section_zig_size: u64 = 0, +func_offset_table: ?OffsetTable = null, + pub const global_symbol_bit: u32 = 0x80000000; pub const symbol_mask: u32 = 0x7fffffff; pub const SHN_ATOM: u16 = 0x100; @@ -128,6 +130,10 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void { if (self.dwarf) |*dw| { dw.deinit(); } + + if (self.func_offset_table) |*ot| { + ot.deinit(allocator); + } } pub fn flushModule(self: *ZigObject, elf_file: *Elf, tid: Zcu.PerThread.Id) !void { @@ -1687,6 +1693,58 @@ const UavTable = std.AutoHashMapUnmanaged(InternPool.Index, AvMetadata); const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata); const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); +pub const OffsetTable = struct { + atom_index: Atom.Index, + entries: std.ArrayListUnmanaged(Symbol.Index) = .{}, + + pub fn deinit(ot: *OffsetTable, allocator: Allocator) void { + ot.entries.deinit(allocator); + } + + pub fn size(ot: OffsetTable, elf_file: *Elf) usize { + return ot.entries.items.len * switch (elf_file.ptr_width) { + .p32 => @as(usize, 4), + .p64 => 8, + }; + } + + const OffsetTableFormatContext = struct { OffsetTable, *ZigObject, *Elf }; + + pub fn format( + ot: OffsetTable, + comptime unused_fmt_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = ot; + _ = unused_fmt_string; + _ = options; + _ = writer; + @compileError("do not format OffsetTable directly"); + } + + pub fn fmt(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) std.fmt.Formatter(format2) { + return .{ .data = .{ ot, zo, elf_file } }; + } + + fn format2( + ctx: OffsetTableFormatContext, + comptime unused_fmt_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = options; + _ = unused_fmt_string; + const ot, const zo, const ef = ctx; + const atom_ptr = zo.atom(ot.atom_index).?; + try writer.print("@{x} : size({x})\n", .{ atom_ptr.address(ef), ot.size(ef) }); + for (ot.entries.items) |sym_index| { + const sym = zo.symbol(sym_index); + try writer.print(" %{d} : {s} : @{x}\n", .{ sym_index, sym.name(ef), sym.value }); + } + } +}; + const assert = std.debug.assert; const builtin = @import("builtin"); const codegen = @import("../../codegen.zig"); -- 2.54.0 From 67e703dc71a30989f9caaa3fa13e141d1e9584c3 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Aug 2024 08:30:56 +0200 Subject: [PATCH 02/39] elf: allocate new offset table via Atom.allocate mechanism --- src/link/Elf.zig | 2 +- src/link/Elf/Symbol.zig | 5 +++ src/link/Elf/ZigObject.zig | 79 +++++++++++++++++++++++++++++++++----- 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 41c4f202d7fb1400ef8b9934baf70675f2c063d0..ffba511e878076baf839d993af89a1574d5ee50e 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -5661,7 +5661,7 @@ fn fmtDumpState( zig_object.fmtAtoms(self), zig_object.fmtSymtab(self), }); - if (zig_object.func_offset_table) |ot| { + if (zig_object.offset_table) |ot| { try writer.print("{}", .{ot.fmt(zig_object, self)}); } try writer.writeByte('\n'); diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index ce6b94a185c4162a7711c4c820b7381f368ce873..f023ad5d915306cf73e82dd93d787115e9af8a8a 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -260,6 +260,7 @@ const AddExtraOpts = struct { gottp: ?u32 = null, tlsdesc: ?u32 = null, zig_got: ?u32 = null, + zig_offset_table: ?u32 = null, }; pub fn addExtra(symbol: *Symbol, opts: AddExtraOpts, elf_file: *Elf) void { @@ -467,6 +468,9 @@ pub const Flags = packed struct { /// Whether the symbol is a merge subsection. merge_subsection: bool = false, + + /// Whether the symbol has __zig_offset_table indirection. + zig_offset_table: bool = false, }; pub const Extra = struct { @@ -481,6 +485,7 @@ pub const Extra = struct { tlsdesc: u32 = 0, zig_got: u32 = 0, merge_section: u32 = 0, + zig_offset_table: u32 = 0, }; pub const Index = u32; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 671dc285e35983267d0436b0706b2b456bef1814..e652afe7279e4e62e99421caff97aa8baf6f2599 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -55,7 +55,10 @@ debug_str_section_zig_size: u64 = 0, debug_aranges_section_zig_size: u64 = 0, debug_line_section_zig_size: u64 = 0, -func_offset_table: ?OffsetTable = null, +/// Function offset table containing pointers to Zig generated functions. +/// The table is used for Zig's incremental compilation and is embedded with +/// the machine code section. +offset_table: ?OffsetTable = null, pub const global_symbol_bit: u32 = 0x80000000; pub const symbol_mask: u32 = 0x7fffffff; @@ -131,7 +134,7 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void { dw.deinit(); } - if (self.func_offset_table) |*ot| { + if (self.offset_table) |*ot| { ot.deinit(allocator); } } @@ -1040,12 +1043,24 @@ pub fn updateFunc( const ip = &zcu.intern_pool; const gpa = elf_file.base.comp.gpa; const func = zcu.funcInfo(func_index); + const offset_table = self.offsetTablePtr() orelse try self.initOffsetTable(gpa, elf_file); log.debug("updateFunc {}({d})", .{ ip.getNav(func.owner_nav).fqn.fmt(ip), func.owner_nav }); const sym_index = try self.getOrCreateMetadataForNav(elf_file, func.owner_nav); self.symbol(sym_index).atom(elf_file).?.freeRelocs(elf_file); + { + const sym = self.symbol(sym_index); + if (!sym.flags.zig_offset_table) { + const index = try offset_table.addSymbol(gpa, sym_index); + sym.flags.zig_offset_table = true; + sym.addExtra(.{ .zig_offset_table = index }, elf_file); + try offset_table.updateSize(self, elf_file); + try self.symbol(offset_table.sym_index).atom(elf_file).?.allocate(elf_file); + } + } + var code_buffer = std.ArrayList(u8).init(gpa); defer code_buffer.deinit(); @@ -1446,6 +1461,27 @@ pub fn getGlobalSymbol(self: *ZigObject, elf_file: *Elf, name: []const u8, lib_n return lookup_gop.value_ptr.*; } +fn offsetTablePtr(self: *ZigObject) ?*OffsetTable { + return if (self.offset_table) |*ot| ot else null; +} + +fn initOffsetTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!*OffsetTable { + const name_off = try self.addString(allocator, "__zig_offset_table"); + const sym_index = try self.newSymbolWithAtom(allocator, name_off); + const sym = self.symbol(sym_index); + const esym = &self.symtab.items(.elf_sym)[sym.esym_index]; + esym.st_info |= elf.STT_OBJECT; + const atom_ptr = sym.atom(elf_file).?; + atom_ptr.alive = true; + atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(switch (elf_file.ptr_width) { + .p32 => 4, + .p64 => 8, + }); + atom_ptr.output_section_index = elf_file.zig_text_section_index.?; + self.offset_table = OffsetTable{ .sym_index = sym_index }; + return &(self.offset_table.?); +} + pub fn asFile(self: *ZigObject) File { return .{ .zig_object = self }; } @@ -1694,18 +1730,39 @@ const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymb const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); pub const OffsetTable = struct { - atom_index: Atom.Index, + sym_index: Symbol.Index, entries: std.ArrayListUnmanaged(Symbol.Index) = .{}, pub fn deinit(ot: *OffsetTable, allocator: Allocator) void { ot.entries.deinit(allocator); } - pub fn size(ot: OffsetTable, elf_file: *Elf) usize { - return ot.entries.items.len * switch (elf_file.ptr_width) { - .p32 => @as(usize, 4), + pub fn addSymbol(ot: *OffsetTable, allocator: Allocator, sym_index: Symbol.Index) !Index { + const index: Index = @intCast(ot.entries.items.len); + try ot.entries.append(allocator, sym_index); + return index; + } + + pub fn address(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) i64 { + const sym = zo.symbol(ot.sym_index); + return sym.address(.{}, elf_file); + } + + pub fn size(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) u64 { + const sym = zo.symbol(ot.sym_index); + return sym.atom(elf_file).?.size; + } + + pub fn updateSize(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) !void { + const ot_size: u64 = @intCast(ot.entries.items.len * switch (elf_file.ptr_width) { + .p32 => @as(u64, 4), .p64 => 8, - }; + }); + const sym = zo.symbol(ot.sym_index); + const esym = &zo.symtab.items(.elf_sym)[sym.esym_index]; + esym.st_size = ot_size; + const atom_ptr = sym.atom(elf_file).?; + atom_ptr.size = ot_size; } const OffsetTableFormatContext = struct { OffsetTable, *ZigObject, *Elf }; @@ -1736,13 +1793,15 @@ pub const OffsetTable = struct { _ = options; _ = unused_fmt_string; const ot, const zo, const ef = ctx; - const atom_ptr = zo.atom(ot.atom_index).?; - try writer.print("@{x} : size({x})\n", .{ atom_ptr.address(ef), ot.size(ef) }); + try writer.writeAll("offset table\n"); + try writer.print(" @{x} : size({x})\n", .{ ot.address(zo, ef), ot.size(zo, ef) }); for (ot.entries.items) |sym_index| { const sym = zo.symbol(sym_index); - try writer.print(" %{d} : {s} : @{x}\n", .{ sym_index, sym.name(ef), sym.value }); + try writer.print(" %{d} : {s} : @{x}\n", .{ sym_index, sym.name(ef), sym.address(.{}, ef) }); } } + + pub const Index = u32; }; const assert = std.debug.assert; -- 2.54.0 From 24b915c9f218d48fa32bf0e8c0a02d274be5ca21 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Aug 2024 21:40:17 +0200 Subject: [PATCH 03/39] elf: write offset table entry if dirty --- src/link/Elf/ZigObject.zig | 97 ++++++++++++++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 9 deletions(-) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index e652afe7279e4e62e99421caff97aa8baf6f2599..17e926417e657d2ab679ad98ddf9b074d7237713 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1101,6 +1101,17 @@ pub fn updateFunc( } // Exports will be updated by `Zcu.processExports` after the update. + + { + const sym = self.symbol(sym_index); + const ot_index = sym.extra(elf_file).zig_offset_table; + var ot_entry = offset_table.entries.get(ot_index); + if (ot_entry.dirty) { + try offset_table.writeEntry(ot_index, self, elf_file); + ot_entry.dirty = false; + } + offset_table.entries.set(ot_index, ot_entry); + } } pub fn updateNav( @@ -1731,15 +1742,15 @@ const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); pub const OffsetTable = struct { sym_index: Symbol.Index, - entries: std.ArrayListUnmanaged(Symbol.Index) = .{}, + entries: std.MultiArrayList(Entry) = .{}, pub fn deinit(ot: *OffsetTable, allocator: Allocator) void { ot.entries.deinit(allocator); } pub fn addSymbol(ot: *OffsetTable, allocator: Allocator, sym_index: Symbol.Index) !Index { - const index: Index = @intCast(ot.entries.items.len); - try ot.entries.append(allocator, sym_index); + const index: Index = @intCast(try ot.entries.addOne(allocator)); + ot.entries.set(index, .{ .sym_index = sym_index }); return index; } @@ -1753,11 +1764,72 @@ pub const OffsetTable = struct { return sym.atom(elf_file).?.size; } + pub fn entryAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { + return ot.address(zo, elf_file) + index * elf_file.archPtrWidthBytes(); + } + + pub fn entryOffset(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 { + const sym = zo.symbol(ot.sym_index); + const atom_ptr = sym.atom(elf_file).?; + const shdr = elf_file.shdrs.items[atom_ptr.output_section_index]; + return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * elf_file.archPtrWidthBytes(); + } + + pub fn targetAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { + const sym_index = ot.entries.items(.sym_index)[index]; + return zo.symbol(sym_index).address(.{}, elf_file); + } + + pub fn writeEntry(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void { + const entry_size: u16 = elf_file.archPtrWidthBytes(); + const target = elf_file.getTarget(); + const endian = target.cpu.arch.endian(); + const fileoff = ot.entryOffset(index, zo, elf_file); + const vaddr: u64 = @intCast(ot.entryAddress(index, zo, elf_file)); + const value = ot.targetAddress(index, zo, elf_file); + switch (entry_size) { + 2 => { + var buf: [2]u8 = undefined; + std.mem.writeInt(u16, &buf, @intCast(value), endian); + try elf_file.base.file.?.pwriteAll(&buf, fileoff); + }, + 4 => { + var buf: [4]u8 = undefined; + std.mem.writeInt(u32, &buf, @intCast(value), endian); + try elf_file.base.file.?.pwriteAll(&buf, fileoff); + }, + 8 => { + var buf: [8]u8 = undefined; + std.mem.writeInt(u64, &buf, @intCast(value), endian); + try elf_file.base.file.?.pwriteAll(&buf, fileoff); + + if (elf_file.base.child_pid) |pid| { + switch (builtin.os.tag) { + .linux => { + var local_vec: [1]std.posix.iovec_const = .{.{ + .base = &buf, + .len = buf.len, + }}; + var remote_vec: [1]std.posix.iovec_const = .{.{ + .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(vaddr)))), + .len = buf.len, + }}; + const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); + switch (std.os.linux.E.init(rc)) { + .SUCCESS => assert(rc == buf.len), + else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), + } + }, + else => return error.HotSwapUnavailableOnHostOperatingSystem, + } + } + }, + else => unreachable, + } + } + pub fn updateSize(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) !void { - const ot_size: u64 = @intCast(ot.entries.items.len * switch (elf_file.ptr_width) { - .p32 => @as(u64, 4), - .p64 => 8, - }); + const ot_size: u64 = @intCast(ot.entries.items(.sym_index).len * elf_file.archPtrWidthBytes()); const sym = zo.symbol(ot.sym_index); const esym = &zo.symtab.items(.elf_sym)[sym.esym_index]; esym.st_size = ot_size; @@ -1795,12 +1867,19 @@ pub const OffsetTable = struct { const ot, const zo, const ef = ctx; try writer.writeAll("offset table\n"); try writer.print(" @{x} : size({x})\n", .{ ot.address(zo, ef), ot.size(zo, ef) }); - for (ot.entries.items) |sym_index| { + for (ot.entries.items(.sym_index), ot.entries.items(.dirty)) |sym_index, dirty| { const sym = zo.symbol(sym_index); - try writer.print(" %{d} : {s} : @{x}\n", .{ sym_index, sym.name(ef), sym.address(.{}, ef) }); + try writer.print(" %{d} : {s} : @{x}", .{ sym_index, sym.name(ef), sym.address(.{}, ef) }); + if (dirty) try writer.writeAll(" : [!]"); + try writer.writeByte('\n'); } } + const Entry = struct { + sym_index: Symbol.Index, + dirty: bool = true, + }; + pub const Index = u32; }; -- 2.54.0 From 97a65ea0d5275602fac39bb08ab23f2908f8845e Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Aug 2024 21:46:19 +0200 Subject: [PATCH 04/39] elf: dirty offset table entry on moving Atom in off/addr space --- src/link/Elf/ZigObject.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 17e926417e657d2ab679ad98ddf9b074d7237713..75a0fd2cbe84b1aebd2cbaab4e736bd3e8b82e6f 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -924,6 +924,10 @@ fn updateNavCode( assert(sym.flags.has_zig_got); const extra = sym.extra(elf_file); try elf_file.zig_got.writeOne(elf_file, extra.zig_got); + if (stt_bits == elf.STT_FUNC) { + const offset_table = self.offsetTablePtr().?; + offset_table.entries.items(.dirty)[extra.zig_offset_table] = true; + } } } } else if (code.len < old_size) { -- 2.54.0 From 7556b32840eabb027945d9a30b57bb9cb46cf0bb Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Aug 2024 23:21:52 +0200 Subject: [PATCH 05/39] elf: indirect via offset table in the linker away from backend --- src/link/Elf/Atom.zig | 12 +++++--- src/link/Elf/Symbol.zig | 8 ++++++ src/link/Elf/ZigObject.zig | 56 ++++++++++++++++++++++++++------------ 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index f2757f570b0c7f95190f62ef09b6fbb2c95e09e0..6fa5d700efba3c15ab1873cf8877748135981016 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -759,13 +759,14 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi // Address of the dynamic thread pointer. const DTP = elf_file.dtpAddress(); - relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ZG({x}) ({s})", .{ + relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ZG({x}) ZG2({x}) ({s})", .{ relocation.fmtRelocType(rel.r_type(), cpu_arch), r_offset, P, S + A, G + GOT + A, ZIG_GOT + A, + target.zigOffsetTableAddress(elf_file) + A, target.name(elf_file), }); @@ -1224,9 +1225,12 @@ const x86_64 = struct { ); }, - .PLT32, - .PC32, - => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), + .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), + + .PC32 => { + const S_ = if (target.flags.zig_offset_table) target.zigOffsetTableAddress(elf_file) else S; + try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); + }, .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little), .GOTPC32 => try cwriter.writeInt(i32, @as(i32, @intCast(GOT + A - P)), .little), diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index f023ad5d915306cf73e82dd93d787115e9af8a8a..cf285b12d0cf68b9011ead2decf0ab42448ec740 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -236,6 +236,14 @@ pub fn zigGotAddress(symbol: Symbol, elf_file: *Elf) i64 { return elf_file.zig_got.entryAddress(extras.zig_got, elf_file); } +pub fn zigOffsetTableAddress(symbol: Symbol, elf_file: *Elf) i64 { + if (!symbol.flags.zig_offset_table) return 0; + const zo = elf_file.zigObjectPtr().?; + const offset_table = zo.offsetTablePtr().?; + const ot_index = symbol.extra(elf_file).zig_offset_table; + return offset_table.entryAddress(ot_index, zo, elf_file); +} + pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 { const file_ptr = symbol.file(elf_file) orelse return 0; assert(file_ptr == .shared_object); diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 75a0fd2cbe84b1aebd2cbaab4e736bd3e8b82e6f..f426e43c0acb635db28f7a4f6a90a3fdff19a19d 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -760,7 +760,9 @@ pub fn getOrCreateMetadataForLazySymbol( const gpa = elf_file.base.comp.gpa; const symbol_index = try self.newSymbolWithAtom(gpa, 0); const sym = self.symbol(symbol_index); - sym.flags.needs_zig_got = true; + if (lazy_sym.kind != .code) { + sym.flags.needs_zig_got = true; + } symbol_index_ptr.* = symbol_index; }, .pending_flush => return symbol_index_ptr.*, @@ -816,7 +818,7 @@ pub fn getOrCreateMetadataForNav( sym.flags.is_tls = true; } } - if (!sym.flags.is_tls) { + if (!sym.flags.is_tls and nav_val.typeOf(zcu).zigTypeTag(zcu) != .Fn) { sym.flags.needs_zig_got = true; } gop.value_ptr.* = .{ .symbol_index = symbol_index }; @@ -919,15 +921,18 @@ fn updateNavCode( sym.value = 0; esym.st_value = 0; - if (!elf_file.base.isRelocatable()) { - log.debug(" (writing new offset table entry)", .{}); - assert(sym.flags.has_zig_got); + if (stt_bits != elf.STT_FUNC) { + if (!elf_file.base.isRelocatable()) { + log.debug(" (writing new offset table entry)", .{}); + assert(sym.flags.has_zig_got); + const extra = sym.extra(elf_file); + try elf_file.zig_got.writeOne(elf_file, extra.zig_got); + } + } + if (stt_bits == elf.STT_FUNC) { const extra = sym.extra(elf_file); - try elf_file.zig_got.writeOne(elf_file, extra.zig_got); - if (stt_bits == elf.STT_FUNC) { - const offset_table = self.offsetTablePtr().?; - offset_table.entries.items(.dirty)[extra.zig_offset_table] = true; - } + const offset_table = self.offsetTablePtr().?; + offset_table.entries.items(.dirty)[extra.zig_offset_table] = true; } } } else if (code.len < old_size) { @@ -938,12 +943,13 @@ fn updateNavCode( errdefer self.freeNavMetadata(elf_file, sym_index); sym.value = 0; - sym.flags.needs_zig_got = true; esym.st_value = 0; - - if (!elf_file.base.isRelocatable()) { - const gop = try sym.getOrCreateZigGotEntry(sym_index, elf_file); - try elf_file.zig_got.writeOne(elf_file, gop.index); + if (stt_bits != elf.STT_FUNC) { + sym.flags.needs_zig_got = true; + if (!elf_file.base.isRelocatable()) { + const gop = try sym.getOrCreateZigGotEntry(sym_index, elf_file); + try elf_file.zig_got.writeOne(elf_file, gop.index); + } } } @@ -1061,7 +1067,12 @@ pub fn updateFunc( sym.flags.zig_offset_table = true; sym.addExtra(.{ .zig_offset_table = index }, elf_file); try offset_table.updateSize(self, elf_file); + const old_vaddr = offset_table.address(self, elf_file); try self.symbol(offset_table.sym_index).atom(elf_file).?.allocate(elf_file); + const new_vaddr = offset_table.address(self, elf_file); + if (old_vaddr != new_vaddr) { + offset_table.dirty = true; + } } } @@ -1106,7 +1117,13 @@ pub fn updateFunc( // Exports will be updated by `Zcu.processExports` after the update. - { + if (offset_table.dirty) { + // TODO write in bulk + for (offset_table.entries.items(.dirty), 0..) |*dirty, i| { + try offset_table.writeEntry(@intCast(i), self, elf_file); + dirty.* = false; + } + } else { const sym = self.symbol(sym_index); const ot_index = sym.extra(elf_file).zig_offset_table; var ot_entry = offset_table.entries.get(ot_index); @@ -1261,7 +1278,9 @@ fn updateLazySymbol( errdefer self.freeNavMetadata(elf_file, symbol_index); local_sym.value = 0; - local_sym.flags.needs_zig_got = true; + if (sym.kind != .code) { + local_sym.flags.needs_zig_got = true; + } local_esym.st_value = 0; if (!elf_file.base.isRelocatable()) { @@ -1476,7 +1495,7 @@ pub fn getGlobalSymbol(self: *ZigObject, elf_file: *Elf, name: []const u8, lib_n return lookup_gop.value_ptr.*; } -fn offsetTablePtr(self: *ZigObject) ?*OffsetTable { +pub fn offsetTablePtr(self: *ZigObject) ?*OffsetTable { return if (self.offset_table) |*ot| ot else null; } @@ -1747,6 +1766,7 @@ const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); pub const OffsetTable = struct { sym_index: Symbol.Index, entries: std.MultiArrayList(Entry) = .{}, + dirty: bool = false, pub fn deinit(ot: *OffsetTable, allocator: Allocator) void { ot.entries.deinit(allocator); -- 2.54.0 From 4c2b34e8abd1fc2091eeb10798658bd9bb3910f5 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 9 Aug 2024 08:35:46 +0200 Subject: [PATCH 06/39] elf: relax R_X86_64_32 into jump table indirection if zig_offset_table set --- src/link/Elf/Atom.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 6fa5d700efba3c15ab1873cf8877748135981016..b5db642c40a6bd5502898d75a2764c828571b9df 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1254,7 +1254,10 @@ const x86_64 = struct { try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little); }, - .@"32" => try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S + A)))), .little), + .@"32" => { + const S_ = if (target.flags.zig_offset_table) target.zigOffsetTableAddress(elf_file) else S; + try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S_ + A)))), .little); + }, .@"32S" => try cwriter.writeInt(i32, @as(i32, @truncate(S + A)), .little), .TPOFF32 => try cwriter.writeInt(i32, @as(i32, @truncate(S + A - TP)), .little), -- 2.54.0 From d328140858ab7f0f3eeb5d53b50bae32ab331686 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 9 Aug 2024 09:10:59 +0200 Subject: [PATCH 07/39] elf: nuke ZigGotSection from existence --- src/arch/x86_64/Emit.zig | 45 ++---- src/arch/x86_64/Lower.zig | 22 +-- src/link/Elf.zig | 78 +---------- src/link/Elf/Atom.zig | 49 ++----- src/link/Elf/Symbol.zig | 25 ---- src/link/Elf/ZigObject.zig | 36 +---- src/link/Elf/relocation.zig | 4 - src/link/Elf/synthetic_sections.zig | 209 ---------------------------- 8 files changed, 30 insertions(+), 438 deletions(-) diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index b6c1fbb968824c7a58ffe0db4172e9500a62b617..b7d0a29c2dc67dcd649ed456c0a66a88e8cc6f68 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -110,21 +110,11 @@ pub fn emitMir(emit: *Emit) Error!void { }); }, .linker_reloc => |data| if (emit.lower.bin_file.cast(.elf)) |elf_file| { - const is_obj_or_static_lib = switch (emit.lower.output_mode) { - .Exe => false, - .Obj => true, - .Lib => emit.lower.link_mode == .static, - }; const zo = elf_file.zigObjectPtr().?; const atom = zo.symbol(data.atom_index).atom(elf_file).?; const sym = zo.symbol(data.sym_index); - if (sym.flags.needs_zig_got and !is_obj_or_static_lib) { - _ = try sym.getOrCreateZigGotEntry(data.sym_index, elf_file); - } if (emit.lower.pic) { - const r_type: u32 = if (sym.flags.needs_zig_got and !is_obj_or_static_lib) - link.File.Elf.R_ZIG_GOTPCREL - else if (sym.flags.needs_got) + const r_type: u32 = if (sym.flags.needs_got) @intFromEnum(std.elf.R_X86_64.GOTPCREL) else @intFromEnum(std.elf.R_X86_64.PC32); @@ -134,28 +124,17 @@ pub fn emitMir(emit: *Emit) Error!void { .r_addend = -4, }); } else { - if (lowered_inst.encoding.mnemonic == .call and sym.flags.needs_zig_got and is_obj_or_static_lib) { - const r_type = @intFromEnum(std.elf.R_X86_64.PC32); - try atom.addReloc(elf_file, .{ - .r_offset = end_offset - 4, - .r_info = (@as(u64, @intCast(data.sym_index)) << 32) | r_type, - .r_addend = -4, - }); - } else { - const r_type: u32 = if (sym.flags.needs_zig_got and !is_obj_or_static_lib) - link.File.Elf.R_ZIG_GOT32 - else if (sym.flags.needs_got) - @intFromEnum(std.elf.R_X86_64.GOT32) - else if (sym.flags.is_tls) - @intFromEnum(std.elf.R_X86_64.TPOFF32) - else - @intFromEnum(std.elf.R_X86_64.@"32"); - try atom.addReloc(elf_file, .{ - .r_offset = end_offset - 4, - .r_info = (@as(u64, @intCast(data.sym_index)) << 32) | r_type, - .r_addend = 0, - }); - } + const r_type: u32 = if (sym.flags.needs_got) + @intFromEnum(std.elf.R_X86_64.GOT32) + else if (sym.flags.is_tls) + @intFromEnum(std.elf.R_X86_64.TPOFF32) + else + @intFromEnum(std.elf.R_X86_64.@"32"); + try atom.addReloc(elf_file, .{ + .r_offset = end_offset - 4, + .r_info = (@as(u64, @intCast(data.sym_index)) << 32) | r_type, + .r_addend = 0, + }); } } else if (emit.lower.bin_file.cast(.macho)) |macho_file| { const is_obj_or_static_lib = switch (emit.lower.output_mode) { diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 11fe279dd9ddc4445b881569db3c276ffecc598d..9ad1579ecceb6a05dc9fd7b34a90fc2bcd53a10f 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -398,30 +398,20 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) _ = lower.reloc(.{ .linker_reloc = sym }); break :op if (lower.pic) switch (mnemonic) { - .lea => { - break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }; - }, - .mov => { - if (is_obj_or_static_lib and elf_sym.flags.needs_zig_got) emit_mnemonic = .lea; - break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }; - }, + .lea => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, + .mov => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, else => unreachable, } else switch (mnemonic) { - .call => break :op if (is_obj_or_static_lib and elf_sym.flags.needs_zig_got) .{ - .imm = Immediate.s(0), - } else .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ + .call => break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ .base = .{ .reg = .ds }, }) }, .lea => { emit_mnemonic = .mov; break :op .{ .imm = Immediate.s(0) }; }, - .mov => { - if (is_obj_or_static_lib and elf_sym.flags.needs_zig_got) emit_mnemonic = .lea; - break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ - .base = .{ .reg = .ds }, - }) }; - }, + .mov => break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ + .base = .{ .reg = .ds }, + }) }, else => unreachable, }; } else if (lower.bin_file.cast(.macho)) |macho_file| { diff --git a/src/link/Elf.zig b/src/link/Elf.zig index ffba511e878076baf839d993af89a1574d5ee50e..ebafd16d6d5e8740543630665b758e8f85e7ee65 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -64,9 +64,6 @@ phdrs: std.ArrayListUnmanaged(elf.Elf64_Phdr) = .{}, /// Tracked loadable segments during incremental linking. /// The index into the program headers of a PT_LOAD program header with Read and Execute flags phdr_zig_load_re_index: ?u16 = null, -/// The index into the program headers of the global offset table. -/// It needs PT_LOAD and Read flags. -phdr_zig_got_index: ?u16 = null, /// The index into the program headers of a PT_LOAD program header with Read flag phdr_zig_load_ro_index: ?u16 = null, /// The index into the program headers of a PT_LOAD program header with Write flag @@ -130,8 +127,6 @@ plt_got: PltGotSection = .{}, copy_rel: CopyRelSection = .{}, /// .rela.plt section rela_plt: std.ArrayListUnmanaged(elf.Elf64_Rela) = .{}, -/// .got.zig section -zig_got: ZigGotSection = .{}, /// SHT_GROUP sections /// Applies only to a relocatable. comdat_group_sections: std.ArrayListUnmanaged(ComdatGroupSection) = .{}, @@ -142,7 +137,6 @@ zig_text_section_index: ?u32 = null, zig_data_rel_ro_section_index: ?u32 = null, zig_data_section_index: ?u32 = null, zig_bss_section_index: ?u32 = null, -zig_got_section_index: ?u32 = null, debug_info_section_index: ?u32 = null, debug_abbrev_section_index: ?u32 = null, @@ -474,7 +468,6 @@ pub fn deinit(self: *Elf) void { self.copy_rel.deinit(gpa); self.rela_dyn.deinit(gpa); self.rela_plt.deinit(gpa); - self.zig_got.deinit(gpa); self.comdat_group_sections.deinit(gpa); } @@ -618,21 +611,6 @@ pub fn initMetadata(self: *Elf, options: InitMetadataOptions) !void { }); } - if (self.phdr_zig_got_index == null) { - const alignment = self.page_size; - const filesz = @as(u64, ptr_size) * options.symbol_count_hint; - const off = self.findFreeSpace(filesz, alignment); - self.phdr_zig_got_index = try self.addPhdr(.{ - .type = elf.PT_LOAD, - .offset = off, - .filesz = filesz, - .addr = if (ptr_bit_width >= 32) 0x4000000 else 0x4000, - .memsz = filesz, - .@"align" = alignment, - .flags = elf.PF_R | elf.PF_W, - }); - } - if (self.phdr_zig_load_ro_index == null) { const alignment = self.page_size; const filesz: u64 = 1024; @@ -701,27 +679,6 @@ pub fn initMetadata(self: *Elf, options: InitMetadataOptions) !void { try self.last_atom_and_free_list_table.putNoClobber(gpa, self.zig_text_section_index.?, .{}); } - if (self.zig_got_section_index == null and !self.base.isRelocatable()) { - self.zig_got_section_index = try self.addSection(.{ - .name = try self.insertShString(".got.zig"), - .type = elf.SHT_PROGBITS, - .addralign = ptr_size, - .flags = elf.SHF_ALLOC | elf.SHF_WRITE, - .offset = std.math.maxInt(u64), - }); - const shdr = &self.shdrs.items[self.zig_got_section_index.?]; - const phndx = self.phdr_zig_got_index.?; - const phdr = self.phdrs.items[phndx]; - shdr.sh_addr = phdr.p_vaddr; - shdr.sh_offset = phdr.p_offset; - shdr.sh_size = phdr.p_memsz; - try self.phdr_to_shdr_table.putNoClobber( - gpa, - self.zig_got_section_index.?, - self.phdr_zig_got_index.?, - ); - } - if (self.zig_data_rel_ro_section_index == null) { self.zig_data_rel_ro_section_index = try self.addSection(.{ .name = try self.insertShString(".data.rel.ro.zig"), @@ -3156,8 +3113,8 @@ fn initSyntheticSections(self: *Elf) !void { }); const needs_rela_dyn = blk: { - if (self.got.flags.needs_rela or self.got.flags.needs_tlsld or - self.zig_got.flags.needs_rela or self.copy_rel.symbols.items.len > 0) break :blk true; + if (self.got.flags.needs_rela or self.got.flags.needs_tlsld or self.copy_rel.symbols.items.len > 0) + break :blk true; if (self.zigObjectPtr()) |zig_object| { if (zig_object.num_dynrelocs > 0) break :blk true; } @@ -3562,7 +3519,6 @@ fn sortPhdrs(self: *Elf) error{OutOfMemory}!void { for (&[_]*?u16{ &self.phdr_zig_load_re_index, - &self.phdr_zig_got_index, &self.phdr_zig_load_ro_index, &self.phdr_zig_load_zerofill_index, &self.phdr_table_index, @@ -3694,7 +3650,6 @@ fn resetShdrIndexes(self: *Elf, backlinks: []const u32) !void { &self.versym_section_index, &self.verneed_section_index, &self.zig_text_section_index, - &self.zig_got_section_index, &self.zig_data_rel_ro_section_index, &self.zig_data_section_index, &self.zig_bss_section_index, @@ -3893,7 +3848,7 @@ fn updateSectionSizes(self: *Elf) !void { } if (self.rela_dyn_section_index) |shndx| { - var num = self.got.numRela(self) + self.copy_rel.numRela() + self.zig_got.numRela(); + var num = self.got.numRela(self) + self.copy_rel.numRela(); if (self.zigObjectPtr()) |zig_object| { num += zig_object.num_dynrelocs; } @@ -4431,15 +4386,6 @@ pub fn updateSymtabSize(self: *Elf) !void { strsize += ctx.strsize; } - if (self.zigObjectPtr()) |_| { - if (self.zig_got_section_index) |_| { - self.zig_got.output_symtab_ctx.ilocal = nlocals + 1; - self.zig_got.updateSymtabSize(self); - nlocals += self.zig_got.output_symtab_ctx.nlocals; - strsize += self.zig_got.output_symtab_ctx.strsize; - } - } - if (self.got_section_index) |_| { self.got.output_symtab_ctx.ilocal = nlocals + 1; self.got.updateSymtabSize(self); @@ -4576,9 +4522,6 @@ fn writeSyntheticSections(self: *Elf) !void { const shdr = self.shdrs.items[shndx]; try self.got.addRela(self); try self.copy_rel.addRela(self); - if (self.zigObjectPtr()) |_| { - try self.zig_got.addRela(self); - } self.sortRelaDyn(); try self.base.file.?.pwriteAll(mem.sliceAsBytes(self.rela_dyn.items), shdr.sh_offset); } @@ -4674,10 +4617,6 @@ pub fn writeSymtab(self: *Elf) !void { obj.asFile().writeSymtab(self); } - if (self.zig_got_section_index) |_| { - self.zig_got.writeSymtab(self); - } - if (self.got_section_index) |_| { self.got.writeSymtab(self); } @@ -5085,7 +5024,6 @@ pub fn isZigSection(self: Elf, shndx: u32) bool { self.zig_data_rel_ro_section_index, self.zig_data_section_index, self.zig_bss_section_index, - self.zig_got_section_index, }) |maybe_index| { if (maybe_index) |index| { if (index == shndx) return true; @@ -5704,7 +5642,6 @@ fn fmtDumpState( } } - try writer.print("{}\n", .{self.zig_got.fmt(self)}); try writer.print("{}\n", .{self.got.fmt(self)}); try writer.print("{}\n", .{self.plt.fmt(self)}); @@ -5995,20 +5932,12 @@ const RelaSection = struct { }; const RelaSectionTable = std.AutoArrayHashMapUnmanaged(u32, RelaSection); -pub const R_ZIG_GOT32: u32 = 0xff00; -pub const R_ZIG_GOTPCREL: u32 = 0xff01; -pub const R_ZIG_GOT_HI20: u32 = 0xff02; -pub const R_ZIG_GOT_LO12: u32 = 0xff03; pub const R_GOT_HI20_STATIC: u32 = 0xff04; pub const R_GOT_LO12_I_STATIC: u32 = 0xff05; // Comptime asserts that no Zig relocs overlap with another ISA's reloc number comptime { const zig_relocs = .{ - R_ZIG_GOT32, - R_ZIG_GOT_HI20, - R_ZIG_GOT_LO12, - R_ZIG_GOTPCREL, R_GOT_HI20_STATIC, R_GOT_LO12_I_STATIC, }; @@ -6099,6 +6028,5 @@ const StringTable = @import("StringTable.zig"); const Thunk = thunks.Thunk; const Value = @import("../Value.zig"); const VerneedSection = synthetic_sections.VerneedSection; -const ZigGotSection = synthetic_sections.ZigGotSection; const ZigObject = @import("Elf/ZigObject.zig"); const riscv = @import("riscv.zig"); diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index b5db642c40a6bd5502898d75a2764c828571b9df..cb505b19e39551cea89b8a23863445d2a166ad3d 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -750,8 +750,8 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi const S = target.address(.{}, elf_file); // Address of the global offset table. const GOT = elf_file.gotAddress(); - // Address of the .zig.got table entry if any. - const ZIG_GOT = target.zigGotAddress(elf_file); + // Address of the offset table entry if any. + const ZIG_GOT = target.zigOffsetTableAddress(elf_file); // Relative offset to the start of the global offset table. const G = target.gotAddress(elf_file) - GOT; // // Address of the thread pointer. @@ -759,14 +759,13 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi // Address of the dynamic thread pointer. const DTP = elf_file.dtpAddress(); - relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ZG({x}) ZG2({x}) ({s})", .{ + relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ZG({x}) ({s})", .{ relocation.fmtRelocType(rel.r_type(), cpu_arch), r_offset, P, S + A, G + GOT + A, ZIG_GOT + A, - target.zigOffsetTableAddress(elf_file) + A, target.name(elf_file), }); @@ -1181,16 +1180,7 @@ const x86_64 = struct { .TLSDESC_CALL, => {}, - else => |x| switch (@intFromEnum(x)) { - // Zig custom relocations - Elf.R_ZIG_GOT32, - Elf.R_ZIG_GOTPCREL, - => { - assert(symbol.flags.has_zig_got); - }, - - else => try atom.reportUnhandledRelocError(rel, elf_file), - }, + else => try atom.reportUnhandledRelocError(rel, elf_file), } } @@ -1228,7 +1218,7 @@ const x86_64 = struct { .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .PC32 => { - const S_ = if (target.flags.zig_offset_table) target.zigOffsetTableAddress(elf_file) else S; + const S_ = if (target.flags.zig_offset_table) ZIG_GOT else S; try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); }, @@ -1255,7 +1245,7 @@ const x86_64 = struct { }, .@"32" => { - const S_ = if (target.flags.zig_offset_table) target.zigOffsetTableAddress(elf_file) else S; + const S_ = if (target.flags.zig_offset_table) ZIG_GOT else S; try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S_ + A)))), .little); }, .@"32S" => try cwriter.writeInt(i32, @as(i32, @truncate(S + A)), .little), @@ -1336,13 +1326,7 @@ const x86_64 = struct { .GOT32 => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A)), .little), - else => |x| switch (@intFromEnum(x)) { - // Zig custom relocations - Elf.R_ZIG_GOT32 => try cwriter.writeInt(u32, @as(u32, @intCast(ZIG_GOT + A)), .little), - Elf.R_ZIG_GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(ZIG_GOT + A - P)), .little), - - else => try atom.reportUnhandledRelocError(rel, elf_file), - }, + else => try atom.reportUnhandledRelocError(rel, elf_file), } } @@ -2006,12 +1990,6 @@ const riscv = struct { => {}, else => |x| switch (@intFromEnum(x)) { - Elf.R_ZIG_GOT_HI20, - Elf.R_ZIG_GOT_LO12, - => { - assert(symbol.flags.has_zig_got); - }, - Elf.R_GOT_HI20_STATIC, Elf.R_GOT_LO12_I_STATIC, => symbol.flags.needs_got = true, @@ -2038,6 +2016,7 @@ const riscv = struct { const P, const A, const S, const GOT, const G, const TP, const DTP, const ZIG_GOT = args; _ = TP; _ = DTP; + _ = ZIG_GOT; switch (r_type) { .NONE => unreachable, @@ -2156,18 +2135,6 @@ const riscv = struct { else => |x| switch (@intFromEnum(x)) { // Zig custom relocations - Elf.R_ZIG_GOT_HI20 => { - assert(target.flags.has_zig_got); - const disp: u32 = @bitCast(math.cast(i32, ZIG_GOT + A) orelse return error.Overflow); - riscv_util.writeInstU(code[r_offset..][0..4], disp); - }, - - Elf.R_ZIG_GOT_LO12 => { - assert(target.flags.has_zig_got); - const value: u32 = @bitCast(math.cast(i32, ZIG_GOT + A) orelse return error.Overflow); - riscv_util.writeInstI(code[r_offset..][0..4], value); - }, - Elf.R_GOT_HI20_STATIC => { assert(target.flags.has_got); const disp: u32 = @bitCast(math.cast(i32, G + GOT + A) orelse return error.Overflow); diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index cf285b12d0cf68b9011ead2decf0ab42448ec740..20697a0426b26e6c3f10ead4054519357281091a 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -217,25 +217,6 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) i64 { return entry.address(elf_file); } -const GetOrCreateZigGotEntryResult = struct { - found_existing: bool, - index: ZigGotSection.Index, -}; - -pub fn getOrCreateZigGotEntry(symbol: *Symbol, symbol_index: Index, elf_file: *Elf) !GetOrCreateZigGotEntryResult { - assert(!elf_file.base.isRelocatable()); - assert(symbol.flags.needs_zig_got); - if (symbol.flags.has_zig_got) return .{ .found_existing = true, .index = symbol.extra(elf_file).zig_got }; - const index = try elf_file.zig_got.addSymbol(symbol_index, elf_file); - return .{ .found_existing = false, .index = index }; -} - -pub fn zigGotAddress(symbol: Symbol, elf_file: *Elf) i64 { - if (!symbol.flags.has_zig_got) return 0; - const extras = symbol.extra(elf_file); - return elf_file.zig_got.entryAddress(extras.zig_got, elf_file); -} - pub fn zigOffsetTableAddress(symbol: Symbol, elf_file: *Elf) i64 { if (!symbol.flags.zig_offset_table) return 0; const zo = elf_file.zigObjectPtr().?; @@ -267,7 +248,6 @@ const AddExtraOpts = struct { tlsgd: ?u32 = null, gottp: ?u32 = null, tlsdesc: ?u32 = null, - zig_got: ?u32 = null, zig_offset_table: ?u32 = null, }; @@ -465,10 +445,6 @@ pub const Flags = packed struct { needs_tlsdesc: bool = false, has_tlsdesc: bool = false, - /// Whether the symbol contains .zig.got indirection. - needs_zig_got: bool = false, - has_zig_got: bool = false, - /// Whether the symbol is a TLS variable. /// TODO this is really not needed if only we operated on esyms between /// codegen and ZigObject. @@ -491,7 +467,6 @@ pub const Extra = struct { tlsgd: u32 = 0, gottp: u32 = 0, tlsdesc: u32 = 0, - zig_got: u32 = 0, merge_section: u32 = 0, zig_offset_table: u32 = 0, }; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index f426e43c0acb635db28f7a4f6a90a3fdff19a19d..107ab783855d4de51e27747ee527873418b4b8e3 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -756,15 +756,7 @@ pub fn getOrCreateMetadataForLazySymbol( .const_data => .{ &gop.value_ptr.rodata_symbol_index, &gop.value_ptr.rodata_state }, }; switch (state_ptr.*) { - .unused => { - const gpa = elf_file.base.comp.gpa; - const symbol_index = try self.newSymbolWithAtom(gpa, 0); - const sym = self.symbol(symbol_index); - if (lazy_sym.kind != .code) { - sym.flags.needs_zig_got = true; - } - symbol_index_ptr.* = symbol_index; - }, + .unused => symbol_index_ptr.* = try self.newSymbolWithAtom(pt.zcu.gpa, 0), .pending_flush => return symbol_index_ptr.*, .flushed => {}, } @@ -818,9 +810,6 @@ pub fn getOrCreateMetadataForNav( sym.flags.is_tls = true; } } - if (!sym.flags.is_tls and nav_val.typeOf(zcu).zigTypeTag(zcu) != .Fn) { - sym.flags.needs_zig_got = true; - } gop.value_ptr.* = .{ .symbol_index = symbol_index }; } return gop.value_ptr.symbol_index; @@ -921,14 +910,6 @@ fn updateNavCode( sym.value = 0; esym.st_value = 0; - if (stt_bits != elf.STT_FUNC) { - if (!elf_file.base.isRelocatable()) { - log.debug(" (writing new offset table entry)", .{}); - assert(sym.flags.has_zig_got); - const extra = sym.extra(elf_file); - try elf_file.zig_got.writeOne(elf_file, extra.zig_got); - } - } if (stt_bits == elf.STT_FUNC) { const extra = sym.extra(elf_file); const offset_table = self.offsetTablePtr().?; @@ -944,13 +925,6 @@ fn updateNavCode( sym.value = 0; esym.st_value = 0; - if (stt_bits != elf.STT_FUNC) { - sym.flags.needs_zig_got = true; - if (!elf_file.base.isRelocatable()) { - const gop = try sym.getOrCreateZigGotEntry(sym_index, elf_file); - try elf_file.zig_got.writeOne(elf_file, gop.index); - } - } } if (elf_file.base.child_pid) |pid| { @@ -1278,16 +1252,8 @@ fn updateLazySymbol( errdefer self.freeNavMetadata(elf_file, symbol_index); local_sym.value = 0; - if (sym.kind != .code) { - local_sym.flags.needs_zig_got = true; - } local_esym.st_value = 0; - if (!elf_file.base.isRelocatable()) { - const gop = try local_sym.getOrCreateZigGotEntry(symbol_index, elf_file); - try elf_file.zig_got.writeOne(elf_file, gop.index); - } - const shdr = elf_file.shdrs.items[output_section_index]; const file_offset = shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)); try elf_file.base.file.?.pwriteAll(code, file_offset); diff --git a/src/link/Elf/relocation.zig b/src/link/Elf/relocation.zig index 5f6810d6f95f7d25c66d80ed35f0271a82a6ad66..887aece8bc96d1c06294de24e743eca03ed77d1e 100644 --- a/src/link/Elf/relocation.zig +++ b/src/link/Elf/relocation.zig @@ -113,10 +113,6 @@ fn formatRelocType( _ = options; const r_type = ctx.r_type; switch (r_type) { - Elf.R_ZIG_GOT32 => try writer.writeAll("R_ZIG_GOT32"), - Elf.R_ZIG_GOTPCREL => try writer.writeAll("R_ZIG_GOTPCREL"), - Elf.R_ZIG_GOT_HI20 => try writer.writeAll("R_ZIG_GOT_HI20"), - Elf.R_ZIG_GOT_LO12 => try writer.writeAll("R_ZIG_GOT_LO12"), Elf.R_GOT_HI20_STATIC => try writer.writeAll("R_GOT_HI20_STATIC"), Elf.R_GOT_LO12_I_STATIC => try writer.writeAll("R_GOT_LO12_I_STATIC"), else => switch (ctx.cpu_arch) { diff --git a/src/link/Elf/synthetic_sections.zig b/src/link/Elf/synthetic_sections.zig index e1ec90139eb52fb3ccc76b87e840c6f6d7ad14da..40cb37b9671f2e97798be28b25856e31b484caf7 100644 --- a/src/link/Elf/synthetic_sections.zig +++ b/src/link/Elf/synthetic_sections.zig @@ -223,215 +223,6 @@ pub const DynamicSection = struct { } }; -pub const ZigGotSection = struct { - entries: std.ArrayListUnmanaged(Symbol.Index) = .{}, - output_symtab_ctx: Elf.SymtabCtx = .{}, - flags: Flags = .{}, - - const Flags = packed struct { - needs_rela: bool = false, - dirty: bool = false, - }; - - pub const Index = u32; - - pub fn deinit(zig_got: *ZigGotSection, allocator: Allocator) void { - zig_got.entries.deinit(allocator); - } - - fn allocateEntry(zig_got: *ZigGotSection, allocator: Allocator) !Index { - try zig_got.entries.ensureUnusedCapacity(allocator, 1); - // TODO add free list - const index = @as(Index, @intCast(zig_got.entries.items.len)); - _ = zig_got.entries.addOneAssumeCapacity(); - zig_got.flags.dirty = true; - return index; - } - - pub fn addSymbol(zig_got: *ZigGotSection, sym_index: Symbol.Index, elf_file: *Elf) !Index { - const comp = elf_file.base.comp; - const gpa = comp.gpa; - const zo = elf_file.zigObjectPtr().?; - const index = try zig_got.allocateEntry(gpa); - const entry = &zig_got.entries.items[index]; - entry.* = sym_index; - const symbol = zo.symbol(sym_index); - symbol.flags.has_zig_got = true; - if (elf_file.isEffectivelyDynLib() or (elf_file.base.isExe() and comp.config.pie)) { - zig_got.flags.needs_rela = true; - } - symbol.addExtra(.{ .zig_got = index }, elf_file); - return index; - } - - pub fn entryOffset(zig_got: ZigGotSection, index: Index, elf_file: *Elf) u64 { - _ = zig_got; - const entry_size = elf_file.archPtrWidthBytes(); - const shdr = elf_file.shdrs.items[elf_file.zig_got_section_index.?]; - return shdr.sh_offset + @as(u64, entry_size) * index; - } - - pub fn entryAddress(zig_got: ZigGotSection, index: Index, elf_file: *Elf) i64 { - _ = zig_got; - const entry_size = elf_file.archPtrWidthBytes(); - const shdr = elf_file.shdrs.items[elf_file.zig_got_section_index.?]; - return @as(i64, @intCast(shdr.sh_addr)) + entry_size * index; - } - - pub fn size(zig_got: ZigGotSection, elf_file: *Elf) usize { - return elf_file.archPtrWidthBytes() * zig_got.entries.items.len; - } - - pub fn writeOne(zig_got: *ZigGotSection, elf_file: *Elf, index: Index) !void { - const zo = elf_file.zigObjectPtr().?; - if (zig_got.flags.dirty) { - const needed_size = zig_got.size(elf_file); - try elf_file.growAllocSection(elf_file.zig_got_section_index.?, needed_size); - zig_got.flags.dirty = false; - } - const entry_size: u16 = elf_file.archPtrWidthBytes(); - const target = elf_file.getTarget(); - const endian = target.cpu.arch.endian(); - const off = zig_got.entryOffset(index, elf_file); - const vaddr: u64 = @intCast(zig_got.entryAddress(index, elf_file)); - const entry = zig_got.entries.items[index]; - const value = zo.symbol(entry).address(.{}, elf_file); - switch (entry_size) { - 2 => { - var buf: [2]u8 = undefined; - std.mem.writeInt(u16, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, off); - }, - 4 => { - var buf: [4]u8 = undefined; - std.mem.writeInt(u32, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, off); - }, - 8 => { - var buf: [8]u8 = undefined; - std.mem.writeInt(u64, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, off); - - if (elf_file.base.child_pid) |pid| { - switch (builtin.os.tag) { - .linux => { - var local_vec: [1]std.posix.iovec_const = .{.{ - .base = &buf, - .len = buf.len, - }}; - var remote_vec: [1]std.posix.iovec_const = .{.{ - .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(vaddr)))), - .len = buf.len, - }}; - const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); - switch (std.os.linux.E.init(rc)) { - .SUCCESS => assert(rc == buf.len), - else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), - } - }, - else => return error.HotSwapUnavailableOnHostOperatingSystem, - } - } - }, - else => unreachable, - } - } - - pub fn writeAll(zig_got: ZigGotSection, elf_file: *Elf, writer: anytype) !void { - const zo = elf_file.zigObjectPtr().?; - for (zig_got.entries.items) |entry| { - const symbol = zo.symbol(entry); - const value = symbol.address(.{ .plt = false }, elf_file); - try writeInt(value, elf_file, writer); - } - } - - pub fn numRela(zig_got: ZigGotSection) usize { - return zig_got.entries.items.len; - } - - pub fn addRela(zig_got: ZigGotSection, elf_file: *Elf) !void { - const comp = elf_file.base.comp; - const gpa = comp.gpa; - const cpu_arch = elf_file.getTarget().cpu.arch; - const zo = elf_file.zigObjectPtr().?; - try elf_file.rela_dyn.ensureUnusedCapacity(gpa, zig_got.numRela()); - for (zig_got.entries.items) |entry| { - const symbol = zo.symbol(entry); - const offset = symbol.zigGotAddress(elf_file); - elf_file.addRelaDynAssumeCapacity(.{ - .offset = @intCast(offset), - .type = relocation.encode(.rel, cpu_arch), - .addend = symbol.address(.{ .plt = false }, elf_file), - }); - } - } - - pub fn updateSymtabSize(zig_got: *ZigGotSection, elf_file: *Elf) void { - const zo = elf_file.zigObjectPtr().?; - zig_got.output_symtab_ctx.nlocals = @as(u32, @intCast(zig_got.entries.items.len)); - for (zig_got.entries.items) |entry| { - const name = zo.symbol(entry).name(elf_file); - zig_got.output_symtab_ctx.strsize += @as(u32, @intCast(name.len + "$ziggot".len)) + 1; - } - } - - pub fn writeSymtab(zig_got: ZigGotSection, elf_file: *Elf) void { - const zo = elf_file.zigObjectPtr().?; - for (zig_got.entries.items, zig_got.output_symtab_ctx.ilocal.., 0..) |entry, ilocal, index| { - const symbol = zo.symbol(entry); - const symbol_name = symbol.name(elf_file); - const st_name = @as(u32, @intCast(elf_file.strtab.items.len)); - elf_file.strtab.appendSliceAssumeCapacity(symbol_name); - elf_file.strtab.appendSliceAssumeCapacity("$ziggot"); - elf_file.strtab.appendAssumeCapacity(0); - const st_value = zig_got.entryAddress(@intCast(index), elf_file); - const st_size = elf_file.archPtrWidthBytes(); - elf_file.symtab.items[ilocal] = .{ - .st_name = st_name, - .st_info = elf.STT_OBJECT, - .st_other = 0, - .st_shndx = @intCast(elf_file.zig_got_section_index.?), - .st_value = @intCast(st_value), - .st_size = st_size, - }; - } - } - - const FormatCtx = struct { - zig_got: ZigGotSection, - elf_file: *Elf, - }; - - pub fn fmt(zig_got: ZigGotSection, elf_file: *Elf) std.fmt.Formatter(format2) { - return .{ .data = .{ .zig_got = zig_got, .elf_file = elf_file } }; - } - - pub fn format2( - ctx: FormatCtx, - comptime unused_fmt_string: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = options; - _ = unused_fmt_string; - const zig_got = ctx.zig_got; - const elf_file = ctx.elf_file; - try writer.writeAll(".zig.got\n"); - for (zig_got.entries.items, 0..) |entry, index| { - const zo = elf_file.zigObjectPtr().?; - const symbol = zo.symbol(entry); - try writer.print(" {d}@0x{x} => {d}@0x{x} ({s})\n", .{ - index, - zig_got.entryAddress(@intCast(index), elf_file), - entry, - symbol.address(.{}, elf_file), - symbol.name(elf_file), - }); - } - } -}; - pub const GotSection = struct { entries: std.ArrayListUnmanaged(Entry) = .{}, output_symtab_ctx: Elf.SymtabCtx = .{}, -- 2.54.0 From d7c5fbce92b0da1bb302db4c4adab21775f3d98b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 12:34:06 +0200 Subject: [PATCH 08/39] elf: emit a jump table in place of offset table for functions --- src/link/Elf/ZigObject.zig | 109 +++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 48 deletions(-) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 107ab783855d4de51e27747ee527873418b4b8e3..f289e0ad19da06aa412e8ddf9b4908302bedcf06 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1473,10 +1473,7 @@ fn initOffsetTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error esym.st_info |= elf.STT_OBJECT; const atom_ptr = sym.atom(elf_file).?; atom_ptr.alive = true; - atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(switch (elf_file.ptr_width) { - .p32 => 4, - .p64 => 8, - }); + atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(OffsetTable.alignment(elf_file.getTarget().cpu.arch)); atom_ptr.output_section_index = elf_file.zig_text_section_index.?; self.offset_table = OffsetTable{ .sym_index = sym_index }; return &(self.offset_table.?); @@ -1754,15 +1751,31 @@ pub const OffsetTable = struct { return sym.atom(elf_file).?.size; } + pub fn alignment(cpu_arch: std.Target.Cpu.Arch) u64 { + return switch (cpu_arch) { + .x86_64 => 1, + else => @panic("TODO implement alignment for this CPU arch"), + }; + } + pub fn entryAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { - return ot.address(zo, elf_file) + index * elf_file.archPtrWidthBytes(); + return ot.address(zo, elf_file) + @as(i64, @intCast(index * entrySize(elf_file.getTarget().cpu.arch))); } pub fn entryOffset(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 { const sym = zo.symbol(ot.sym_index); const atom_ptr = sym.atom(elf_file).?; const shdr = elf_file.shdrs.items[atom_ptr.output_section_index]; - return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * elf_file.archPtrWidthBytes(); + return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * entrySize(elf_file.getTarget().cpu.arch); + } + + pub fn entrySize(cpu_arch: std.Target.Cpu.Arch) u64 { + const seq_len = switch (cpu_arch) { + .x86_64 => 5, // jmp rel32 + else => @panic("TODO implement entry size for this CPU arch"), + }; + comptime assert(seq_len <= max_jump_seq_len); + return seq_len; } pub fn targetAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { @@ -1770,56 +1783,43 @@ pub const OffsetTable = struct { return zo.symbol(sym_index).address(.{}, elf_file); } + const max_jump_seq_len = 12; + pub fn writeEntry(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void { - const entry_size: u16 = elf_file.archPtrWidthBytes(); - const target = elf_file.getTarget(); - const endian = target.cpu.arch.endian(); const fileoff = ot.entryOffset(index, zo, elf_file); - const vaddr: u64 = @intCast(ot.entryAddress(index, zo, elf_file)); - const value = ot.targetAddress(index, zo, elf_file); - switch (entry_size) { - 2 => { - var buf: [2]u8 = undefined; - std.mem.writeInt(u16, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, fileoff); - }, - 4 => { - var buf: [4]u8 = undefined; - std.mem.writeInt(u32, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, fileoff); - }, - 8 => { - var buf: [8]u8 = undefined; - std.mem.writeInt(u64, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, fileoff); + const source_addr = ot.entryAddress(index, zo, elf_file); + const target_addr = @as(i64, @intCast(ot.targetAddress(index, zo, elf_file))); + var buf: [max_jump_seq_len]u8 = undefined; + const out = switch (elf_file.getTarget().cpu.arch) { + .x86_64 => try x86_64.writeEntry(source_addr, target_addr, &buf), + else => @panic("TODO implement write entry for this CPU arch"), + }; + try elf_file.base.file.?.pwriteAll(out, fileoff); - if (elf_file.base.child_pid) |pid| { - switch (builtin.os.tag) { - .linux => { - var local_vec: [1]std.posix.iovec_const = .{.{ - .base = &buf, - .len = buf.len, - }}; - var remote_vec: [1]std.posix.iovec_const = .{.{ - .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(vaddr)))), - .len = buf.len, - }}; - const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); - switch (std.os.linux.E.init(rc)) { - .SUCCESS => assert(rc == buf.len), - else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), - } - }, - else => return error.HotSwapUnavailableOnHostOperatingSystem, + if (elf_file.base.child_pid) |pid| { + switch (builtin.os.tag) { + .linux => { + var local_vec: [1]std.posix.iovec_const = .{.{ + .base = out.ptr, + .len = out.len, + }}; + var remote_vec: [1]std.posix.iovec_const = .{.{ + .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(source_addr)))), + .len = out.len, + }}; + const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); + switch (std.os.linux.E.init(rc)) { + .SUCCESS => assert(rc == out.len), + else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), } - } - }, - else => unreachable, + }, + else => return error.HotSwapUnavailableOnHostOperatingSystem, + } } } pub fn updateSize(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) !void { - const ot_size: u64 = @intCast(ot.entries.items(.sym_index).len * elf_file.archPtrWidthBytes()); + const ot_size: u64 = @intCast(ot.entries.items(.sym_index).len * entrySize(elf_file.getTarget().cpu.arch)); const sym = zo.symbol(ot.sym_index); const esym = &zo.symtab.items(.elf_sym)[sym.esym_index]; esym.st_size = ot_size; @@ -1871,6 +1871,19 @@ pub const OffsetTable = struct { }; pub const Index = u32; + + const x86_64 = struct { + fn writeEntry(source_addr: i64, target_addr: i64, buf: *[max_jump_seq_len]u8) ![]u8 { + const disp = @as(i64, @intCast(target_addr)) - source_addr - 4; + var bytes = [_]u8{ + 0xe8, 0x00, 0x00, 0x00, 0x00, // jmp rel32 + }; + assert(bytes.len == entrySize(.x86_64)); + mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little); + @memcpy(buf[0..bytes.len], &bytes); + return buf[0..bytes.len]; + } + }; }; const assert = std.debug.assert; -- 2.54.0 From 5fd53dc36fb019a0930a9be0700411bb04c688a1 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 17:40:41 +0200 Subject: [PATCH 09/39] x86_64: start converting away from .got.zig knowledge --- src/arch/x86_64/CodeGen.zig | 48 +++++++++++++++++++++---------------- src/arch/x86_64/Lower.zig | 1 + 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index a36bd981bcc8b33594927bcbf2c41239eed527d4..cfeca4c390ba50d5c83cc43d15407c8d6ca1f949 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -12311,25 +12311,33 @@ fn genCall(self: *Self, info: union(enum) { const zo = elf_file.zigObjectPtr().?; const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); if (self.mod.pic) { - const callee_reg: Register = switch (resolved_cc) { - .SysV => callee: { - if (!fn_info.is_var_args) break :callee .rax; - const param_regs = abi.getCAbiIntParamRegs(resolved_cc); - break :callee if (call_info.gp_count < param_regs.len) - param_regs[call_info.gp_count] - else - .r10; - }, - .Win64 => .rax, - else => unreachable, - }; - try self.genSetReg( - callee_reg, - Type.usize, - .{ .load_symbol = .{ .sym = sym_index } }, - .{}, - ); - try self.asmRegister(.{ ._, .call }, callee_reg); + // const callee_reg: Register = switch (resolved_cc) { + // .SysV => callee: { + // if (!fn_info.is_var_args) break :callee .rax; + // const param_regs = abi.getCAbiIntParamRegs(resolved_cc); + // break :callee if (call_info.gp_count < param_regs.len) + // param_regs[call_info.gp_count] + // else + // .r10; + // }, + // .Win64 => .rax, + // else => unreachable, + // }; + // TODO convert to near jump + try self.asmMemory(.{ ._, .call }, .{ + .base = .{ .reloc = .{ + .atom_index = try self.owner.getSymbolIndex(self), + .sym_index = sym_index, + } }, + .mod = .{ .rm = .{ .size = .qword } }, + }); + // try self.genSetReg( + // callee_reg, + // Type.usize, + // .{ .load_symbol = .{ .sym = sym_index } }, + // .{}, + // ); + // try self.asmRegister(.{ ._, .call }, callee_reg); } else try self.asmMemory(.{ ._, .call }, .{ .base = .{ .reloc = .{ .atom_index = try self.owner.getSymbolIndex(self), @@ -14097,7 +14105,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { else return self.fail("invalid modifier: '{s}'", .{modifier}), .load_symbol => |sym_off| if (mem.eql(u8, modifier, "P")) - .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .load_symbol = sym_off }) } + .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_symbol = sym_off }) } else return self.fail("invalid modifier: '{s}'", .{modifier}), else => return self.fail("invalid constraint: '{s}'", .{op_str}), diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 9ad1579ecceb6a05dc9fd7b34a90fc2bcd53a10f..2cb874f003561fc529234ae63a314403e4b3840b 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -398,6 +398,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) _ = lower.reloc(.{ .linker_reloc = sym }); break :op if (lower.pic) switch (mnemonic) { + .call => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, .lea => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, .mov => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, else => unreachable, -- 2.54.0 From f968dd0cb1fc9197fc6483ce235b037f6c86c8ce Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 21:57:14 +0200 Subject: [PATCH 10/39] comp: actually report codegen errors --- src/Compilation.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Compilation.zig b/src/Compilation.zig index d7dabd51640efd9ffa15520dcac59a130746010a..0beb960d022a0f182e6c07061d2a71a2e1814dfb 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -3092,6 +3092,10 @@ pub fn totalErrorCount(comp: *Compilation) u32 { if (zcu.intern_pool.global_error_set.getNamesFromMainThread().len > zcu.error_limit) { total += 1; } + + for (zcu.failed_codegen.keys()) |_| { + total += 1; + } } // The "no entry point found" error only counts if there are no semantic analysis errors. @@ -3237,6 +3241,9 @@ pub fn getAllErrorsAlloc(comp: *Compilation) !ErrorBundle { } } } + for (zcu.failed_codegen.values()) |error_msg| { + try addModuleErrorMsg(zcu, &bundle, error_msg.*, &all_references); + } for (zcu.failed_exports.values()) |value| { try addModuleErrorMsg(zcu, &bundle, value.*, &all_references); } -- 2.54.0 From 16abf51ceef2ba2ebb38f27270af676387435f05 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 22:04:18 +0200 Subject: [PATCH 11/39] x86_64: handle lea_symbol returned by genNavRef --- src/arch/x86_64/CodeGen.zig | 3 ++- src/codegen.zig | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index cfeca4c390ba50d5c83cc43d15407c8d6ca1f949..dede09fc71876d982c901c4f1f1792ac29f5f275 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -14104,7 +14104,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_got = sym_index }) } else return self.fail("invalid modifier: '{s}'", .{modifier}), - .load_symbol => |sym_off| if (mem.eql(u8, modifier, "P")) + .lea_symbol => |sym_off| if (mem.eql(u8, modifier, "P")) .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_symbol = sym_off }) } else return self.fail("invalid modifier: '{s}'", .{modifier}), @@ -18798,6 +18798,7 @@ fn genTypedValue(self: *Self, val: Value) InnerError!MCValue { .immediate => |imm| .{ .immediate = imm }, .memory => |addr| .{ .memory = addr }, .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } }, + .lea_symbol => |sym_index| .{ .lea_symbol = .{ .sym = sym_index } }, .load_direct => |sym_index| .{ .load_direct = sym_index }, .load_got => |sym_index| .{ .lea_got = sym_index }, .load_tlv => |sym_index| .{ .lea_tlv = sym_index }, diff --git a/src/codegen.zig b/src/codegen.zig index f2fa60fdf8be31805a2ff6f3cb444e44e8c7e24a..679ee03c27d533869e70d98dc01eb982b2daaeb2 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -828,6 +828,9 @@ pub const GenResult = union(enum) { /// Reference to memory location but deferred until linker allocated the Decl in memory. /// Traditionally, this corresponds to emitting a relocation in a relocatable object file. load_symbol: u32, + /// Reference to memory location but deferred until linker allocated the Decl in memory. + /// Traditionally, this corresponds to emitting a relocation in a relocatable object file. + lea_symbol: u32, }; fn mcv(val: MCValue) GenResult { @@ -904,7 +907,7 @@ fn genNavRef( if (!single_threaded and is_threadlocal) { return GenResult.mcv(.{ .load_tlv = sym_index }); } - return GenResult.mcv(.{ .load_symbol = sym_index }); + return GenResult.mcv(.{ .lea_symbol = sym_index }); } else if (lf.cast(.macho)) |macho_file| { const zo = macho_file.getZigObject().?; if (is_extern) { -- 2.54.0 From e3f6ebaea94bb11c2e990e1fa17b48e51e598b5c Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 22:51:33 +0200 Subject: [PATCH 12/39] x86_64+elf: fix jump table indirection for functions --- src/arch/x86_64/CodeGen.zig | 46 +++++++++++++++---------------------- src/link/Elf/ZigObject.zig | 26 ++++++++------------- 2 files changed, 29 insertions(+), 43 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index dede09fc71876d982c901c4f1f1792ac29f5f275..f6fbc67ff11118d1b83f144779f1ae32e707685e 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -12311,33 +12311,25 @@ fn genCall(self: *Self, info: union(enum) { const zo = elf_file.zigObjectPtr().?; const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); if (self.mod.pic) { - // const callee_reg: Register = switch (resolved_cc) { - // .SysV => callee: { - // if (!fn_info.is_var_args) break :callee .rax; - // const param_regs = abi.getCAbiIntParamRegs(resolved_cc); - // break :callee if (call_info.gp_count < param_regs.len) - // param_regs[call_info.gp_count] - // else - // .r10; - // }, - // .Win64 => .rax, - // else => unreachable, - // }; - // TODO convert to near jump - try self.asmMemory(.{ ._, .call }, .{ - .base = .{ .reloc = .{ - .atom_index = try self.owner.getSymbolIndex(self), - .sym_index = sym_index, - } }, - .mod = .{ .rm = .{ .size = .qword } }, - }); - // try self.genSetReg( - // callee_reg, - // Type.usize, - // .{ .load_symbol = .{ .sym = sym_index } }, - // .{}, - // ); - // try self.asmRegister(.{ ._, .call }, callee_reg); + const callee_reg: Register = switch (resolved_cc) { + .SysV => callee: { + if (!fn_info.is_var_args) break :callee .rax; + const param_regs = abi.getCAbiIntParamRegs(resolved_cc); + break :callee if (call_info.gp_count < param_regs.len) + param_regs[call_info.gp_count] + else + .r10; + }, + .Win64 => .rax, + else => unreachable, + }; + try self.genSetReg( + callee_reg, + Type.usize, + .{ .lea_symbol = .{ .sym = sym_index } }, + .{}, + ); + try self.asmRegister(.{ ._, .call }, callee_reg); } else try self.asmMemory(.{ ._, .call }, .{ .base = .{ .reloc = .{ .atom_index = try self.owner.getSymbolIndex(self), diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index f289e0ad19da06aa412e8ddf9b4908302bedcf06..0cb47d74735b560711a303c612ad082f679e3376 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -107,23 +107,17 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void { } self.relocs.deinit(allocator); - { - var it = self.navs.iterator(); - while (it.next()) |entry| { - entry.value_ptr.exports.deinit(allocator); - } - self.navs.deinit(allocator); + for (self.navs.values()) |*meta| { + meta.exports.deinit(allocator); } + self.navs.deinit(allocator); self.lazy_syms.deinit(allocator); - { - var it = self.uavs.iterator(); - while (it.next()) |entry| { - entry.value_ptr.exports.deinit(allocator); - } - self.uavs.deinit(allocator); + for (self.uavs.values()) |*meta| { + meta.exports.deinit(allocator); } + self.uavs.deinit(allocator); for (self.tls_variables.values()) |*tlv| { tlv.deinit(allocator); @@ -1721,8 +1715,8 @@ const TlsVariable = struct { }; const AtomList = std.ArrayListUnmanaged(Atom.Index); -const NavTable = std.AutoHashMapUnmanaged(InternPool.Nav.Index, AvMetadata); -const UavTable = std.AutoHashMapUnmanaged(InternPool.Index, AvMetadata); +const NavTable = std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, AvMetadata); +const UavTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, AvMetadata); const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata); const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); @@ -1874,9 +1868,9 @@ pub const OffsetTable = struct { const x86_64 = struct { fn writeEntry(source_addr: i64, target_addr: i64, buf: *[max_jump_seq_len]u8) ![]u8 { - const disp = @as(i64, @intCast(target_addr)) - source_addr - 4; + const disp = @as(i64, @intCast(target_addr)) - source_addr - 5; var bytes = [_]u8{ - 0xe8, 0x00, 0x00, 0x00, 0x00, // jmp rel32 + 0xe9, 0x00, 0x00, 0x00, 0x00, // jmp rel32 }; assert(bytes.len == entrySize(.x86_64)); mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little); -- 2.54.0 From e1ce9a7065e92e3e9b61229c7903f82c3684f489 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 22:52:11 +0200 Subject: [PATCH 13/39] elf: add poorman's reporting tool for unallocated NAVs/UAVs --- src/link/Elf/ZigObject.zig | 46 +++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 0cb47d74735b560711a303c612ad082f679e3376..441518059784787e793e026a4cd6921f2c34a7c1 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -164,6 +164,16 @@ pub fn flushModule(self: *ZigObject, elf_file: *Elf, tid: Zcu.PerThread.Id) !voi if (metadata.rodata_state != .unused) metadata.rodata_state = .flushed; } + if (build_options.enable_logging) { + const pt: Zcu.PerThread = .{ .zcu = elf_file.base.comp.module.?, .tid = tid }; + for (self.navs.keys(), self.navs.values()) |nav_index, meta| { + checkNavAllocated(pt, nav_index, meta); + } + for (self.uavs.keys(), self.uavs.values()) |uav_index, meta| { + checkUavAllocated(pt, uav_index, meta); + } + } + if (self.dwarf) |*dw| { const pt: Zcu.PerThread = .{ .zcu = elf_file.base.comp.module.?, .tid = tid }; try dw.flushModule(pt); @@ -701,6 +711,7 @@ pub fn lowerUav( else => explicit_alignment, }; if (self.uavs.get(uav)) |metadata| { + assert(metadata.allocated); const sym = self.symbol(metadata.symbol_index); const existing_alignment = sym.atom(elf_file).?.alignment; if (uav_alignment.order(existing_alignment).compare(.lte)) @@ -732,7 +743,7 @@ pub fn lowerUav( .ok => |sym_index| sym_index, .fail => |em| return .{ .fail = em }, }; - try self.uavs.put(gpa, uav, .{ .symbol_index = sym_index }); + try self.uavs.put(gpa, uav, .{ .symbol_index = sym_index, .allocated = true }); return .{ .mcv = .{ .load_symbol = sym_index } }; } @@ -921,6 +932,8 @@ fn updateNavCode( esym.st_value = 0; } + self.navs.getPtr(nav_index).?.allocated = true; + if (elf_file.base.child_pid) |pid| { switch (builtin.os.tag) { .linux => { @@ -988,6 +1001,8 @@ fn updateTlv( atom_ptr.alignment = required_alignment; atom_ptr.size = code.len; + self.navs.getPtr(nav_index).?.allocated = true; + { const gop = try self.tls_variables.getOrPut(gpa, atom_ptr.atom_index); assert(!gop.found_existing); // TODO incremental updates @@ -1695,6 +1710,8 @@ const AvMetadata = struct { symbol_index: Symbol.Index, /// A list of all exports aliases of this Av. exports: std.ArrayListUnmanaged(Symbol.Index) = .{}, + /// Set to true if the AV has been initialized and allocated. + allocated: bool = false, fn @"export"(m: AvMetadata, zig_object: *ZigObject, name: []const u8) ?*u32 { for (m.exports.items) |*exp| { @@ -1705,6 +1722,32 @@ const AvMetadata = struct { } }; +fn checkNavAllocated(pt: Zcu.PerThread, index: InternPool.Nav.Index, meta: AvMetadata) void { + if (!meta.allocated) { + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const nav = ip.getNav(index); + log.err("NAV {}({d}) assigned symbol {d} but not allocated!", .{ + nav.fqn.fmt(ip), + index, + meta.symbol_index, + }); + } +} + +fn checkUavAllocated(pt: Zcu.PerThread, index: InternPool.Index, meta: AvMetadata) void { + if (!meta.allocated) { + const zcu = pt.zcu; + const uav = Value.fromInterned(index); + const ty = uav.typeOf(zcu); + log.err("UAV {}({d}) assigned symbol {d} but not allocated!", .{ + ty.fmt(pt), + index, + meta.symbol_index, + }); + } +} + const TlsVariable = struct { symbol_index: Symbol.Index, code: []const u8 = &[0]u8{}, @@ -1881,6 +1924,7 @@ pub const OffsetTable = struct { }; const assert = std.debug.assert; +const build_options = @import("build_options"); const builtin = @import("builtin"); const codegen = @import("../../codegen.zig"); const elf = std.elf; -- 2.54.0 From 78b1c736026443c937145ba85e1de52c11c9a44e Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 23:23:39 +0200 Subject: [PATCH 14/39] elf: rename OffsetTable to JumpTable --- src/link/Elf.zig | 4 +- src/link/Elf/Atom.zig | 24 +++---- src/link/Elf/Symbol.zig | 18 ++--- src/link/Elf/ZigObject.zig | 136 ++++++++++++++++++------------------- 4 files changed, 91 insertions(+), 91 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index ebafd16d6d5e8740543630665b758e8f85e7ee65..f4003ff38fbd94e72efb7a3220e05e909a4f83d2 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -5599,8 +5599,8 @@ fn fmtDumpState( zig_object.fmtAtoms(self), zig_object.fmtSymtab(self), }); - if (zig_object.offset_table) |ot| { - try writer.print("{}", .{ot.fmt(zig_object, self)}); + if (zig_object.jump_table) |jt| { + try writer.print("{}", .{jt.fmt(zig_object, self)}); } try writer.writeByte('\n'); } diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index cb505b19e39551cea89b8a23863445d2a166ad3d..31ba0d642e22cf79bd140b487575ce40484c2c37 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -750,8 +750,8 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi const S = target.address(.{}, elf_file); // Address of the global offset table. const GOT = elf_file.gotAddress(); - // Address of the offset table entry if any. - const ZIG_GOT = target.zigOffsetTableAddress(elf_file); + // Address of the zig jump table entry if any. + const ZJT = target.zigJumpTableAddress(elf_file); // Relative offset to the start of the global offset table. const G = target.gotAddress(elf_file) - GOT; // // Address of the thread pointer. @@ -759,19 +759,19 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi // Address of the dynamic thread pointer. const DTP = elf_file.dtpAddress(); - relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ZG({x}) ({s})", .{ + relocs_log.debug(" {s}: {x}: [{x} => {x}] GOT({x}) ZJT({x}) ({s})", .{ relocation.fmtRelocType(rel.r_type(), cpu_arch), r_offset, P, S + A, G + GOT + A, - ZIG_GOT + A, + ZJT + A, target.name(elf_file), }); try stream.seekTo(r_offset); - const args = ResolveArgs{ P, A, S, GOT, G, TP, DTP, ZIG_GOT }; + const args = ResolveArgs{ P, A, S, GOT, G, TP, DTP, ZJT }; switch (cpu_arch) { .x86_64 => x86_64.resolveRelocAlloc(self, elf_file, rel, target, args, &it, code, &stream) catch |err| switch (err) { @@ -1200,7 +1200,7 @@ const x86_64 = struct { const cwriter = stream.writer(); - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZIG_GOT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; switch (r_type) { .NONE => unreachable, @@ -1218,7 +1218,7 @@ const x86_64 = struct { .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .PC32 => { - const S_ = if (target.flags.zig_offset_table) ZIG_GOT else S; + const S_ = if (target.flags.zig_jump_table) ZJT else S; try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); }, @@ -1245,7 +1245,7 @@ const x86_64 = struct { }, .@"32" => { - const S_ = if (target.flags.zig_offset_table) ZIG_GOT else S; + const S_ = if (target.flags.zig_jump_table) ZJT else S; try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S_ + A)))), .little); }, .@"32S" => try cwriter.writeInt(i32, @as(i32, @truncate(S + A)), .little), @@ -1729,9 +1729,9 @@ const aarch64 = struct { const code = code_buffer[r_offset..][0..4]; const file_ptr = atom.file(elf_file).?; - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZIG_GOT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; _ = DTP; - _ = ZIG_GOT; + _ = ZJT; switch (r_type) { .NONE => unreachable, @@ -2013,10 +2013,10 @@ const riscv = struct { const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow; const cwriter = stream.writer(); - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZIG_GOT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; _ = TP; _ = DTP; - _ = ZIG_GOT; + _ = ZJT; switch (r_type) { .NONE => unreachable, diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index 20697a0426b26e6c3f10ead4054519357281091a..38ef6bb8f7826483bd2e6093884f1c748fac9469 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -217,12 +217,12 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) i64 { return entry.address(elf_file); } -pub fn zigOffsetTableAddress(symbol: Symbol, elf_file: *Elf) i64 { - if (!symbol.flags.zig_offset_table) return 0; +pub fn zigJumpTableAddress(symbol: Symbol, elf_file: *Elf) i64 { + if (!symbol.flags.zig_jump_table) return 0; const zo = elf_file.zigObjectPtr().?; - const offset_table = zo.offsetTablePtr().?; - const ot_index = symbol.extra(elf_file).zig_offset_table; - return offset_table.entryAddress(ot_index, zo, elf_file); + const jump_table = zo.jumpTablePtr().?; + const jt_index = symbol.extra(elf_file).zig_jump_table; + return jump_table.entryAddress(jt_index, zo, elf_file); } pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 { @@ -248,7 +248,7 @@ const AddExtraOpts = struct { tlsgd: ?u32 = null, gottp: ?u32 = null, tlsdesc: ?u32 = null, - zig_offset_table: ?u32 = null, + zig_jump_table: ?u32 = null, }; pub fn addExtra(symbol: *Symbol, opts: AddExtraOpts, elf_file: *Elf) void { @@ -453,8 +453,8 @@ pub const Flags = packed struct { /// Whether the symbol is a merge subsection. merge_subsection: bool = false, - /// Whether the symbol has __zig_offset_table indirection. - zig_offset_table: bool = false, + /// Whether the symbol has __zig_jump_table indirection. + zig_jump_table: bool = false, }; pub const Extra = struct { @@ -468,7 +468,7 @@ pub const Extra = struct { gottp: u32 = 0, tlsdesc: u32 = 0, merge_section: u32 = 0, - zig_offset_table: u32 = 0, + zig_jump_table: u32 = 0, }; pub const Index = u32; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 441518059784787e793e026a4cd6921f2c34a7c1..596ed781ca96deccb89b3db64653267488364a96 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -55,10 +55,10 @@ debug_str_section_zig_size: u64 = 0, debug_aranges_section_zig_size: u64 = 0, debug_line_section_zig_size: u64 = 0, -/// Function offset table containing pointers to Zig generated functions. +/// Function jump table containing trampolines to Zcu functions. /// The table is used for Zig's incremental compilation and is embedded with /// the machine code section. -offset_table: ?OffsetTable = null, +jump_table: ?JumpTable = null, pub const global_symbol_bit: u32 = 0x80000000; pub const symbol_mask: u32 = 0x7fffffff; @@ -128,8 +128,8 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void { dw.deinit(); } - if (self.offset_table) |*ot| { - ot.deinit(allocator); + if (self.jump_table) |*jt| { + jt.deinit(allocator); } } @@ -917,8 +917,8 @@ fn updateNavCode( if (stt_bits == elf.STT_FUNC) { const extra = sym.extra(elf_file); - const offset_table = self.offsetTablePtr().?; - offset_table.entries.items(.dirty)[extra.zig_offset_table] = true; + const jump_table = self.jumpTablePtr().?; + jump_table.entries.items(.dirty)[extra.zig_jump_table] = true; } } } else if (code.len < old_size) { @@ -1036,7 +1036,7 @@ pub fn updateFunc( const ip = &zcu.intern_pool; const gpa = elf_file.base.comp.gpa; const func = zcu.funcInfo(func_index); - const offset_table = self.offsetTablePtr() orelse try self.initOffsetTable(gpa, elf_file); + const jump_table = self.jumpTablePtr() orelse try self.initJumpTable(gpa, elf_file); log.debug("updateFunc {}({d})", .{ ip.getNav(func.owner_nav).fqn.fmt(ip), func.owner_nav }); @@ -1045,16 +1045,16 @@ pub fn updateFunc( { const sym = self.symbol(sym_index); - if (!sym.flags.zig_offset_table) { - const index = try offset_table.addSymbol(gpa, sym_index); - sym.flags.zig_offset_table = true; - sym.addExtra(.{ .zig_offset_table = index }, elf_file); - try offset_table.updateSize(self, elf_file); - const old_vaddr = offset_table.address(self, elf_file); - try self.symbol(offset_table.sym_index).atom(elf_file).?.allocate(elf_file); - const new_vaddr = offset_table.address(self, elf_file); + if (!sym.flags.zig_jump_table) { + const index = try jump_table.addSymbol(gpa, sym_index); + sym.flags.zig_jump_table = true; + sym.addExtra(.{ .zig_jump_table = index }, elf_file); + try jump_table.updateSize(self, elf_file); + const old_vaddr = jump_table.address(self, elf_file); + try self.symbol(jump_table.sym_index).atom(elf_file).?.allocate(elf_file); + const new_vaddr = jump_table.address(self, elf_file); if (old_vaddr != new_vaddr) { - offset_table.dirty = true; + jump_table.dirty = true; } } } @@ -1100,21 +1100,21 @@ pub fn updateFunc( // Exports will be updated by `Zcu.processExports` after the update. - if (offset_table.dirty) { + if (jump_table.dirty) { // TODO write in bulk - for (offset_table.entries.items(.dirty), 0..) |*dirty, i| { - try offset_table.writeEntry(@intCast(i), self, elf_file); + for (jump_table.entries.items(.dirty), 0..) |*dirty, i| { + try jump_table.writeEntry(@intCast(i), self, elf_file); dirty.* = false; } } else { const sym = self.symbol(sym_index); - const ot_index = sym.extra(elf_file).zig_offset_table; - var ot_entry = offset_table.entries.get(ot_index); - if (ot_entry.dirty) { - try offset_table.writeEntry(ot_index, self, elf_file); - ot_entry.dirty = false; + const jt_index = sym.extra(elf_file).zig_jump_table; + var jt_entry = jump_table.entries.get(jt_index); + if (jt_entry.dirty) { + try jump_table.writeEntry(jt_index, self, elf_file); + jt_entry.dirty = false; } - offset_table.entries.set(ot_index, ot_entry); + jump_table.entries.set(jt_index, jt_entry); } } @@ -1470,22 +1470,22 @@ pub fn getGlobalSymbol(self: *ZigObject, elf_file: *Elf, name: []const u8, lib_n return lookup_gop.value_ptr.*; } -pub fn offsetTablePtr(self: *ZigObject) ?*OffsetTable { - return if (self.offset_table) |*ot| ot else null; +pub fn jumpTablePtr(self: *ZigObject) ?*JumpTable { + return if (self.jump_table) |*jt| jt else null; } -fn initOffsetTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!*OffsetTable { - const name_off = try self.addString(allocator, "__zig_offset_table"); +fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!*JumpTable { + const name_off = try self.addString(allocator, "__zig_jump_table"); const sym_index = try self.newSymbolWithAtom(allocator, name_off); const sym = self.symbol(sym_index); const esym = &self.symtab.items(.elf_sym)[sym.esym_index]; esym.st_info |= elf.STT_OBJECT; const atom_ptr = sym.atom(elf_file).?; atom_ptr.alive = true; - atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(OffsetTable.alignment(elf_file.getTarget().cpu.arch)); + atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(JumpTable.alignment(elf_file.getTarget().cpu.arch)); atom_ptr.output_section_index = elf_file.zig_text_section_index.?; - self.offset_table = OffsetTable{ .sym_index = sym_index }; - return &(self.offset_table.?); + self.jump_table = JumpTable{ .sym_index = sym_index }; + return &(self.jump_table.?); } pub fn asFile(self: *ZigObject) File { @@ -1763,28 +1763,28 @@ const UavTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, AvMetadata); const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata); const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); -pub const OffsetTable = struct { +pub const JumpTable = struct { sym_index: Symbol.Index, entries: std.MultiArrayList(Entry) = .{}, dirty: bool = false, - pub fn deinit(ot: *OffsetTable, allocator: Allocator) void { - ot.entries.deinit(allocator); + pub fn deinit(jt: *JumpTable, allocator: Allocator) void { + jt.entries.deinit(allocator); } - pub fn addSymbol(ot: *OffsetTable, allocator: Allocator, sym_index: Symbol.Index) !Index { - const index: Index = @intCast(try ot.entries.addOne(allocator)); - ot.entries.set(index, .{ .sym_index = sym_index }); + pub fn addSymbol(jt: *JumpTable, allocator: Allocator, sym_index: Symbol.Index) !Index { + const index: Index = @intCast(try jt.entries.addOne(allocator)); + jt.entries.set(index, .{ .sym_index = sym_index }); return index; } - pub fn address(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) i64 { - const sym = zo.symbol(ot.sym_index); + pub fn address(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) i64 { + const sym = zo.symbol(jt.sym_index); return sym.address(.{}, elf_file); } - pub fn size(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) u64 { - const sym = zo.symbol(ot.sym_index); + pub fn size(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) u64 { + const sym = zo.symbol(jt.sym_index); return sym.atom(elf_file).?.size; } @@ -1795,12 +1795,12 @@ pub const OffsetTable = struct { }; } - pub fn entryAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { - return ot.address(zo, elf_file) + @as(i64, @intCast(index * entrySize(elf_file.getTarget().cpu.arch))); + pub fn entryAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { + return jt.address(zo, elf_file) + @as(i64, @intCast(index * entrySize(elf_file.getTarget().cpu.arch))); } - pub fn entryOffset(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 { - const sym = zo.symbol(ot.sym_index); + pub fn entryOffset(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 { + const sym = zo.symbol(jt.sym_index); const atom_ptr = sym.atom(elf_file).?; const shdr = elf_file.shdrs.items[atom_ptr.output_section_index]; return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * entrySize(elf_file.getTarget().cpu.arch); @@ -1815,17 +1815,17 @@ pub const OffsetTable = struct { return seq_len; } - pub fn targetAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { - const sym_index = ot.entries.items(.sym_index)[index]; + pub fn targetAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { + const sym_index = jt.entries.items(.sym_index)[index]; return zo.symbol(sym_index).address(.{}, elf_file); } const max_jump_seq_len = 12; - pub fn writeEntry(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void { - const fileoff = ot.entryOffset(index, zo, elf_file); - const source_addr = ot.entryAddress(index, zo, elf_file); - const target_addr = @as(i64, @intCast(ot.targetAddress(index, zo, elf_file))); + pub fn writeEntry(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void { + const fileoff = jt.entryOffset(index, zo, elf_file); + const source_addr = jt.entryAddress(index, zo, elf_file); + const target_addr = @as(i64, @intCast(jt.targetAddress(index, zo, elf_file))); var buf: [max_jump_seq_len]u8 = undefined; const out = switch (elf_file.getTarget().cpu.arch) { .x86_64 => try x86_64.writeEntry(source_addr, target_addr, &buf), @@ -1855,46 +1855,46 @@ pub const OffsetTable = struct { } } - pub fn updateSize(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) !void { - const ot_size: u64 = @intCast(ot.entries.items(.sym_index).len * entrySize(elf_file.getTarget().cpu.arch)); - const sym = zo.symbol(ot.sym_index); + pub fn updateSize(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) !void { + const jt_size: u64 = @intCast(jt.entries.items(.sym_index).len * entrySize(elf_file.getTarget().cpu.arch)); + const sym = zo.symbol(jt.sym_index); const esym = &zo.symtab.items(.elf_sym)[sym.esym_index]; - esym.st_size = ot_size; + esym.st_size = jt_size; const atom_ptr = sym.atom(elf_file).?; - atom_ptr.size = ot_size; + atom_ptr.size = jt_size; } - const OffsetTableFormatContext = struct { OffsetTable, *ZigObject, *Elf }; + const JumpTableFormatContext = struct { JumpTable, *ZigObject, *Elf }; pub fn format( - ot: OffsetTable, + jt: JumpTable, comptime unused_fmt_string: []const u8, options: std.fmt.FormatOptions, writer: anytype, ) !void { - _ = ot; + _ = jt; _ = unused_fmt_string; _ = options; _ = writer; - @compileError("do not format OffsetTable directly"); + @compileError("do not format JumpTable directly"); } - pub fn fmt(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) std.fmt.Formatter(format2) { - return .{ .data = .{ ot, zo, elf_file } }; + pub fn fmt(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) std.fmt.Formatter(format2) { + return .{ .data = .{ jt, zo, elf_file } }; } fn format2( - ctx: OffsetTableFormatContext, + ctx: JumpTableFormatContext, comptime unused_fmt_string: []const u8, options: std.fmt.FormatOptions, writer: anytype, ) !void { _ = options; _ = unused_fmt_string; - const ot, const zo, const ef = ctx; - try writer.writeAll("offset table\n"); - try writer.print(" @{x} : size({x})\n", .{ ot.address(zo, ef), ot.size(zo, ef) }); - for (ot.entries.items(.sym_index), ot.entries.items(.dirty)) |sym_index, dirty| { + const jt, const zo, const ef = ctx; + try writer.writeAll("__zig_jump_table\n"); + try writer.print(" @{x} : size({x})\n", .{ jt.address(zo, ef), jt.size(zo, ef) }); + for (jt.entries.items(.sym_index), jt.entries.items(.dirty)) |sym_index, dirty| { const sym = zo.symbol(sym_index); try writer.print(" %{d} : {s} : @{x}", .{ sym_index, sym.name(ef), sym.address(.{}, ef) }); if (dirty) try writer.writeAll(" : [!]"); -- 2.54.0 From ffcf0478fe646569465ac509b3e60faadd9b3697 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 23:42:37 +0200 Subject: [PATCH 15/39] x86_64: remove handling of .call since it's unused for now --- src/arch/x86_64/Lower.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 2cb874f003561fc529234ae63a314403e4b3840b..9ad1579ecceb6a05dc9fd7b34a90fc2bcd53a10f 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -398,7 +398,6 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) _ = lower.reloc(.{ .linker_reloc = sym }); break :op if (lower.pic) switch (mnemonic) { - .call => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, .lea => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, .mov => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, else => unreachable, -- 2.54.0 From d25c93a868585e4e2f5941fcff15fa49c90510e9 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 09:56:17 +0200 Subject: [PATCH 16/39] x86_64: emit call rel32 for near calls with linker reloc --- src/arch/x86_64/CodeGen.zig | 60 ++++++-------- src/arch/x86_64/Disassembler.zig | 2 +- src/arch/x86_64/Lower.zig | 6 +- src/arch/x86_64/Mir.zig | 2 +- src/arch/x86_64/bits.zig | 45 +++-------- src/arch/x86_64/encoder.zig | 134 +++++++++++++++++++++---------- src/link/Elf/Atom.zig | 7 +- 7 files changed, 138 insertions(+), 118 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index f6fbc67ff11118d1b83f144779f1ae32e707685e..ea4c7ae358241197e539677952ecd211e7dd493e 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1379,14 +1379,19 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void { .ops = switch (imm) { .signed => .i_s, .unsigned => .i_u, + .reloc => .rel, + }, + .data = switch (imm) { + .reloc => |x| .{ .reloc = x }, + .signed, .unsigned => .{ .i = .{ + .fixes = tag[0], + .i = switch (imm) { + .signed => |s| @bitCast(s), + .unsigned => |u| @intCast(u), + .reloc => unreachable, + }, + } }, }, - .data = .{ .i = .{ - .fixes = tag[0], - .i = switch (imm) { - .signed => |s| @bitCast(s), - .unsigned => |u| @intCast(u), - }, - } }, }); } @@ -1406,6 +1411,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: const ops: Mir.Inst.Ops = switch (imm) { .signed => .ri_s, .unsigned => |u| if (math.cast(u32, u)) |_| .ri_u else .ri64, + .reloc => unreachable, }; _ = try self.addInst(.{ .tag = tag[1], @@ -1417,6 +1423,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: .i = switch (imm) { .signed => |s| @bitCast(s), .unsigned => |u| @intCast(u), + .reloc => unreachable, }, } }, .ri64 => .{ .rx = .{ @@ -1488,6 +1495,7 @@ fn asmRegisterRegisterRegisterImmediate( .i = switch (imm) { .signed => |s| @bitCast(@as(i8, @intCast(s))), .unsigned => |u| @intCast(u), + .reloc => unreachable, }, } }, }); @@ -1505,6 +1513,7 @@ fn asmRegisterRegisterImmediate( .ops = switch (imm) { .signed => .rri_s, .unsigned => .rri_u, + .reloc => unreachable, }, .data = .{ .rri = .{ .fixes = tag[0], @@ -1513,6 +1522,7 @@ fn asmRegisterRegisterImmediate( .i = switch (imm) { .signed => |s| @bitCast(s), .unsigned => |u| @intCast(u), + .reloc => unreachable, }, } }, }); @@ -1610,6 +1620,7 @@ fn asmRegisterMemoryImmediate( if (switch (imm) { .signed => |s| if (math.cast(i16, s)) |x| @as(u16, @bitCast(x)) else null, .unsigned => |u| math.cast(u16, u), + .reloc => unreachable, }) |small_imm| { _ = try self.addInst(.{ .tag = tag[1], @@ -1625,6 +1636,7 @@ fn asmRegisterMemoryImmediate( const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) { .signed => |s| @bitCast(s), .unsigned => unreachable, + .reloc => unreachable, } }); assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m))); _ = try self.addInst(.{ @@ -1632,6 +1644,7 @@ fn asmRegisterMemoryImmediate( .ops = switch (imm) { .signed => .rmi_s, .unsigned => .rmi_u, + .reloc => unreachable, }, .data = .{ .rx = .{ .fixes = tag[0], @@ -1679,6 +1692,7 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) { .signed => |s| @bitCast(s), .unsigned => |u| @intCast(u), + .reloc => unreachable, } }); assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m))); _ = try self.addInst(.{ @@ -1686,6 +1700,7 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed .ops = switch (imm) { .signed => .mi_s, .unsigned => .mi_u, + .reloc => unreachable, }, .data = .{ .x = .{ .fixes = tag[0], @@ -12310,33 +12325,10 @@ fn genCall(self: *Self, info: union(enum) { if (self.bin_file.cast(.elf)) |elf_file| { const zo = elf_file.zigObjectPtr().?; const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); - if (self.mod.pic) { - const callee_reg: Register = switch (resolved_cc) { - .SysV => callee: { - if (!fn_info.is_var_args) break :callee .rax; - const param_regs = abi.getCAbiIntParamRegs(resolved_cc); - break :callee if (call_info.gp_count < param_regs.len) - param_regs[call_info.gp_count] - else - .r10; - }, - .Win64 => .rax, - else => unreachable, - }; - try self.genSetReg( - callee_reg, - Type.usize, - .{ .lea_symbol = .{ .sym = sym_index } }, - .{}, - ); - try self.asmRegister(.{ ._, .call }, callee_reg); - } else try self.asmMemory(.{ ._, .call }, .{ - .base = .{ .reloc = .{ - .atom_index = try self.owner.getSymbolIndex(self), - .sym_index = sym_index, - } }, - .mod = .{ .rm = .{ .size = .qword } }, - }); + try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ + .atom_index = try self.owner.getSymbolIndex(self), + .sym_index = sym_index, + })); } else if (self.bin_file.cast(.coff)) |coff_file| { const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav); const sym_index = coff_file.getAtom(atom).getSymbolIndex().?; diff --git a/src/arch/x86_64/Disassembler.zig b/src/arch/x86_64/Disassembler.zig index e0117fa17b7894079aecc95a90c1775aba90bd2e..d21cec28ab20fa4c080b1801050673d7e51a6d91 100644 --- a/src/arch/x86_64/Disassembler.zig +++ b/src/arch/x86_64/Disassembler.zig @@ -8,7 +8,7 @@ const bits = @import("bits.zig"); const encoder = @import("encoder.zig"); const Encoding = @import("Encoding.zig"); -const Immediate = bits.Immediate; +const Immediate = Instruction.Immediate; const Instruction = encoder.Instruction; const LegacyPrefixes = encoder.LegacyPrefixes; const Memory = Instruction.Memory; diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 9ad1579ecceb6a05dc9fd7b34a90fc2bcd53a10f..56cd2669d67439670f63495204b3a37668030894 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -475,7 +475,7 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void { .rrmi => inst.data.rrix.fixes, .mi_u, .mi_s => inst.data.x.fixes, .m => inst.data.x.fixes, - .extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._, + .extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc, .rel => ._, else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}), }; try lower.emit(switch (fixes) { @@ -607,7 +607,7 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void { .{ .mem = lower.mem(inst.data.rrix.payload) }, .{ .imm = lower.imm(inst.ops, inst.data.rrix.i) }, }, - .extern_fn_reloc => &.{ + .extern_fn_reloc, .rel => &.{ .{ .imm = lower.reloc(.{ .linker_extern_fn = inst.data.reloc }) }, }, .got_reloc, .direct_reloc, .import_reloc => ops: { @@ -650,7 +650,7 @@ const std = @import("std"); const Air = @import("../../Air.zig"); const Allocator = std.mem.Allocator; const ErrorMsg = Zcu.ErrorMsg; -const Immediate = bits.Immediate; +const Immediate = Instruction.Immediate; const Instruction = encoder.Instruction; const Lower = @This(); const Memory = Instruction.Memory; diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index d2dd6237a5e662a0c007145f14bc212e92327582..08f2cc3b253a736e706d31e525b7ba41ea99ead0 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -769,7 +769,7 @@ pub const Inst = struct { /// Uses `imm` payload. i_u, /// Relative displacement operand. - /// Uses `imm` payload. + /// Uses `reloc` payload. rel, /// Register, memory operands. /// Uses `rx` payload with extra data of type `Memory`. diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 6896d6c2e72a00a949c739beff4184c17fe5f94f..79f9b20924bf7f1220227f77cf18f68737a2b407 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -569,6 +569,7 @@ pub const Memory = struct { pub const Immediate = union(enum) { signed: i32, unsigned: u64, + reloc: Symbol, pub fn u(x: u64) Immediate { return .{ .unsigned = x }; @@ -578,39 +579,19 @@ pub const Immediate = union(enum) { return .{ .signed = x }; } - pub fn asSigned(imm: Immediate, bit_size: u64) i64 { - return switch (imm) { - .signed => |x| switch (bit_size) { - 1, 8 => @as(i8, @intCast(x)), - 16 => @as(i16, @intCast(x)), - 32, 64 => x, - else => unreachable, - }, - .unsigned => |x| switch (bit_size) { - 1, 8 => @as(i8, @bitCast(@as(u8, @intCast(x)))), - 16 => @as(i16, @bitCast(@as(u16, @intCast(x)))), - 32 => @as(i32, @bitCast(@as(u32, @intCast(x)))), - 64 => @bitCast(x), - else => unreachable, - }, - }; + pub fn rel(symbol: Symbol) Immediate { + return .{ .reloc = symbol }; } - pub fn asUnsigned(imm: Immediate, bit_size: u64) u64 { - return switch (imm) { - .signed => |x| switch (bit_size) { - 1, 8 => @as(u8, @bitCast(@as(i8, @intCast(x)))), - 16 => @as(u16, @bitCast(@as(i16, @intCast(x)))), - 32, 64 => @as(u32, @bitCast(x)), - else => unreachable, - }, - .unsigned => |x| switch (bit_size) { - 1, 8 => @as(u8, @intCast(x)), - 16 => @as(u16, @intCast(x)), - 32 => @as(u32, @intCast(x)), - 64 => x, - else => unreachable, - }, - }; + pub fn format( + imm: Immediate, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + switch (imm) { + .reloc => |x| try std.fmt.formatType(x, fmt, options, writer, 0), + inline else => |x| try writer.print("{d}", .{x}), + } } }; diff --git a/src/arch/x86_64/encoder.zig b/src/arch/x86_64/encoder.zig index 5f49443934faaad47d898881dbc3af74cea72cad..b525b0e11e8e00c01dddb15007a19cba4865e461 100644 --- a/src/arch/x86_64/encoder.zig +++ b/src/arch/x86_64/encoder.zig @@ -7,7 +7,6 @@ const testing = std.testing; const bits = @import("bits.zig"); const Encoding = @import("Encoding.zig"); const FrameIndex = bits.FrameIndex; -const Immediate = bits.Immediate; const Register = bits.Register; const Symbol = bits.Symbol; @@ -28,6 +27,55 @@ pub const Instruction = struct { repnz, }; + pub const Immediate = union(enum) { + signed: i32, + unsigned: u64, + + pub fn u(x: u64) Immediate { + return .{ .unsigned = x }; + } + + pub fn s(x: i32) Immediate { + return .{ .signed = x }; + } + + pub fn asSigned(imm: Immediate, bit_size: u64) i64 { + return switch (imm) { + .signed => |x| switch (bit_size) { + 1, 8 => @as(i8, @intCast(x)), + 16 => @as(i16, @intCast(x)), + 32, 64 => x, + else => unreachable, + }, + .unsigned => |x| switch (bit_size) { + 1, 8 => @as(i8, @bitCast(@as(u8, @intCast(x)))), + 16 => @as(i16, @bitCast(@as(u16, @intCast(x)))), + 32 => @as(i32, @bitCast(@as(u32, @intCast(x)))), + 64 => @bitCast(x), + else => unreachable, + }, + }; + } + + pub fn asUnsigned(imm: Immediate, bit_size: u64) u64 { + return switch (imm) { + .signed => |x| switch (bit_size) { + 1, 8 => @as(u8, @bitCast(@as(i8, @intCast(x)))), + 16 => @as(u16, @bitCast(@as(i16, @intCast(x)))), + 32, 64 => @as(u32, @bitCast(x)), + else => unreachable, + }, + .unsigned => |x| switch (bit_size) { + 1, 8 => @as(u8, @intCast(x)), + 16 => @as(u16, @intCast(x)), + 32 => @as(u32, @intCast(x)), + 64 => x, + else => unreachable, + }, + }; + } + }; + pub const Memory = union(enum) { sib: Sib, rip: Rip, @@ -1119,7 +1167,7 @@ test "encode" { const inst = try Instruction.new(.none, .mov, &.{ .{ .reg = .rbx }, - .{ .imm = Immediate.u(4) }, + .{ .imm = Instruction.Immediate.u(4) }, }); try inst.encode(buf.writer(), .{}); try testing.expectEqualSlices(u8, &.{ 0x48, 0xc7, 0xc3, 0x4, 0x0, 0x0, 0x0 }, buf.items); @@ -1129,47 +1177,47 @@ test "lower I encoding" { var enc = TestEncode{}; try enc.encode(.push, &.{ - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x6A\x10", enc.code(), "push 0x10"); try enc.encode(.push, &.{ - .{ .imm = Immediate.u(0x1000) }, + .{ .imm = Instruction.Immediate.u(0x1000) }, }); try expectEqualHexStrings("\x66\x68\x00\x10", enc.code(), "push 0x1000"); try enc.encode(.push, &.{ - .{ .imm = Immediate.u(0x10000000) }, + .{ .imm = Instruction.Immediate.u(0x10000000) }, }); try expectEqualHexStrings("\x68\x00\x00\x00\x10", enc.code(), "push 0x10000000"); try enc.encode(.adc, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x10000000) }, + .{ .imm = Instruction.Immediate.u(0x10000000) }, }); try expectEqualHexStrings("\x48\x15\x00\x00\x00\x10", enc.code(), "adc rax, 0x10000000"); try enc.encode(.add, &.{ .{ .reg = .al }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x04\x10", enc.code(), "add al, 0x10"); try enc.encode(.add, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10"); try enc.encode(.sbb, &.{ .{ .reg = .ax }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x66\x1D\x10\x00", enc.code(), "sbb ax, 0x10"); try enc.encode(.xor, &.{ .{ .reg = .al }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x34\x10", enc.code(), "xor al, 0x10"); } @@ -1179,43 +1227,43 @@ test "lower MI encoding" { try enc.encode(.mov, &.{ .{ .reg = .r12 }, - .{ .imm = Immediate.u(0x1000) }, + .{ .imm = Instruction.Immediate.u(0x1000) }, }); try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.sib(.byte, .{ .base = .{ .reg = .r12 } }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10"); try enc.encode(.mov, &.{ .{ .reg = .r12 }, - .{ .imm = Immediate.u(0x1000) }, + .{ .imm = Instruction.Immediate.u(0x1000) }, }); try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); try enc.encode(.mov, &.{ .{ .reg = .r12 }, - .{ .imm = Immediate.u(0x1000) }, + .{ .imm = Instruction.Immediate.u(0x1000) }, }); try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); try enc.encode(.mov, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10"); try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .r11 } }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10"); try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.rip(.qword, 0x10) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00", @@ -1225,19 +1273,19 @@ test "lower MI encoding" { try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.sib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -8 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\xc7\x45\xf8\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rbp - 8], 0x10"); try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.sib(.word, .{ .base = .{ .reg = .rbp }, .disp = -2 }) }, - .{ .imm = Immediate.s(-16) }, + .{ .imm = Instruction.Immediate.s(-16) }, }); try expectEqualHexStrings("\x66\xC7\x45\xFE\xF0\xFF", enc.code(), "mov WORD PTR [rbp - 2], -16"); try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.sib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -1 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\xC6\x45\xFF\x10", enc.code(), "mov BYTE PTR [rbp - 1], 0x10"); @@ -1247,7 +1295,7 @@ test "lower MI encoding" { .disp = 0x10000000, .scale_index = .{ .scale = 2, .index = .rcx }, }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00", @@ -1257,43 +1305,43 @@ test "lower MI encoding" { try enc.encode(.adc, &.{ .{ .mem = Instruction.Memory.sib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x80\x55\xF0\x10", enc.code(), "adc BYTE PTR [rbp - 0x10], 0x10"); try enc.encode(.adc, &.{ .{ .mem = Instruction.Memory.rip(.qword, 0) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\x83\x15\x00\x00\x00\x00\x10", enc.code(), "adc QWORD PTR [rip], 0x10"); try enc.encode(.adc, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\x83\xD0\x10", enc.code(), "adc rax, 0x10"); try enc.encode(.add, &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .rdx }, .disp = -8 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x83\x42\xF8\x10", enc.code(), "add DWORD PTR [rdx - 8], 0x10"); try enc.encode(.add, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10"); try enc.encode(.add, &.{ .{ .mem = Instruction.Memory.sib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) }, - .{ .imm = Immediate.s(-0x10) }, + .{ .imm = Instruction.Immediate.s(-0x10) }, }); try expectEqualHexStrings("\x48\x83\x45\xF0\xF0", enc.code(), "add QWORD PTR [rbp - 0x10], -0x10"); try enc.encode(.@"and", &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x83\x24\x25\x00\x00\x00\x10\x10", @@ -1303,7 +1351,7 @@ test "lower MI encoding" { try enc.encode(.@"and", &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .es }, .disp = 0x10000000 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x26\x83\x24\x25\x00\x00\x00\x10\x10", @@ -1313,7 +1361,7 @@ test "lower MI encoding" { try enc.encode(.@"and", &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .r12 }, .disp = 0x10000000 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x41\x83\xA4\x24\x00\x00\x00\x10\x10", @@ -1323,7 +1371,7 @@ test "lower MI encoding" { try enc.encode(.sub, &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .r11 }, .disp = 0x10000000 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x41\x83\xAB\x00\x00\x00\x10\x10", @@ -1542,14 +1590,14 @@ test "lower RMI encoding" { try enc.encode(.imul, &.{ .{ .reg = .r11 }, .{ .reg = .r12 }, - .{ .imm = Immediate.s(-2) }, + .{ .imm = Instruction.Immediate.s(-2) }, }); try expectEqualHexStrings("\x4D\x6B\xDC\xFE", enc.code(), "imul r11, r12, -2"); try enc.encode(.imul, &.{ .{ .reg = .r11 }, .{ .mem = Instruction.Memory.rip(.qword, -16) }, - .{ .imm = Immediate.s(-1024) }, + .{ .imm = Instruction.Immediate.s(-1024) }, }); try expectEqualHexStrings( "\x4C\x69\x1D\xF0\xFF\xFF\xFF\x00\xFC\xFF\xFF", @@ -1560,7 +1608,7 @@ test "lower RMI encoding" { try enc.encode(.imul, &.{ .{ .reg = .bx }, .{ .mem = Instruction.Memory.sib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) }, - .{ .imm = Immediate.s(-1024) }, + .{ .imm = Instruction.Immediate.s(-1024) }, }); try expectEqualHexStrings( "\x66\x69\x5D\xF0\x00\xFC", @@ -1571,7 +1619,7 @@ test "lower RMI encoding" { try enc.encode(.imul, &.{ .{ .reg = .bx }, .{ .mem = Instruction.Memory.sib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) }, - .{ .imm = Immediate.u(1024) }, + .{ .imm = Instruction.Immediate.u(1024) }, }); try expectEqualHexStrings( "\x66\x69\x5D\xF0\x00\x04", @@ -1687,7 +1735,7 @@ test "lower M encoding" { try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0"); try enc.encode(.call, &.{ - .{ .imm = Immediate.s(0) }, + .{ .imm = Instruction.Immediate.s(0) }, }); try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0"); @@ -1746,7 +1794,7 @@ test "lower OI encoding" { try enc.encode(.mov, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x1000000000000000) }, + .{ .imm = Instruction.Immediate.u(0x1000000000000000) }, }); try expectEqualHexStrings( "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10", @@ -1756,7 +1804,7 @@ test "lower OI encoding" { try enc.encode(.mov, &.{ .{ .reg = .r11 }, - .{ .imm = Immediate.u(0x1000000000000000) }, + .{ .imm = Instruction.Immediate.u(0x1000000000000000) }, }); try expectEqualHexStrings( "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10", @@ -1766,19 +1814,19 @@ test "lower OI encoding" { try enc.encode(.mov, &.{ .{ .reg = .r11d }, - .{ .imm = Immediate.u(0x10000000) }, + .{ .imm = Instruction.Immediate.u(0x10000000) }, }); try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", enc.code(), "mov r11d, 0x10000000"); try enc.encode(.mov, &.{ .{ .reg = .r11w }, - .{ .imm = Immediate.u(0x1000) }, + .{ .imm = Instruction.Immediate.u(0x1000) }, }); try expectEqualHexStrings("\x66\x41\xBB\x00\x10", enc.code(), "mov r11w, 0x1000"); try enc.encode(.mov, &.{ .{ .reg = .r11b }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x41\xB3\x10", enc.code(), "mov r11b, 0x10"); } @@ -1900,7 +1948,7 @@ test "invalid instruction" { .{ .reg = .r12d }, }); try invalidInstruction(.push, &.{ - .{ .imm = Immediate.u(0x1000000000000000) }, + .{ .imm = Instruction.Immediate.u(0x1000000000000000) }, }); } @@ -2213,7 +2261,7 @@ const Assembler = struct { .immediate => { const is_neg = if (as.expect(.minus)) |_| true else |_| false; const imm_tok = try as.expect(.numeral); - const imm: Immediate = if (is_neg) blk: { + const imm: Instruction.Immediate = if (is_neg) blk: { const imm = try std.fmt.parseInt(i32, as.source(imm_tok), 0); break :blk .{ .signed = imm * -1 }; } else .{ .unsigned = try std.fmt.parseInt(u64, as.source(imm_tok), 0) }; diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 31ba0d642e22cf79bd140b487575ce40484c2c37..32583b39d7797fa6682c5353622cc2e72f5c414d 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1215,12 +1215,11 @@ const x86_64 = struct { ); }, - .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), - - .PC32 => { + .PLT32 => { const S_ = if (target.flags.zig_jump_table) ZJT else S; try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); }, + .PC32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little), .GOTPC32 => try cwriter.writeInt(i32, @as(i32, @intCast(GOT + A - P)), .little), @@ -1620,7 +1619,7 @@ const x86_64 = struct { const bits = @import("../../arch/x86_64/bits.zig"); const encoder = @import("../../arch/x86_64/encoder.zig"); const Disassembler = @import("../../arch/x86_64/Disassembler.zig"); - const Immediate = bits.Immediate; + const Immediate = Instruction.Immediate; const Instruction = encoder.Instruction; }; -- 2.54.0 From ce8886d57d1593a7eb4909e518fc49e1c95360dd Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 10:28:01 +0200 Subject: [PATCH 17/39] elf: make zig jump table indirection implicit via Symbol.address --- src/link/Elf/Atom.zig | 35 ++++++++++++----------------------- src/link/Elf/Symbol.zig | 21 ++++++++++++--------- src/link/Elf/ZigObject.zig | 17 +++++++++++------ 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 32583b39d7797fa6682c5353622cc2e72f5c414d..e067e53c10cb0a6284f5469096b2b321d5cb9488 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -750,8 +750,6 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi const S = target.address(.{}, elf_file); // Address of the global offset table. const GOT = elf_file.gotAddress(); - // Address of the zig jump table entry if any. - const ZJT = target.zigJumpTableAddress(elf_file); // Relative offset to the start of the global offset table. const G = target.gotAddress(elf_file) - GOT; // // Address of the thread pointer. @@ -759,19 +757,18 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi // Address of the dynamic thread pointer. const DTP = elf_file.dtpAddress(); - relocs_log.debug(" {s}: {x}: [{x} => {x}] GOT({x}) ZJT({x}) ({s})", .{ + relocs_log.debug(" {s}: {x}: [{x} => {x}] GOT({x}) ({s})", .{ relocation.fmtRelocType(rel.r_type(), cpu_arch), r_offset, P, S + A, G + GOT + A, - ZJT + A, target.name(elf_file), }); try stream.seekTo(r_offset); - const args = ResolveArgs{ P, A, S, GOT, G, TP, DTP, ZJT }; + const args = ResolveArgs{ P, A, S, GOT, G, TP, DTP }; switch (cpu_arch) { .x86_64 => x86_64.resolveRelocAlloc(self, elf_file, rel, target, args, &it, code, &stream) catch |err| switch (err) { @@ -956,7 +953,7 @@ pub fn resolveRelocsNonAlloc(self: Atom, elf_file: *Elf, code: []u8, undefs: any // Address of the dynamic thread pointer. const DTP = elf_file.dtpAddress(); - const args = ResolveArgs{ P, A, S, GOT, 0, 0, DTP, 0 }; + const args = ResolveArgs{ P, A, S, GOT, 0, 0, DTP }; relocs_log.debug(" {}: {x}: [{x} => {x}] ({s})", .{ relocation.fmtRelocType(rel.r_type(), cpu_arch), @@ -1200,7 +1197,7 @@ const x86_64 = struct { const cwriter = stream.writer(); - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP = args; switch (r_type) { .NONE => unreachable, @@ -1215,10 +1212,7 @@ const x86_64 = struct { ); }, - .PLT32 => { - const S_ = if (target.flags.zig_jump_table) ZJT else S; - try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); - }, + .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .PC32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little), @@ -1243,10 +1237,7 @@ const x86_64 = struct { try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little); }, - .@"32" => { - const S_ = if (target.flags.zig_jump_table) ZJT else S; - try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S_ + A)))), .little); - }, + .@"32" => try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S + A)))), .little), .@"32S" => try cwriter.writeInt(i32, @as(i32, @truncate(S + A)), .little), .TPOFF32 => try cwriter.writeInt(i32, @as(i32, @truncate(S + A - TP)), .little), @@ -1345,7 +1336,7 @@ const x86_64 = struct { const r_type: elf.R_X86_64 = @enumFromInt(rel.r_type()); const cwriter = stream.writer(); - _, const A, const S, const GOT, _, _, const DTP, _ = args; + _, const A, const S, const GOT, _, _, const DTP = args; switch (r_type) { .NONE => unreachable, @@ -1728,9 +1719,8 @@ const aarch64 = struct { const code = code_buffer[r_offset..][0..4]; const file_ptr = atom.file(elf_file).?; - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP = args; _ = DTP; - _ = ZJT; switch (r_type) { .NONE => unreachable, @@ -1932,7 +1922,7 @@ const aarch64 = struct { const r_type: elf.R_AARCH64 = @enumFromInt(rel.r_type()); const cwriter = stream.writer(); - _, const A, const S, _, _, _, _, _ = args; + _, const A, const S, _, _, _, _ = args; switch (r_type) { .NONE => unreachable, @@ -2012,10 +2002,9 @@ const riscv = struct { const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow; const cwriter = stream.writer(); - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP = args; _ = TP; _ = DTP; - _ = ZJT; switch (r_type) { .NONE => unreachable, @@ -2167,7 +2156,7 @@ const riscv = struct { const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow; const cwriter = stream.writer(); - _, const A, const S, const GOT, _, _, const DTP, _ = args; + _, const A, const S, const GOT, _, _, const DTP = args; _ = GOT; _ = DTP; @@ -2203,7 +2192,7 @@ const riscv = struct { const riscv_util = @import("../riscv.zig"); }; -const ResolveArgs = struct { i64, i64, i64, i64, i64, i64, i64, i64 }; +const ResolveArgs = struct { i64, i64, i64, i64, i64, i64, i64 }; const RelocError = error{ Overflow, diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index 38ef6bb8f7826483bd2e6093884f1c748fac9469..8b30f0ab036441c674d7a2d551a2d0a8c4dd55b1 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -101,7 +101,7 @@ pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 { return file_ptr.symbolRank(sym, in_archive); } -pub fn address(symbol: Symbol, opts: struct { plt: bool = true }, elf_file: *Elf) i64 { +pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = true }, elf_file: *Elf) i64 { if (symbol.mergeSubsection(elf_file)) |msub| { if (!msub.alive) return 0; return msub.address(elf_file) + symbol.value; @@ -109,6 +109,9 @@ pub fn address(symbol: Symbol, opts: struct { plt: bool = true }, elf_file: *Elf if (symbol.flags.has_copy_rel) { return symbol.copyRelAddress(elf_file); } + if (symbol.flags.has_zjt and opts.zjt) { + return symbol.zjtAddress(elf_file); + } if (symbol.flags.has_plt and opts.plt) { if (!symbol.flags.is_canonical and symbol.flags.has_got) { // We have a non-lazy bound function pointer, use that! @@ -217,12 +220,12 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) i64 { return entry.address(elf_file); } -pub fn zigJumpTableAddress(symbol: Symbol, elf_file: *Elf) i64 { - if (!symbol.flags.zig_jump_table) return 0; +pub fn zjtAddress(symbol: Symbol, elf_file: *Elf) i64 { + if (!symbol.flags.has_zjt) return 0; const zo = elf_file.zigObjectPtr().?; const jump_table = zo.jumpTablePtr().?; - const jt_index = symbol.extra(elf_file).zig_jump_table; - return jump_table.entryAddress(jt_index, zo, elf_file); + const index = symbol.extra(elf_file).zjt; + return jump_table.entryAddress(index, zo, elf_file); } pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 { @@ -248,7 +251,7 @@ const AddExtraOpts = struct { tlsgd: ?u32 = null, gottp: ?u32 = null, tlsdesc: ?u32 = null, - zig_jump_table: ?u32 = null, + zjt: ?u32 = null, }; pub fn addExtra(symbol: *Symbol, opts: AddExtraOpts, elf_file: *Elf) void { @@ -377,7 +380,7 @@ fn format2( try writer.print("%{d} : {s} : @{x}", .{ symbol.esym_index, symbol.fmtName(elf_file), - symbol.address(.{}, elf_file), + symbol.address(.{ .plt = false, .zjt = false }, elf_file), }); if (symbol.file(elf_file)) |file_ptr| { if (symbol.isAbs(elf_file)) { @@ -454,7 +457,7 @@ pub const Flags = packed struct { merge_subsection: bool = false, /// Whether the symbol has __zig_jump_table indirection. - zig_jump_table: bool = false, + has_zjt: bool = false, }; pub const Extra = struct { @@ -468,7 +471,7 @@ pub const Extra = struct { gottp: u32 = 0, tlsdesc: u32 = 0, merge_section: u32 = 0, - zig_jump_table: u32 = 0, + zjt: u32 = 0, }; pub const Index = u32; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 596ed781ca96deccb89b3db64653267488364a96..9069e180a387b02325cf1302ac06e1723b7aca41 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -918,7 +918,7 @@ fn updateNavCode( if (stt_bits == elf.STT_FUNC) { const extra = sym.extra(elf_file); const jump_table = self.jumpTablePtr().?; - jump_table.entries.items(.dirty)[extra.zig_jump_table] = true; + jump_table.entries.items(.dirty)[extra.zjt] = true; } } } else if (code.len < old_size) { @@ -1045,10 +1045,10 @@ pub fn updateFunc( { const sym = self.symbol(sym_index); - if (!sym.flags.zig_jump_table) { + if (!sym.flags.has_zjt) { const index = try jump_table.addSymbol(gpa, sym_index); - sym.flags.zig_jump_table = true; - sym.addExtra(.{ .zig_jump_table = index }, elf_file); + sym.flags.has_zjt = true; + sym.addExtra(.{ .zjt = index }, elf_file); try jump_table.updateSize(self, elf_file); const old_vaddr = jump_table.address(self, elf_file); try self.symbol(jump_table.sym_index).atom(elf_file).?.allocate(elf_file); @@ -1108,7 +1108,7 @@ pub fn updateFunc( } } else { const sym = self.symbol(sym_index); - const jt_index = sym.extra(elf_file).zig_jump_table; + const jt_index = sym.extra(elf_file).zjt; var jt_entry = jump_table.entries.get(jt_index); if (jt_entry.dirty) { try jump_table.writeEntry(jt_index, self, elf_file); @@ -1896,7 +1896,12 @@ pub const JumpTable = struct { try writer.print(" @{x} : size({x})\n", .{ jt.address(zo, ef), jt.size(zo, ef) }); for (jt.entries.items(.sym_index), jt.entries.items(.dirty)) |sym_index, dirty| { const sym = zo.symbol(sym_index); - try writer.print(" %{d} : {s} : @{x}", .{ sym_index, sym.name(ef), sym.address(.{}, ef) }); + try writer.print(" {x} => {x} : %{d} : {s}", .{ + sym.address(.{}, ef), + sym.address(.{ .zjt = false }, ef), + sym_index, + sym.name(ef), + }); if (dirty) try writer.writeAll(" : [!]"); try writer.writeByte('\n'); } -- 2.54.0 From 57f72095082a5fadd6419d8176475f296a65bdbc Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 10:39:49 +0200 Subject: [PATCH 18/39] elf: replace use of linker_extern_fn with more generic Immediate.reloc --- src/arch/x86_64/CodeGen.zig | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index ea4c7ae358241197e539677952ecd211e7dd493e..116911c4098cd8b2ac7d47f9bd94ab95e52519ca 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -12357,7 +12357,16 @@ fn genCall(self: *Self, info: union(enum) { }); } else unreachable; }, - .@"extern" => |@"extern"| try self.genExternSymbolRef( + .@"extern" => |@"extern"| if (self.bin_file.cast(.elf)) |elf_file| { + const target_sym_index = try elf_file.getGlobalSymbol( + @"extern".name.toSlice(ip), + @"extern".lib_name.toSlice(ip), + ); + try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ + .atom_index = try self.owner.getSymbolIndex(self), + .sym_index = target_sym_index, + })); + } else try self.genExternSymbolRef( .call, @"extern".lib_name.toSlice(ip), @"extern".name.toSlice(ip), @@ -12369,7 +12378,13 @@ fn genCall(self: *Self, info: union(enum) { try self.genSetReg(.rax, Type.usize, .{ .air_ref = callee }, .{}); try self.asmRegister(.{ ._, .call }, .rax); }, - .lib => |lib| try self.genExternSymbolRef(.call, lib.lib, lib.callee), + .lib => |lib| if (self.bin_file.cast(.elf)) |elf_file| { + const target_sym_index = try elf_file.getGlobalSymbol(lib.callee, lib.lib); + try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ + .atom_index = try self.owner.getSymbolIndex(self), + .sym_index = target_sym_index, + })); + } else try self.genExternSymbolRef(.call, lib.lib, lib.callee), } return call_info.return_value.short; } @@ -15245,16 +15260,7 @@ fn genExternSymbolRef( callee: []const u8, ) InnerError!void { const atom_index = try self.owner.getSymbolIndex(self); - if (self.bin_file.cast(.elf)) |elf_file| { - _ = try self.addInst(.{ - .tag = tag, - .ops = .extern_fn_reloc, - .data = .{ .reloc = .{ - .atom_index = atom_index, - .sym_index = try elf_file.getGlobalSymbol(callee, lib), - } }, - }); - } else if (self.bin_file.cast(.coff)) |coff_file| { + if (self.bin_file.cast(.coff)) |coff_file| { const global_index = try coff_file.getGlobalSymbol(callee, lib); _ = try self.addInst(.{ .tag = .mov, -- 2.54.0 From edd72c75eb4164f5799d56128ce6e17668efb0e2 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 10:41:57 +0200 Subject: [PATCH 19/39] elf: fix circular dep loop in zig jump table --- src/link/Elf/ZigObject.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 9069e180a387b02325cf1302ac06e1723b7aca41..70f6a2500da8388c1cc1e5254db4f07fd6f16f7e 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1817,7 +1817,7 @@ pub const JumpTable = struct { pub fn targetAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { const sym_index = jt.entries.items(.sym_index)[index]; - return zo.symbol(sym_index).address(.{}, elf_file); + return zo.symbol(sym_index).address(.{ .zjt = false }, elf_file); } const max_jump_seq_len = 12; -- 2.54.0 From afaec5c3e477a66695fdfca30bcc662fd1c73cd4 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 11:11:16 +0200 Subject: [PATCH 20/39] x86_64: fix generating lazy symbol refs --- src/arch/x86_64/CodeGen.zig | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 116911c4098cd8b2ac7d47f9bd94ab95e52519ca..1edf208d4c9c3f56b171c21d4f14e09c29e9b03d 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -15304,7 +15304,7 @@ fn genLazySymbolRef( if (self.mod.pic) { switch (tag) { .lea, .call => try self.genSetReg(reg, Type.usize, .{ - .load_symbol = .{ .sym = sym_index }, + .lea_symbol = .{ .sym = sym_index }, }, .{}), .mov => try self.genSetReg(reg, Type.usize, .{ .load_symbol = .{ .sym = sym_index }, @@ -15322,14 +15322,11 @@ fn genLazySymbolRef( .sym_index = sym_index, }; switch (tag) { - .lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), .{ - .base = .{ .reloc = reloc }, - .mod = .{ .rm = .{ .size = .qword } }, - }), - .call => try self.asmMemory(.{ ._, .call }, .{ + .lea, .mov => try self.asmRegisterMemory(.{ ._, tag }, reg.to64(), .{ .base = .{ .reloc = reloc }, .mod = .{ .rm = .{ .size = .qword } }, }), + .call => try self.asmImmediate(.{ ._, .call }, Immediate.rel(reloc)), else => unreachable, } } -- 2.54.0 From 39ee346681eac850f2f1f9bcb8ca6114dbad581a Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 11:11:54 +0200 Subject: [PATCH 21/39] elf: make .text.zig bigger now that jump table is part of it --- src/link/Elf.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index f4003ff38fbd94e72efb7a3220e05e909a4f83d2..dce5d45c809847859557522484a8b5f332f95801 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -604,7 +604,7 @@ pub fn initMetadata(self: *Elf, options: InitMetadataOptions) !void { .type = elf.PT_LOAD, .offset = off, .filesz = filesz, - .addr = if (ptr_bit_width >= 32) 0x8000000 else 0x8000, + .addr = if (ptr_bit_width >= 32) 0x4000000 else 0x4000, .memsz = filesz, .@"align" = self.page_size, .flags = elf.PF_X | elf.PF_R | elf.PF_W, -- 2.54.0 From df80ccf7600818faafd485b4c01bd4fe0f26a8e5 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 12:01:56 +0200 Subject: [PATCH 22/39] elf: pretty print alingment when pretty printing atoms --- src/link/Elf/Atom.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index e067e53c10cb0a6284f5469096b2b321d5cb9488..6e68b849526d5ee27eb711fcf5fc4fbe7643bbaf 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1022,7 +1022,7 @@ pub fn format( _ = unused_fmt_string; _ = options; _ = writer; - @compileError("do not format symbols directly"); + @compileError("do not format Atom directly"); } pub fn fmt(atom: Atom, elf_file: *Elf) std.fmt.Formatter(format2) { @@ -1048,8 +1048,8 @@ fn format2( const atom = ctx.atom; const elf_file = ctx.elf_file; try writer.print("atom({d}) : {s} : @{x} : shdr({d}) : align({x}) : size({x})", .{ - atom.atom_index, atom.name(elf_file), atom.address(elf_file), - atom.output_section_index, atom.alignment, atom.size, + atom.atom_index, atom.name(elf_file), atom.address(elf_file), + atom.output_section_index, atom.alignment.toByteUnits() orelse 0, atom.size, }); if (atom.fdes(elf_file).len > 0) { try writer.writeAll(" : fdes{ "); -- 2.54.0 From 9daf5e81c4bdc413a3bb11c1bab207c19e6ba547 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 12:08:21 +0200 Subject: [PATCH 23/39] elf: commit non-indirected symbol address to symtab --- src/link/Elf/Symbol.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index 8b30f0ab036441c674d7a2d551a2d0a8c4dd55b1..1f81b3ff98b19825d9e094daa5db09ea6a61d561 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -300,11 +300,11 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void { if (symbol.flags.is_canonical) break :blk symbol.address(.{}, elf_file); break :blk 0; } - if (st_shndx == elf.SHN_ABS or st_shndx == elf.SHN_COMMON) break :blk symbol.address(.{ .plt = false }, elf_file); + if (st_shndx == elf.SHN_ABS or st_shndx == elf.SHN_COMMON) break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file); const shdr = elf_file.shdrs.items[st_shndx]; if (shdr.sh_flags & elf.SHF_TLS != 0 and file_ptr != .linker_defined) - break :blk symbol.address(.{ .plt = false }, elf_file) - elf_file.tlsAddress(); - break :blk symbol.address(.{ .plt = false }, elf_file); + break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file) - elf_file.tlsAddress(); + break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file); }; out.st_info = (st_bind << 4) | st_type; out.st_other = esym.st_other; @@ -323,7 +323,7 @@ pub fn format( _ = unused_fmt_string; _ = options; _ = writer; - @compileError("do not format symbols directly"); + @compileError("do not format Symbol directly"); } const FormatContext = struct { -- 2.54.0 From 49d78cc793f6dd987d2a51c7a96333713264691f Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 12:50:53 +0200 Subject: [PATCH 24/39] elf: only apply zig jump table indirection to function calls (PLT32) --- src/link/Elf.zig | 2 +- src/link/Elf/Atom.zig | 7 +++++-- src/link/Elf/Symbol.zig | 10 +++++----- src/link/Elf/ZigObject.zig | 12 ++++++------ 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index dce5d45c809847859557522484a8b5f332f95801..50dfeffa172a8d39a1c61714be445b723e2e4561 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -2798,7 +2798,7 @@ pub fn writeElfHeader(self: *Elf) !void { const e_entry: u64 = if (self.linkerDefinedPtr()) |obj| blk: { const entry_sym = obj.entrySymbol(self) orelse break :blk 0; - break :blk @intCast(entry_sym.address(.{}, self)); + break :blk @intCast(entry_sym.address(.{ .zjt = true }, self)); } else 0; const phdr_table_offset = if (self.phdr_table_index) |phndx| self.phdrs.items[phndx].p_offset else 0; switch (self.ptr_width) { diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 6e68b849526d5ee27eb711fcf5fc4fbe7643bbaf..7338927ebd28589585741540720d9952031e8abd 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -747,7 +747,7 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi // Addend from the relocation. const A = rel.r_addend; // Address of the target symbol - can be address of the symbol within an atom or address of PLT stub. - const S = target.address(.{}, elf_file); + const S = target.address(.{ .zjt = false }, elf_file); // Address of the global offset table. const GOT = elf_file.gotAddress(); // Relative offset to the start of the global offset table. @@ -1212,7 +1212,10 @@ const x86_64 = struct { ); }, - .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), + .PLT32 => { + const S_ = if (target.flags.has_zjt) target.address(.{ .zjt = true }, elf_file) else S; + try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); + }, .PC32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little), diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index 1f81b3ff98b19825d9e094daa5db09ea6a61d561..d946e95511741c5ebe7192173a061179b6710705 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -101,7 +101,7 @@ pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 { return file_ptr.symbolRank(sym, in_archive); } -pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = true }, elf_file: *Elf) i64 { +pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = false }, elf_file: *Elf) i64 { if (symbol.mergeSubsection(elf_file)) |msub| { if (!msub.alive) return 0; return msub.address(elf_file) + symbol.value; @@ -300,11 +300,11 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void { if (symbol.flags.is_canonical) break :blk symbol.address(.{}, elf_file); break :blk 0; } - if (st_shndx == elf.SHN_ABS or st_shndx == elf.SHN_COMMON) break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file); + if (st_shndx == elf.SHN_ABS or st_shndx == elf.SHN_COMMON) break :blk symbol.address(.{ .plt = false }, elf_file); const shdr = elf_file.shdrs.items[st_shndx]; if (shdr.sh_flags & elf.SHF_TLS != 0 and file_ptr != .linker_defined) - break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file) - elf_file.tlsAddress(); - break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file); + break :blk symbol.address(.{ .plt = false }, elf_file) - elf_file.tlsAddress(); + break :blk symbol.address(.{ .plt = false }, elf_file); }; out.st_info = (st_bind << 4) | st_type; out.st_other = esym.st_other; @@ -380,7 +380,7 @@ fn format2( try writer.print("%{d} : {s} : @{x}", .{ symbol.esym_index, symbol.fmtName(elf_file), - symbol.address(.{ .plt = false, .zjt = false }, elf_file), + symbol.address(.{ .plt = false }, elf_file), }); if (symbol.file(elf_file)) |file_ptr| { if (symbol.isAbs(elf_file)) { diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 70f6a2500da8388c1cc1e5254db4f07fd6f16f7e..865f4118d360f6563ac3ef5d5b9891b6330e0865 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -665,7 +665,7 @@ pub fn getNavVAddr( else => try self.getOrCreateMetadataForNav(elf_file, nav_index), }; const this_sym = self.symbol(this_sym_index); - const vaddr = this_sym.address(.{}, elf_file); + const vaddr = this_sym.address(.{ .zjt = true }, elf_file); const parent_atom = self.symbol(reloc_info.parent_atom_index).atom(elf_file).?; const r_type = relocation.encode(.abs, elf_file.getTarget().cpu.arch); try parent_atom.addReloc(elf_file, .{ @@ -942,7 +942,7 @@ fn updateNavCode( .len = code.len, }}; var remote_vec: [1]std.posix.iovec_const = .{.{ - .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{}, elf_file))))), + .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{ .zjt = true }, elf_file))))), .len = code.len, }}; const rc = std.os.linux.process_vm_writev(pid, &code_vec, &remote_vec, 0); @@ -1092,7 +1092,7 @@ pub fn updateFunc( try self.dwarf.?.commitNavState( pt, func.owner_nav, - @intCast(sym.address(.{}, elf_file)), + @intCast(sym.address(.{ .zjt = true }, elf_file)), sym.atom(elf_file).?.size, ds, ); @@ -1189,7 +1189,7 @@ pub fn updateNav( try self.dwarf.?.commitNavState( pt, nav_index, - @intCast(sym.address(.{}, elf_file)), + @intCast(sym.address(.{ .zjt = true }, elf_file)), sym.atom(elf_file).?.size, ns, ); @@ -1817,7 +1817,7 @@ pub const JumpTable = struct { pub fn targetAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { const sym_index = jt.entries.items(.sym_index)[index]; - return zo.symbol(sym_index).address(.{ .zjt = false }, elf_file); + return zo.symbol(sym_index).address(.{}, elf_file); } const max_jump_seq_len = 12; @@ -1897,8 +1897,8 @@ pub const JumpTable = struct { for (jt.entries.items(.sym_index), jt.entries.items(.dirty)) |sym_index, dirty| { const sym = zo.symbol(sym_index); try writer.print(" {x} => {x} : %{d} : {s}", .{ + sym.address(.{ .zjt = true }, ef), sym.address(.{}, ef), - sym.address(.{ .zjt = false }, ef), sym_index, sym.name(ef), }); -- 2.54.0 From 97ab420dcf9bdfe954d6bff730a7492abaeb90bb Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 12:51:17 +0200 Subject: [PATCH 25/39] elf: do not emit zig jump table in relocatables --- src/link/Elf/ZigObject.zig | 39 ++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 865f4118d360f6563ac3ef5d5b9891b6330e0865..4a3bdd604965b56e8049ce4f942dac3c75ec90f9 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1036,14 +1036,16 @@ pub fn updateFunc( const ip = &zcu.intern_pool; const gpa = elf_file.base.comp.gpa; const func = zcu.funcInfo(func_index); - const jump_table = self.jumpTablePtr() orelse try self.initJumpTable(gpa, elf_file); + if (elf_file.base.isRelocatable() and self.jumpTablePtr() == null) { + try self.initJumpTable(gpa, elf_file); + } log.debug("updateFunc {}({d})", .{ ip.getNav(func.owner_nav).fqn.fmt(ip), func.owner_nav }); const sym_index = try self.getOrCreateMetadataForNav(elf_file, func.owner_nav); self.symbol(sym_index).atom(elf_file).?.freeRelocs(elf_file); - { + if (self.jumpTablePtr()) |jump_table| { const sym = self.symbol(sym_index); if (!sym.flags.has_zjt) { const index = try jump_table.addSymbol(gpa, sym_index); @@ -1100,21 +1102,23 @@ pub fn updateFunc( // Exports will be updated by `Zcu.processExports` after the update. - if (jump_table.dirty) { - // TODO write in bulk - for (jump_table.entries.items(.dirty), 0..) |*dirty, i| { - try jump_table.writeEntry(@intCast(i), self, elf_file); - dirty.* = false; + if (self.jumpTablePtr()) |jump_table| { + if (jump_table.dirty) { + // TODO write in bulk + for (jump_table.entries.items(.dirty), 0..) |*dirty, i| { + try jump_table.writeEntry(@intCast(i), self, elf_file); + dirty.* = false; + } + } else { + const sym = self.symbol(sym_index); + const jt_index = sym.extra(elf_file).zjt; + var jt_entry = jump_table.entries.get(jt_index); + if (jt_entry.dirty) { + try jump_table.writeEntry(jt_index, self, elf_file); + jt_entry.dirty = false; + } + jump_table.entries.set(jt_index, jt_entry); } - } else { - const sym = self.symbol(sym_index); - const jt_index = sym.extra(elf_file).zjt; - var jt_entry = jump_table.entries.get(jt_index); - if (jt_entry.dirty) { - try jump_table.writeEntry(jt_index, self, elf_file); - jt_entry.dirty = false; - } - jump_table.entries.set(jt_index, jt_entry); } } @@ -1474,7 +1478,7 @@ pub fn jumpTablePtr(self: *ZigObject) ?*JumpTable { return if (self.jump_table) |*jt| jt else null; } -fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!*JumpTable { +fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!void { const name_off = try self.addString(allocator, "__zig_jump_table"); const sym_index = try self.newSymbolWithAtom(allocator, name_off); const sym = self.symbol(sym_index); @@ -1485,7 +1489,6 @@ fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{O atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(JumpTable.alignment(elf_file.getTarget().cpu.arch)); atom_ptr.output_section_index = elf_file.zig_text_section_index.?; self.jump_table = JumpTable{ .sym_index = sym_index }; - return &(self.jump_table.?); } pub fn asFile(self: *ZigObject) File { -- 2.54.0 From 1bd54a55fa40db9f91d8a0c1cf5244ff51be6081 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 21:52:40 +0200 Subject: [PATCH 26/39] fix compile errors in other codegen backends --- src/arch/aarch64/CodeGen.zig | 41 ++++++------------------------------ src/arch/arm/CodeGen.zig | 20 +++--------------- src/arch/riscv64/CodeGen.zig | 1 + src/arch/riscv64/Emit.zig | 13 +----------- src/arch/sparc64/CodeGen.zig | 32 +++------------------------- src/link/Elf/ZigObject.zig | 2 +- 6 files changed, 16 insertions(+), 93 deletions(-) diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index 2810b6b521895012fe2c23c1fa0320286448b449..231db8e98c5327f2014e4748ef7242581af62885 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -4352,24 +4352,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier // on linking. if (try self.air.value(callee, pt)) |func_value| switch (ip.indexToKey(func_value.toIntern())) { .func => |func| { - if (self.bin_file.cast(.elf)) |elf_file| { - const zo = elf_file.zigObjectPtr().?; - const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); - const sym = zo.symbol(sym_index); - _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); - const got_addr = @as(u32, @intCast(sym.zigGotAddress(elf_file))); - try self.genSetReg(Type.usize, .x30, .{ .memory = got_addr }); - } else if (self.bin_file.cast(.macho)) |macho_file| { - _ = macho_file; - @panic("TODO airCall"); - // const atom = try macho_file.getOrCreateAtomForNav(func.owner_nav); - // const sym_index = macho_file.getAtom(atom).getSymbolIndex().?; - // try self.genSetReg(Type.u64, .x30, .{ - // .linker_load = .{ - // .type = .got, - // .sym_index = sym_index, - // }, - // }); + if (self.bin_file.cast(.elf)) |_| { + return self.fail("TODO implement calling functions for Elf", .{}); + } else if (self.bin_file.cast(.macho)) |_| { + return self.fail("TODO implement calling functions for MachO", .{}); } else if (self.bin_file.cast(.coff)) |coff_file| { const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav); const sym_index = coff_file.getAtom(atom).getSymbolIndex().?; @@ -4393,21 +4379,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier .@"extern" => |@"extern"| { const nav_name = ip.getNav(@"extern".owner_nav).name.toSlice(ip); const lib_name = @"extern".lib_name.toSlice(ip); - if (self.bin_file.cast(.macho)) |macho_file| { - _ = macho_file; - @panic("TODO airCall"); - // const sym_index = try macho_file.getGlobalSymbol(nav_name, lib_name); - // const atom = try macho_file.getOrCreateAtomForNav(self.owner_nav); - // const atom_index = macho_file.getAtom(atom).getSymbolIndex().?; - // _ = try self.addInst(.{ - // .tag = .call_extern, - // .data = .{ - // .relocation = .{ - // .atom_index = atom_index, - // .sym_index = sym_index, - // }, - // }, - // }); + if (self.bin_file.cast(.macho)) |_| { + return self.fail("TODO implement calling extern functions for MachO", .{}); } else if (self.bin_file.cast(.coff)) |coff_file| { const sym_index = try coff_file.getGlobalSymbol(nav_name, lib_name); try self.genSetReg(Type.u64, .x30, .{ @@ -6234,7 +6207,7 @@ fn genTypedValue(self: *Self, val: Value) InnerError!MCValue { .memory => |addr| .{ .memory = addr }, .load_got => |sym_index| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } }, .load_direct => |sym_index| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } }, - .load_symbol, .load_tlv => unreachable, // TODO + .load_symbol, .load_tlv, .lea_symbol => unreachable, // TODO }, .fail => |msg| { self.err_msg = msg; diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig index e4d106921ed45d413a110b7e67b1839752bc8116..1c0369955208c1d56a9c8ac58c1b05fdea7a374a 100644 --- a/src/arch/arm/CodeGen.zig +++ b/src/arch/arm/CodeGen.zig @@ -4333,22 +4333,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier // Due to incremental compilation, how function calls are generated depends // on linking. if (try self.air.value(callee, pt)) |func_value| switch (ip.indexToKey(func_value.toIntern())) { - .func => |func| { - if (self.bin_file.cast(.elf)) |elf_file| { - const zo = elf_file.zigObjectPtr().?; - const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); - const sym = zo.symbol(sym_index); - _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); - const got_addr: u32 = @intCast(sym.zigGotAddress(elf_file)); - try self.genSetReg(Type.usize, .lr, .{ .memory = got_addr }); - } else if (self.bin_file.cast(.macho)) |_| { - unreachable; // unsupported architecture for MachO - } else { - return self.fail("TODO implement call on {s} for {s}", .{ - @tagName(self.bin_file.tag), - @tagName(self.target.cpu.arch), - }); - } + .func => { + return self.fail("TODO implement calling functions", .{}); }, .@"extern" => { return self.fail("TODO implement calling extern functions", .{}); @@ -6184,7 +6170,7 @@ fn genTypedValue(self: *Self, val: Value) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => .undef, - .load_got, .load_symbol, .load_direct, .load_tlv => unreachable, // TODO + .load_got, .load_symbol, .load_direct, .load_tlv, .lea_symbol => unreachable, // TODO .immediate => |imm| .{ .immediate = @truncate(imm) }, .memory => |addr| .{ .memory = addr }, }, diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index b5384542e73fa766ddaae411cc56aa3fa192c14f..e9f593485f910f7672e991a3ba94213fd71590fb 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -8026,6 +8026,7 @@ fn genTypedValue(func: *Func, val: Value) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => unreachable, + .lea_symbol => |sym_index| .{ .lea_symbol = .{ .sym = sym_index } }, .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } }, .load_tlv => |sym_index| .{ .lea_tlv = sym_index }, .immediate => |imm| .{ .immediate = imm }, diff --git a/src/arch/riscv64/Emit.zig b/src/arch/riscv64/Emit.zig index 137bc395721ef8e83eac521a28cd9e8c7a965eeb..c1e7565c0bb5426c738a969d018b0efb66e02c87 100644 --- a/src/arch/riscv64/Emit.zig +++ b/src/arch/riscv64/Emit.zig @@ -43,12 +43,6 @@ pub fn emitMir(emit: *Emit) Error!void { .fmt = std.meta.activeTag(lowered_inst), }), .load_symbol_reloc => |symbol| { - const is_obj_or_static_lib = switch (emit.lower.output_mode) { - .Exe => false, - .Obj => true, - .Lib => emit.lower.link_mode == .static, - }; - const elf_file = emit.bin_file.cast(.elf).?; const zo = elf_file.zigObjectPtr().?; @@ -58,12 +52,7 @@ pub fn emitMir(emit: *Emit) Error!void { var hi_r_type: u32 = @intFromEnum(std.elf.R_RISCV.HI20); var lo_r_type: u32 = @intFromEnum(std.elf.R_RISCV.LO12_I); - if (sym.flags.needs_zig_got and !is_obj_or_static_lib) { - _ = try sym.getOrCreateZigGotEntry(symbol.sym_index, elf_file); - - hi_r_type = Elf.R_ZIG_GOT_HI20; - lo_r_type = Elf.R_ZIG_GOT_LO12; - } else if (sym.flags.needs_got) { + if (sym.flags.needs_got) { hi_r_type = Elf.R_GOT_HI20_STATIC; // TODO: rework this #20887 lo_r_type = Elf.R_GOT_LO12_I_STATIC; // TODO: rework this #20887 } diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig index bba8cdec3481678a2e899dc9f0d9b6a8fdf6680e..d0b0eedc57565873c50192760b5fc0c33247216e 100644 --- a/src/arch/sparc64/CodeGen.zig +++ b/src/arch/sparc64/CodeGen.zig @@ -1349,34 +1349,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier // Due to incremental compilation, how function calls are generated depends // on linking. if (try self.air.value(callee, pt)) |func_value| switch (ip.indexToKey(func_value.toIntern())) { - .func => |func| { - const got_addr = if (self.bin_file.cast(.elf)) |elf_file| blk: { - const zo = elf_file.zigObjectPtr().?; - const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); - const sym = zo.symbol(sym_index); - _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); - break :blk @as(u32, @intCast(sym.zigGotAddress(elf_file))); - } else @panic("TODO SPARCv9 currently does not support non-ELF binaries"); - - try self.genSetReg(Type.usize, .o7, .{ .memory = got_addr }); - - _ = try self.addInst(.{ - .tag = .jmpl, - .data = .{ - .arithmetic_3op = .{ - .is_imm = false, - .rd = .o7, - .rs1 = .o7, - .rs2_or_imm = .{ .rs2 = .g0 }, - }, - }, - }); - - // TODO Find a way to fill this delay slot - _ = try self.addInst(.{ - .tag = .nop, - .data = .{ .nop = {} }, - }); + .func => { + return self.fail("TODO implement calling functions", .{}); }, .@"extern" => { return self.fail("TODO implement calling extern functions", .{}); @@ -4153,7 +4127,7 @@ fn genTypedValue(self: *Self, val: Value) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => .undef, - .load_got, .load_symbol, .load_direct, .load_tlv => unreachable, // TODO + .load_got, .load_symbol, .load_direct, .load_tlv, .lea_symbol => unreachable, // TODO .immediate => |imm| .{ .immediate = imm }, .memory => |addr| .{ .memory = addr }, }, diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 4a3bdd604965b56e8049ce4f942dac3c75ec90f9..2adf2532c76e18599561ce7b043bfb0c2939910c 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1036,7 +1036,7 @@ pub fn updateFunc( const ip = &zcu.intern_pool; const gpa = elf_file.base.comp.gpa; const func = zcu.funcInfo(func_index); - if (elf_file.base.isRelocatable() and self.jumpTablePtr() == null) { + if (!elf_file.base.isRelocatable() and self.jumpTablePtr() == null) { try self.initJumpTable(gpa, elf_file); } -- 2.54.0 From f26573fddfcebe44d4d69ea0e01d345fc1b15835 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 14 Aug 2024 12:10:28 +0200 Subject: [PATCH 27/39] elf: re-use old atom slot for a trampoline to that atom This is the initial implementation of Jacob Young's idea of re-using old function slots as trampolines for new function's location. This way the trampoline is guaranteed to be aligned to the function's alignment. The only edge case is if an incremental update further overaligns the function in which case we skip/delete the trampoline and re-evaluate all references. --- src/link/Elf.zig | 5 +- src/link/Elf/Atom.zig | 9 +- src/link/Elf/Symbol.zig | 27 ++-- src/link/Elf/ZigObject.zig | 323 +++++++++++-------------------------- 4 files changed, 112 insertions(+), 252 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 50dfeffa172a8d39a1c61714be445b723e2e4561..1c2f90d14ccf86a10cb910f70a6866f2f44f620d 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -2798,7 +2798,7 @@ pub fn writeElfHeader(self: *Elf) !void { const e_entry: u64 = if (self.linkerDefinedPtr()) |obj| blk: { const entry_sym = obj.entrySymbol(self) orelse break :blk 0; - break :blk @intCast(entry_sym.address(.{ .zjt = true }, self)); + break :blk @intCast(entry_sym.address(.{}, self)); } else 0; const phdr_table_offset = if (self.phdr_table_index) |phndx| self.phdrs.items[phndx].p_offset else 0; switch (self.ptr_width) { @@ -5599,9 +5599,6 @@ fn fmtDumpState( zig_object.fmtAtoms(self), zig_object.fmtSymtab(self), }); - if (zig_object.jump_table) |jt| { - try writer.print("{}", .{jt.fmt(zig_object, self)}); - } try writer.writeByte('\n'); } diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 7338927ebd28589585741540720d9952031e8abd..47bb21608011c67987a6a863d8f93b52dbc3a9e7 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -746,8 +746,8 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi const P = self.address(elf_file) + @as(i64, @intCast(rel.r_offset)); // Addend from the relocation. const A = rel.r_addend; - // Address of the target symbol - can be address of the symbol within an atom or address of PLT stub. - const S = target.address(.{ .zjt = false }, elf_file); + // Address of the target symbol - can be address of the symbol within an atom or address of PLT stub, or address of a Zig trampoline. + const S = target.address(.{}, elf_file); // Address of the global offset table. const GOT = elf_file.gotAddress(); // Relative offset to the start of the global offset table. @@ -1212,10 +1212,7 @@ const x86_64 = struct { ); }, - .PLT32 => { - const S_ = if (target.flags.has_zjt) target.address(.{ .zjt = true }, elf_file) else S; - try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); - }, + .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .PC32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little), diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index d946e95511741c5ebe7192173a061179b6710705..4beb6a0918d6de4b53b017b85d8b9803b62b7716 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -101,7 +101,7 @@ pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 { return file_ptr.symbolRank(sym, in_archive); } -pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = false }, elf_file: *Elf) i64 { +pub fn address(symbol: Symbol, opts: struct { plt: bool = true, trampoline: bool = true }, elf_file: *Elf) i64 { if (symbol.mergeSubsection(elf_file)) |msub| { if (!msub.alive) return 0; return msub.address(elf_file) + symbol.value; @@ -109,8 +109,8 @@ pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = fals if (symbol.flags.has_copy_rel) { return symbol.copyRelAddress(elf_file); } - if (symbol.flags.has_zjt and opts.zjt) { - return symbol.zjtAddress(elf_file); + if (symbol.flags.has_trampoline and opts.trampoline) { + return symbol.trampolineAddress(elf_file); } if (symbol.flags.has_plt and opts.plt) { if (!symbol.flags.is_canonical and symbol.flags.has_got) { @@ -220,12 +220,11 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) i64 { return entry.address(elf_file); } -pub fn zjtAddress(symbol: Symbol, elf_file: *Elf) i64 { - if (!symbol.flags.has_zjt) return 0; +pub fn trampolineAddress(symbol: Symbol, elf_file: *Elf) i64 { + if (!symbol.flags.has_trampoline) return 0; const zo = elf_file.zigObjectPtr().?; - const jump_table = zo.jumpTablePtr().?; - const index = symbol.extra(elf_file).zjt; - return jump_table.entryAddress(index, zo, elf_file); + const index = symbol.extra(elf_file).trampoline; + return zo.symbol(index).address(.{}, elf_file); } pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 { @@ -251,7 +250,7 @@ const AddExtraOpts = struct { tlsgd: ?u32 = null, gottp: ?u32 = null, tlsdesc: ?u32 = null, - zjt: ?u32 = null, + trampoline: ?u32 = null, }; pub fn addExtra(symbol: *Symbol, opts: AddExtraOpts, elf_file: *Elf) void { @@ -304,7 +303,7 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void { const shdr = elf_file.shdrs.items[st_shndx]; if (shdr.sh_flags & elf.SHF_TLS != 0 and file_ptr != .linker_defined) break :blk symbol.address(.{ .plt = false }, elf_file) - elf_file.tlsAddress(); - break :blk symbol.address(.{ .plt = false }, elf_file); + break :blk symbol.address(.{ .plt = false, .trampoline = false }, elf_file); }; out.st_info = (st_bind << 4) | st_type; out.st_other = esym.st_other; @@ -380,7 +379,7 @@ fn format2( try writer.print("%{d} : {s} : @{x}", .{ symbol.esym_index, symbol.fmtName(elf_file), - symbol.address(.{ .plt = false }, elf_file), + symbol.address(.{ .plt = false, .trampoline = false }, elf_file), }); if (symbol.file(elf_file)) |file_ptr| { if (symbol.isAbs(elf_file)) { @@ -456,8 +455,8 @@ pub const Flags = packed struct { /// Whether the symbol is a merge subsection. merge_subsection: bool = false, - /// Whether the symbol has __zig_jump_table indirection. - has_zjt: bool = false, + /// Whether the symbol has a trampoline. + has_trampoline: bool = false, }; pub const Extra = struct { @@ -471,7 +470,7 @@ pub const Extra = struct { gottp: u32 = 0, tlsdesc: u32 = 0, merge_section: u32 = 0, - zjt: u32 = 0, + trampoline: u32 = 0, }; pub const Index = u32; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 2adf2532c76e18599561ce7b043bfb0c2939910c..a472c6fad0d8940de7b9cb9ca7904cd80adcda5e 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -55,11 +55,6 @@ debug_str_section_zig_size: u64 = 0, debug_aranges_section_zig_size: u64 = 0, debug_line_section_zig_size: u64 = 0, -/// Function jump table containing trampolines to Zcu functions. -/// The table is used for Zig's incremental compilation and is embedded with -/// the machine code section. -jump_table: ?JumpTable = null, - pub const global_symbol_bit: u32 = 0x80000000; pub const symbol_mask: u32 = 0x7fffffff; pub const SHN_ATOM: u16 = 0x100; @@ -127,10 +122,6 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void { if (self.dwarf) |*dw| { dw.deinit(); } - - if (self.jump_table) |*jt| { - jt.deinit(allocator); - } } pub fn flushModule(self: *ZigObject, elf_file: *Elf, tid: Zcu.PerThread.Id) !void { @@ -665,7 +656,7 @@ pub fn getNavVAddr( else => try self.getOrCreateMetadataForNav(elf_file, nav_index), }; const this_sym = self.symbol(this_sym_index); - const vaddr = this_sym.address(.{ .zjt = true }, elf_file); + const vaddr = this_sym.address(.{}, elf_file); const parent_atom = self.symbol(reloc_info.parent_atom_index).atom(elf_file).?; const r_type = relocation.encode(.abs, elf_file.getTarget().cpu.arch); try parent_atom.addReloc(elf_file, .{ @@ -914,12 +905,6 @@ fn updateNavCode( if (old_vaddr != atom_ptr.value) { sym.value = 0; esym.st_value = 0; - - if (stt_bits == elf.STT_FUNC) { - const extra = sym.extra(elf_file); - const jump_table = self.jumpTablePtr().?; - jump_table.entries.items(.dirty)[extra.zjt] = true; - } } } else if (code.len < old_size) { atom_ptr.shrink(elf_file); @@ -942,7 +927,7 @@ fn updateNavCode( .len = code.len, }}; var remote_vec: [1]std.posix.iovec_const = .{.{ - .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{ .zjt = true }, elf_file))))), + .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{}, elf_file))))), .len = code.len, }}; const rc = std.os.linux.process_vm_writev(pid, &code_vec, &remote_vec, 0); @@ -1036,31 +1021,12 @@ pub fn updateFunc( const ip = &zcu.intern_pool; const gpa = elf_file.base.comp.gpa; const func = zcu.funcInfo(func_index); - if (!elf_file.base.isRelocatable() and self.jumpTablePtr() == null) { - try self.initJumpTable(gpa, elf_file); - } log.debug("updateFunc {}({d})", .{ ip.getNav(func.owner_nav).fqn.fmt(ip), func.owner_nav }); const sym_index = try self.getOrCreateMetadataForNav(elf_file, func.owner_nav); self.symbol(sym_index).atom(elf_file).?.freeRelocs(elf_file); - if (self.jumpTablePtr()) |jump_table| { - const sym = self.symbol(sym_index); - if (!sym.flags.has_zjt) { - const index = try jump_table.addSymbol(gpa, sym_index); - sym.flags.has_zjt = true; - sym.addExtra(.{ .zjt = index }, elf_file); - try jump_table.updateSize(self, elf_file); - const old_vaddr = jump_table.address(self, elf_file); - try self.symbol(jump_table.sym_index).atom(elf_file).?.allocate(elf_file); - const new_vaddr = jump_table.address(self, elf_file); - if (old_vaddr != new_vaddr) { - jump_table.dirty = true; - } - } - } - var code_buffer = std.ArrayList(u8).init(gpa); defer code_buffer.deinit(); @@ -1087,14 +1053,22 @@ pub fn updateFunc( }; const shndx = try self.getNavShdrIndex(elf_file, zcu, func.owner_nav, code); + const old_rva, const old_alignment = blk: { + const atom_ptr = self.symbol(sym_index).atom(elf_file).?; + break :blk .{ atom_ptr.value, atom_ptr.alignment }; + }; try self.updateNavCode(elf_file, pt, func.owner_nav, sym_index, shndx, code, elf.STT_FUNC); + const new_rva, const new_alignment = blk: { + const atom_ptr = self.symbol(sym_index).atom(elf_file).?; + break :blk .{ atom_ptr.value, atom_ptr.alignment }; + }; if (dwarf_state) |*ds| { const sym = self.symbol(sym_index); try self.dwarf.?.commitNavState( pt, func.owner_nav, - @intCast(sym.address(.{ .zjt = true }, elf_file)), + @intCast(sym.address(.{}, elf_file)), sym.atom(elf_file).?.size, ds, ); @@ -1102,23 +1076,39 @@ pub fn updateFunc( // Exports will be updated by `Zcu.processExports` after the update. - if (self.jumpTablePtr()) |jump_table| { - if (jump_table.dirty) { - // TODO write in bulk - for (jump_table.entries.items(.dirty), 0..) |*dirty, i| { - try jump_table.writeEntry(@intCast(i), self, elf_file); - dirty.* = false; - } - } else { - const sym = self.symbol(sym_index); - const jt_index = sym.extra(elf_file).zjt; - var jt_entry = jump_table.entries.get(jt_index); - if (jt_entry.dirty) { - try jump_table.writeEntry(jt_index, self, elf_file); - jt_entry.dirty = false; - } - jump_table.entries.set(jt_index, jt_entry); + if (old_rva != new_rva and old_rva > 0) { + // If we had to reallocate the function, we re-use the existing slot for a trampoline. + // In the rare case that the function has been further overaligned we skip creating a + // trampoline and update all symbols referring this function. + if (old_alignment.order(new_alignment) == .lt) { + @panic("TODO update all symbols referring this function"); } + + // Create a trampoline to the new location at `old_rva`. + if (!self.symbol(sym_index).flags.has_trampoline) { + const name = try std.fmt.allocPrint(gpa, "{s}$trampoline", .{ + self.symbol(sym_index).name(elf_file), + }); + defer gpa.free(name); + const name_off = try self.addString(gpa, name); + const tr_size = trampolineSize(elf_file.getTarget().cpu.arch); + const tr_sym_index = try self.newSymbolWithAtom(gpa, name_off); + const tr_sym = self.symbol(tr_sym_index); + const tr_esym = &self.symtab.items(.elf_sym)[tr_sym.esym_index]; + tr_esym.st_info |= elf.STT_OBJECT; + tr_esym.st_size = tr_size; + const tr_atom_ptr = tr_sym.atom(elf_file).?; + tr_atom_ptr.value = old_rva; + tr_atom_ptr.alive = true; + tr_atom_ptr.alignment = old_alignment; + tr_atom_ptr.output_section_index = elf_file.zig_text_section_index.?; + tr_atom_ptr.size = tr_size; + const target_sym = self.symbol(sym_index); + target_sym.addExtra(.{ .trampoline = tr_sym_index }, elf_file); + target_sym.flags.has_trampoline = true; + } + const target_sym = self.symbol(sym_index); + try writeTrampoline(self.symbol(target_sym.extra(elf_file).trampoline).*, target_sym.*, elf_file); } } @@ -1193,7 +1183,7 @@ pub fn updateNav( try self.dwarf.?.commitNavState( pt, nav_index, - @intCast(sym.address(.{ .zjt = true }, elf_file)), + @intCast(sym.address(.{}, elf_file)), sym.atom(elf_file).?.size, ns, ); @@ -1474,21 +1464,50 @@ pub fn getGlobalSymbol(self: *ZigObject, elf_file: *Elf, name: []const u8, lib_n return lookup_gop.value_ptr.*; } -pub fn jumpTablePtr(self: *ZigObject) ?*JumpTable { - return if (self.jump_table) |*jt| jt else null; +const max_trampoline_len = 12; + +fn trampolineSize(cpu_arch: std.Target.Cpu.Arch) u64 { + const len = switch (cpu_arch) { + .x86_64 => 5, // jmp rel32 + else => @panic("TODO implement trampoline size for this CPU arch"), + }; + comptime assert(len <= max_trampoline_len); + return len; } -fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!void { - const name_off = try self.addString(allocator, "__zig_jump_table"); - const sym_index = try self.newSymbolWithAtom(allocator, name_off); - const sym = self.symbol(sym_index); - const esym = &self.symtab.items(.elf_sym)[sym.esym_index]; - esym.st_info |= elf.STT_OBJECT; - const atom_ptr = sym.atom(elf_file).?; - atom_ptr.alive = true; - atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(JumpTable.alignment(elf_file.getTarget().cpu.arch)); - atom_ptr.output_section_index = elf_file.zig_text_section_index.?; - self.jump_table = JumpTable{ .sym_index = sym_index }; +fn writeTrampoline(tr_sym: Symbol, target: Symbol, elf_file: *Elf) !void { + const atom_ptr = tr_sym.atom(elf_file).?; + const shdr = elf_file.shdrs.items[atom_ptr.output_section_index]; + const fileoff = shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)); + const source_addr = tr_sym.address(.{}, elf_file); + const target_addr = target.address(.{ .trampoline = false }, elf_file); + var buf: [max_trampoline_len]u8 = undefined; + const out = switch (elf_file.getTarget().cpu.arch) { + .x86_64 => try x86_64.writeTrampolineCode(source_addr, target_addr, &buf), + else => @panic("TODO implement write trampoline for this CPU arch"), + }; + try elf_file.base.file.?.pwriteAll(out, fileoff); + + if (elf_file.base.child_pid) |pid| { + switch (builtin.os.tag) { + .linux => { + var local_vec: [1]std.posix.iovec_const = .{.{ + .base = out.ptr, + .len = out.len, + }}; + var remote_vec: [1]std.posix.iovec_const = .{.{ + .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(source_addr)))), + .len = out.len, + }}; + const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); + switch (std.os.linux.E.init(rc)) { + .SUCCESS => assert(rc == out.len), + else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), + } + }, + else => return error.HotSwapUnavailableOnHostOperatingSystem, + } + } } pub fn asFile(self: *ZigObject) File { @@ -1766,169 +1785,17 @@ const UavTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, AvMetadata); const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata); const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); -pub const JumpTable = struct { - sym_index: Symbol.Index, - entries: std.MultiArrayList(Entry) = .{}, - dirty: bool = false, - - pub fn deinit(jt: *JumpTable, allocator: Allocator) void { - jt.entries.deinit(allocator); - } - - pub fn addSymbol(jt: *JumpTable, allocator: Allocator, sym_index: Symbol.Index) !Index { - const index: Index = @intCast(try jt.entries.addOne(allocator)); - jt.entries.set(index, .{ .sym_index = sym_index }); - return index; - } - - pub fn address(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) i64 { - const sym = zo.symbol(jt.sym_index); - return sym.address(.{}, elf_file); - } - - pub fn size(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) u64 { - const sym = zo.symbol(jt.sym_index); - return sym.atom(elf_file).?.size; - } - - pub fn alignment(cpu_arch: std.Target.Cpu.Arch) u64 { - return switch (cpu_arch) { - .x86_64 => 1, - else => @panic("TODO implement alignment for this CPU arch"), - }; - } - - pub fn entryAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { - return jt.address(zo, elf_file) + @as(i64, @intCast(index * entrySize(elf_file.getTarget().cpu.arch))); - } - - pub fn entryOffset(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 { - const sym = zo.symbol(jt.sym_index); - const atom_ptr = sym.atom(elf_file).?; - const shdr = elf_file.shdrs.items[atom_ptr.output_section_index]; - return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * entrySize(elf_file.getTarget().cpu.arch); - } - - pub fn entrySize(cpu_arch: std.Target.Cpu.Arch) u64 { - const seq_len = switch (cpu_arch) { - .x86_64 => 5, // jmp rel32 - else => @panic("TODO implement entry size for this CPU arch"), +const x86_64 = struct { + fn writeTrampolineCode(source_addr: i64, target_addr: i64, buf: *[max_trampoline_len]u8) ![]u8 { + const disp = @as(i64, @intCast(target_addr)) - source_addr - 5; + var bytes = [_]u8{ + 0xe9, 0x00, 0x00, 0x00, 0x00, // jmp rel32 }; - comptime assert(seq_len <= max_jump_seq_len); - return seq_len; - } - - pub fn targetAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { - const sym_index = jt.entries.items(.sym_index)[index]; - return zo.symbol(sym_index).address(.{}, elf_file); - } - - const max_jump_seq_len = 12; - - pub fn writeEntry(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void { - const fileoff = jt.entryOffset(index, zo, elf_file); - const source_addr = jt.entryAddress(index, zo, elf_file); - const target_addr = @as(i64, @intCast(jt.targetAddress(index, zo, elf_file))); - var buf: [max_jump_seq_len]u8 = undefined; - const out = switch (elf_file.getTarget().cpu.arch) { - .x86_64 => try x86_64.writeEntry(source_addr, target_addr, &buf), - else => @panic("TODO implement write entry for this CPU arch"), - }; - try elf_file.base.file.?.pwriteAll(out, fileoff); - - if (elf_file.base.child_pid) |pid| { - switch (builtin.os.tag) { - .linux => { - var local_vec: [1]std.posix.iovec_const = .{.{ - .base = out.ptr, - .len = out.len, - }}; - var remote_vec: [1]std.posix.iovec_const = .{.{ - .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(source_addr)))), - .len = out.len, - }}; - const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); - switch (std.os.linux.E.init(rc)) { - .SUCCESS => assert(rc == out.len), - else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), - } - }, - else => return error.HotSwapUnavailableOnHostOperatingSystem, - } - } - } - - pub fn updateSize(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) !void { - const jt_size: u64 = @intCast(jt.entries.items(.sym_index).len * entrySize(elf_file.getTarget().cpu.arch)); - const sym = zo.symbol(jt.sym_index); - const esym = &zo.symtab.items(.elf_sym)[sym.esym_index]; - esym.st_size = jt_size; - const atom_ptr = sym.atom(elf_file).?; - atom_ptr.size = jt_size; - } - - const JumpTableFormatContext = struct { JumpTable, *ZigObject, *Elf }; - - pub fn format( - jt: JumpTable, - comptime unused_fmt_string: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = jt; - _ = unused_fmt_string; - _ = options; - _ = writer; - @compileError("do not format JumpTable directly"); - } - - pub fn fmt(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) std.fmt.Formatter(format2) { - return .{ .data = .{ jt, zo, elf_file } }; - } - - fn format2( - ctx: JumpTableFormatContext, - comptime unused_fmt_string: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = options; - _ = unused_fmt_string; - const jt, const zo, const ef = ctx; - try writer.writeAll("__zig_jump_table\n"); - try writer.print(" @{x} : size({x})\n", .{ jt.address(zo, ef), jt.size(zo, ef) }); - for (jt.entries.items(.sym_index), jt.entries.items(.dirty)) |sym_index, dirty| { - const sym = zo.symbol(sym_index); - try writer.print(" {x} => {x} : %{d} : {s}", .{ - sym.address(.{ .zjt = true }, ef), - sym.address(.{}, ef), - sym_index, - sym.name(ef), - }); - if (dirty) try writer.writeAll(" : [!]"); - try writer.writeByte('\n'); - } + assert(bytes.len == trampolineSize(.x86_64)); + mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little); + @memcpy(buf[0..bytes.len], &bytes); + return buf[0..bytes.len]; } - - const Entry = struct { - sym_index: Symbol.Index, - dirty: bool = true, - }; - - pub const Index = u32; - - const x86_64 = struct { - fn writeEntry(source_addr: i64, target_addr: i64, buf: *[max_jump_seq_len]u8) ![]u8 { - const disp = @as(i64, @intCast(target_addr)) - source_addr - 5; - var bytes = [_]u8{ - 0xe9, 0x00, 0x00, 0x00, 0x00, // jmp rel32 - }; - assert(bytes.len == entrySize(.x86_64)); - mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little); - @memcpy(buf[0..bytes.len], &bytes); - return buf[0..bytes.len]; - } - }; }; const assert = std.debug.assert; -- 2.54.0 From b8203fac1b272d4f13f668797cf356d995b7967f Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 08:37:13 +0200 Subject: [PATCH 28/39] elf: check for relocs before deciding on shndx in getNavShdrIndex --- src/link/Elf.zig | 5 +++++ src/link/Elf/ZigObject.zig | 32 +++++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 1c2f90d14ccf86a10cb910f70a6866f2f44f620d..3dcfb0bc6aba719f693e9b62ef3210ab05934f5a 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -857,6 +857,11 @@ pub fn growAllocSection(self: *Elf, shdr_index: u32, needed_size: u64) !void { const shdr = &self.shdrs.items[shdr_index]; const maybe_phdr = if (self.phdr_to_shdr_table.get(shdr_index)) |phndx| &self.phdrs.items[phndx] else null; const is_zerofill = shdr.sh_type == elf.SHT_NOBITS; + log.debug("allocated size {x} of {s}, needed size {x}", .{ + self.allocatedSize(shdr.sh_offset), + self.getShString(shdr.sh_name), + needed_size, + }); if (needed_size > self.allocatedSize(shdr.sh_offset) and !is_zerofill) { const existing_size = shdr.sh_size; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index a472c6fad0d8940de7b9cb9ca7904cd80adcda5e..4c05c0b98557d9207e9d2dc6bafd076eaa66277d 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -816,9 +816,9 @@ fn getNavShdrIndex( elf_file: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index, + sym_index: Symbol.Index, code: []const u8, ) error{OutOfMemory}!u32 { - _ = self; const ip = &zcu.intern_pool; const any_non_single_threaded = elf_file.base.comp.config.any_non_single_threaded; const nav_val = zcu.navValue(nav_index); @@ -828,10 +828,12 @@ fn getNavShdrIndex( .@"extern" => |@"extern"| .{ @"extern".is_const, @"extern".is_threadlocal, .none }, else => .{ true, false, nav_val.toIntern() }, }; + const has_relocs = self.symbol(sym_index).atom(elf_file).?.relocs(elf_file).len > 0; if (any_non_single_threaded and is_threadlocal) { - for (code) |byte| { - if (byte != 0) break; - } else return elf_file.sectionByName(".tbss") orelse try elf_file.addSection(.{ + const is_bss = !has_relocs and for (code) |byte| { + if (byte != 0) break false; + } else true; + if (is_bss) return elf_file.sectionByName(".tbss") orelse try elf_file.addSection(.{ .type = elf.SHT_NOBITS, .flags = elf.SHF_ALLOC | elf.SHF_WRITE | elf.SHF_TLS, .name = try elf_file.insertShString(".tbss"), @@ -850,9 +852,10 @@ fn getNavShdrIndex( .Debug, .ReleaseSafe => elf_file.zig_data_section_index.?, .ReleaseFast, .ReleaseSmall => elf_file.zig_bss_section_index.?, }; - for (code) |byte| { - if (byte != 0) break; - } else return elf_file.zig_bss_section_index.?; + const is_bss = !has_relocs and for (code) |byte| { + if (byte != 0) break false; + } else true; + if (is_bss) return elf_file.zig_bss_section_index.?; return elf_file.zig_data_section_index.?; } @@ -944,6 +947,7 @@ fn updateNavCode( if (shdr.sh_type != elf.SHT_NOBITS) { const file_offset = shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)); try elf_file.base.file.?.pwriteAll(code, file_offset); + log.debug("writing {} from 0x{x} to 0x{x}", .{ nav.fqn.fmt(ip), file_offset, file_offset + code.len }); } } @@ -1052,7 +1056,12 @@ pub fn updateFunc( }, }; - const shndx = try self.getNavShdrIndex(elf_file, zcu, func.owner_nav, code); + const shndx = try self.getNavShdrIndex(elf_file, zcu, func.owner_nav, sym_index, code); + log.debug("setting shdr({x},{s}) for {}", .{ + shndx, + elf_file.getShString(elf_file.shdrs.items[shndx].sh_name), + ip.getNav(func.owner_nav).fqn.fmt(ip), + }); const old_rva, const old_alignment = blk: { const atom_ptr = self.symbol(sym_index).atom(elf_file).?; break :blk .{ atom_ptr.value, atom_ptr.alignment }; @@ -1172,7 +1181,12 @@ pub fn updateNav( }, }; - const shndx = try self.getNavShdrIndex(elf_file, zcu, nav_index, code); + const shndx = try self.getNavShdrIndex(elf_file, zcu, nav_index, sym_index, code); + log.debug("setting shdr({x},{s}) for {}", .{ + shndx, + elf_file.getShString(elf_file.shdrs.items[shndx].sh_name), + nav.fqn.fmt(ip), + }); if (elf_file.shdrs.items[shndx].sh_flags & elf.SHF_TLS != 0) try self.updateTlv(elf_file, pt, nav_index, sym_index, shndx, code) else -- 2.54.0 From c9db8878006fe53eb438bc6a6f2d2416cc5ef412 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 08:38:08 +0200 Subject: [PATCH 29/39] builtin: init test_functions to empty slice to avoid coercing to undefined --- src/Builtin.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Builtin.zig b/src/Builtin.zig index 6e573d843f5f5b2987c5455ac7688a5b79912816..58abcb7a4bec8f873e084b4b9628246d0d423640 100644 --- a/src/Builtin.zig +++ b/src/Builtin.zig @@ -218,7 +218,7 @@ pub fn append(opts: @This(), buffer: *std.ArrayList(u8)) Allocator.Error!void { if (opts.is_test) { try buffer.appendSlice( - \\pub var test_functions: []const std.builtin.TestFn = undefined; // overwritten later + \\pub var test_functions: []const std.builtin.TestFn = &.{}; // overwritten later \\ ); } -- 2.54.0 From 2e8351cc9e30c28a9415821ed99d664b76eb2062 Mon Sep 17 00:00:00 2001 From: David Rubin Date: Wed, 14 Aug 2024 17:17:13 -0700 Subject: [PATCH 30/39] elf: fix up riscv for `.got.zig` rewrite --- src/arch/riscv64/CodeGen.zig | 8 ++++---- test/behavior/basic.zig | 1 - test/behavior/export_builtin.zig | 2 -- test/behavior/extern.zig | 1 - test/behavior/fn.zig | 1 - test/behavior/pointers.zig | 1 - 6 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index e9f593485f910f7672e991a3ba94213fd71590fb..6531bf34d99c7fc2ce6d7a3f04fb08e9c2521d82 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -4937,7 +4937,7 @@ fn genCall( if (func.mod.pic) { return func.fail("TODO: genCall pic", .{}); } else { - try func.genSetReg(Type.u64, .ra, .{ .load_symbol = .{ .sym = sym_index } }); + try func.genSetReg(Type.u64, .ra, .{ .lea_symbol = .{ .sym = sym_index } }); _ = try func.addInst(.{ .tag = .jalr, .data = .{ .i_type = .{ @@ -6120,7 +6120,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { arg_map.get(op_str["%[".len .. mod_index orelse op_str.len - "]".len]) orelse return func.fail("no matching constraint: '{s}'", .{op_str}) ]) { - .load_symbol => |sym_off| if (mem.eql(u8, modifier, "plt")) blk: { + .lea_symbol => |sym_off| if (mem.eql(u8, modifier, "plt")) blk: { assert(sym_off.off == 0); break :blk .{ .sym = sym_off }; } else return func.fail("invalid modifier: '{s}'", .{modifier}), @@ -6388,7 +6388,7 @@ fn genCopy(func: *Func, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { ty, src_mcv, ), - .load_tlv => { + .load_symbol, .load_tlv => { const addr_reg, const addr_lock = try func.allocReg(.int); defer func.register_manager.unlockReg(addr_lock); @@ -6433,7 +6433,7 @@ fn genCopy(func: *Func, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { part_disp += @intCast(dst_ty.abiSize(func.pt)); } }, - else => return func.fail("TODO: genCopy to {s} from {s}", .{ @tagName(dst_mcv), @tagName(src_mcv) }), + else => return std.debug.panic("TODO: genCopy to {s} from {s}", .{ @tagName(dst_mcv), @tagName(src_mcv) }), } } diff --git a/test/behavior/basic.zig b/test/behavior/basic.zig index 90d12e6858bf18a39000f96f5f943b8c67ee3ac0..9d3474260468c7985b8507e4161f82869755cf67 100644 --- a/test/behavior/basic.zig +++ b/test/behavior/basic.zig @@ -733,7 +733,6 @@ test "extern variable with non-pointer opaque type" { if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest; if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; @export(var_to_export, .{ .name = "opaque_extern_var" }); try expect(@as(*align(1) u32, @ptrCast(&opaque_extern_var)).* == 42); diff --git a/test/behavior/export_builtin.zig b/test/behavior/export_builtin.zig index 547a9b990af8884ef5cae625427b9fa37a64bdf6..25b6e2527ec3475876bff70c9b84d8e3c5868cd2 100644 --- a/test/behavior/export_builtin.zig +++ b/test/behavior/export_builtin.zig @@ -48,7 +48,6 @@ test "exporting using field access" { test "exporting comptime-known value" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_x86_64 and (builtin.target.ofmt != .elf and builtin.target.ofmt != .macho and @@ -68,7 +67,6 @@ test "exporting comptime-known value" { test "exporting comptime var" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_x86_64 and (builtin.target.ofmt != .elf and builtin.target.ofmt != .macho and diff --git a/test/behavior/extern.zig b/test/behavior/extern.zig index cd80c545ce145fc8df3dc3de8420007b6ce3f1f1..0ef3e49353372d3599f6ce2a94db6329f0b488c6 100644 --- a/test/behavior/extern.zig +++ b/test/behavior/extern.zig @@ -7,7 +7,6 @@ test "anyopaque extern symbol" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest; if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; const a = @extern(*anyopaque, .{ .name = "a_mystery_symbol" }); const b: *i32 = @alignCast(@ptrCast(a)); diff --git a/test/behavior/fn.zig b/test/behavior/fn.zig index 1e4039c1bb0741b42656375f2d5bb5f8b255da93..ab7aca6ed6b464d8524d5ba38303d27be091c049 100644 --- a/test/behavior/fn.zig +++ b/test/behavior/fn.zig @@ -429,7 +429,6 @@ test "implicit cast function to function ptr" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; const S1 = struct { export fn someFunctionThatReturnsAValue() c_int { diff --git a/test/behavior/pointers.zig b/test/behavior/pointers.zig index 42e3ea0ae970093e9f838beb12a1d04202344d6a..f8260d1f0d83237e50fc45b3c4e787bec31a1c6a 100644 --- a/test/behavior/pointers.zig +++ b/test/behavior/pointers.zig @@ -45,7 +45,6 @@ test "pointer-integer arithmetic" { test "pointer subtraction" { if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; { const a: *u8 = @ptrFromInt(100); -- 2.54.0 From 8a0cb7002e6b8f7b50fdf7ee40311ce93fbef009 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 10:05:41 +0200 Subject: [PATCH 31/39] elf: introduce Symbol.flags.is_extern_ptr for refs potentially needing GOT --- src/arch/x86_64/Emit.zig | 4 ++-- src/codegen.zig | 3 +-- src/link/Elf/Symbol.zig | 12 +++++++----- src/link/Elf/ZigObject.zig | 3 +-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index b7d0a29c2dc67dcd649ed456c0a66a88e8cc6f68..fd1389eb3ba715141309130f6be4dbc6ad388f5a 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -114,7 +114,7 @@ pub fn emitMir(emit: *Emit) Error!void { const atom = zo.symbol(data.atom_index).atom(elf_file).?; const sym = zo.symbol(data.sym_index); if (emit.lower.pic) { - const r_type: u32 = if (sym.flags.needs_got) + const r_type: u32 = if (sym.flags.is_extern_ptr) @intFromEnum(std.elf.R_X86_64.GOTPCREL) else @intFromEnum(std.elf.R_X86_64.PC32); @@ -124,7 +124,7 @@ pub fn emitMir(emit: *Emit) Error!void { .r_addend = -4, }); } else { - const r_type: u32 = if (sym.flags.needs_got) + const r_type: u32 = if (sym.flags.is_extern_ptr) @intFromEnum(std.elf.R_X86_64.GOT32) else if (sym.flags.is_tls) @intFromEnum(std.elf.R_X86_64.TPOFF32) diff --git a/src/codegen.zig b/src/codegen.zig index 679ee03c27d533869e70d98dc01eb982b2daaeb2..3df7ce37aba51dd336f75cd2c4c82de536ed04cb 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -898,9 +898,8 @@ fn genNavRef( if (lf.cast(.elf)) |elf_file| { const zo = elf_file.zigObjectPtr().?; if (is_extern) { - // TODO audit this const sym_index = try elf_file.getGlobalSymbol(name.toSlice(ip), lib_name.toSlice(ip)); - zo.symbol(sym_index).flags.needs_got = true; + zo.symbol(sym_index).flags.is_extern_ptr = true; return GenResult.mcv(.{ .load_symbol = sym_index }); } const sym_index = try zo.getOrCreateMetadataForNav(elf_file, nav_index); diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index 4beb6a0918d6de4b53b017b85d8b9803b62b7716..4f6c2b8c7e34c98162f8d1c7a01baa038d5a8fee 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -447,16 +447,18 @@ pub const Flags = packed struct { needs_tlsdesc: bool = false, has_tlsdesc: bool = false, - /// Whether the symbol is a TLS variable. - /// TODO this is really not needed if only we operated on esyms between - /// codegen and ZigObject. - is_tls: bool = false, - /// Whether the symbol is a merge subsection. merge_subsection: bool = false, + /// ZigObject specific flags /// Whether the symbol has a trampoline. has_trampoline: bool = false, + + /// Whether the symbol is a TLS variable. + is_tls: bool = false, + + /// Whether the symbol is an extern pointer (as opposed to function). + is_extern_ptr: bool = false, }; pub const Extra = struct { diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 4c05c0b98557d9207e9d2dc6bafd076eaa66277d..b72946c0f9ab81305723b37f470be7cfbf3d1f9b 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1141,13 +1141,12 @@ pub fn updateNav( .variable => |variable| Value.fromInterned(variable.init), .@"extern" => |@"extern"| { if (ip.isFunctionType(@"extern".ty)) return; - // Extern variable gets a .got entry only. const sym_index = try self.getGlobalSymbol( elf_file, nav.name.toSlice(ip), @"extern".lib_name.toSlice(ip), ); - self.symbol(sym_index).flags.needs_got = true; + self.symbol(sym_index).flags.is_extern_ptr = true; return; }, else => nav_val, -- 2.54.0 From 0fd0b11bc43a6053c68f649e1e1a1e23b995a266 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 10:21:20 +0200 Subject: [PATCH 32/39] riscv: do not emit GOT relocations for special linker symbols --- src/arch/riscv64/Emit.zig | 15 +++++++++------ src/codegen.zig | 2 +- src/link/Elf.zig | 27 --------------------------- src/link/Elf/Atom.zig | 25 ++----------------------- src/link/Elf/relocation.zig | 14 +++++--------- 5 files changed, 17 insertions(+), 66 deletions(-) diff --git a/src/arch/riscv64/Emit.zig b/src/arch/riscv64/Emit.zig index c1e7565c0bb5426c738a969d018b0efb66e02c87..8b32f1e5809172e2d657fc879ca97f54d44295d3 100644 --- a/src/arch/riscv64/Emit.zig +++ b/src/arch/riscv64/Emit.zig @@ -49,14 +49,17 @@ pub fn emitMir(emit: *Emit) Error!void { const atom_ptr = zo.symbol(symbol.atom_index).atom(elf_file).?; const sym = zo.symbol(symbol.sym_index); - var hi_r_type: u32 = @intFromEnum(std.elf.R_RISCV.HI20); - var lo_r_type: u32 = @intFromEnum(std.elf.R_RISCV.LO12_I); - - if (sym.flags.needs_got) { - hi_r_type = Elf.R_GOT_HI20_STATIC; // TODO: rework this #20887 - lo_r_type = Elf.R_GOT_LO12_I_STATIC; // TODO: rework this #20887 + if (sym.flags.is_extern_ptr) blk: { + const name = sym.name(elf_file); + if (mem.eql(u8, "__init_array_start", name) or mem.eql(u8, "__init_array_end", name) or + mem.eql(u8, "__fini_array_start", name) or mem.eql(u8, "__fini_array_end", name)) + break :blk; + return emit.fail("emit GOT relocation for symbol '{s}'", .{name}); } + const hi_r_type: u32 = @intFromEnum(std.elf.R_RISCV.HI20); + const lo_r_type: u32 = @intFromEnum(std.elf.R_RISCV.LO12_I); + try atom_ptr.addReloc(elf_file, .{ .r_offset = start_offset, .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | hi_r_type, diff --git a/src/codegen.zig b/src/codegen.zig index 3df7ce37aba51dd336f75cd2c4c82de536ed04cb..89b4a8a29174d0342751282d8ca1f600ec3efc23 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -900,7 +900,7 @@ fn genNavRef( if (is_extern) { const sym_index = try elf_file.getGlobalSymbol(name.toSlice(ip), lib_name.toSlice(ip)); zo.symbol(sym_index).flags.is_extern_ptr = true; - return GenResult.mcv(.{ .load_symbol = sym_index }); + return GenResult.mcv(.{ .lea_symbol = sym_index }); } const sym_index = try zo.getOrCreateMetadataForNav(elf_file, nav_index); if (!single_threaded and is_threadlocal) { diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 3dcfb0bc6aba719f693e9b62ef3210ab05934f5a..0ef1e9862d4acd820c80c4d49900b41dbc71f522 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -5934,33 +5934,6 @@ const RelaSection = struct { }; const RelaSectionTable = std.AutoArrayHashMapUnmanaged(u32, RelaSection); -pub const R_GOT_HI20_STATIC: u32 = 0xff04; -pub const R_GOT_LO12_I_STATIC: u32 = 0xff05; - -// Comptime asserts that no Zig relocs overlap with another ISA's reloc number -comptime { - const zig_relocs = .{ - R_GOT_HI20_STATIC, - R_GOT_LO12_I_STATIC, - }; - - const other_relocs = .{ - elf.R_X86_64, - elf.R_AARCH64, - elf.R_RISCV, - elf.R_PPC64, - }; - - @setEvalBranchQuota(@min(other_relocs.len * zig_relocs.len * 256, 6200)); - for (other_relocs) |relocs| { - for (@typeInfo(relocs).Enum.fields) |reloc| { - for (zig_relocs) |zig_reloc| { - assert(reloc.value != zig_reloc); - } - } - } -} - fn defaultEntrySymbolName(cpu_arch: std.Target.Cpu.Arch) []const u8 { return switch (cpu_arch) { .mips, .mipsel, .mips64, .mips64el => "__start", diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 47bb21608011c67987a6a863d8f93b52dbc3a9e7..c214b0791413f8249ff8f494047ae6ee6385cc2d 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1978,13 +1978,7 @@ const riscv = struct { .SUB32, => {}, - else => |x| switch (@intFromEnum(x)) { - Elf.R_GOT_HI20_STATIC, - Elf.R_GOT_LO12_I_STATIC, - => symbol.flags.needs_got = true, - - else => try atom.reportUnhandledRelocError(rel, elf_file), - }, + else => try atom.reportUnhandledRelocError(rel, elf_file), } } @@ -2121,22 +2115,7 @@ const riscv = struct { // TODO: annotates an ADD instruction that can be removed when TPREL is relaxed }, - else => |x| switch (@intFromEnum(x)) { - // Zig custom relocations - Elf.R_GOT_HI20_STATIC => { - assert(target.flags.has_got); - const disp: u32 = @bitCast(math.cast(i32, G + GOT + A) orelse return error.Overflow); - riscv_util.writeInstU(code[r_offset..][0..4], disp); - }, - - Elf.R_GOT_LO12_I_STATIC => { - assert(target.flags.has_got); - const disp: u32 = @bitCast(math.cast(i32, G + GOT + A) orelse return error.Overflow); - riscv_util.writeInstI(code[r_offset..][0..4], disp); - }, - - else => try atom.reportUnhandledRelocError(rel, elf_file), - }, + else => try atom.reportUnhandledRelocError(rel, elf_file), } } diff --git a/src/link/Elf/relocation.zig b/src/link/Elf/relocation.zig index 887aece8bc96d1c06294de24e743eca03ed77d1e..e1a317f67d466f34ebd31ce40f03a18d5f360d49 100644 --- a/src/link/Elf/relocation.zig +++ b/src/link/Elf/relocation.zig @@ -112,15 +112,11 @@ fn formatRelocType( _ = unused_fmt_string; _ = options; const r_type = ctx.r_type; - switch (r_type) { - Elf.R_GOT_HI20_STATIC => try writer.writeAll("R_GOT_HI20_STATIC"), - Elf.R_GOT_LO12_I_STATIC => try writer.writeAll("R_GOT_LO12_I_STATIC"), - else => switch (ctx.cpu_arch) { - .x86_64 => try writer.print("R_X86_64_{s}", .{@tagName(@as(elf.R_X86_64, @enumFromInt(r_type)))}), - .aarch64 => try writer.print("R_AARCH64_{s}", .{@tagName(@as(elf.R_AARCH64, @enumFromInt(r_type)))}), - .riscv64 => try writer.print("R_RISCV_{s}", .{@tagName(@as(elf.R_RISCV, @enumFromInt(r_type)))}), - else => unreachable, - }, + switch (ctx.cpu_arch) { + .x86_64 => try writer.print("R_X86_64_{s}", .{@tagName(@as(elf.R_X86_64, @enumFromInt(r_type)))}), + .aarch64 => try writer.print("R_AARCH64_{s}", .{@tagName(@as(elf.R_AARCH64, @enumFromInt(r_type)))}), + .riscv64 => try writer.print("R_RISCV_{s}", .{@tagName(@as(elf.R_RISCV, @enumFromInt(r_type)))}), + else => unreachable, } } -- 2.54.0 From f0df0acd70637eddbec559df20e8ea11429c26ed Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 10:33:30 +0200 Subject: [PATCH 33/39] x86_64: fix handling on externs in lower/emit --- src/arch/x86_64/Emit.zig | 4 +--- src/arch/x86_64/Lower.zig | 9 ++++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index fd1389eb3ba715141309130f6be4dbc6ad388f5a..693dca1da65c0efff679b520fd2054c1a8bf8dd3 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -124,9 +124,7 @@ pub fn emitMir(emit: *Emit) Error!void { .r_addend = -4, }); } else { - const r_type: u32 = if (sym.flags.is_extern_ptr) - @intFromEnum(std.elf.R_X86_64.GOT32) - else if (sym.flags.is_tls) + const r_type: u32 = if (sym.flags.is_tls) @intFromEnum(std.elf.R_X86_64.TPOFF32) else @intFromEnum(std.elf.R_X86_64.@"32"); diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 56cd2669d67439670f63495204b3a37668030894..4fab1b806d3a811fdc32c1b16852a085e9a0672f 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -397,8 +397,11 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) } _ = lower.reloc(.{ .linker_reloc = sym }); - break :op if (lower.pic) switch (mnemonic) { - .lea => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, + if (lower.pic) switch (mnemonic) { + .lea => { + if (elf_sym.flags.is_extern_ptr) emit_mnemonic = .mov; + break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }; + }, .mov => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, else => unreachable, } else switch (mnemonic) { @@ -413,7 +416,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) .base = .{ .reg = .ds }, }) }, else => unreachable, - }; + } } else if (lower.bin_file.cast(.macho)) |macho_file| { const zo = macho_file.getZigObject().?; const macho_sym = zo.symbols.items[sym.sym_index]; -- 2.54.0 From c79290467f4908c1fc4833d82d397829e92cf838 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 10:36:37 +0200 Subject: [PATCH 34/39] elf: fix GOT32 reloc resolution logic --- src/link/Elf/Atom.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index c214b0791413f8249ff8f494047ae6ee6385cc2d..fe31712f30becfa8032d76c658d13624185197fb 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1314,7 +1314,7 @@ const x86_64 = struct { } }, - .GOT32 => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A)), .little), + .GOT32 => try cwriter.writeInt(i32, @as(i32, @intCast(G + A)), .little), else => try atom.reportUnhandledRelocError(rel, elf_file), } -- 2.54.0 From 79418fa0abe4b19a2fd2fb10cf3c1144b927d10f Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 10:52:06 +0200 Subject: [PATCH 35/39] riscv: remove redundant by-symbol-name check; just check for PIC and extern ptr --- src/arch/riscv64/Emit.zig | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/arch/riscv64/Emit.zig b/src/arch/riscv64/Emit.zig index 8b32f1e5809172e2d657fc879ca97f54d44295d3..a4fadad5265531411629e2a9398d4fe17a6c596d 100644 --- a/src/arch/riscv64/Emit.zig +++ b/src/arch/riscv64/Emit.zig @@ -49,12 +49,8 @@ pub fn emitMir(emit: *Emit) Error!void { const atom_ptr = zo.symbol(symbol.atom_index).atom(elf_file).?; const sym = zo.symbol(symbol.sym_index); - if (sym.flags.is_extern_ptr) blk: { - const name = sym.name(elf_file); - if (mem.eql(u8, "__init_array_start", name) or mem.eql(u8, "__init_array_end", name) or - mem.eql(u8, "__fini_array_start", name) or mem.eql(u8, "__fini_array_end", name)) - break :blk; - return emit.fail("emit GOT relocation for symbol '{s}'", .{name}); + if (sym.flags.is_extern_ptr and emit.lower.pic) { + return emit.fail("emit GOT relocation for symbol '{s}'", .{sym.name(elf_file)}); } const hi_r_type: u32 = @intFromEnum(std.elf.R_RISCV.HI20); -- 2.54.0 From 4d5bf0f09a29eeb0eb7f3602825f1a0095b8427e Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 14:23:36 +0200 Subject: [PATCH 36/39] x86_64: deref GOT pointer when requesting var value --- src/arch/x86_64/Lower.zig | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 4fab1b806d3a811fdc32c1b16852a085e9a0672f..15322ce4f6b602c206f3ecd9519ad0ad4bca0588 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -402,7 +402,21 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) if (elf_sym.flags.is_extern_ptr) emit_mnemonic = .mov; break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }; }, - .mov => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, + .mov => { + if (elf_sym.flags.is_extern_ptr) { + const reg = ops[0].reg; + lower.result_insts[lower.result_insts_len] = + try Instruction.new(.none, .mov, &[_]Operand{ + .{ .reg = reg.to64() }, + .{ .mem = Memory.rip(.qword, 0) }, + }); + lower.result_insts_len += 1; + break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ .base = .{ + .reg = reg.to64(), + } }) }; + } + break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }; + }, else => unreachable, } else switch (mnemonic) { .call => break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ -- 2.54.0 From 9473d764498fbb4eef3e2bec42e44a3f93a5a56b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 21:47:34 +0200 Subject: [PATCH 37/39] test/elf: enhance testImportingDataDynamic --- test/link/elf.zig | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/test/link/elf.zig b/test/link/elf.zig index 71b722a9f983fef707652ac26117c516c7252007..5539638ba7b60f569143cf9fad4a1ac3765a8f03 100644 --- a/test/link/elf.zig +++ b/test/link/elf.zig @@ -1773,25 +1773,41 @@ fn testImportingDataDynamic(b: *Build, opts: Options) *Step { .use_llvm = true, }, .{ .name = "a", - .c_source_bytes = "int foo = 42;", + .c_source_bytes = + \\#include + \\int foo = 42; + \\void printFoo() { fprintf(stderr, "lib foo=%d\n", foo); } + , }); + dso.linkLibC(); const main = addExecutable(b, opts, .{ .name = "main", .zig_source_bytes = + \\const std = @import("std"); \\extern var foo: i32; + \\extern fn printFoo() void; \\pub fn main() void { - \\ @import("std").debug.print("{d}\n", .{foo}); + \\ std.debug.print("exe foo={d}\n", .{foo}); + \\ printFoo(); + \\ foo += 1; + \\ std.debug.print("exe foo={d}\n", .{foo}); + \\ printFoo(); \\} , .strip = true, // TODO temp hack }); main.pie = true; main.linkLibrary(dso); - main.linkLibC(); const run = addRunArtifact(main); - run.expectStdErrEqual("42\n"); + run.expectStdErrEqual( + \\exe foo=42 + \\lib foo=42 + \\exe foo=43 + \\lib foo=43 + \\ + ); test_step.dependOn(&run.step); return test_step; -- 2.54.0 From 624016e8f39f291e92bb2e0319b2cdf2ce4b15c9 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Fri, 16 Aug 2024 05:42:50 -0400 Subject: [PATCH 38/39] riscv64: fix incorrect branch target --- src/arch/riscv64/CodeGen.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index 6531bf34d99c7fc2ce6d7a3f04fb08e9c2521d82..942c56249652d5eeb39aa318f17f59bc24826a7f 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -6594,7 +6594,7 @@ fn genInlineMemset( .tag = .beq, .data = .{ .b_type = .{ - .inst = @intCast(func.mir_instructions.len + 4), // points after the last inst + .inst = @intCast(func.mir_instructions.len + 3), // points after the last inst .rs1 = count, .rs2 = .zero, }, -- 2.54.0 From 73f385eec57d4fc28069b60f059fa8553d3a0c1b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 16 Aug 2024 11:49:23 +0200 Subject: [PATCH 39/39] Update src/arch/x86_64/CodeGen.zig Co-authored-by: Jacob Young --- src/arch/x86_64/CodeGen.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 1edf208d4c9c3f56b171c21d4f14e09c29e9b03d..bfcc29348cb9440f71a3d1926197ce4dca65c3bb 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1382,7 +1382,10 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void { .reloc => .rel, }, .data = switch (imm) { - .reloc => |x| .{ .reloc = x }, + .reloc => |x| reloc: { + assert(tag[0] == ._); + break :reloc .{ .reloc = x }; + }, .signed, .unsigned => .{ .i = .{ .fixes = tag[0], .i = switch (imm) { -- 2.54.0