authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2024-08-14 12:10:28+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2024-08-14 12:10:40+02:00
logf26573fddfcebe44d4d69ea0e01d345fc1b15835
tree9dbe0f94673f8df0f3d3b0c939db2ffa70dbcad8
parent1bd54a55fa40db9f91d8a0c1cf5244ff51be6081

elf: re-use old atom slot for a trampoline to that atom

This is the initial implementation of Jacob Young's idea of re-using old function slots as trampolines for new function's location. This way the trampoline is guaranteed to be aligned to the function's alignment. The only edge case is if an incremental update further overaligns the function in which case we skip/delete the trampoline and re-evaluate all references.

4 files changed, 113 insertions(+), 253 deletions(-)

src/link/Elf.zig+1-4
...@@ -2798,7 +2798,7 @@ pub fn writeElfHeader(self: *Elf) !void {...@@ -2798,7 +2798,7 @@ pub fn writeElfHeader(self: *Elf) !void {
27982798
2799 const e_entry: u64 = if (self.linkerDefinedPtr()) |obj| blk: {2799 const e_entry: u64 = if (self.linkerDefinedPtr()) |obj| blk: {
2800 const entry_sym = obj.entrySymbol(self) orelse break :blk 0;2800 const entry_sym = obj.entrySymbol(self) orelse break :blk 0;
2801 break :blk @intCast(entry_sym.address(.{ .zjt = true }, self));2801 break :blk @intCast(entry_sym.address(.{}, self));
2802 } else 0;2802 } else 0;
2803 const phdr_table_offset = if (self.phdr_table_index) |phndx| self.phdrs.items[phndx].p_offset else 0;2803 const phdr_table_offset = if (self.phdr_table_index) |phndx| self.phdrs.items[phndx].p_offset else 0;
2804 switch (self.ptr_width) {2804 switch (self.ptr_width) {
...@@ -5599,9 +5599,6 @@ fn fmtDumpState(...@@ -5599,9 +5599,6 @@ fn fmtDumpState(
5599 zig_object.fmtAtoms(self),5599 zig_object.fmtAtoms(self),
5600 zig_object.fmtSymtab(self),5600 zig_object.fmtSymtab(self),
5601 });5601 });
5602 if (zig_object.jump_table) |jt| {
5603 try writer.print("{}", .{jt.fmt(zig_object, self)});
5604 }
5605 try writer.writeByte('\n');5602 try writer.writeByte('\n');
5606 }5603 }
56075604
src/link/Elf/Atom.zig+3-6
...@@ -746,8 +746,8 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi...@@ -746,8 +746,8 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi
746 const P = self.address(elf_file) + @as(i64, @intCast(rel.r_offset));746 const P = self.address(elf_file) + @as(i64, @intCast(rel.r_offset));
747 // Addend from the relocation.747 // Addend from the relocation.
748 const A = rel.r_addend;748 const A = rel.r_addend;
749 // Address of the target symbol - can be address of the symbol within an atom or address of PLT stub.749 // Address of the target symbol - can be address of the symbol within an atom or address of PLT stub, or address of a Zig trampoline.
750 const S = target.address(.{ .zjt = false }, elf_file);750 const S = target.address(.{}, elf_file);
751 // Address of the global offset table.751 // Address of the global offset table.
752 const GOT = elf_file.gotAddress();752 const GOT = elf_file.gotAddress();
753 // Relative offset to the start of the global offset table.753 // Relative offset to the start of the global offset table.
...@@ -1212,10 +1212,7 @@ const x86_64 = struct {...@@ -1212,10 +1212,7 @@ const x86_64 = struct {
1212 );1212 );
1213 },1213 },
12141214
1215 .PLT32 => {1215 .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little),
1216 const S_ = if (target.flags.has_zjt) target.address(.{ .zjt = true }, elf_file) else S;
1217 try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little);
1218 },
1219 .PC32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little),1216 .PC32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little),
12201217
1221 .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little),1218 .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little),
src/link/Elf/Symbol.zig+13-14
...@@ -101,7 +101,7 @@ pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 {...@@ -101,7 +101,7 @@ pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 {
101 return file_ptr.symbolRank(sym, in_archive);101 return file_ptr.symbolRank(sym, in_archive);
102}102}
103103
104pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = false }, elf_file: *Elf) i64 {104pub fn address(symbol: Symbol, opts: struct { plt: bool = true, trampoline: bool = true }, elf_file: *Elf) i64 {
105 if (symbol.mergeSubsection(elf_file)) |msub| {105 if (symbol.mergeSubsection(elf_file)) |msub| {
106 if (!msub.alive) return 0;106 if (!msub.alive) return 0;
107 return msub.address(elf_file) + symbol.value;107 return msub.address(elf_file) + symbol.value;
...@@ -109,8 +109,8 @@ pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = fals...@@ -109,8 +109,8 @@ pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = fals
109 if (symbol.flags.has_copy_rel) {109 if (symbol.flags.has_copy_rel) {
110 return symbol.copyRelAddress(elf_file);110 return symbol.copyRelAddress(elf_file);
111 }111 }
112 if (symbol.flags.has_zjt and opts.zjt) {112 if (symbol.flags.has_trampoline and opts.trampoline) {
113 return symbol.zjtAddress(elf_file);113 return symbol.trampolineAddress(elf_file);
114 }114 }
115 if (symbol.flags.has_plt and opts.plt) {115 if (symbol.flags.has_plt and opts.plt) {
116 if (!symbol.flags.is_canonical and symbol.flags.has_got) {116 if (!symbol.flags.is_canonical and symbol.flags.has_got) {
...@@ -220,12 +220,11 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) i64 {...@@ -220,12 +220,11 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) i64 {
220 return entry.address(elf_file);220 return entry.address(elf_file);
221}221}
222222
223pub fn zjtAddress(symbol: Symbol, elf_file: *Elf) i64 {223pub fn trampolineAddress(symbol: Symbol, elf_file: *Elf) i64 {
224 if (!symbol.flags.has_zjt) return 0;224 if (!symbol.flags.has_trampoline) return 0;
225 const zo = elf_file.zigObjectPtr().?;225 const zo = elf_file.zigObjectPtr().?;
226 const jump_table = zo.jumpTablePtr().?;226 const index = symbol.extra(elf_file).trampoline;
227 const index = symbol.extra(elf_file).zjt;227 return zo.symbol(index).address(.{}, elf_file);
228 return jump_table.entryAddress(index, zo, elf_file);
229}228}
230229
231pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 {230pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 {
...@@ -251,7 +250,7 @@ const AddExtraOpts = struct {...@@ -251,7 +250,7 @@ const AddExtraOpts = struct {
251 tlsgd: ?u32 = null,250 tlsgd: ?u32 = null,
252 gottp: ?u32 = null,251 gottp: ?u32 = null,
253 tlsdesc: ?u32 = null,252 tlsdesc: ?u32 = null,
254 zjt: ?u32 = null,253 trampoline: ?u32 = null,
255};254};
256255
257pub fn addExtra(symbol: *Symbol, opts: AddExtraOpts, elf_file: *Elf) void {256pub fn addExtra(symbol: *Symbol, opts: AddExtraOpts, elf_file: *Elf) void {
...@@ -304,7 +303,7 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void {...@@ -304,7 +303,7 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void {
304 const shdr = elf_file.shdrs.items[st_shndx];303 const shdr = elf_file.shdrs.items[st_shndx];
305 if (shdr.sh_flags & elf.SHF_TLS != 0 and file_ptr != .linker_defined)304 if (shdr.sh_flags & elf.SHF_TLS != 0 and file_ptr != .linker_defined)
306 break :blk symbol.address(.{ .plt = false }, elf_file) - elf_file.tlsAddress();305 break :blk symbol.address(.{ .plt = false }, elf_file) - elf_file.tlsAddress();
307 break :blk symbol.address(.{ .plt = false }, elf_file);306 break :blk symbol.address(.{ .plt = false, .trampoline = false }, elf_file);
308 };307 };
309 out.st_info = (st_bind << 4) | st_type;308 out.st_info = (st_bind << 4) | st_type;
310 out.st_other = esym.st_other;309 out.st_other = esym.st_other;
...@@ -380,7 +379,7 @@ fn format2(...@@ -380,7 +379,7 @@ fn format2(
380 try writer.print("%{d} : {s} : @{x}", .{379 try writer.print("%{d} : {s} : @{x}", .{
381 symbol.esym_index,380 symbol.esym_index,
382 symbol.fmtName(elf_file),381 symbol.fmtName(elf_file),
383 symbol.address(.{ .plt = false }, elf_file),382 symbol.address(.{ .plt = false, .trampoline = false }, elf_file),
384 });383 });
385 if (symbol.file(elf_file)) |file_ptr| {384 if (symbol.file(elf_file)) |file_ptr| {
386 if (symbol.isAbs(elf_file)) {385 if (symbol.isAbs(elf_file)) {
...@@ -456,8 +455,8 @@ pub const Flags = packed struct {...@@ -456,8 +455,8 @@ pub const Flags = packed struct {
456 /// Whether the symbol is a merge subsection.455 /// Whether the symbol is a merge subsection.
457 merge_subsection: bool = false,456 merge_subsection: bool = false,
458457
459 /// Whether the symbol has __zig_jump_table indirection.458 /// Whether the symbol has a trampoline.
460 has_zjt: bool = false,459 has_trampoline: bool = false,
461};460};
462461
463pub const Extra = struct {462pub const Extra = struct {
...@@ -471,7 +470,7 @@ pub const Extra = struct {...@@ -471,7 +470,7 @@ pub const Extra = struct {
471 gottp: u32 = 0,470 gottp: u32 = 0,
472 tlsdesc: u32 = 0,471 tlsdesc: u32 = 0,
473 merge_section: u32 = 0,472 merge_section: u32 = 0,
474 zjt: u32 = 0,473 trampoline: u32 = 0,
475};474};
476475
477pub const Index = u32;476pub const Index = u32;
src/link/Elf/ZigObject.zig+96-229
...@@ -55,11 +55,6 @@ debug_str_section_zig_size: u64 = 0,...@@ -55,11 +55,6 @@ debug_str_section_zig_size: u64 = 0,
55debug_aranges_section_zig_size: u64 = 0,55debug_aranges_section_zig_size: u64 = 0,
56debug_line_section_zig_size: u64 = 0,56debug_line_section_zig_size: u64 = 0,
5757
58/// Function jump table containing trampolines to Zcu functions.
59/// The table is used for Zig's incremental compilation and is embedded with
60/// the machine code section.
61jump_table: ?JumpTable = null,
62
63pub const global_symbol_bit: u32 = 0x80000000;58pub const global_symbol_bit: u32 = 0x80000000;
64pub const symbol_mask: u32 = 0x7fffffff;59pub const symbol_mask: u32 = 0x7fffffff;
65pub const SHN_ATOM: u16 = 0x100;60pub const SHN_ATOM: u16 = 0x100;
...@@ -127,10 +122,6 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void {...@@ -127,10 +122,6 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void {
127 if (self.dwarf) |*dw| {122 if (self.dwarf) |*dw| {
128 dw.deinit();123 dw.deinit();
129 }124 }
130
131 if (self.jump_table) |*jt| {
132 jt.deinit(allocator);
133 }
134}125}
135126
136pub fn flushModule(self: *ZigObject, elf_file: *Elf, tid: Zcu.PerThread.Id) !void {127pub fn flushModule(self: *ZigObject, elf_file: *Elf, tid: Zcu.PerThread.Id) !void {
...@@ -665,7 +656,7 @@ pub fn getNavVAddr(...@@ -665,7 +656,7 @@ pub fn getNavVAddr(
665 else => try self.getOrCreateMetadataForNav(elf_file, nav_index),656 else => try self.getOrCreateMetadataForNav(elf_file, nav_index),
666 };657 };
667 const this_sym = self.symbol(this_sym_index);658 const this_sym = self.symbol(this_sym_index);
668 const vaddr = this_sym.address(.{ .zjt = true }, elf_file);659 const vaddr = this_sym.address(.{}, elf_file);
669 const parent_atom = self.symbol(reloc_info.parent_atom_index).atom(elf_file).?;660 const parent_atom = self.symbol(reloc_info.parent_atom_index).atom(elf_file).?;
670 const r_type = relocation.encode(.abs, elf_file.getTarget().cpu.arch);661 const r_type = relocation.encode(.abs, elf_file.getTarget().cpu.arch);
671 try parent_atom.addReloc(elf_file, .{662 try parent_atom.addReloc(elf_file, .{
...@@ -914,12 +905,6 @@ fn updateNavCode(...@@ -914,12 +905,6 @@ fn updateNavCode(
914 if (old_vaddr != atom_ptr.value) {905 if (old_vaddr != atom_ptr.value) {
915 sym.value = 0;906 sym.value = 0;
916 esym.st_value = 0;907 esym.st_value = 0;
917
918 if (stt_bits == elf.STT_FUNC) {
919 const extra = sym.extra(elf_file);
920 const jump_table = self.jumpTablePtr().?;
921 jump_table.entries.items(.dirty)[extra.zjt] = true;
922 }
923 }908 }
924 } else if (code.len < old_size) {909 } else if (code.len < old_size) {
925 atom_ptr.shrink(elf_file);910 atom_ptr.shrink(elf_file);
...@@ -942,7 +927,7 @@ fn updateNavCode(...@@ -942,7 +927,7 @@ fn updateNavCode(
942 .len = code.len,927 .len = code.len,
943 }};928 }};
944 var remote_vec: [1]std.posix.iovec_const = .{.{929 var remote_vec: [1]std.posix.iovec_const = .{.{
945 .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{ .zjt = true }, elf_file))))),930 .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{}, elf_file))))),
946 .len = code.len,931 .len = code.len,
947 }};932 }};
948 const rc = std.os.linux.process_vm_writev(pid, &code_vec, &remote_vec, 0);933 const rc = std.os.linux.process_vm_writev(pid, &code_vec, &remote_vec, 0);
...@@ -1036,31 +1021,12 @@ pub fn updateFunc(...@@ -1036,31 +1021,12 @@ pub fn updateFunc(
1036 const ip = &zcu.intern_pool;1021 const ip = &zcu.intern_pool;
1037 const gpa = elf_file.base.comp.gpa;1022 const gpa = elf_file.base.comp.gpa;
1038 const func = zcu.funcInfo(func_index);1023 const func = zcu.funcInfo(func_index);
1039 if (!elf_file.base.isRelocatable() and self.jumpTablePtr() == null) {
1040 try self.initJumpTable(gpa, elf_file);
1041 }
10421024
1043 log.debug("updateFunc {}({d})", .{ ip.getNav(func.owner_nav).fqn.fmt(ip), func.owner_nav });1025 log.debug("updateFunc {}({d})", .{ ip.getNav(func.owner_nav).fqn.fmt(ip), func.owner_nav });
10441026
1045 const sym_index = try self.getOrCreateMetadataForNav(elf_file, func.owner_nav);1027 const sym_index = try self.getOrCreateMetadataForNav(elf_file, func.owner_nav);
1046 self.symbol(sym_index).atom(elf_file).?.freeRelocs(elf_file);1028 self.symbol(sym_index).atom(elf_file).?.freeRelocs(elf_file);
10471029
1048 if (self.jumpTablePtr()) |jump_table| {
1049 const sym = self.symbol(sym_index);
1050 if (!sym.flags.has_zjt) {
1051 const index = try jump_table.addSymbol(gpa, sym_index);
1052 sym.flags.has_zjt = true;
1053 sym.addExtra(.{ .zjt = index }, elf_file);
1054 try jump_table.updateSize(self, elf_file);
1055 const old_vaddr = jump_table.address(self, elf_file);
1056 try self.symbol(jump_table.sym_index).atom(elf_file).?.allocate(elf_file);
1057 const new_vaddr = jump_table.address(self, elf_file);
1058 if (old_vaddr != new_vaddr) {
1059 jump_table.dirty = true;
1060 }
1061 }
1062 }
1063
1064 var code_buffer = std.ArrayList(u8).init(gpa);1030 var code_buffer = std.ArrayList(u8).init(gpa);
1065 defer code_buffer.deinit();1031 defer code_buffer.deinit();
10661032
...@@ -1087,14 +1053,22 @@ pub fn updateFunc(...@@ -1087,14 +1053,22 @@ pub fn updateFunc(
1087 };1053 };
10881054
1089 const shndx = try self.getNavShdrIndex(elf_file, zcu, func.owner_nav, code);1055 const shndx = try self.getNavShdrIndex(elf_file, zcu, func.owner_nav, code);
1056 const old_rva, const old_alignment = blk: {
1057 const atom_ptr = self.symbol(sym_index).atom(elf_file).?;
1058 break :blk .{ atom_ptr.value, atom_ptr.alignment };
1059 };
1090 try self.updateNavCode(elf_file, pt, func.owner_nav, sym_index, shndx, code, elf.STT_FUNC);1060 try self.updateNavCode(elf_file, pt, func.owner_nav, sym_index, shndx, code, elf.STT_FUNC);
1061 const new_rva, const new_alignment = blk: {
1062 const atom_ptr = self.symbol(sym_index).atom(elf_file).?;
1063 break :blk .{ atom_ptr.value, atom_ptr.alignment };
1064 };
10911065
1092 if (dwarf_state) |*ds| {1066 if (dwarf_state) |*ds| {
1093 const sym = self.symbol(sym_index);1067 const sym = self.symbol(sym_index);
1094 try self.dwarf.?.commitNavState(1068 try self.dwarf.?.commitNavState(
1095 pt,1069 pt,
1096 func.owner_nav,1070 func.owner_nav,
1097 @intCast(sym.address(.{ .zjt = true }, elf_file)),1071 @intCast(sym.address(.{}, elf_file)),
1098 sym.atom(elf_file).?.size,1072 sym.atom(elf_file).?.size,
1099 ds,1073 ds,
1100 );1074 );
...@@ -1102,23 +1076,39 @@ pub fn updateFunc(...@@ -1102,23 +1076,39 @@ pub fn updateFunc(
11021076
1103 // Exports will be updated by `Zcu.processExports` after the update.1077 // Exports will be updated by `Zcu.processExports` after the update.
11041078
1105 if (self.jumpTablePtr()) |jump_table| {1079 if (old_rva != new_rva and old_rva > 0) {
1106 if (jump_table.dirty) {1080 // If we had to reallocate the function, we re-use the existing slot for a trampoline.
1107 // TODO write in bulk1081 // In the rare case that the function has been further overaligned we skip creating a
1108 for (jump_table.entries.items(.dirty), 0..) |*dirty, i| {1082 // trampoline and update all symbols referring this function.
1109 try jump_table.writeEntry(@intCast(i), self, elf_file);1083 if (old_alignment.order(new_alignment) == .lt) {
1110 dirty.* = false;1084 @panic("TODO update all symbols referring this function");
1111 }1085 }
1112 } else {1086
1113 const sym = self.symbol(sym_index);1087 // Create a trampoline to the new location at `old_rva`.
1114 const jt_index = sym.extra(elf_file).zjt;1088 if (!self.symbol(sym_index).flags.has_trampoline) {
1115 var jt_entry = jump_table.entries.get(jt_index);1089 const name = try std.fmt.allocPrint(gpa, "{s}$trampoline", .{
1116 if (jt_entry.dirty) {1090 self.symbol(sym_index).name(elf_file),
1117 try jump_table.writeEntry(jt_index, self, elf_file);1091 });
1118 jt_entry.dirty = false;1092 defer gpa.free(name);
1119 }1093 const name_off = try self.addString(gpa, name);
1120 jump_table.entries.set(jt_index, jt_entry);1094 const tr_size = trampolineSize(elf_file.getTarget().cpu.arch);
1095 const tr_sym_index = try self.newSymbolWithAtom(gpa, name_off);
1096 const tr_sym = self.symbol(tr_sym_index);
1097 const tr_esym = &self.symtab.items(.elf_sym)[tr_sym.esym_index];
1098 tr_esym.st_info |= elf.STT_OBJECT;
1099 tr_esym.st_size = tr_size;
1100 const tr_atom_ptr = tr_sym.atom(elf_file).?;
1101 tr_atom_ptr.value = old_rva;
1102 tr_atom_ptr.alive = true;
1103 tr_atom_ptr.alignment = old_alignment;
1104 tr_atom_ptr.output_section_index = elf_file.zig_text_section_index.?;
1105 tr_atom_ptr.size = tr_size;
1106 const target_sym = self.symbol(sym_index);
1107 target_sym.addExtra(.{ .trampoline = tr_sym_index }, elf_file);
1108 target_sym.flags.has_trampoline = true;
1121 }1109 }
1110 const target_sym = self.symbol(sym_index);
1111 try writeTrampoline(self.symbol(target_sym.extra(elf_file).trampoline).*, target_sym.*, elf_file);
1122 }1112 }
1123}1113}
11241114
...@@ -1193,7 +1183,7 @@ pub fn updateNav(...@@ -1193,7 +1183,7 @@ pub fn updateNav(
1193 try self.dwarf.?.commitNavState(1183 try self.dwarf.?.commitNavState(
1194 pt,1184 pt,
1195 nav_index,1185 nav_index,
1196 @intCast(sym.address(.{ .zjt = true }, elf_file)),1186 @intCast(sym.address(.{}, elf_file)),
1197 sym.atom(elf_file).?.size,1187 sym.atom(elf_file).?.size,
1198 ns,1188 ns,
1199 );1189 );
...@@ -1474,21 +1464,50 @@ pub fn getGlobalSymbol(self: *ZigObject, elf_file: *Elf, name: []const u8, lib_n...@@ -1474,21 +1464,50 @@ pub fn getGlobalSymbol(self: *ZigObject, elf_file: *Elf, name: []const u8, lib_n
1474 return lookup_gop.value_ptr.*;1464 return lookup_gop.value_ptr.*;
1475}1465}
14761466
1477pub fn jumpTablePtr(self: *ZigObject) ?*JumpTable {1467const max_trampoline_len = 12;
1478 return if (self.jump_table) |*jt| jt else null;
1479}
14801468
1481fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!void {1469fn trampolineSize(cpu_arch: std.Target.Cpu.Arch) u64 {
1482 const name_off = try self.addString(allocator, "__zig_jump_table");1470 const len = switch (cpu_arch) {
1483 const sym_index = try self.newSymbolWithAtom(allocator, name_off);1471 .x86_64 => 5, // jmp rel32
1484 const sym = self.symbol(sym_index);1472 else => @panic("TODO implement trampoline size for this CPU arch"),
1485 const esym = &self.symtab.items(.elf_sym)[sym.esym_index];1473 };
1486 esym.st_info |= elf.STT_OBJECT;1474 comptime assert(len <= max_trampoline_len);
1487 const atom_ptr = sym.atom(elf_file).?;1475 return len;
1488 atom_ptr.alive = true;1476}
1489 atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(JumpTable.alignment(elf_file.getTarget().cpu.arch));1477
1490 atom_ptr.output_section_index = elf_file.zig_text_section_index.?;1478fn writeTrampoline(tr_sym: Symbol, target: Symbol, elf_file: *Elf) !void {
1491 self.jump_table = JumpTable{ .sym_index = sym_index };1479 const atom_ptr = tr_sym.atom(elf_file).?;
1480 const shdr = elf_file.shdrs.items[atom_ptr.output_section_index];
1481 const fileoff = shdr.sh_offset + @as(u64, @intCast(atom_ptr.value));
1482 const source_addr = tr_sym.address(.{}, elf_file);
1483 const target_addr = target.address(.{ .trampoline = false }, elf_file);
1484 var buf: [max_trampoline_len]u8 = undefined;
1485 const out = switch (elf_file.getTarget().cpu.arch) {
1486 .x86_64 => try x86_64.writeTrampolineCode(source_addr, target_addr, &buf),
1487 else => @panic("TODO implement write trampoline for this CPU arch"),
1488 };
1489 try elf_file.base.file.?.pwriteAll(out, fileoff);
1490
1491 if (elf_file.base.child_pid) |pid| {
1492 switch (builtin.os.tag) {
1493 .linux => {
1494 var local_vec: [1]std.posix.iovec_const = .{.{
1495 .base = out.ptr,
1496 .len = out.len,
1497 }};
1498 var remote_vec: [1]std.posix.iovec_const = .{.{
1499 .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(source_addr)))),
1500 .len = out.len,
1501 }};
1502 const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0);
1503 switch (std.os.linux.E.init(rc)) {
1504 .SUCCESS => assert(rc == out.len),
1505 else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}),
1506 }
1507 },
1508 else => return error.HotSwapUnavailableOnHostOperatingSystem,
1509 }
1510 }
1492}1511}
14931512
1494pub fn asFile(self: *ZigObject) File {1513pub fn asFile(self: *ZigObject) File {
...@@ -1766,169 +1785,17 @@ const UavTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, AvMetadata);...@@ -1766,169 +1785,17 @@ const UavTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, AvMetadata);
1766const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata);1785const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata);
1767const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable);1786const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable);
17681787
1769pub const JumpTable = struct {1788const x86_64 = struct {
1770 sym_index: Symbol.Index,1789 fn writeTrampolineCode(source_addr: i64, target_addr: i64, buf: *[max_trampoline_len]u8) ![]u8 {
1771 entries: std.MultiArrayList(Entry) = .{},1790 const disp = @as(i64, @intCast(target_addr)) - source_addr - 5;
1772 dirty: bool = false,1791 var bytes = [_]u8{
17731792 0xe9, 0x00, 0x00, 0x00, 0x00, // jmp rel32
1774 pub fn deinit(jt: *JumpTable, allocator: Allocator) void {
1775 jt.entries.deinit(allocator);
1776 }
1777
1778 pub fn addSymbol(jt: *JumpTable, allocator: Allocator, sym_index: Symbol.Index) !Index {
1779 const index: Index = @intCast(try jt.entries.addOne(allocator));
1780 jt.entries.set(index, .{ .sym_index = sym_index });
1781 return index;
1782 }
1783
1784 pub fn address(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) i64 {
1785 const sym = zo.symbol(jt.sym_index);
1786 return sym.address(.{}, elf_file);
1787 }
1788
1789 pub fn size(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) u64 {
1790 const sym = zo.symbol(jt.sym_index);
1791 return sym.atom(elf_file).?.size;
1792 }
1793
1794 pub fn alignment(cpu_arch: std.Target.Cpu.Arch) u64 {
1795 return switch (cpu_arch) {
1796 .x86_64 => 1,
1797 else => @panic("TODO implement alignment for this CPU arch"),
1798 };
1799 }
1800
1801 pub fn entryAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 {
1802 return jt.address(zo, elf_file) + @as(i64, @intCast(index * entrySize(elf_file.getTarget().cpu.arch)));
1803 }
1804
1805 pub fn entryOffset(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 {
1806 const sym = zo.symbol(jt.sym_index);
1807 const atom_ptr = sym.atom(elf_file).?;
1808 const shdr = elf_file.shdrs.items[atom_ptr.output_section_index];
1809 return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * entrySize(elf_file.getTarget().cpu.arch);
1810 }
1811
1812 pub fn entrySize(cpu_arch: std.Target.Cpu.Arch) u64 {
1813 const seq_len = switch (cpu_arch) {
1814 .x86_64 => 5, // jmp rel32
1815 else => @panic("TODO implement entry size for this CPU arch"),
1816 };1793 };
1817 comptime assert(seq_len <= max_jump_seq_len);1794 assert(bytes.len == trampolineSize(.x86_64));
1818 return seq_len;1795 mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little);
1796 @memcpy(buf[0..bytes.len], &bytes);
1797 return buf[0..bytes.len];
1819 }1798 }
1820
1821 pub fn targetAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 {
1822 const sym_index = jt.entries.items(.sym_index)[index];
1823 return zo.symbol(sym_index).address(.{}, elf_file);
1824 }
1825
1826 const max_jump_seq_len = 12;
1827
1828 pub fn writeEntry(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void {
1829 const fileoff = jt.entryOffset(index, zo, elf_file);
1830 const source_addr = jt.entryAddress(index, zo, elf_file);
1831 const target_addr = @as(i64, @intCast(jt.targetAddress(index, zo, elf_file)));
1832 var buf: [max_jump_seq_len]u8 = undefined;
1833 const out = switch (elf_file.getTarget().cpu.arch) {
1834 .x86_64 => try x86_64.writeEntry(source_addr, target_addr, &buf),
1835 else => @panic("TODO implement write entry for this CPU arch"),
1836 };
1837 try elf_file.base.file.?.pwriteAll(out, fileoff);
1838
1839 if (elf_file.base.child_pid) |pid| {
1840 switch (builtin.os.tag) {
1841 .linux => {
1842 var local_vec: [1]std.posix.iovec_const = .{.{
1843 .base = out.ptr,
1844 .len = out.len,
1845 }};
1846 var remote_vec: [1]std.posix.iovec_const = .{.{
1847 .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(source_addr)))),
1848 .len = out.len,
1849 }};
1850 const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0);
1851 switch (std.os.linux.E.init(rc)) {
1852 .SUCCESS => assert(rc == out.len),
1853 else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}),
1854 }
1855 },
1856 else => return error.HotSwapUnavailableOnHostOperatingSystem,
1857 }
1858 }
1859 }
1860
1861 pub fn updateSize(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) !void {
1862 const jt_size: u64 = @intCast(jt.entries.items(.sym_index).len * entrySize(elf_file.getTarget().cpu.arch));
1863 const sym = zo.symbol(jt.sym_index);
1864 const esym = &zo.symtab.items(.elf_sym)[sym.esym_index];
1865 esym.st_size = jt_size;
1866 const atom_ptr = sym.atom(elf_file).?;
1867 atom_ptr.size = jt_size;
1868 }
1869
1870 const JumpTableFormatContext = struct { JumpTable, *ZigObject, *Elf };
1871
1872 pub fn format(
1873 jt: JumpTable,
1874 comptime unused_fmt_string: []const u8,
1875 options: std.fmt.FormatOptions,
1876 writer: anytype,
1877 ) !void {
1878 _ = jt;
1879 _ = unused_fmt_string;
1880 _ = options;
1881 _ = writer;
1882 @compileError("do not format JumpTable directly");
1883 }
1884
1885 pub fn fmt(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) std.fmt.Formatter(format2) {
1886 return .{ .data = .{ jt, zo, elf_file } };
1887 }
1888
1889 fn format2(
1890 ctx: JumpTableFormatContext,
1891 comptime unused_fmt_string: []const u8,
1892 options: std.fmt.FormatOptions,
1893 writer: anytype,
1894 ) !void {
1895 _ = options;
1896 _ = unused_fmt_string;
1897 const jt, const zo, const ef = ctx;
1898 try writer.writeAll("__zig_jump_table\n");
1899 try writer.print(" @{x} : size({x})\n", .{ jt.address(zo, ef), jt.size(zo, ef) });
1900 for (jt.entries.items(.sym_index), jt.entries.items(.dirty)) |sym_index, dirty| {
1901 const sym = zo.symbol(sym_index);
1902 try writer.print(" {x} => {x} : %{d} : {s}", .{
1903 sym.address(.{ .zjt = true }, ef),
1904 sym.address(.{}, ef),
1905 sym_index,
1906 sym.name(ef),
1907 });
1908 if (dirty) try writer.writeAll(" : [!]");
1909 try writer.writeByte('\n');
1910 }
1911 }
1912
1913 const Entry = struct {
1914 sym_index: Symbol.Index,
1915 dirty: bool = true,
1916 };
1917
1918 pub const Index = u32;
1919
1920 const x86_64 = struct {
1921 fn writeEntry(source_addr: i64, target_addr: i64, buf: *[max_jump_seq_len]u8) ![]u8 {
1922 const disp = @as(i64, @intCast(target_addr)) - source_addr - 5;
1923 var bytes = [_]u8{
1924 0xe9, 0x00, 0x00, 0x00, 0x00, // jmp rel32
1925 };
1926 assert(bytes.len == entrySize(.x86_64));
1927 mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little);
1928 @memcpy(buf[0..bytes.len], &bytes);
1929 return buf[0..bytes.len];
1930 }
1931 };
1932};1799};
19331800
1934const assert = std.debug.assert;1801const assert = std.debug.assert;