authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2024-08-14 12:10:28+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2024-08-14 12:10:40+02:00
logf26573fddfcebe44d4d69ea0e01d345fc1b15835
tree9dbe0f94673f8df0f3d3b0c939db2ffa70dbcad8
parent1bd54a55fa40db9f91d8a0c1cf5244ff51be6081

elf: re-use old atom slot for a trampoline to that atom

This is the initial implementation of Jacob Young's idea of re-using old function slots as trampolines for new function's location. This way the trampoline is guaranteed to be aligned to the function's alignment. The only edge case is if an incremental update further overaligns the function in which case we skip/delete the trampoline and re-evaluate all references.

4 files changed, 113 insertions(+), 253 deletions(-)

src/link/Elf.zig+1-4
......@@ -2798,7 +2798,7 @@ pub fn writeElfHeader(self: *Elf) !void {
27982798
27992799 const e_entry: u64 = if (self.linkerDefinedPtr()) |obj| blk: {
28002800 const entry_sym = obj.entrySymbol(self) orelse break :blk 0;
2801 break :blk @intCast(entry_sym.address(.{ .zjt = true }, self));
2801 break :blk @intCast(entry_sym.address(.{}, self));
28022802 } else 0;
28032803 const phdr_table_offset = if (self.phdr_table_index) |phndx| self.phdrs.items[phndx].p_offset else 0;
28042804 switch (self.ptr_width) {
......@@ -5599,9 +5599,6 @@ fn fmtDumpState(
55995599 zig_object.fmtAtoms(self),
56005600 zig_object.fmtSymtab(self),
56015601 });
5602 if (zig_object.jump_table) |jt| {
5603 try writer.print("{}", .{jt.fmt(zig_object, self)});
5604 }
56055602 try writer.writeByte('\n');
56065603 }
56075604
src/link/Elf/Atom.zig+3-6
......@@ -746,8 +746,8 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi
746746 const P = self.address(elf_file) + @as(i64, @intCast(rel.r_offset));
747747 // Addend from the relocation.
748748 const A = rel.r_addend;
749 // Address of the target symbol - can be address of the symbol within an atom or address of PLT stub.
750 const S = target.address(.{ .zjt = false }, elf_file);
749 // Address of the target symbol - can be address of the symbol within an atom or address of PLT stub, or address of a Zig trampoline.
750 const S = target.address(.{}, elf_file);
751751 // Address of the global offset table.
752752 const GOT = elf_file.gotAddress();
753753 // Relative offset to the start of the global offset table.
......@@ -1212,10 +1212,7 @@ const x86_64 = struct {
12121212 );
12131213 },
12141214
1215 .PLT32 => {
1216 const S_ = if (target.flags.has_zjt) target.address(.{ .zjt = true }, elf_file) else S;
1217 try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little);
1218 },
1215 .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little),
12191216 .PC32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little),
12201217
12211218 .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little),
src/link/Elf/Symbol.zig+13-14
......@@ -101,7 +101,7 @@ pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 {
101101 return file_ptr.symbolRank(sym, in_archive);
102102}
103103
104pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = false }, elf_file: *Elf) i64 {
104pub fn address(symbol: Symbol, opts: struct { plt: bool = true, trampoline: bool = true }, elf_file: *Elf) i64 {
105105 if (symbol.mergeSubsection(elf_file)) |msub| {
106106 if (!msub.alive) return 0;
107107 return msub.address(elf_file) + symbol.value;
......@@ -109,8 +109,8 @@ pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = fals
109109 if (symbol.flags.has_copy_rel) {
110110 return symbol.copyRelAddress(elf_file);
111111 }
112 if (symbol.flags.has_zjt and opts.zjt) {
113 return symbol.zjtAddress(elf_file);
112 if (symbol.flags.has_trampoline and opts.trampoline) {
113 return symbol.trampolineAddress(elf_file);
114114 }
115115 if (symbol.flags.has_plt and opts.plt) {
116116 if (!symbol.flags.is_canonical and symbol.flags.has_got) {
......@@ -220,12 +220,11 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) i64 {
220220 return entry.address(elf_file);
221221}
222222
223pub fn zjtAddress(symbol: Symbol, elf_file: *Elf) i64 {
224 if (!symbol.flags.has_zjt) return 0;
223pub fn trampolineAddress(symbol: Symbol, elf_file: *Elf) i64 {
224 if (!symbol.flags.has_trampoline) return 0;
225225 const zo = elf_file.zigObjectPtr().?;
226 const jump_table = zo.jumpTablePtr().?;
227 const index = symbol.extra(elf_file).zjt;
228 return jump_table.entryAddress(index, zo, elf_file);
226 const index = symbol.extra(elf_file).trampoline;
227 return zo.symbol(index).address(.{}, elf_file);
229228}
230229
231230pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 {
......@@ -251,7 +250,7 @@ const AddExtraOpts = struct {
251250 tlsgd: ?u32 = null,
252251 gottp: ?u32 = null,
253252 tlsdesc: ?u32 = null,
254 zjt: ?u32 = null,
253 trampoline: ?u32 = null,
255254};
256255
257256pub fn addExtra(symbol: *Symbol, opts: AddExtraOpts, elf_file: *Elf) void {
......@@ -304,7 +303,7 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void {
304303 const shdr = elf_file.shdrs.items[st_shndx];
305304 if (shdr.sh_flags & elf.SHF_TLS != 0 and file_ptr != .linker_defined)
306305 break :blk symbol.address(.{ .plt = false }, elf_file) - elf_file.tlsAddress();
307 break :blk symbol.address(.{ .plt = false }, elf_file);
306 break :blk symbol.address(.{ .plt = false, .trampoline = false }, elf_file);
308307 };
309308 out.st_info = (st_bind << 4) | st_type;
310309 out.st_other = esym.st_other;
......@@ -380,7 +379,7 @@ fn format2(
380379 try writer.print("%{d} : {s} : @{x}", .{
381380 symbol.esym_index,
382381 symbol.fmtName(elf_file),
383 symbol.address(.{ .plt = false }, elf_file),
382 symbol.address(.{ .plt = false, .trampoline = false }, elf_file),
384383 });
385384 if (symbol.file(elf_file)) |file_ptr| {
386385 if (symbol.isAbs(elf_file)) {
......@@ -456,8 +455,8 @@ pub const Flags = packed struct {
456455 /// Whether the symbol is a merge subsection.
457456 merge_subsection: bool = false,
458457
459 /// Whether the symbol has __zig_jump_table indirection.
460 has_zjt: bool = false,
458 /// Whether the symbol has a trampoline.
459 has_trampoline: bool = false,
461460};
462461
463462pub const Extra = struct {
......@@ -471,7 +470,7 @@ pub const Extra = struct {
471470 gottp: u32 = 0,
472471 tlsdesc: u32 = 0,
473472 merge_section: u32 = 0,
474 zjt: u32 = 0,
473 trampoline: u32 = 0,
475474};
476475
477476pub const Index = u32;
src/link/Elf/ZigObject.zig+96-229
......@@ -55,11 +55,6 @@ debug_str_section_zig_size: u64 = 0,
5555debug_aranges_section_zig_size: u64 = 0,
5656debug_line_section_zig_size: u64 = 0,
5757
58/// Function jump table containing trampolines to Zcu functions.
59/// The table is used for Zig's incremental compilation and is embedded with
60/// the machine code section.
61jump_table: ?JumpTable = null,
62
6358pub const global_symbol_bit: u32 = 0x80000000;
6459pub const symbol_mask: u32 = 0x7fffffff;
6560pub const SHN_ATOM: u16 = 0x100;
......@@ -127,10 +122,6 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void {
127122 if (self.dwarf) |*dw| {
128123 dw.deinit();
129124 }
130
131 if (self.jump_table) |*jt| {
132 jt.deinit(allocator);
133 }
134125}
135126
136127pub fn flushModule(self: *ZigObject, elf_file: *Elf, tid: Zcu.PerThread.Id) !void {
......@@ -665,7 +656,7 @@ pub fn getNavVAddr(
665656 else => try self.getOrCreateMetadataForNav(elf_file, nav_index),
666657 };
667658 const this_sym = self.symbol(this_sym_index);
668 const vaddr = this_sym.address(.{ .zjt = true }, elf_file);
659 const vaddr = this_sym.address(.{}, elf_file);
669660 const parent_atom = self.symbol(reloc_info.parent_atom_index).atom(elf_file).?;
670661 const r_type = relocation.encode(.abs, elf_file.getTarget().cpu.arch);
671662 try parent_atom.addReloc(elf_file, .{
......@@ -914,12 +905,6 @@ fn updateNavCode(
914905 if (old_vaddr != atom_ptr.value) {
915906 sym.value = 0;
916907 esym.st_value = 0;
917
918 if (stt_bits == elf.STT_FUNC) {
919 const extra = sym.extra(elf_file);
920 const jump_table = self.jumpTablePtr().?;
921 jump_table.entries.items(.dirty)[extra.zjt] = true;
922 }
923908 }
924909 } else if (code.len < old_size) {
925910 atom_ptr.shrink(elf_file);
......@@ -942,7 +927,7 @@ fn updateNavCode(
942927 .len = code.len,
943928 }};
944929 var remote_vec: [1]std.posix.iovec_const = .{.{
945 .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{ .zjt = true }, elf_file))))),
930 .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{}, elf_file))))),
946931 .len = code.len,
947932 }};
948933 const rc = std.os.linux.process_vm_writev(pid, &code_vec, &remote_vec, 0);
......@@ -1036,31 +1021,12 @@ pub fn updateFunc(
10361021 const ip = &zcu.intern_pool;
10371022 const gpa = elf_file.base.comp.gpa;
10381023 const func = zcu.funcInfo(func_index);
1039 if (!elf_file.base.isRelocatable() and self.jumpTablePtr() == null) {
1040 try self.initJumpTable(gpa, elf_file);
1041 }
10421024
10431025 log.debug("updateFunc {}({d})", .{ ip.getNav(func.owner_nav).fqn.fmt(ip), func.owner_nav });
10441026
10451027 const sym_index = try self.getOrCreateMetadataForNav(elf_file, func.owner_nav);
10461028 self.symbol(sym_index).atom(elf_file).?.freeRelocs(elf_file);
10471029
1048 if (self.jumpTablePtr()) |jump_table| {
1049 const sym = self.symbol(sym_index);
1050 if (!sym.flags.has_zjt) {
1051 const index = try jump_table.addSymbol(gpa, sym_index);
1052 sym.flags.has_zjt = true;
1053 sym.addExtra(.{ .zjt = index }, elf_file);
1054 try jump_table.updateSize(self, elf_file);
1055 const old_vaddr = jump_table.address(self, elf_file);
1056 try self.symbol(jump_table.sym_index).atom(elf_file).?.allocate(elf_file);
1057 const new_vaddr = jump_table.address(self, elf_file);
1058 if (old_vaddr != new_vaddr) {
1059 jump_table.dirty = true;
1060 }
1061 }
1062 }
1063
10641030 var code_buffer = std.ArrayList(u8).init(gpa);
10651031 defer code_buffer.deinit();
10661032
......@@ -1087,14 +1053,22 @@ pub fn updateFunc(
10871053 };
10881054
10891055 const shndx = try self.getNavShdrIndex(elf_file, zcu, func.owner_nav, code);
1056 const old_rva, const old_alignment = blk: {
1057 const atom_ptr = self.symbol(sym_index).atom(elf_file).?;
1058 break :blk .{ atom_ptr.value, atom_ptr.alignment };
1059 };
10901060 try self.updateNavCode(elf_file, pt, func.owner_nav, sym_index, shndx, code, elf.STT_FUNC);
1061 const new_rva, const new_alignment = blk: {
1062 const atom_ptr = self.symbol(sym_index).atom(elf_file).?;
1063 break :blk .{ atom_ptr.value, atom_ptr.alignment };
1064 };
10911065
10921066 if (dwarf_state) |*ds| {
10931067 const sym = self.symbol(sym_index);
10941068 try self.dwarf.?.commitNavState(
10951069 pt,
10961070 func.owner_nav,
1097 @intCast(sym.address(.{ .zjt = true }, elf_file)),
1071 @intCast(sym.address(.{}, elf_file)),
10981072 sym.atom(elf_file).?.size,
10991073 ds,
11001074 );
......@@ -1102,23 +1076,39 @@ pub fn updateFunc(
11021076
11031077 // Exports will be updated by `Zcu.processExports` after the update.
11041078
1105 if (self.jumpTablePtr()) |jump_table| {
1106 if (jump_table.dirty) {
1107 // TODO write in bulk
1108 for (jump_table.entries.items(.dirty), 0..) |*dirty, i| {
1109 try jump_table.writeEntry(@intCast(i), self, elf_file);
1110 dirty.* = false;
1111 }
1112 } else {
1113 const sym = self.symbol(sym_index);
1114 const jt_index = sym.extra(elf_file).zjt;
1115 var jt_entry = jump_table.entries.get(jt_index);
1116 if (jt_entry.dirty) {
1117 try jump_table.writeEntry(jt_index, self, elf_file);
1118 jt_entry.dirty = false;
1119 }
1120 jump_table.entries.set(jt_index, jt_entry);
1079 if (old_rva != new_rva and old_rva > 0) {
1080 // If we had to reallocate the function, we re-use the existing slot for a trampoline.
1081 // In the rare case that the function has been further overaligned we skip creating a
1082 // trampoline and update all symbols referring this function.
1083 if (old_alignment.order(new_alignment) == .lt) {
1084 @panic("TODO update all symbols referring this function");
1085 }
1086
1087 // Create a trampoline to the new location at `old_rva`.
1088 if (!self.symbol(sym_index).flags.has_trampoline) {
1089 const name = try std.fmt.allocPrint(gpa, "{s}$trampoline", .{
1090 self.symbol(sym_index).name(elf_file),
1091 });
1092 defer gpa.free(name);
1093 const name_off = try self.addString(gpa, name);
1094 const tr_size = trampolineSize(elf_file.getTarget().cpu.arch);
1095 const tr_sym_index = try self.newSymbolWithAtom(gpa, name_off);
1096 const tr_sym = self.symbol(tr_sym_index);
1097 const tr_esym = &self.symtab.items(.elf_sym)[tr_sym.esym_index];
1098 tr_esym.st_info |= elf.STT_OBJECT;
1099 tr_esym.st_size = tr_size;
1100 const tr_atom_ptr = tr_sym.atom(elf_file).?;
1101 tr_atom_ptr.value = old_rva;
1102 tr_atom_ptr.alive = true;
1103 tr_atom_ptr.alignment = old_alignment;
1104 tr_atom_ptr.output_section_index = elf_file.zig_text_section_index.?;
1105 tr_atom_ptr.size = tr_size;
1106 const target_sym = self.symbol(sym_index);
1107 target_sym.addExtra(.{ .trampoline = tr_sym_index }, elf_file);
1108 target_sym.flags.has_trampoline = true;
11211109 }
1110 const target_sym = self.symbol(sym_index);
1111 try writeTrampoline(self.symbol(target_sym.extra(elf_file).trampoline).*, target_sym.*, elf_file);
11221112 }
11231113}
11241114
......@@ -1193,7 +1183,7 @@ pub fn updateNav(
11931183 try self.dwarf.?.commitNavState(
11941184 pt,
11951185 nav_index,
1196 @intCast(sym.address(.{ .zjt = true }, elf_file)),
1186 @intCast(sym.address(.{}, elf_file)),
11971187 sym.atom(elf_file).?.size,
11981188 ns,
11991189 );
......@@ -1474,21 +1464,50 @@ pub fn getGlobalSymbol(self: *ZigObject, elf_file: *Elf, name: []const u8, lib_n
14741464 return lookup_gop.value_ptr.*;
14751465}
14761466
1477pub fn jumpTablePtr(self: *ZigObject) ?*JumpTable {
1478 return if (self.jump_table) |*jt| jt else null;
1479}
1467const max_trampoline_len = 12;
14801468
1481fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!void {
1482 const name_off = try self.addString(allocator, "__zig_jump_table");
1483 const sym_index = try self.newSymbolWithAtom(allocator, name_off);
1484 const sym = self.symbol(sym_index);
1485 const esym = &self.symtab.items(.elf_sym)[sym.esym_index];
1486 esym.st_info |= elf.STT_OBJECT;
1487 const atom_ptr = sym.atom(elf_file).?;
1488 atom_ptr.alive = true;
1489 atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(JumpTable.alignment(elf_file.getTarget().cpu.arch));
1490 atom_ptr.output_section_index = elf_file.zig_text_section_index.?;
1491 self.jump_table = JumpTable{ .sym_index = sym_index };
1469fn trampolineSize(cpu_arch: std.Target.Cpu.Arch) u64 {
1470 const len = switch (cpu_arch) {
1471 .x86_64 => 5, // jmp rel32
1472 else => @panic("TODO implement trampoline size for this CPU arch"),
1473 };
1474 comptime assert(len <= max_trampoline_len);
1475 return len;
1476}
1477
1478fn writeTrampoline(tr_sym: Symbol, target: Symbol, elf_file: *Elf) !void {
1479 const atom_ptr = tr_sym.atom(elf_file).?;
1480 const shdr = elf_file.shdrs.items[atom_ptr.output_section_index];
1481 const fileoff = shdr.sh_offset + @as(u64, @intCast(atom_ptr.value));
1482 const source_addr = tr_sym.address(.{}, elf_file);
1483 const target_addr = target.address(.{ .trampoline = false }, elf_file);
1484 var buf: [max_trampoline_len]u8 = undefined;
1485 const out = switch (elf_file.getTarget().cpu.arch) {
1486 .x86_64 => try x86_64.writeTrampolineCode(source_addr, target_addr, &buf),
1487 else => @panic("TODO implement write trampoline for this CPU arch"),
1488 };
1489 try elf_file.base.file.?.pwriteAll(out, fileoff);
1490
1491 if (elf_file.base.child_pid) |pid| {
1492 switch (builtin.os.tag) {
1493 .linux => {
1494 var local_vec: [1]std.posix.iovec_const = .{.{
1495 .base = out.ptr,
1496 .len = out.len,
1497 }};
1498 var remote_vec: [1]std.posix.iovec_const = .{.{
1499 .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(source_addr)))),
1500 .len = out.len,
1501 }};
1502 const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0);
1503 switch (std.os.linux.E.init(rc)) {
1504 .SUCCESS => assert(rc == out.len),
1505 else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}),
1506 }
1507 },
1508 else => return error.HotSwapUnavailableOnHostOperatingSystem,
1509 }
1510 }
14921511}
14931512
14941513pub fn asFile(self: *ZigObject) File {
......@@ -1766,169 +1785,17 @@ const UavTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, AvMetadata);
17661785const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata);
17671786const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable);
17681787
1769pub const JumpTable = struct {
1770 sym_index: Symbol.Index,
1771 entries: std.MultiArrayList(Entry) = .{},
1772 dirty: bool = false,
1773
1774 pub fn deinit(jt: *JumpTable, allocator: Allocator) void {
1775 jt.entries.deinit(allocator);
1776 }
1777
1778 pub fn addSymbol(jt: *JumpTable, allocator: Allocator, sym_index: Symbol.Index) !Index {
1779 const index: Index = @intCast(try jt.entries.addOne(allocator));
1780 jt.entries.set(index, .{ .sym_index = sym_index });
1781 return index;
1782 }
1783
1784 pub fn address(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) i64 {
1785 const sym = zo.symbol(jt.sym_index);
1786 return sym.address(.{}, elf_file);
1787 }
1788
1789 pub fn size(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) u64 {
1790 const sym = zo.symbol(jt.sym_index);
1791 return sym.atom(elf_file).?.size;
1792 }
1793
1794 pub fn alignment(cpu_arch: std.Target.Cpu.Arch) u64 {
1795 return switch (cpu_arch) {
1796 .x86_64 => 1,
1797 else => @panic("TODO implement alignment for this CPU arch"),
1798 };
1799 }
1800
1801 pub fn entryAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 {
1802 return jt.address(zo, elf_file) + @as(i64, @intCast(index * entrySize(elf_file.getTarget().cpu.arch)));
1803 }
1804
1805 pub fn entryOffset(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 {
1806 const sym = zo.symbol(jt.sym_index);
1807 const atom_ptr = sym.atom(elf_file).?;
1808 const shdr = elf_file.shdrs.items[atom_ptr.output_section_index];
1809 return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * entrySize(elf_file.getTarget().cpu.arch);
1810 }
1811
1812 pub fn entrySize(cpu_arch: std.Target.Cpu.Arch) u64 {
1813 const seq_len = switch (cpu_arch) {
1814 .x86_64 => 5, // jmp rel32
1815 else => @panic("TODO implement entry size for this CPU arch"),
1788const x86_64 = struct {
1789 fn writeTrampolineCode(source_addr: i64, target_addr: i64, buf: *[max_trampoline_len]u8) ![]u8 {
1790 const disp = @as(i64, @intCast(target_addr)) - source_addr - 5;
1791 var bytes = [_]u8{
1792 0xe9, 0x00, 0x00, 0x00, 0x00, // jmp rel32
18161793 };
1817 comptime assert(seq_len <= max_jump_seq_len);
1818 return seq_len;
1794 assert(bytes.len == trampolineSize(.x86_64));
1795 mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little);
1796 @memcpy(buf[0..bytes.len], &bytes);
1797 return buf[0..bytes.len];
18191798 }
1820
1821 pub fn targetAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 {
1822 const sym_index = jt.entries.items(.sym_index)[index];
1823 return zo.symbol(sym_index).address(.{}, elf_file);
1824 }
1825
1826 const max_jump_seq_len = 12;
1827
1828 pub fn writeEntry(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void {
1829 const fileoff = jt.entryOffset(index, zo, elf_file);
1830 const source_addr = jt.entryAddress(index, zo, elf_file);
1831 const target_addr = @as(i64, @intCast(jt.targetAddress(index, zo, elf_file)));
1832 var buf: [max_jump_seq_len]u8 = undefined;
1833 const out = switch (elf_file.getTarget().cpu.arch) {
1834 .x86_64 => try x86_64.writeEntry(source_addr, target_addr, &buf),
1835 else => @panic("TODO implement write entry for this CPU arch"),
1836 };
1837 try elf_file.base.file.?.pwriteAll(out, fileoff);
1838
1839 if (elf_file.base.child_pid) |pid| {
1840 switch (builtin.os.tag) {
1841 .linux => {
1842 var local_vec: [1]std.posix.iovec_const = .{.{
1843 .base = out.ptr,
1844 .len = out.len,
1845 }};
1846 var remote_vec: [1]std.posix.iovec_const = .{.{
1847 .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(source_addr)))),
1848 .len = out.len,
1849 }};
1850 const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0);
1851 switch (std.os.linux.E.init(rc)) {
1852 .SUCCESS => assert(rc == out.len),
1853 else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}),
1854 }
1855 },
1856 else => return error.HotSwapUnavailableOnHostOperatingSystem,
1857 }
1858 }
1859 }
1860
1861 pub fn updateSize(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) !void {
1862 const jt_size: u64 = @intCast(jt.entries.items(.sym_index).len * entrySize(elf_file.getTarget().cpu.arch));
1863 const sym = zo.symbol(jt.sym_index);
1864 const esym = &zo.symtab.items(.elf_sym)[sym.esym_index];
1865 esym.st_size = jt_size;
1866 const atom_ptr = sym.atom(elf_file).?;
1867 atom_ptr.size = jt_size;
1868 }
1869
1870 const JumpTableFormatContext = struct { JumpTable, *ZigObject, *Elf };
1871
1872 pub fn format(
1873 jt: JumpTable,
1874 comptime unused_fmt_string: []const u8,
1875 options: std.fmt.FormatOptions,
1876 writer: anytype,
1877 ) !void {
1878 _ = jt;
1879 _ = unused_fmt_string;
1880 _ = options;
1881 _ = writer;
1882 @compileError("do not format JumpTable directly");
1883 }
1884
1885 pub fn fmt(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) std.fmt.Formatter(format2) {
1886 return .{ .data = .{ jt, zo, elf_file } };
1887 }
1888
1889 fn format2(
1890 ctx: JumpTableFormatContext,
1891 comptime unused_fmt_string: []const u8,
1892 options: std.fmt.FormatOptions,
1893 writer: anytype,
1894 ) !void {
1895 _ = options;
1896 _ = unused_fmt_string;
1897 const jt, const zo, const ef = ctx;
1898 try writer.writeAll("__zig_jump_table\n");
1899 try writer.print(" @{x} : size({x})\n", .{ jt.address(zo, ef), jt.size(zo, ef) });
1900 for (jt.entries.items(.sym_index), jt.entries.items(.dirty)) |sym_index, dirty| {
1901 const sym = zo.symbol(sym_index);
1902 try writer.print(" {x} => {x} : %{d} : {s}", .{
1903 sym.address(.{ .zjt = true }, ef),
1904 sym.address(.{}, ef),
1905 sym_index,
1906 sym.name(ef),
1907 });
1908 if (dirty) try writer.writeAll(" : [!]");
1909 try writer.writeByte('\n');
1910 }
1911 }
1912
1913 const Entry = struct {
1914 sym_index: Symbol.Index,
1915 dirty: bool = true,
1916 };
1917
1918 pub const Index = u32;
1919
1920 const x86_64 = struct {
1921 fn writeEntry(source_addr: i64, target_addr: i64, buf: *[max_jump_seq_len]u8) ![]u8 {
1922 const disp = @as(i64, @intCast(target_addr)) - source_addr - 5;
1923 var bytes = [_]u8{
1924 0xe9, 0x00, 0x00, 0x00, 0x00, // jmp rel32
1925 };
1926 assert(bytes.len == entrySize(.x86_64));
1927 mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little);
1928 @memcpy(buf[0..bytes.len], &bytes);
1929 return buf[0..bytes.len];
1930 }
1931 };
19321799};
19331800
19341801const assert = std.debug.assert;