authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2024-01-21 10:32:06+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2024-01-24 12:34:42+01:00
log3a6410959ca6df6f020547c58845730753dc9e97
treea8e3f8b35f617fcd903d97be7c478527edb65632
parent411c7f6669ed2eb758f371dfde59e03abf05aa0a

macho: actually lower TLS variables

5 files changed, 163 insertions(+), 37 deletions(-)

src/arch/x86_64/Lower.zig+1-2
......@@ -439,10 +439,9 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
439439 .{ .mem = Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }) },
440440 });
441441 lower.result_insts_len += 1;
442 emit_mnemonic = .lea;
442 emit_mnemonic = .mov;
443443 break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{
444444 .base = .{ .reg = .rax },
445 .disp = std.math.minInt(i32),
446445 }) };
447446 }
448447
src/link/MachO.zig+14-3
......@@ -606,7 +606,10 @@ pub fn flushModule(self: *MachO, arena: Allocator, prog_node: *std.Progress.Node
606606 if (!atom.flags.alive) continue;
607607 const sect = &self.sections.items(.header)[atom.out_n_sect];
608608 if (sect.isZerofill()) continue;
609 const code = zo.getAtomDataAlloc(self, atom.*) catch |err| switch (err) {
609 if (mem.indexOf(u8, sect.segName(), "ZIG") == null) continue; // Non-Zig sections are handled separately
610 // TODO: we will resolve and write ZigObject's TLS data twice:
611 // once here, and once in writeAtoms
612 const code = zo.getAtomDataAlloc(self, gpa, atom.*) catch |err| switch (err) {
610613 error.InputOutput => {
611614 try self.reportUnexpectedError("fetching code for '{s}' failed", .{
612615 atom.getName(self),
......@@ -1806,7 +1809,7 @@ fn initOutputSections(self: *MachO) !void {
18061809 .aarch64 => 2,
18071810 else => unreachable,
18081811 },
1809 .flags = macho.S_SYMBOL_STUBS |
1812 .flags = macho.S_REGULAR |
18101813 macho.S_ATTR_PURE_INSTRUCTIONS | macho.S_ATTR_SOME_INSTRUCTIONS,
18111814 });
18121815 }
......@@ -2545,6 +2548,9 @@ fn writeAtoms(self: *MachO) !void {
25452548 defer tracy.end();
25462549
25472550 const gpa = self.base.comp.gpa;
2551 var arena = std.heap.ArenaAllocator.init(gpa);
2552 defer arena.deinit();
2553
25482554 const cpu_arch = self.getTarget().cpu.arch;
25492555 const slice = self.sections.slice();
25502556
......@@ -2562,7 +2568,12 @@ fn writeAtoms(self: *MachO) !void {
25622568 const atom = self.getAtom(atom_index).?;
25632569 assert(atom.flags.alive);
25642570 const off = atom.value - header.addr;
2565 @memcpy(buffer[off..][0..atom.size], atom.getFile(self).object.getAtomData(atom.*));
2571 const data = switch (atom.getFile(self)) {
2572 .object => |x| x.getAtomData(atom.*),
2573 .zig_object => |x| try x.getAtomDataAlloc(self, arena.allocator(), atom.*),
2574 else => unreachable,
2575 };
2576 @memcpy(buffer[off..][0..atom.size], data);
25662577 atom.resolveRelocs(self, buffer[off..][0..atom.size]) catch |err| switch (err) {
25672578 error.ResolveFailed => has_resolve_error = true,
25682579 else => |e| return e,
src/link/MachO/ZigObject.zig+138-30
......@@ -38,8 +38,8 @@ unnamed_consts: UnnamedConstTable = .{},
3838/// Table of tracked AnonDecls.
3939anon_decls: AnonDeclTable = .{},
4040
41/// TLS variables indexed by Atom.Index.
42tls_variables: TlsTable = .{},
41/// TLV initializers indexed by Atom.Index.
42tlv_initializers: TlvInitializerTable = .{},
4343
4444/// A table of relocations.
4545relocs: RelocationTable = .{},
......@@ -91,10 +91,10 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void {
9191 }
9292 self.relocs.deinit(allocator);
9393
94 for (self.tls_variables.values()) |*tlv| {
95 tlv.deinit(allocator);
94 for (self.tlv_initializers.values()) |*tlv_init| {
95 tlv_init.deinit(allocator);
9696 }
97 self.tls_variables.deinit(allocator);
97 self.tlv_initializers.deinit(allocator);
9898}
9999
100100fn addNlist(self: *ZigObject, allocator: Allocator) !Symbol.Index {
......@@ -137,25 +137,35 @@ pub fn addAtom(self: *ZigObject, macho_file: *MachO) !Symbol.Index {
137137}
138138
139139/// Caller owns the memory.
140pub fn getAtomDataAlloc(self: ZigObject, macho_file: *MachO, atom: Atom) ![]u8 {
141 const gpa = macho_file.base.comp.gpa;
140pub fn getAtomDataAlloc(
141 self: ZigObject,
142 macho_file: *MachO,
143 allocator: Allocator,
144 atom: Atom,
145) ![]u8 {
142146 assert(atom.file == self.index);
143147 const sect = macho_file.sections.items(.header)[atom.out_n_sect];
148 assert(!sect.isZerofill());
144149
145150 switch (sect.type()) {
146151 macho.S_THREAD_LOCAL_REGULAR => {
147 const tlv = self.tls_variables.get(atom.atom_index).?;
148 const code = try gpa.dupe(u8, tlv.code);
149 return code;
152 const tlv = self.tlv_initializers.get(atom.atom_index).?;
153 const data = try allocator.dupe(u8, tlv.data);
154 return data;
155 },
156 macho.S_THREAD_LOCAL_VARIABLES => {
157 const data = try allocator.alloc(u8, atom.size);
158 @memset(data, 0);
159 return data;
150160 },
151161 else => {
152162 const file_offset = sect.offset + atom.value - sect.addr;
153163 const size = std.math.cast(usize, atom.size) orelse return error.Overflow;
154 const code = try gpa.alloc(u8, size);
155 errdefer gpa.free(code);
156 const amt = try macho_file.base.file.?.preadAll(code, file_offset);
157 if (amt != code.len) return error.InputOutput;
158 return code;
164 const data = try allocator.alloc(u8, size);
165 errdefer allocator.free(data);
166 const amt = try macho_file.base.file.?.preadAll(data, file_offset);
167 if (amt != data.len) return error.InputOutput;
168 return data;
159169 },
160170 }
161171}
......@@ -421,7 +431,7 @@ pub fn lowerAnonDecl(
421431 self: *ZigObject,
422432 macho_file: *MachO,
423433 decl_val: InternPool.Index,
424 explicit_alignment: InternPool.Alignment,
434 explicit_alignment: Atom.Alignment,
425435 src_loc: Module.SrcLoc,
426436) !codegen.Result {
427437 const gpa = macho_file.base.comp.gpa;
......@@ -732,6 +742,9 @@ fn updateDeclCode(
732742 }
733743}
734744
745/// Lowering a TLV on macOS involves two stages:
746/// 1. first we lower the initializer into appopriate section (__thread_data or __thread_bss)
747/// 2. next, we create a corresponding threadlocal variable descriptor in __thread_vars
735748fn updateTlv(
736749 self: *ZigObject,
737750 macho_file: *MachO,
......@@ -740,9 +753,7 @@ fn updateTlv(
740753 sect_index: u8,
741754 code: []const u8,
742755) !void {
743 const comp = macho_file.base.comp;
744 const gpa = comp.gpa;
745 const mod = comp.module.?;
756 const mod = macho_file.base.comp.module.?;
746757 const decl = mod.declPtr(decl_index);
747758 const decl_name = mod.intern_pool.stringToSlice(try decl.getFullyQualifiedName(mod));
748759
......@@ -750,6 +761,32 @@ fn updateTlv(
750761
751762 const required_alignment = decl.getAlignment(mod);
752763
764 // 1. Lower TLV initializer
765 const init_sym_index = try self.createTlvInitializer(
766 macho_file,
767 decl_name,
768 required_alignment,
769 sect_index,
770 code,
771 );
772
773 // 2. Create TLV descriptor
774 try self.createTlvDescriptor(macho_file, sym_index, init_sym_index, decl_name);
775}
776
777fn createTlvInitializer(
778 self: *ZigObject,
779 macho_file: *MachO,
780 name: []const u8,
781 alignment: Atom.Alignment,
782 sect_index: u8,
783 code: []const u8,
784) !Symbol.Index {
785 const gpa = macho_file.base.comp.gpa;
786 const sym_name = try std.fmt.allocPrint(gpa, "{s}$tlv$init", .{name});
787 defer gpa.free(sym_name);
788
789 const sym_index = try self.addAtom(macho_file);
753790 const sym = macho_file.getSymbol(sym_index);
754791 const nlist = &self.symtab.items(.nlist)[sym.nlist_idx];
755792 const atom = sym.getAtom(macho_file).?;
......@@ -758,7 +795,7 @@ fn updateTlv(
758795 atom.out_n_sect = sect_index;
759796
760797 sym.value = 0;
761 sym.name = try macho_file.strings.insert(gpa, decl_name);
798 sym.name = try macho_file.strings.insert(gpa, sym_name);
762799 atom.flags.alive = true;
763800 atom.name = sym.name;
764801 nlist.n_strx = sym.name;
......@@ -767,23 +804,94 @@ fn updateTlv(
767804 nlist.n_value = 0;
768805 self.symtab.items(.size)[sym.nlist_idx] = code.len;
769806
770 atom.alignment = required_alignment;
807 atom.alignment = alignment;
771808 atom.size = code.len;
772809
773810 const slice = macho_file.sections.slice();
774811 const header = slice.items(.header)[sect_index];
775812 const atoms = &slice.items(.atoms)[sect_index];
776813
777 const gop = try self.tls_variables.getOrPut(gpa, atom.atom_index);
814 const gop = try self.tlv_initializers.getOrPut(gpa, atom.atom_index);
778815 assert(!gop.found_existing); // TODO incremental updates
779816 gop.value_ptr.* = .{ .symbol_index = sym_index };
780817
781818 // We only store the data for the TLV if it's non-zerofill.
782819 if (!header.isZerofill()) {
783 gop.value_ptr.code = try gpa.dupe(u8, code);
820 gop.value_ptr.data = try gpa.dupe(u8, code);
784821 }
785822
786823 try atoms.append(gpa, atom.atom_index);
824
825 return sym_index;
826}
827
828fn createTlvDescriptor(
829 self: *ZigObject,
830 macho_file: *MachO,
831 sym_index: Symbol.Index,
832 init_sym_index: Symbol.Index,
833 name: []const u8,
834) !void {
835 const gpa = macho_file.base.comp.gpa;
836
837 const sym = macho_file.getSymbol(sym_index);
838 const nlist = &self.symtab.items(.nlist)[sym.nlist_idx];
839 const atom = sym.getAtom(macho_file).?;
840 const alignment = Atom.Alignment.fromNonzeroByteUnits(@alignOf(u64));
841 const size: u64 = @sizeOf(u64) * 3;
842
843 const sect_index = macho_file.getSectionByName("__DATA", "__thread_vars") orelse
844 try macho_file.addSection("__DATA", "__thread_vars", .{
845 .flags = macho.S_THREAD_LOCAL_VARIABLES,
846 });
847 sym.out_n_sect = sect_index;
848 atom.out_n_sect = sect_index;
849
850 sym.value = 0;
851 sym.name = try macho_file.strings.insert(gpa, name);
852 atom.flags.alive = true;
853 atom.name = sym.name;
854 nlist.n_strx = sym.name;
855 nlist.n_sect = sect_index + 1;
856 nlist.n_type = macho.N_SECT;
857 nlist.n_value = 0;
858 self.symtab.items(.size)[sym.nlist_idx] = size;
859
860 atom.alignment = alignment;
861 atom.size = size;
862
863 const tlv_bootstrap_index = blk: {
864 const index = try self.getGlobalSymbol(macho_file, "_tlv_bootstrap", null);
865 break :blk self.symbols.items[index];
866 };
867 try atom.addReloc(macho_file, .{
868 .tag = .@"extern",
869 .offset = 0,
870 .target = tlv_bootstrap_index,
871 .addend = 0,
872 .type = .unsigned,
873 .meta = .{
874 .pcrel = false,
875 .has_subtractor = false,
876 .length = 3,
877 .symbolnum = 0,
878 },
879 });
880 try atom.addReloc(macho_file, .{
881 .tag = .@"extern",
882 .offset = 16,
883 .target = init_sym_index,
884 .addend = 0,
885 .type = .unsigned,
886 .meta = .{
887 .pcrel = false,
888 .has_subtractor = false,
889 .length = 3,
890 .symbolnum = 0,
891 },
892 });
893
894 try macho_file.sections.items(.atoms)[sect_index].append(gpa, atom.atom_index);
787895}
788896
789897fn getDeclOutputSection(
......@@ -888,7 +996,7 @@ fn lowerConst(
888996 macho_file: *MachO,
889997 name: []const u8,
890998 tv: TypedValue,
891 required_alignment: InternPool.Alignment,
999 required_alignment: Atom.Alignment,
8921000 output_section_index: u8,
8931001 src_loc: Module.SrcLoc,
8941002) !LowerConstResult {
......@@ -1040,7 +1148,7 @@ fn updateLazySymbol(
10401148 const gpa = macho_file.base.comp.gpa;
10411149 const mod = macho_file.base.comp.module.?;
10421150
1043 var required_alignment: InternPool.Alignment = .none;
1151 var required_alignment: Atom.Alignment = .none;
10441152 var code_buffer = std.ArrayList(u8).init(gpa);
10451153 defer code_buffer.deinit();
10461154
......@@ -1315,12 +1423,12 @@ const LazySymbolMetadata = struct {
13151423 const_state: State = .unused,
13161424};
13171425
1318const TlsVariable = struct {
1426const TlvInitializer = struct {
13191427 symbol_index: Symbol.Index,
1320 code: []const u8 = &[0]u8{},
1428 data: []const u8 = &[0]u8{},
13211429
1322 fn deinit(tlv: *TlsVariable, allocator: Allocator) void {
1323 allocator.free(tlv.code);
1430 fn deinit(tlv_init: *TlvInitializer, allocator: Allocator) void {
1431 allocator.free(tlv_init.data);
13241432 }
13251433};
13261434
......@@ -1329,7 +1437,7 @@ const UnnamedConstTable = std.AutoHashMapUnmanaged(InternPool.DeclIndex, std.Arr
13291437const AnonDeclTable = std.AutoHashMapUnmanaged(InternPool.Index, DeclMetadata);
13301438const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.OptionalDeclIndex, LazySymbolMetadata);
13311439const RelocationTable = std.ArrayListUnmanaged(std.ArrayListUnmanaged(Relocation));
1332const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable);
1440const TlvInitializerTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlvInitializer);
13331441
13341442const assert = std.debug.assert;
13351443const builtin = @import("builtin");
src/link/MachO/dyld_info/Rebase.zig+3-1
......@@ -3,7 +3,7 @@ const Rebase = @This();
33const std = @import("std");
44const assert = std.debug.assert;
55const leb = std.leb;
6const log = std.log.scoped(.dyld_info);
6const log = std.log.scoped(.link_dyld_info);
77const macho = std.macho;
88const testing = std.testing;
99
......@@ -39,6 +39,8 @@ pub fn finalize(rebase: *Rebase, gpa: Allocator) !void {
3939
4040 const writer = rebase.buffer.writer(gpa);
4141
42 log.debug("rebase opcodes", .{});
43
4244 std.mem.sort(Entry, rebase.entries.items, {}, Entry.lessThan);
4345
4446 try setTypePointer(writer);
src/link/MachO/dyld_info/bind.zig+7-1
......@@ -1,7 +1,7 @@
11const std = @import("std");
22const assert = std.debug.assert;
33const leb = std.leb;
4const log = std.log.scoped(.dyld_info);
4const log = std.log.scoped(.link_dyld_info);
55const macho = std.macho;
66const testing = std.testing;
77
......@@ -48,6 +48,8 @@ pub const Bind = struct {
4848
4949 const writer = self.buffer.writer(gpa);
5050
51 log.debug("bind opcodes", .{});
52
5153 std.mem.sort(Entry, self.entries.items, ctx, Entry.lessThan);
5254
5355 var start: usize = 0;
......@@ -201,6 +203,8 @@ pub const WeakBind = struct {
201203
202204 const writer = self.buffer.writer(gpa);
203205
206 log.debug("weak bind opcodes", .{});
207
204208 std.mem.sort(Entry, self.entries.items, ctx, Entry.lessThan);
205209
206210 var start: usize = 0;
......@@ -348,6 +352,8 @@ pub const LazyBind = struct {
348352 var cwriter = std.io.countingWriter(self.buffer.writer(gpa));
349353 const writer = cwriter.writer();
350354
355 log.debug("lazy bind opcodes", .{});
356
351357 var addend: i64 = 0;
352358
353359 for (self.entries.items) |entry| {