authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2024-01-21 10:32:06+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2024-01-24 12:34:42+01:00
log3a6410959ca6df6f020547c58845730753dc9e97
treea8e3f8b35f617fcd903d97be7c478527edb65632
parent411c7f6669ed2eb758f371dfde59e03abf05aa0a

macho: actually lower TLS variables

5 files changed, 163 insertions(+), 37 deletions(-)

src/arch/x86_64/Lower.zig+1-2
...@@ -439,10 +439,9 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)...@@ -439,10 +439,9 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
439 .{ .mem = Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }) },439 .{ .mem = Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }) },
440 });440 });
441 lower.result_insts_len += 1;441 lower.result_insts_len += 1;
442 emit_mnemonic = .lea;442 emit_mnemonic = .mov;
443 break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{443 break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{
444 .base = .{ .reg = .rax },444 .base = .{ .reg = .rax },
445 .disp = std.math.minInt(i32),
446 }) };445 }) };
447 }446 }
448447
src/link/MachO.zig+14-3
...@@ -606,7 +606,10 @@ pub fn flushModule(self: *MachO, arena: Allocator, prog_node: *std.Progress.Node...@@ -606,7 +606,10 @@ pub fn flushModule(self: *MachO, arena: Allocator, prog_node: *std.Progress.Node
606 if (!atom.flags.alive) continue;606 if (!atom.flags.alive) continue;
607 const sect = &self.sections.items(.header)[atom.out_n_sect];607 const sect = &self.sections.items(.header)[atom.out_n_sect];
608 if (sect.isZerofill()) continue;608 if (sect.isZerofill()) continue;
609 const code = zo.getAtomDataAlloc(self, atom.*) catch |err| switch (err) {609 if (mem.indexOf(u8, sect.segName(), "ZIG") == null) continue; // Non-Zig sections are handled separately
610 // TODO: we will resolve and write ZigObject's TLS data twice:
611 // once here, and once in writeAtoms
612 const code = zo.getAtomDataAlloc(self, gpa, atom.*) catch |err| switch (err) {
610 error.InputOutput => {613 error.InputOutput => {
611 try self.reportUnexpectedError("fetching code for '{s}' failed", .{614 try self.reportUnexpectedError("fetching code for '{s}' failed", .{
612 atom.getName(self),615 atom.getName(self),
...@@ -1806,7 +1809,7 @@ fn initOutputSections(self: *MachO) !void {...@@ -1806,7 +1809,7 @@ fn initOutputSections(self: *MachO) !void {
1806 .aarch64 => 2,1809 .aarch64 => 2,
1807 else => unreachable,1810 else => unreachable,
1808 },1811 },
1809 .flags = macho.S_SYMBOL_STUBS |1812 .flags = macho.S_REGULAR |
1810 macho.S_ATTR_PURE_INSTRUCTIONS | macho.S_ATTR_SOME_INSTRUCTIONS,1813 macho.S_ATTR_PURE_INSTRUCTIONS | macho.S_ATTR_SOME_INSTRUCTIONS,
1811 });1814 });
1812 }1815 }
...@@ -2545,6 +2548,9 @@ fn writeAtoms(self: *MachO) !void {...@@ -2545,6 +2548,9 @@ fn writeAtoms(self: *MachO) !void {
2545 defer tracy.end();2548 defer tracy.end();
25462549
2547 const gpa = self.base.comp.gpa;2550 const gpa = self.base.comp.gpa;
2551 var arena = std.heap.ArenaAllocator.init(gpa);
2552 defer arena.deinit();
2553
2548 const cpu_arch = self.getTarget().cpu.arch;2554 const cpu_arch = self.getTarget().cpu.arch;
2549 const slice = self.sections.slice();2555 const slice = self.sections.slice();
25502556
...@@ -2562,7 +2568,12 @@ fn writeAtoms(self: *MachO) !void {...@@ -2562,7 +2568,12 @@ fn writeAtoms(self: *MachO) !void {
2562 const atom = self.getAtom(atom_index).?;2568 const atom = self.getAtom(atom_index).?;
2563 assert(atom.flags.alive);2569 assert(atom.flags.alive);
2564 const off = atom.value - header.addr;2570 const off = atom.value - header.addr;
2565 @memcpy(buffer[off..][0..atom.size], atom.getFile(self).object.getAtomData(atom.*));2571 const data = switch (atom.getFile(self)) {
2572 .object => |x| x.getAtomData(atom.*),
2573 .zig_object => |x| try x.getAtomDataAlloc(self, arena.allocator(), atom.*),
2574 else => unreachable,
2575 };
2576 @memcpy(buffer[off..][0..atom.size], data);
2566 atom.resolveRelocs(self, buffer[off..][0..atom.size]) catch |err| switch (err) {2577 atom.resolveRelocs(self, buffer[off..][0..atom.size]) catch |err| switch (err) {
2567 error.ResolveFailed => has_resolve_error = true,2578 error.ResolveFailed => has_resolve_error = true,
2568 else => |e| return e,2579 else => |e| return e,
src/link/MachO/ZigObject.zig+138-30
...@@ -38,8 +38,8 @@ unnamed_consts: UnnamedConstTable = .{},...@@ -38,8 +38,8 @@ unnamed_consts: UnnamedConstTable = .{},
38/// Table of tracked AnonDecls.38/// Table of tracked AnonDecls.
39anon_decls: AnonDeclTable = .{},39anon_decls: AnonDeclTable = .{},
4040
41/// TLS variables indexed by Atom.Index.41/// TLV initializers indexed by Atom.Index.
42tls_variables: TlsTable = .{},42tlv_initializers: TlvInitializerTable = .{},
4343
44/// A table of relocations.44/// A table of relocations.
45relocs: RelocationTable = .{},45relocs: RelocationTable = .{},
...@@ -91,10 +91,10 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void {...@@ -91,10 +91,10 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void {
91 }91 }
92 self.relocs.deinit(allocator);92 self.relocs.deinit(allocator);
9393
94 for (self.tls_variables.values()) |*tlv| {94 for (self.tlv_initializers.values()) |*tlv_init| {
95 tlv.deinit(allocator);95 tlv_init.deinit(allocator);
96 }96 }
97 self.tls_variables.deinit(allocator);97 self.tlv_initializers.deinit(allocator);
98}98}
9999
100fn addNlist(self: *ZigObject, allocator: Allocator) !Symbol.Index {100fn addNlist(self: *ZigObject, allocator: Allocator) !Symbol.Index {
...@@ -137,25 +137,35 @@ pub fn addAtom(self: *ZigObject, macho_file: *MachO) !Symbol.Index {...@@ -137,25 +137,35 @@ pub fn addAtom(self: *ZigObject, macho_file: *MachO) !Symbol.Index {
137}137}
138138
139/// Caller owns the memory.139/// Caller owns the memory.
140pub fn getAtomDataAlloc(self: ZigObject, macho_file: *MachO, atom: Atom) ![]u8 {140pub fn getAtomDataAlloc(
141 const gpa = macho_file.base.comp.gpa;141 self: ZigObject,
142 macho_file: *MachO,
143 allocator: Allocator,
144 atom: Atom,
145) ![]u8 {
142 assert(atom.file == self.index);146 assert(atom.file == self.index);
143 const sect = macho_file.sections.items(.header)[atom.out_n_sect];147 const sect = macho_file.sections.items(.header)[atom.out_n_sect];
148 assert(!sect.isZerofill());
144149
145 switch (sect.type()) {150 switch (sect.type()) {
146 macho.S_THREAD_LOCAL_REGULAR => {151 macho.S_THREAD_LOCAL_REGULAR => {
147 const tlv = self.tls_variables.get(atom.atom_index).?;152 const tlv = self.tlv_initializers.get(atom.atom_index).?;
148 const code = try gpa.dupe(u8, tlv.code);153 const data = try allocator.dupe(u8, tlv.data);
149 return code;154 return data;
155 },
156 macho.S_THREAD_LOCAL_VARIABLES => {
157 const data = try allocator.alloc(u8, atom.size);
158 @memset(data, 0);
159 return data;
150 },160 },
151 else => {161 else => {
152 const file_offset = sect.offset + atom.value - sect.addr;162 const file_offset = sect.offset + atom.value - sect.addr;
153 const size = std.math.cast(usize, atom.size) orelse return error.Overflow;163 const size = std.math.cast(usize, atom.size) orelse return error.Overflow;
154 const code = try gpa.alloc(u8, size);164 const data = try allocator.alloc(u8, size);
155 errdefer gpa.free(code);165 errdefer allocator.free(data);
156 const amt = try macho_file.base.file.?.preadAll(code, file_offset);166 const amt = try macho_file.base.file.?.preadAll(data, file_offset);
157 if (amt != code.len) return error.InputOutput;167 if (amt != data.len) return error.InputOutput;
158 return code;168 return data;
159 },169 },
160 }170 }
161}171}
...@@ -421,7 +431,7 @@ pub fn lowerAnonDecl(...@@ -421,7 +431,7 @@ pub fn lowerAnonDecl(
421 self: *ZigObject,431 self: *ZigObject,
422 macho_file: *MachO,432 macho_file: *MachO,
423 decl_val: InternPool.Index,433 decl_val: InternPool.Index,
424 explicit_alignment: InternPool.Alignment,434 explicit_alignment: Atom.Alignment,
425 src_loc: Module.SrcLoc,435 src_loc: Module.SrcLoc,
426) !codegen.Result {436) !codegen.Result {
427 const gpa = macho_file.base.comp.gpa;437 const gpa = macho_file.base.comp.gpa;
...@@ -732,6 +742,9 @@ fn updateDeclCode(...@@ -732,6 +742,9 @@ fn updateDeclCode(
732 }742 }
733}743}
734744
745/// Lowering a TLV on macOS involves two stages:
746/// 1. first we lower the initializer into appopriate section (__thread_data or __thread_bss)
747/// 2. next, we create a corresponding threadlocal variable descriptor in __thread_vars
735fn updateTlv(748fn updateTlv(
736 self: *ZigObject,749 self: *ZigObject,
737 macho_file: *MachO,750 macho_file: *MachO,
...@@ -740,9 +753,7 @@ fn updateTlv(...@@ -740,9 +753,7 @@ fn updateTlv(
740 sect_index: u8,753 sect_index: u8,
741 code: []const u8,754 code: []const u8,
742) !void {755) !void {
743 const comp = macho_file.base.comp;756 const mod = macho_file.base.comp.module.?;
744 const gpa = comp.gpa;
745 const mod = comp.module.?;
746 const decl = mod.declPtr(decl_index);757 const decl = mod.declPtr(decl_index);
747 const decl_name = mod.intern_pool.stringToSlice(try decl.getFullyQualifiedName(mod));758 const decl_name = mod.intern_pool.stringToSlice(try decl.getFullyQualifiedName(mod));
748759
...@@ -750,6 +761,32 @@ fn updateTlv(...@@ -750,6 +761,32 @@ fn updateTlv(
750761
751 const required_alignment = decl.getAlignment(mod);762 const required_alignment = decl.getAlignment(mod);
752763
764 // 1. Lower TLV initializer
765 const init_sym_index = try self.createTlvInitializer(
766 macho_file,
767 decl_name,
768 required_alignment,
769 sect_index,
770 code,
771 );
772
773 // 2. Create TLV descriptor
774 try self.createTlvDescriptor(macho_file, sym_index, init_sym_index, decl_name);
775}
776
777fn createTlvInitializer(
778 self: *ZigObject,
779 macho_file: *MachO,
780 name: []const u8,
781 alignment: Atom.Alignment,
782 sect_index: u8,
783 code: []const u8,
784) !Symbol.Index {
785 const gpa = macho_file.base.comp.gpa;
786 const sym_name = try std.fmt.allocPrint(gpa, "{s}$tlv$init", .{name});
787 defer gpa.free(sym_name);
788
789 const sym_index = try self.addAtom(macho_file);
753 const sym = macho_file.getSymbol(sym_index);790 const sym = macho_file.getSymbol(sym_index);
754 const nlist = &self.symtab.items(.nlist)[sym.nlist_idx];791 const nlist = &self.symtab.items(.nlist)[sym.nlist_idx];
755 const atom = sym.getAtom(macho_file).?;792 const atom = sym.getAtom(macho_file).?;
...@@ -758,7 +795,7 @@ fn updateTlv(...@@ -758,7 +795,7 @@ fn updateTlv(
758 atom.out_n_sect = sect_index;795 atom.out_n_sect = sect_index;
759796
760 sym.value = 0;797 sym.value = 0;
761 sym.name = try macho_file.strings.insert(gpa, decl_name);798 sym.name = try macho_file.strings.insert(gpa, sym_name);
762 atom.flags.alive = true;799 atom.flags.alive = true;
763 atom.name = sym.name;800 atom.name = sym.name;
764 nlist.n_strx = sym.name;801 nlist.n_strx = sym.name;
...@@ -767,23 +804,94 @@ fn updateTlv(...@@ -767,23 +804,94 @@ fn updateTlv(
767 nlist.n_value = 0;804 nlist.n_value = 0;
768 self.symtab.items(.size)[sym.nlist_idx] = code.len;805 self.symtab.items(.size)[sym.nlist_idx] = code.len;
769806
770 atom.alignment = required_alignment;807 atom.alignment = alignment;
771 atom.size = code.len;808 atom.size = code.len;
772809
773 const slice = macho_file.sections.slice();810 const slice = macho_file.sections.slice();
774 const header = slice.items(.header)[sect_index];811 const header = slice.items(.header)[sect_index];
775 const atoms = &slice.items(.atoms)[sect_index];812 const atoms = &slice.items(.atoms)[sect_index];
776813
777 const gop = try self.tls_variables.getOrPut(gpa, atom.atom_index);814 const gop = try self.tlv_initializers.getOrPut(gpa, atom.atom_index);
778 assert(!gop.found_existing); // TODO incremental updates815 assert(!gop.found_existing); // TODO incremental updates
779 gop.value_ptr.* = .{ .symbol_index = sym_index };816 gop.value_ptr.* = .{ .symbol_index = sym_index };
780817
781 // We only store the data for the TLV if it's non-zerofill.818 // We only store the data for the TLV if it's non-zerofill.
782 if (!header.isZerofill()) {819 if (!header.isZerofill()) {
783 gop.value_ptr.code = try gpa.dupe(u8, code);820 gop.value_ptr.data = try gpa.dupe(u8, code);
784 }821 }
785822
786 try atoms.append(gpa, atom.atom_index);823 try atoms.append(gpa, atom.atom_index);
824
825 return sym_index;
826}
827
828fn createTlvDescriptor(
829 self: *ZigObject,
830 macho_file: *MachO,
831 sym_index: Symbol.Index,
832 init_sym_index: Symbol.Index,
833 name: []const u8,
834) !void {
835 const gpa = macho_file.base.comp.gpa;
836
837 const sym = macho_file.getSymbol(sym_index);
838 const nlist = &self.symtab.items(.nlist)[sym.nlist_idx];
839 const atom = sym.getAtom(macho_file).?;
840 const alignment = Atom.Alignment.fromNonzeroByteUnits(@alignOf(u64));
841 const size: u64 = @sizeOf(u64) * 3;
842
843 const sect_index = macho_file.getSectionByName("__DATA", "__thread_vars") orelse
844 try macho_file.addSection("__DATA", "__thread_vars", .{
845 .flags = macho.S_THREAD_LOCAL_VARIABLES,
846 });
847 sym.out_n_sect = sect_index;
848 atom.out_n_sect = sect_index;
849
850 sym.value = 0;
851 sym.name = try macho_file.strings.insert(gpa, name);
852 atom.flags.alive = true;
853 atom.name = sym.name;
854 nlist.n_strx = sym.name;
855 nlist.n_sect = sect_index + 1;
856 nlist.n_type = macho.N_SECT;
857 nlist.n_value = 0;
858 self.symtab.items(.size)[sym.nlist_idx] = size;
859
860 atom.alignment = alignment;
861 atom.size = size;
862
863 const tlv_bootstrap_index = blk: {
864 const index = try self.getGlobalSymbol(macho_file, "_tlv_bootstrap", null);
865 break :blk self.symbols.items[index];
866 };
867 try atom.addReloc(macho_file, .{
868 .tag = .@"extern",
869 .offset = 0,
870 .target = tlv_bootstrap_index,
871 .addend = 0,
872 .type = .unsigned,
873 .meta = .{
874 .pcrel = false,
875 .has_subtractor = false,
876 .length = 3,
877 .symbolnum = 0,
878 },
879 });
880 try atom.addReloc(macho_file, .{
881 .tag = .@"extern",
882 .offset = 16,
883 .target = init_sym_index,
884 .addend = 0,
885 .type = .unsigned,
886 .meta = .{
887 .pcrel = false,
888 .has_subtractor = false,
889 .length = 3,
890 .symbolnum = 0,
891 },
892 });
893
894 try macho_file.sections.items(.atoms)[sect_index].append(gpa, atom.atom_index);
787}895}
788896
789fn getDeclOutputSection(897fn getDeclOutputSection(
...@@ -888,7 +996,7 @@ fn lowerConst(...@@ -888,7 +996,7 @@ fn lowerConst(
888 macho_file: *MachO,996 macho_file: *MachO,
889 name: []const u8,997 name: []const u8,
890 tv: TypedValue,998 tv: TypedValue,
891 required_alignment: InternPool.Alignment,999 required_alignment: Atom.Alignment,
892 output_section_index: u8,1000 output_section_index: u8,
893 src_loc: Module.SrcLoc,1001 src_loc: Module.SrcLoc,
894) !LowerConstResult {1002) !LowerConstResult {
...@@ -1040,7 +1148,7 @@ fn updateLazySymbol(...@@ -1040,7 +1148,7 @@ fn updateLazySymbol(
1040 const gpa = macho_file.base.comp.gpa;1148 const gpa = macho_file.base.comp.gpa;
1041 const mod = macho_file.base.comp.module.?;1149 const mod = macho_file.base.comp.module.?;
10421150
1043 var required_alignment: InternPool.Alignment = .none;1151 var required_alignment: Atom.Alignment = .none;
1044 var code_buffer = std.ArrayList(u8).init(gpa);1152 var code_buffer = std.ArrayList(u8).init(gpa);
1045 defer code_buffer.deinit();1153 defer code_buffer.deinit();
10461154
...@@ -1315,12 +1423,12 @@ const LazySymbolMetadata = struct {...@@ -1315,12 +1423,12 @@ const LazySymbolMetadata = struct {
1315 const_state: State = .unused,1423 const_state: State = .unused,
1316};1424};
13171425
1318const TlsVariable = struct {1426const TlvInitializer = struct {
1319 symbol_index: Symbol.Index,1427 symbol_index: Symbol.Index,
1320 code: []const u8 = &[0]u8{},1428 data: []const u8 = &[0]u8{},
13211429
1322 fn deinit(tlv: *TlsVariable, allocator: Allocator) void {1430 fn deinit(tlv_init: *TlvInitializer, allocator: Allocator) void {
1323 allocator.free(tlv.code);1431 allocator.free(tlv_init.data);
1324 }1432 }
1325};1433};
13261434
...@@ -1329,7 +1437,7 @@ const UnnamedConstTable = std.AutoHashMapUnmanaged(InternPool.DeclIndex, std.Arr...@@ -1329,7 +1437,7 @@ const UnnamedConstTable = std.AutoHashMapUnmanaged(InternPool.DeclIndex, std.Arr
1329const AnonDeclTable = std.AutoHashMapUnmanaged(InternPool.Index, DeclMetadata);1437const AnonDeclTable = std.AutoHashMapUnmanaged(InternPool.Index, DeclMetadata);
1330const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.OptionalDeclIndex, LazySymbolMetadata);1438const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.OptionalDeclIndex, LazySymbolMetadata);
1331const RelocationTable = std.ArrayListUnmanaged(std.ArrayListUnmanaged(Relocation));1439const RelocationTable = std.ArrayListUnmanaged(std.ArrayListUnmanaged(Relocation));
1332const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable);1440const TlvInitializerTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlvInitializer);
13331441
1334const assert = std.debug.assert;1442const assert = std.debug.assert;
1335const builtin = @import("builtin");1443const builtin = @import("builtin");
src/link/MachO/dyld_info/Rebase.zig+3-1
...@@ -3,7 +3,7 @@ const Rebase = @This();...@@ -3,7 +3,7 @@ const Rebase = @This();
3const std = @import("std");3const std = @import("std");
4const assert = std.debug.assert;4const assert = std.debug.assert;
5const leb = std.leb;5const leb = std.leb;
6const log = std.log.scoped(.dyld_info);6const log = std.log.scoped(.link_dyld_info);
7const macho = std.macho;7const macho = std.macho;
8const testing = std.testing;8const testing = std.testing;
99
...@@ -39,6 +39,8 @@ pub fn finalize(rebase: *Rebase, gpa: Allocator) !void {...@@ -39,6 +39,8 @@ pub fn finalize(rebase: *Rebase, gpa: Allocator) !void {
3939
40 const writer = rebase.buffer.writer(gpa);40 const writer = rebase.buffer.writer(gpa);
4141
42 log.debug("rebase opcodes", .{});
43
42 std.mem.sort(Entry, rebase.entries.items, {}, Entry.lessThan);44 std.mem.sort(Entry, rebase.entries.items, {}, Entry.lessThan);
4345
44 try setTypePointer(writer);46 try setTypePointer(writer);
src/link/MachO/dyld_info/bind.zig+7-1
...@@ -1,7 +1,7 @@...@@ -1,7 +1,7 @@
1const std = @import("std");1const std = @import("std");
2const assert = std.debug.assert;2const assert = std.debug.assert;
3const leb = std.leb;3const leb = std.leb;
4const log = std.log.scoped(.dyld_info);4const log = std.log.scoped(.link_dyld_info);
5const macho = std.macho;5const macho = std.macho;
6const testing = std.testing;6const testing = std.testing;
77
...@@ -48,6 +48,8 @@ pub const Bind = struct {...@@ -48,6 +48,8 @@ pub const Bind = struct {
4848
49 const writer = self.buffer.writer(gpa);49 const writer = self.buffer.writer(gpa);
5050
51 log.debug("bind opcodes", .{});
52
51 std.mem.sort(Entry, self.entries.items, ctx, Entry.lessThan);53 std.mem.sort(Entry, self.entries.items, ctx, Entry.lessThan);
5254
53 var start: usize = 0;55 var start: usize = 0;
...@@ -201,6 +203,8 @@ pub const WeakBind = struct {...@@ -201,6 +203,8 @@ pub const WeakBind = struct {
201203
202 const writer = self.buffer.writer(gpa);204 const writer = self.buffer.writer(gpa);
203205
206 log.debug("weak bind opcodes", .{});
207
204 std.mem.sort(Entry, self.entries.items, ctx, Entry.lessThan);208 std.mem.sort(Entry, self.entries.items, ctx, Entry.lessThan);
205209
206 var start: usize = 0;210 var start: usize = 0;
...@@ -348,6 +352,8 @@ pub const LazyBind = struct {...@@ -348,6 +352,8 @@ pub const LazyBind = struct {
348 var cwriter = std.io.countingWriter(self.buffer.writer(gpa));352 var cwriter = std.io.countingWriter(self.buffer.writer(gpa));
349 const writer = cwriter.writer();353 const writer = cwriter.writer();
350354
355 log.debug("lazy bind opcodes", .{});
356
351 var addend: i64 = 0;357 var addend: i64 = 0;
352358
353 for (self.entries.items) |entry| {359 for (self.entries.items) |entry| {