authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-11-10 19:14:42+01:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-11-12 18:12:40-05:00
logf34247c4bc3a400d23b99c8921c9b358bf5a3250
treedc4eca3ed433b40ef8160ee18137dca8dde6de77
parent70d8baaec11ca370b73fce72d7f3dfce2277455b

elf: lower TLS data into appropriate TLS section

4 files changed, 135 insertions(+), 78 deletions(-)

src/link/Elf.zig+12-3
...@@ -4512,7 +4512,10 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void {...@@ -4512,7 +4512,10 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void {
45124512
4513 for (cover.items) |shndx| {4513 for (cover.items) |shndx| {
4514 const shdr = &self.shdrs.items[shndx];4514 const shdr = &self.shdrs.items[shndx];
4515 if (shdr.sh_type == elf.SHT_NOBITS) continue;4515 if (shdr.sh_type == elf.SHT_NOBITS) {
4516 shdr.sh_offset = 0;
4517 continue;
4518 }
4516 off = alignment.@"align"(shndx, shdr.sh_addralign, off);4519 off = alignment.@"align"(shndx, shdr.sh_addralign, off);
4517 shdr.sh_offset = off;4520 shdr.sh_offset = off;
4518 off += shdr.sh_size;4521 off += shdr.sh_size;
...@@ -4640,6 +4643,9 @@ fn allocateSpecialPhdrs(self: *Elf) void {...@@ -4640,6 +4643,9 @@ fn allocateSpecialPhdrs(self: *Elf) void {
4640}4643}
46414644
4642fn allocateAtoms(self: *Elf) void {4645fn allocateAtoms(self: *Elf) void {
4646 if (self.zigObjectPtr()) |zig_object| {
4647 zig_object.allocateTlvAtoms(self);
4648 }
4643 for (self.objects.items) |index| {4649 for (self.objects.items) |index| {
4644 self.file(index).?.object.allocateAtoms(self);4650 self.file(index).?.object.allocateAtoms(self);
4645 }4651 }
...@@ -4698,7 +4704,6 @@ fn writeAtoms(self: *Elf) !void {...@@ -4698,7 +4704,6 @@ fn writeAtoms(self: *Elf) !void {
4698 const atom_ptr = self.atom(atom_index).?;4704 const atom_ptr = self.atom(atom_index).?;
4699 assert(atom_ptr.flags.alive);4705 assert(atom_ptr.flags.alive);
47004706
4701 const object = atom_ptr.file(self).?.object;
4702 const offset = math.cast(usize, atom_ptr.value - shdr.sh_addr - base_offset) orelse4707 const offset = math.cast(usize, atom_ptr.value - shdr.sh_addr - base_offset) orelse
4703 return error.Overflow;4708 return error.Overflow;
4704 const size = math.cast(usize, atom_ptr.size) orelse return error.Overflow;4709 const size = math.cast(usize, atom_ptr.size) orelse return error.Overflow;
...@@ -4707,7 +4712,11 @@ fn writeAtoms(self: *Elf) !void {...@@ -4707,7 +4712,11 @@ fn writeAtoms(self: *Elf) !void {
47074712
4708 // TODO decompress directly into provided buffer4713 // TODO decompress directly into provided buffer
4709 const out_code = buffer[offset..][0..size];4714 const out_code = buffer[offset..][0..size];
4710 const in_code = try object.codeDecompressAlloc(self, atom_index);4715 const in_code = switch (atom_ptr.file(self).?) {
4716 .object => |x| try x.codeDecompressAlloc(self, atom_index),
4717 .zig_object => |x| try x.codeAlloc(self, atom_index),
4718 else => unreachable,
4719 };
4711 defer gpa.free(in_code);4720 defer gpa.free(in_code);
4712 @memcpy(out_code, in_code);4721 @memcpy(out_code, in_code);
47134722
src/link/Elf/Object.zig+1
...@@ -251,6 +251,7 @@ fn initOutputSection(self: Object, elf_file: *Elf, shdr: ElfShdr) error{OutOfMem...@@ -251,6 +251,7 @@ fn initOutputSection(self: Object, elf_file: *Elf, shdr: ElfShdr) error{OutOfMem
251 .type = @"type",251 .type = @"type",
252 .flags = flags,252 .flags = flags,
253 .name = name,253 .name = name,
254 .offset = std.math.maxInt(u32),
254 });255 });
255 return out_shndx;256 return out_shndx;
256}257}
src/link/Elf/ZigObject.zig+121-74
...@@ -29,6 +29,9 @@ lazy_syms: LazySymbolTable = .{},...@@ -29,6 +29,9 @@ lazy_syms: LazySymbolTable = .{},
29/// Table of tracked Decls.29/// Table of tracked Decls.
30decls: DeclTable = .{},30decls: DeclTable = .{},
3131
32/// TLS variables indexed by Atom.Index.
33tls_variables: TlsTable = .{},
34
32/// Table of unnamed constants associated with a parent `Decl`.35/// Table of unnamed constants associated with a parent `Decl`.
33/// We store them here so that we can free the constants whenever the `Decl`36/// We store them here so that we can free the constants whenever the `Decl`
34/// needs updating or is freed.37/// needs updating or is freed.
...@@ -137,6 +140,11 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void {...@@ -137,6 +140,11 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void {
137 self.anon_decls.deinit(allocator);140 self.anon_decls.deinit(allocator);
138 }141 }
139142
143 for (self.tls_variables.values()) |*tlv| {
144 tlv.deinit(allocator);
145 }
146 self.tls_variables.deinit(allocator);
147
140 if (self.dwarf) |*dw| {148 if (self.dwarf) |*dw| {
141 dw.deinit();149 dw.deinit();
142 }150 }
...@@ -212,8 +220,6 @@ pub fn flushModule(self: *ZigObject, elf_file: *Elf) !void {...@@ -212,8 +220,6 @@ pub fn flushModule(self: *ZigObject, elf_file: *Elf) !void {
212 self.saveDebugSectionsSizes(elf_file);220 self.saveDebugSectionsSizes(elf_file);
213 }221 }
214222
215 try self.sortSymbols(elf_file);
216
217 // The point of flushModule() is to commit changes, so in theory, nothing should223 // The point of flushModule() is to commit changes, so in theory, nothing should
218 // be dirty after this. However, it is possible for some things to remain224 // be dirty after this. However, it is possible for some things to remain
219 // dirty because they fail to be written in the event of compile errors,225 // dirty because they fail to be written in the event of compile errors,
...@@ -388,6 +394,19 @@ pub fn claimUnresolvedObject(self: ZigObject, elf_file: *Elf) void {...@@ -388,6 +394,19 @@ pub fn claimUnresolvedObject(self: ZigObject, elf_file: *Elf) void {
388 }394 }
389}395}
390396
397pub fn allocateTlvAtoms(self: ZigObject, elf_file: *Elf) void {
398 for (self.tls_variables.keys(), self.tls_variables.values()) |atom_index, tlv| {
399 const atom = elf_file.atom(atom_index) orelse continue;
400 if (!atom.flags.alive) continue;
401 const local = elf_file.symbol(tlv.symbol_index);
402 const shdr = elf_file.shdrs.items[atom.output_section_index];
403 atom.value += shdr.sh_addr;
404 local.value += shdr.sh_addr;
405
406 // TODO exported TLS vars
407 }
408}
409
391pub fn scanRelocs(self: *ZigObject, elf_file: *Elf, undefs: anytype) !void {410pub fn scanRelocs(self: *ZigObject, elf_file: *Elf, undefs: anytype) !void {
392 for (self.atoms.items) |atom_index| {411 for (self.atoms.items) |atom_index| {
393 const atom = elf_file.atom(atom_index) orelse continue;412 const atom = elf_file.atom(atom_index) orelse continue;
...@@ -421,72 +440,6 @@ pub fn markLive(self: *ZigObject, elf_file: *Elf) void {...@@ -421,72 +440,6 @@ pub fn markLive(self: *ZigObject, elf_file: *Elf) void {
421 }440 }
422}441}
423442
424fn sortSymbols(self: *ZigObject, elf_file: *Elf) error{OutOfMemory}!void {
425 _ = self;
426 _ = elf_file;
427 // const Entry = struct {
428 // index: Symbol.Index,
429
430 // const Ctx = struct {
431 // zobj: ZigObject,
432 // efile: *Elf,
433 // };
434
435 // pub fn lessThan(ctx: Ctx, lhs: @This(), rhs: @This()) bool {
436 // const lhs_sym = ctx.efile.symbol(zobj.symbol(lhs.index));
437 // const rhs_sym = ctx.efile.symbol(zobj.symbol(rhs.index));
438 // if (lhs_sym.outputShndx() != null and rhs_sym.outputShndx() != null) {
439 // if (lhs_sym.output_section_index == rhs_sym.output_section_index) {
440 // if (lhs_sym.value == rhs_sym.value) {
441 // return lhs_sym.name_offset < rhs_sym.name_offset;
442 // }
443 // return lhs_sym.value < rhs_sym.value;
444 // }
445 // return lhs_sym.output_section_index < rhs_sym.output_section_index;
446 // }
447 // if (lhs_sym.outputShndx() != null) {
448 // if (rhs_sym.isAbs(ctx.efile)) return false;
449 // return true;
450 // }
451 // return false;
452 // }
453 // };
454
455 // const gpa = elf_file.base.allocator;
456
457 // {
458 // const sorted = try gpa.alloc(Entry, self.local_symbols.items.len);
459 // defer gpa.free(sorted);
460 // for (0..self.local_symbols.items.len) |index| {
461 // sorted[i] = .{ .index = @as(Symbol.Index, @intCast(index)) };
462 // }
463 // mem.sort(Entry, sorted, .{ .zobj = self, .efile = elf_file }, Entry.lessThan);
464
465 // const backlinks = try gpa.alloc(Symbol.Index, sorted.len);
466 // defer gpa.free(backlinks);
467 // for (sorted, 0..) |entry, i| {
468 // backlinks[entry.index] = @as(Symbol.Index, @intCast(i));
469 // }
470
471 // const local_symbols = try self.local_symbols.toOwnedSlice(gpa);
472 // defer gpa.free(local_symbols);
473
474 // try self.local_symbols.ensureTotalCapacityPrecise(gpa, local_symbols.len);
475 // for (sorted) |entry| {
476 // self.local_symbols.appendAssumeCapacity(local_symbols[entry.index]);
477 // }
478
479 // for (self.)
480 // }
481
482 // const sorted_globals = try gpa.alloc(Entry, self.global_symbols.items.len);
483 // defer gpa.free(sorted_globals);
484 // for (self.global_symbols.items, 0..) |index, i| {
485 // sorted_globals[i] = .{ .index = index };
486 // }
487 // mem.sort(Entry, sorted_globals, elf_file, Entry.lessThan);
488}
489
490pub fn updateArSymtab(self: ZigObject, ar_symtab: *Archive.ArSymtab, elf_file: *Elf) error{OutOfMemory}!void {443pub fn updateArSymtab(self: ZigObject, ar_symtab: *Archive.ArSymtab, elf_file: *Elf) error{OutOfMemory}!void {
491 const gpa = elf_file.base.allocator;444 const gpa = elf_file.base.allocator;
492445
...@@ -583,6 +536,13 @@ pub fn codeAlloc(self: ZigObject, elf_file: *Elf, atom_index: Atom.Index) ![]u8...@@ -583,6 +536,13 @@ pub fn codeAlloc(self: ZigObject, elf_file: *Elf, atom_index: Atom.Index) ![]u8
583 const atom = elf_file.atom(atom_index).?;536 const atom = elf_file.atom(atom_index).?;
584 assert(atom.file_index == self.index);537 assert(atom.file_index == self.index);
585 const shdr = &elf_file.shdrs.items[atom.outputShndx().?];538 const shdr = &elf_file.shdrs.items[atom.outputShndx().?];
539
540 if (shdr.sh_flags & elf.SHF_TLS != 0) {
541 const tlv = self.tls_variables.get(atom_index).?;
542 const code = try gpa.dupe(u8, tlv.code);
543 return code;
544 }
545
586 const file_offset = shdr.sh_offset + atom.value - shdr.sh_addr;546 const file_offset = shdr.sh_offset + atom.value - shdr.sh_addr;
587 const size = std.math.cast(usize, atom.size) orelse return error.Overflow;547 const size = std.math.cast(usize, atom.size) orelse return error.Overflow;
588 const code = try gpa.alloc(u8, size);548 const code = try gpa.alloc(u8, size);
...@@ -765,15 +725,37 @@ pub fn getOrCreateMetadataForDecl(...@@ -765,15 +725,37 @@ pub fn getOrCreateMetadataForDecl(
765 return gop.value_ptr.symbol_index;725 return gop.value_ptr.symbol_index;
766}726}
767727
768fn getDeclShdrIndex(self: *ZigObject, elf_file: *Elf, decl_index: Module.Decl.Index, code: []const u8) u16 {728fn getDeclShdrIndex(
729 self: *ZigObject,
730 elf_file: *Elf,
731 decl: *const Module.Decl,
732 code: []const u8,
733) error{OutOfMemory}!u16 {
769 _ = self;734 _ = self;
770 const mod = elf_file.base.options.module.?;735 const mod = elf_file.base.options.module.?;
771 const decl = mod.declPtr(decl_index);736 const single_threaded = elf_file.base.options.single_threaded;
772 const shdr_index = switch (decl.ty.zigTypeTag(mod)) {737 const shdr_index = switch (decl.ty.zigTypeTag(mod)) {
773 // TODO: what if this is a function pointer?
774 .Fn => elf_file.zig_text_section_index.?,738 .Fn => elf_file.zig_text_section_index.?,
775 else => blk: {739 else => blk: {
776 if (decl.getOwnedVariable(mod)) |variable| {740 if (decl.getOwnedVariable(mod)) |variable| {
741 if (variable.is_threadlocal and !single_threaded) {
742 const is_all_zeroes = for (code) |byte| {
743 if (byte != 0) break false;
744 } else true;
745 if (is_all_zeroes) break :blk elf_file.sectionByName(".tbss") orelse try elf_file.addSection(.{
746 .type = elf.SHT_NOBITS,
747 .flags = elf.SHF_ALLOC | elf.SHF_WRITE | elf.SHF_TLS,
748 .name = ".tbss",
749 .offset = std.math.maxInt(u32),
750 });
751
752 break :blk elf_file.sectionByName(".tdata") orelse try elf_file.addSection(.{
753 .type = elf.SHT_PROGBITS,
754 .flags = elf.SHF_ALLOC | elf.SHF_WRITE | elf.SHF_TLS,
755 .name = ".tdata",
756 .offset = std.math.maxInt(u32),
757 });
758 }
777 if (variable.is_const) break :blk elf_file.zig_data_rel_ro_section_index.?;759 if (variable.is_const) break :blk elf_file.zig_data_rel_ro_section_index.?;
778 if (variable.init.toValue().isUndefDeep(mod)) {760 if (variable.init.toValue().isUndefDeep(mod)) {
779 const mode = elf_file.base.options.optimize_mode;761 const mode = elf_file.base.options.optimize_mode;
...@@ -799,6 +781,7 @@ fn updateDeclCode(...@@ -799,6 +781,7 @@ fn updateDeclCode(
799 elf_file: *Elf,781 elf_file: *Elf,
800 decl_index: Module.Decl.Index,782 decl_index: Module.Decl.Index,
801 sym_index: Symbol.Index,783 sym_index: Symbol.Index,
784 shdr_index: u16,
802 code: []const u8,785 code: []const u8,
803 stt_bits: u8,786 stt_bits: u8,
804) !void {787) !void {
...@@ -815,7 +798,6 @@ fn updateDeclCode(...@@ -815,7 +798,6 @@ fn updateDeclCode(
815 const esym = &self.local_esyms.items(.elf_sym)[sym.esym_index];798 const esym = &self.local_esyms.items(.elf_sym)[sym.esym_index];
816 const atom_ptr = sym.atom(elf_file).?;799 const atom_ptr = sym.atom(elf_file).?;
817800
818 const shdr_index = self.getDeclShdrIndex(elf_file, decl_index, code);
819 sym.output_section_index = shdr_index;801 sym.output_section_index = shdr_index;
820 atom_ptr.output_section_index = shdr_index;802 atom_ptr.output_section_index = shdr_index;
821803
...@@ -893,6 +875,53 @@ fn updateDeclCode(...@@ -893,6 +875,53 @@ fn updateDeclCode(
893 }875 }
894}876}
895877
878fn updateTlv(
879 self: *ZigObject,
880 elf_file: *Elf,
881 decl_index: Module.Decl.Index,
882 sym_index: Symbol.Index,
883 shndx: u16,
884 code: []const u8,
885) !void {
886 const gpa = elf_file.base.allocator;
887 const mod = elf_file.base.options.module.?;
888 const decl = mod.declPtr(decl_index);
889 const decl_name = mod.intern_pool.stringToSlice(try decl.getFullyQualifiedName(mod));
890
891 log.debug("updateTlv {s}{*}", .{ decl_name, decl });
892
893 const required_alignment = decl.getAlignment(mod);
894
895 const sym = elf_file.symbol(sym_index);
896 const esym = &self.local_esyms.items(.elf_sym)[sym.esym_index];
897 const atom_ptr = sym.atom(elf_file).?;
898
899 sym.output_section_index = shndx;
900 atom_ptr.output_section_index = shndx;
901
902 sym.name_offset = try self.strtab.insert(gpa, decl_name);
903 atom_ptr.flags.alive = true;
904 atom_ptr.name_offset = sym.name_offset;
905 esym.st_name = sym.name_offset;
906 esym.st_info |= elf.STT_TLS;
907 esym.st_size = code.len;
908
909 atom_ptr.alignment = required_alignment;
910 atom_ptr.size = code.len;
911
912 {
913 const gop = try self.tls_variables.getOrPut(gpa, atom_ptr.atom_index);
914 assert(!gop.found_existing); // TODO incremental updates
915 gop.value_ptr.* = .{ .symbol_index = sym_index, .code = try gpa.dupe(u8, code) };
916 }
917
918 {
919 const gop = try elf_file.output_sections.getOrPut(gpa, atom_ptr.output_section_index);
920 if (!gop.found_existing) gop.value_ptr.* = .{};
921 try gop.value_ptr.append(gpa, atom_ptr.atom_index);
922 }
923}
924
896pub fn updateFunc(925pub fn updateFunc(
897 self: *ZigObject,926 self: *ZigObject,
898 elf_file: *Elf,927 elf_file: *Elf,
...@@ -947,7 +976,10 @@ pub fn updateFunc(...@@ -947,7 +976,10 @@ pub fn updateFunc(
947 return;976 return;
948 },977 },
949 };978 };
950 try self.updateDeclCode(elf_file, decl_index, sym_index, code, elf.STT_FUNC);979
980 const shndx = try self.getDeclShdrIndex(elf_file, decl, code);
981 try self.updateDeclCode(elf_file, decl_index, sym_index, shndx, code, elf.STT_FUNC);
982
951 if (decl_state) |*ds| {983 if (decl_state) |*ds| {
952 const sym = elf_file.symbol(sym_index);984 const sym = elf_file.symbol(sym_index);
953 try self.dwarf.?.commitDeclState(985 try self.dwarf.?.commitDeclState(
...@@ -1026,7 +1058,12 @@ pub fn updateDecl(...@@ -1026,7 +1058,12 @@ pub fn updateDecl(
1026 },1058 },
1027 };1059 };
10281060
1029 try self.updateDeclCode(elf_file, decl_index, sym_index, code, elf.STT_OBJECT);1061 const shndx = try self.getDeclShdrIndex(elf_file, decl, code);
1062 if (elf_file.shdrs.items[shndx].sh_flags & elf.SHF_TLS != 0)
1063 try self.updateTlv(elf_file, decl_index, sym_index, shndx, code)
1064 else
1065 try self.updateDeclCode(elf_file, decl_index, sym_index, shndx, code, elf.STT_OBJECT);
1066
1030 if (decl_state) |*ds| {1067 if (decl_state) |*ds| {
1031 const sym = elf_file.symbol(sym_index);1068 const sym = elf_file.symbol(sym_index);
1032 try self.dwarf.?.commitDeclState(1069 try self.dwarf.?.commitDeclState(
...@@ -1454,11 +1491,21 @@ const DeclMetadata = struct {...@@ -1454,11 +1491,21 @@ const DeclMetadata = struct {
1454 }1491 }
1455};1492};
14561493
1494const TlsVariable = struct {
1495 symbol_index: Symbol.Index,
1496 code: []const u8,
1497
1498 fn deinit(tlv: *TlsVariable, allocator: Allocator) void {
1499 allocator.free(tlv.code);
1500 }
1501};
1502
1457const AtomList = std.ArrayListUnmanaged(Atom.Index);1503const AtomList = std.ArrayListUnmanaged(Atom.Index);
1458const UnnamedConstTable = std.AutoHashMapUnmanaged(Module.Decl.Index, std.ArrayListUnmanaged(Symbol.Index));1504const UnnamedConstTable = std.AutoHashMapUnmanaged(Module.Decl.Index, std.ArrayListUnmanaged(Symbol.Index));
1459const DeclTable = std.AutoHashMapUnmanaged(Module.Decl.Index, DeclMetadata);1505const DeclTable = std.AutoHashMapUnmanaged(Module.Decl.Index, DeclMetadata);
1460const AnonDeclTable = std.AutoHashMapUnmanaged(InternPool.Index, DeclMetadata);1506const AnonDeclTable = std.AutoHashMapUnmanaged(InternPool.Index, DeclMetadata);
1461const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata);1507const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata);
1508const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable);
14621509
1463const assert = std.debug.assert;1510const assert = std.debug.assert;
1464const builtin = @import("builtin");1511const builtin = @import("builtin");
src/target.zig+1-1
...@@ -654,7 +654,7 @@ pub fn supportsTailCall(target: std.Target, backend: std.builtin.CompilerBackend...@@ -654,7 +654,7 @@ pub fn supportsTailCall(target: std.Target, backend: std.builtin.CompilerBackend
654654
655pub fn supportsThreads(target: std.Target, backend: std.builtin.CompilerBackend) bool {655pub fn supportsThreads(target: std.Target, backend: std.builtin.CompilerBackend) bool {
656 return switch (backend) {656 return switch (backend) {
657 .stage2_x86_64 => target.ofmt == .macho,657 .stage2_x86_64 => target.ofmt == .macho or target.ofmt == .elf,
658 else => true,658 else => true,
659 };659 };
660}660}