authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-09-30 08:43:33+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-09-30 08:43:33+02:00
log873c695c41dffd89ba7ef1b3ed6662e429bfa00d
treeb2131a824259cf307d2e626b0f38941336af97a8
parent101df768a06ef85753efdd6dc558bca68d50d1a5
parente72fd185e01aac14d7962f2eeb718653dc0c8e68
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17319 from ziglang/elf-tls

elf: add basic TLS segment handling

7 files changed, 486 insertions(+), 143 deletions(-)

src/link/Elf.zig+184-86
...@@ -43,6 +43,12 @@ phdr_load_ro_index: ?u16 = null,...@@ -43,6 +43,12 @@ phdr_load_ro_index: ?u16 = null,
43phdr_load_rw_index: ?u16 = null,43phdr_load_rw_index: ?u16 = null,
44/// The index into the program headers of a PT_LOAD program header with zerofill data.44/// The index into the program headers of a PT_LOAD program header with zerofill data.
45phdr_load_zerofill_index: ?u16 = null,45phdr_load_zerofill_index: ?u16 = null,
46/// The index into the program headers of the PT_TLS program header.
47phdr_tls_index: ?u16 = null,
48/// The index into the program headers of a PT_LOAD program header with TLS data.
49phdr_load_tls_data_index: ?u16 = null,
50/// The index into the program headers of a PT_LOAD program header with TLS zerofill data.
51phdr_load_tls_zerofill_index: ?u16 = null,
4652
47entry_addr: ?u64 = null,53entry_addr: ?u64 = null,
48page_size: u32,54page_size: u32,
...@@ -56,10 +62,13 @@ strtab: StringTable(.strtab) = .{},...@@ -56,10 +62,13 @@ strtab: StringTable(.strtab) = .{},
56/// Representation of the GOT table as committed to the file.62/// Representation of the GOT table as committed to the file.
57got: GotSection = .{},63got: GotSection = .{},
5864
65/// Tracked section headers
59text_section_index: ?u16 = null,66text_section_index: ?u16 = null,
60rodata_section_index: ?u16 = null,67rodata_section_index: ?u16 = null,
61data_section_index: ?u16 = null,68data_section_index: ?u16 = null,
62bss_section_index: ?u16 = null,69bss_section_index: ?u16 = null,
70tdata_section_index: ?u16 = null,
71tbss_section_index: ?u16 = null,
63eh_frame_section_index: ?u16 = null,72eh_frame_section_index: ?u16 = null,
64eh_frame_hdr_section_index: ?u16 = null,73eh_frame_hdr_section_index: ?u16 = null,
65dynamic_section_index: ?u16 = null,74dynamic_section_index: ?u16 = null,
...@@ -238,7 +247,8 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Elf {...@@ -238,7 +247,8 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Elf {
238 else247 else
239 elf.VER_NDX_LOCAL;248 elf.VER_NDX_LOCAL;
240249
241 var dwarf: ?Dwarf = if (!options.strip and options.module != null)250 const use_llvm = options.use_llvm;
251 var dwarf: ?Dwarf = if (!options.strip and options.module != null and !use_llvm)
242 Dwarf.init(gpa, &self.base, options.target)252 Dwarf.init(gpa, &self.base, options.target)
243 else253 else
244 null;254 null;
...@@ -255,7 +265,6 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Elf {...@@ -255,7 +265,6 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Elf {
255 .page_size = page_size,265 .page_size = page_size,
256 .default_sym_version = default_sym_version,266 .default_sym_version = default_sym_version,
257 };267 };
258 const use_llvm = options.use_llvm;
259 if (use_llvm and options.module != null) {268 if (use_llvm and options.module != null) {
260 self.llvm_object = try LlvmObject.create(gpa, options);269 self.llvm_object = try LlvmObject.create(gpa, options);
261 }270 }
...@@ -358,10 +367,12 @@ fn detectAllocCollision(self: *Elf, start: u64, size: u64) ?u64 {...@@ -358,10 +367,12 @@ fn detectAllocCollision(self: *Elf, start: u64, size: u64) ?u64 {
358 }367 }
359 }368 }
360369
361 for (self.shdrs.items) |section| {370 for (self.shdrs.items) |shdr| {
362 const increased_size = padToIdeal(section.sh_size);371 // SHT_NOBITS takes no physical space in the output file so set its size to 0.
363 const test_end = section.sh_offset + increased_size;372 const sh_size = if (shdr.sh_type == elf.SHT_NOBITS) 0 else shdr.sh_size;
364 if (end > section.sh_offset and start < test_end) {373 const increased_size = padToIdeal(sh_size);
374 const test_end = shdr.sh_offset + increased_size;
375 if (end > shdr.sh_offset and start < test_end) {
365 return test_end;376 return test_end;
366 }377 }
367 }378 }
...@@ -429,15 +440,15 @@ pub fn allocateSegment(self: *Elf, opts: AllocateSegmentOpts) error{OutOfMemory}...@@ -429,15 +440,15 @@ pub fn allocateSegment(self: *Elf, opts: AllocateSegmentOpts) error{OutOfMemory}
429 const addr = opts.addr orelse blk: {440 const addr = opts.addr orelse blk: {
430 const reserved_capacity = self.calcImageBase() * 4;441 const reserved_capacity = self.calcImageBase() * 4;
431 // Calculate largest VM address442 // Calculate largest VM address
432 const count = self.phdrs.items.len;
433 var addresses = std.ArrayList(u64).init(gpa);443 var addresses = std.ArrayList(u64).init(gpa);
434 defer addresses.deinit();444 defer addresses.deinit();
435 try addresses.ensureTotalCapacityPrecise(count);445 try addresses.ensureTotalCapacityPrecise(self.phdrs.items.len);
436 for (self.phdrs.items) |phdr| {446 for (self.phdrs.items) |phdr| {
447 if (phdr.p_type != elf.PT_LOAD) continue;
437 addresses.appendAssumeCapacity(phdr.p_vaddr + reserved_capacity);448 addresses.appendAssumeCapacity(phdr.p_vaddr + reserved_capacity);
438 }449 }
439 mem.sort(u64, addresses.items, {}, std.sort.asc(u64));450 mem.sort(u64, addresses.items, {}, std.sort.asc(u64));
440 break :blk mem.alignForward(u64, addresses.items[count - 1], opts.alignment);451 break :blk mem.alignForward(u64, addresses.pop(), opts.alignment);
441 };452 };
442 log.debug("allocating phdr({d})({c}{c}{c}) from 0x{x} to 0x{x} (0x{x} - 0x{x})", .{453 log.debug("allocating phdr({d})({c}{c}{c}) from 0x{x} to 0x{x} (0x{x} - 0x{x})", .{
443 index,454 index,
...@@ -492,7 +503,7 @@ pub fn allocateAllocSection(self: *Elf, opts: AllocateAllocSectionOpts) error{Ou...@@ -492,7 +503,7 @@ pub fn allocateAllocSection(self: *Elf, opts: AllocateAllocSectionOpts) error{Ou
492 .sh_flags = opts.flags,503 .sh_flags = opts.flags,
493 .sh_addr = phdr.p_vaddr,504 .sh_addr = phdr.p_vaddr,
494 .sh_offset = phdr.p_offset,505 .sh_offset = phdr.p_offset,
495 .sh_size = phdr.p_filesz,506 .sh_size = phdr.p_memsz,
496 .sh_link = 0,507 .sh_link = 0,
497 .sh_info = 0,508 .sh_info = 0,
498 .sh_addralign = opts.alignment,509 .sh_addralign = opts.alignment,
...@@ -543,7 +554,6 @@ pub fn populateMissingMetadata(self: *Elf) !void {...@@ -543,7 +554,6 @@ pub fn populateMissingMetadata(self: *Elf) !void {
543 };554 };
544 const ptr_size: u8 = self.ptrWidthBytes();555 const ptr_size: u8 = self.ptrWidthBytes();
545 const is_linux = self.base.options.target.os.tag == .linux;556 const is_linux = self.base.options.target.os.tag == .linux;
546 const large_addrspace = self.base.options.target.ptrBitWidth() >= 32;
547 const image_base = self.calcImageBase();557 const image_base = self.calcImageBase();
548558
549 if (self.phdr_table_index == null) {559 if (self.phdr_table_index == null) {
...@@ -566,23 +576,16 @@ pub fn populateMissingMetadata(self: *Elf) !void {...@@ -566,23 +576,16 @@ pub fn populateMissingMetadata(self: *Elf) !void {
566 }576 }
567577
568 if (self.phdr_table_load_index == null) {578 if (self.phdr_table_load_index == null) {
569 self.phdr_table_load_index = @intCast(self.phdrs.items.len);579 self.phdr_table_load_index = try self.allocateSegment(.{
570 try self.phdrs.append(gpa, .{580 .addr = image_base,
571 .p_type = elf.PT_LOAD,581 .size = 0,
572 .p_offset = 0,582 .alignment = self.page_size,
573 .p_filesz = 0,
574 .p_vaddr = image_base,
575 .p_paddr = image_base,
576 .p_memsz = 0,
577 .p_align = self.page_size,
578 .p_flags = elf.PF_R,
579 });583 });
580 self.phdr_table_dirty = true;584 self.phdr_table_dirty = true;
581 }585 }
582586
583 if (self.phdr_load_re_index == null) {587 if (self.phdr_load_re_index == null) {
584 self.phdr_load_re_index = try self.allocateSegment(.{588 self.phdr_load_re_index = try self.allocateSegment(.{
585 .addr = self.defaultEntryAddress(),
586 .size = self.base.options.program_code_size_hint,589 .size = self.base.options.program_code_size_hint,
587 .alignment = self.page_size,590 .alignment = self.page_size,
588 .flags = elf.PF_X | elf.PF_R | elf.PF_W,591 .flags = elf.PF_X | elf.PF_R | elf.PF_W,
...@@ -591,12 +594,10 @@ pub fn populateMissingMetadata(self: *Elf) !void {...@@ -591,12 +594,10 @@ pub fn populateMissingMetadata(self: *Elf) !void {
591 }594 }
592595
593 if (self.phdr_got_index == null) {596 if (self.phdr_got_index == null) {
594 const addr: u64 = if (large_addrspace) 0x4000000 else 0x8000;
595 // We really only need ptr alignment but since we are using PROGBITS, linux requires597 // We really only need ptr alignment but since we are using PROGBITS, linux requires
596 // page align.598 // page align.
597 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);599 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
598 self.phdr_got_index = try self.allocateSegment(.{600 self.phdr_got_index = try self.allocateSegment(.{
599 .addr = addr,
600 .size = @as(u64, ptr_size) * self.base.options.symbol_count_hint,601 .size = @as(u64, ptr_size) * self.base.options.symbol_count_hint,
601 .alignment = alignment,602 .alignment = alignment,
602 .flags = elf.PF_R | elf.PF_W,603 .flags = elf.PF_R | elf.PF_W,
...@@ -604,10 +605,8 @@ pub fn populateMissingMetadata(self: *Elf) !void {...@@ -604,10 +605,8 @@ pub fn populateMissingMetadata(self: *Elf) !void {
604 }605 }
605606
606 if (self.phdr_load_ro_index == null) {607 if (self.phdr_load_ro_index == null) {
607 const addr: u64 = if (large_addrspace) 0xc000000 else 0xa000;
608 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);608 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
609 self.phdr_load_ro_index = try self.allocateSegment(.{609 self.phdr_load_ro_index = try self.allocateSegment(.{
610 .addr = addr,
611 .size = 1024,610 .size = 1024,
612 .alignment = alignment,611 .alignment = alignment,
613 .flags = elf.PF_R | elf.PF_W,612 .flags = elf.PF_R | elf.PF_W,
...@@ -615,10 +614,8 @@ pub fn populateMissingMetadata(self: *Elf) !void {...@@ -615,10 +614,8 @@ pub fn populateMissingMetadata(self: *Elf) !void {
615 }614 }
616615
617 if (self.phdr_load_rw_index == null) {616 if (self.phdr_load_rw_index == null) {
618 const addr: u64 = if (large_addrspace) 0x10000000 else 0xc000;
619 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);617 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
620 self.phdr_load_rw_index = try self.allocateSegment(.{618 self.phdr_load_rw_index = try self.allocateSegment(.{
621 .addr = addr,
622 .size = 1024,619 .size = 1024,
623 .alignment = alignment,620 .alignment = alignment,
624 .flags = elf.PF_R | elf.PF_W,621 .flags = elf.PF_R | elf.PF_W,
...@@ -626,10 +623,8 @@ pub fn populateMissingMetadata(self: *Elf) !void {...@@ -626,10 +623,8 @@ pub fn populateMissingMetadata(self: *Elf) !void {
626 }623 }
627624
628 if (self.phdr_load_zerofill_index == null) {625 if (self.phdr_load_zerofill_index == null) {
629 const addr: u64 = if (large_addrspace) 0x14000000 else 0xf000;
630 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);626 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
631 self.phdr_load_zerofill_index = try self.allocateSegment(.{627 self.phdr_load_zerofill_index = try self.allocateSegment(.{
632 .addr = addr,
633 .size = 0,628 .size = 0,
634 .alignment = alignment,629 .alignment = alignment,
635 .flags = elf.PF_R | elf.PF_W,630 .flags = elf.PF_R | elf.PF_W,
...@@ -639,6 +634,53 @@ pub fn populateMissingMetadata(self: *Elf) !void {...@@ -639,6 +634,53 @@ pub fn populateMissingMetadata(self: *Elf) !void {
639 phdr.p_memsz = 1024;634 phdr.p_memsz = 1024;
640 }635 }
641636
637 if (!self.base.options.single_threaded) {
638 if (self.phdr_load_tls_data_index == null) {
639 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
640 self.phdr_load_tls_data_index = try self.allocateSegment(.{
641 .size = 1024,
642 .alignment = alignment,
643 .flags = elf.PF_R | elf.PF_W,
644 });
645 }
646
647 if (self.phdr_load_tls_zerofill_index == null) {
648 // TODO .tbss doesn't need any physical or memory representation (aka a loadable segment)
649 // since the loader only cares about the PT_TLS to work out TLS size. However, when
650 // relocating we need to have .tdata and .tbss contiguously laid out so that we can
651 // work out correct offsets to the start/end of the TLS segment. I am thinking that
652 // perhaps it's possible to completely spoof it by having an abstracted mechanism
653 // for this that wouldn't require us to explicitly track .tbss. Anyhow, for now,
654 // we go the savage route of treating .tbss like .bss.
655 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
656 self.phdr_load_tls_zerofill_index = try self.allocateSegment(.{
657 .size = 0,
658 .alignment = alignment,
659 .flags = elf.PF_R | elf.PF_W,
660 });
661 const phdr = &self.phdrs.items[self.phdr_load_tls_zerofill_index.?];
662 phdr.p_offset = self.phdrs.items[self.phdr_load_tls_data_index.?].p_offset; // .tbss overlaps .tdata
663 phdr.p_memsz = 1024;
664 }
665
666 if (self.phdr_tls_index == null) {
667 self.phdr_tls_index = @intCast(self.phdrs.items.len);
668 const phdr_tdata = &self.phdrs.items[self.phdr_load_tls_data_index.?];
669 const phdr_tbss = &self.phdrs.items[self.phdr_load_tls_zerofill_index.?];
670 try self.phdrs.append(gpa, .{
671 .p_type = elf.PT_TLS,
672 .p_offset = phdr_tdata.p_offset,
673 .p_vaddr = phdr_tdata.p_vaddr,
674 .p_paddr = phdr_tdata.p_paddr,
675 .p_filesz = phdr_tdata.p_filesz,
676 .p_memsz = phdr_tbss.p_vaddr + phdr_tbss.p_memsz - phdr_tdata.p_vaddr,
677 .p_align = ptr_size,
678 .p_flags = elf.PF_R,
679 });
680 self.phdr_table_dirty = true;
681 }
682 }
683
642 if (self.shstrtab_section_index == null) {684 if (self.shstrtab_section_index == null) {
643 assert(self.shstrtab.buffer.items.len == 0);685 assert(self.shstrtab.buffer.items.len == 0);
644 try self.shstrtab.buffer.append(gpa, 0); // need a 0 at position 0686 try self.shstrtab.buffer.append(gpa, 0); // need a 0 at position 0
...@@ -707,6 +749,31 @@ pub fn populateMissingMetadata(self: *Elf) !void {...@@ -707,6 +749,31 @@ pub fn populateMissingMetadata(self: *Elf) !void {
707 try self.last_atom_and_free_list_table.putNoClobber(gpa, self.bss_section_index.?, .{});749 try self.last_atom_and_free_list_table.putNoClobber(gpa, self.bss_section_index.?, .{});
708 }750 }
709751
752 if (self.phdr_load_tls_data_index) |phdr_index| {
753 if (self.tdata_section_index == null) {
754 self.tdata_section_index = try self.allocateAllocSection(.{
755 .name = ".tdata",
756 .phdr_index = phdr_index,
757 .alignment = ptr_size,
758 .flags = elf.SHF_ALLOC | elf.SHF_WRITE | elf.SHF_TLS,
759 });
760 try self.last_atom_and_free_list_table.putNoClobber(gpa, self.tdata_section_index.?, .{});
761 }
762 }
763
764 if (self.phdr_load_tls_zerofill_index) |phdr_index| {
765 if (self.tbss_section_index == null) {
766 self.tbss_section_index = try self.allocateAllocSection(.{
767 .name = ".tbss",
768 .phdr_index = phdr_index,
769 .alignment = ptr_size,
770 .flags = elf.SHF_ALLOC | elf.SHF_WRITE | elf.SHF_TLS,
771 .type = elf.SHT_NOBITS,
772 });
773 try self.last_atom_and_free_list_table.putNoClobber(gpa, self.tbss_section_index.?, .{});
774 }
775 }
776
710 if (self.symtab_section_index == null) {777 if (self.symtab_section_index == null) {
711 const min_align: u16 = if (small_ptr) @alignOf(elf.Elf32_Sym) else @alignOf(elf.Elf64_Sym);778 const min_align: u16 = if (small_ptr) @alignOf(elf.Elf32_Sym) else @alignOf(elf.Elf64_Sym);
712 const each_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Sym) else @sizeOf(elf.Elf64_Sym);779 const each_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Sym) else @sizeOf(elf.Elf64_Sym);
...@@ -844,10 +911,7 @@ pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void {...@@ -844,10 +911,7 @@ pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void {
844 if (needed_size > self.allocatedSize(shdr.sh_offset) and !is_zerofill) {911 if (needed_size > self.allocatedSize(shdr.sh_offset) and !is_zerofill) {
845 // Must move the entire section.912 // Must move the entire section.
846 const new_offset = self.findFreeSpace(needed_size, self.page_size);913 const new_offset = self.findFreeSpace(needed_size, self.page_size);
847 const existing_size = if (self.last_atom_and_free_list_table.get(shdr_index)) |meta| blk: {914 const existing_size = shdr.sh_size;
848 const last = self.atom(meta.last_atom_index) orelse break :blk 0;
849 break :blk (last.value + last.size) - phdr.p_vaddr;
850 } else shdr.sh_size;
851 shdr.sh_size = 0;915 shdr.sh_size = 0;
852916
853 log.debug("new '{s}' file offset 0x{x} to 0x{x}", .{917 log.debug("new '{s}' file offset 0x{x} to 0x{x}", .{
...@@ -857,12 +921,18 @@ pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void {...@@ -857,12 +921,18 @@ pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void {
857 });921 });
858922
859 const amt = try self.base.file.?.copyRangeAll(shdr.sh_offset, self.base.file.?, new_offset, existing_size);923 const amt = try self.base.file.?.copyRangeAll(shdr.sh_offset, self.base.file.?, new_offset, existing_size);
924 // TODO figure out what to about this error condition - how to communicate it up.
860 if (amt != existing_size) return error.InputOutput;925 if (amt != existing_size) return error.InputOutput;
861926
862 shdr.sh_offset = new_offset;927 shdr.sh_offset = new_offset;
863 phdr.p_offset = new_offset;928 phdr.p_offset = new_offset;
864 }929 }
865930
931 shdr.sh_size = needed_size;
932 if (!is_zerofill) {
933 phdr.p_filesz = needed_size;
934 }
935
866 const mem_capacity = self.allocatedVirtualSize(phdr.p_vaddr);936 const mem_capacity = self.allocatedVirtualSize(phdr.p_vaddr);
867 if (needed_size > mem_capacity) {937 if (needed_size > mem_capacity) {
868 // We are exceeding our allocated VM capacity so we need to shift everything in memory938 // We are exceeding our allocated VM capacity so we need to shift everything in memory
...@@ -889,13 +959,8 @@ pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void {...@@ -889,13 +959,8 @@ pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void {
889 }959 }
890 }960 }
891961
892 shdr.sh_size = needed_size;
893 phdr.p_memsz = needed_size;962 phdr.p_memsz = needed_size;
894963
895 if (!is_zerofill) {
896 phdr.p_filesz = needed_size;
897 }
898
899 self.markDirty(shdr_index, phdr_index);964 self.markDirty(shdr_index, phdr_index);
900}965}
901966
...@@ -965,21 +1030,15 @@ pub fn growNonAllocSection(...@@ -965,21 +1030,15 @@ pub fn growNonAllocSection(
965 const shdr = &self.shdrs.items[shdr_index];1030 const shdr = &self.shdrs.items[shdr_index];
9661031
967 if (needed_size > self.allocatedSize(shdr.sh_offset)) {1032 if (needed_size > self.allocatedSize(shdr.sh_offset)) {
968 const existing_size = if (self.symtab_section_index.? == shdr_index) blk: {1033 const existing_size = shdr.sh_size;
969 const sym_size: u64 = switch (self.ptr_width) {
970 .p32 => @sizeOf(elf.Elf32_Sym),
971 .p64 => @sizeOf(elf.Elf64_Sym),
972 };
973 break :blk @as(u64, shdr.sh_info) * sym_size;
974 } else shdr.sh_size;
975 shdr.sh_size = 0;1034 shdr.sh_size = 0;
976 // Move all the symbols to a new file location.1035 // Move all the symbols to a new file location.
977 const new_offset = self.findFreeSpace(needed_size, min_alignment);1036 const new_offset = self.findFreeSpace(needed_size, min_alignment);
9781037
979 log.debug("moving '{?s}' from 0x{x} to 0x{x}", .{1038 log.debug("new '{s}' file offset 0x{x} to 0x{x}", .{
980 self.shstrtab.get(shdr.sh_name),1039 self.shstrtab.getAssumeExists(shdr.sh_name),
981 shdr.sh_offset,
982 new_offset,1040 new_offset,
1041 new_offset + existing_size,
983 });1042 });
9841043
985 if (requires_file_copy) {1044 if (requires_file_copy) {
...@@ -1223,19 +1282,48 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node...@@ -1223,19 +1282,48 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
1223 try self.allocateObjects();1282 try self.allocateObjects();
1224 self.allocateLinkerDefinedSymbols();1283 self.allocateLinkerDefinedSymbols();
12251284
1285 // .bss always overlaps .data in file offset, but is zero-sized in file so it doesn't
1286 // get mapped by the loader
1287 if (self.data_section_index) |data_shndx| blk: {
1288 const bss_shndx = self.bss_section_index orelse break :blk;
1289 const data_phndx = self.phdr_to_shdr_table.get(data_shndx).?;
1290 const bss_phndx = self.phdr_to_shdr_table.get(bss_shndx).?;
1291 self.shdrs.items[bss_shndx].sh_offset = self.shdrs.items[data_shndx].sh_offset;
1292 self.phdrs.items[bss_phndx].p_offset = self.phdrs.items[data_phndx].p_offset;
1293 }
1294
1295 // Same treatment for .tbss section.
1296 if (self.tdata_section_index) |tdata_shndx| blk: {
1297 const tbss_shndx = self.tbss_section_index orelse break :blk;
1298 const tdata_phndx = self.phdr_to_shdr_table.get(tdata_shndx).?;
1299 const tbss_phndx = self.phdr_to_shdr_table.get(tbss_shndx).?;
1300 self.shdrs.items[tbss_shndx].sh_offset = self.shdrs.items[tdata_shndx].sh_offset;
1301 self.phdrs.items[tbss_phndx].p_offset = self.phdrs.items[tdata_phndx].p_offset;
1302 }
1303
1304 if (self.phdr_tls_index) |tls_index| {
1305 const tdata_phdr = &self.phdrs.items[self.phdr_load_tls_data_index.?];
1306 const tbss_phdr = &self.phdrs.items[self.phdr_load_tls_zerofill_index.?];
1307 const phdr = &self.phdrs.items[tls_index];
1308 phdr.p_offset = tdata_phdr.p_offset;
1309 phdr.p_filesz = tdata_phdr.p_filesz;
1310 phdr.p_vaddr = tdata_phdr.p_vaddr;
1311 phdr.p_paddr = tdata_phdr.p_vaddr;
1312 phdr.p_memsz = tbss_phdr.p_vaddr + tbss_phdr.p_memsz - tdata_phdr.p_vaddr;
1313 }
1314
1226 // Beyond this point, everything has been allocated a virtual address and we can resolve1315 // Beyond this point, everything has been allocated a virtual address and we can resolve
1227 // the relocations, and commit objects to file.1316 // the relocations, and commit objects to file.
1228 if (self.zig_module_index) |index| {1317 if (self.zig_module_index) |index| {
1229 for (self.file(index).?.zig_module.atoms.keys()) |atom_index| {1318 const zig_module = self.file(index).?.zig_module;
1319 for (zig_module.atoms.keys()) |atom_index| {
1230 const atom_ptr = self.atom(atom_index).?;1320 const atom_ptr = self.atom(atom_index).?;
1231 if (!atom_ptr.flags.alive) continue;1321 if (!atom_ptr.flags.alive) continue;
1232 const shdr = &self.shdrs.items[atom_ptr.outputShndx().?];1322 const shdr = &self.shdrs.items[atom_ptr.outputShndx().?];
1233 const file_offset = shdr.sh_offset + atom_ptr.value - shdr.sh_addr;1323 if (shdr.sh_type == elf.SHT_NOBITS) continue;
1234 const size = math.cast(usize, atom_ptr.size) orelse return error.Overflow;1324 const code = try zig_module.codeAlloc(self, atom_index);
1235 const code = try gpa.alloc(u8, size);
1236 defer gpa.free(code);1325 defer gpa.free(code);
1237 const amt = try self.base.file.?.preadAll(code, file_offset);1326 const file_offset = shdr.sh_offset + atom_ptr.value - shdr.sh_addr;
1238 if (amt != code.len) return error.InputOutput;
1239 try atom_ptr.resolveRelocs(self, code);1327 try atom_ptr.resolveRelocs(self, code);
1240 try self.base.file.?.pwriteAll(code, file_offset);1328 try self.base.file.?.pwriteAll(code, file_offset);
1241 }1329 }
...@@ -1268,22 +1356,6 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node...@@ -1268,22 +1356,6 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
1268 try self.updateSymtabSize();1356 try self.updateSymtabSize();
1269 try self.writeSymtab();1357 try self.writeSymtab();
12701358
1271 // .bss always overlaps .data in file offset, but is zero-sized in file so it doesn't
1272 // get mapped by the loader
1273 if (self.data_section_index) |data_shndx| blk: {
1274 const bss_shndx = self.bss_section_index orelse break :blk;
1275 const data_phndx = self.phdr_to_shdr_table.get(data_shndx).?;
1276 const bss_phndx = self.phdr_to_shdr_table.get(bss_shndx).?;
1277 self.shdrs.items[bss_shndx].sh_offset = self.shdrs.items[data_shndx].sh_offset;
1278 self.phdrs.items[bss_phndx].p_offset = self.phdrs.items[data_phndx].p_offset;
1279 }
1280
1281 // Dump the state for easy debugging.
1282 // State can be dumped via `--debug-log link_state`.
1283 if (build_options.enable_logging) {
1284 state_log.debug("{}", .{self.dumpState()});
1285 }
1286
1287 if (self.dwarf) |*dw| {1359 if (self.dwarf) |*dw| {
1288 if (self.debug_abbrev_section_dirty) {1360 if (self.debug_abbrev_section_dirty) {
1289 try dw.writeDbgAbbrev();1361 try dw.writeDbgAbbrev();
...@@ -1470,6 +1542,12 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node...@@ -1470,6 +1542,12 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
1470 try self.writeElfHeader();1542 try self.writeElfHeader();
1471 }1543 }
14721544
1545 // Dump the state for easy debugging.
1546 // State can be dumped via `--debug-log link_state`.
1547 if (build_options.enable_logging) {
1548 state_log.debug("{}", .{self.dumpState()});
1549 }
1550
1473 // The point of flush() is to commit changes, so in theory, nothing should1551 // The point of flush() is to commit changes, so in theory, nothing should
1474 // be dirty after this. However, it is possible for some things to remain1552 // be dirty after this. However, it is possible for some things to remain
1475 // dirty because they fail to be written in the event of compile errors,1553 // dirty because they fail to be written in the event of compile errors,
...@@ -1779,7 +1857,7 @@ fn writeObjects(self: *Elf) !void {...@@ -1779,7 +1857,7 @@ fn writeObjects(self: *Elf) !void {
17791857
1780 const file_offset = shdr.sh_offset + atom_ptr.value - shdr.sh_addr;1858 const file_offset = shdr.sh_offset + atom_ptr.value - shdr.sh_addr;
1781 log.debug("writing atom({d}) at 0x{x}", .{ atom_ptr.atom_index, file_offset });1859 log.debug("writing atom({d}) at 0x{x}", .{ atom_ptr.atom_index, file_offset });
1782 const code = try atom_ptr.codeInObjectUncompressAlloc(self);1860 const code = try object.codeDecompressAlloc(self, atom_ptr.atom_index);
1783 defer gpa.free(code);1861 defer gpa.free(code);
17841862
1785 try atom_ptr.resolveRelocs(self, code);1863 try atom_ptr.resolveRelocs(self, code);
...@@ -2785,10 +2863,6 @@ fn updateDeclCode(...@@ -2785,10 +2863,6 @@ fn updateDeclCode(
2785 try self.got.writeEntry(self, gop.index);2863 try self.got.writeEntry(self, gop.index);
2786 }2864 }
27872865
2788 const phdr_index = self.phdr_to_shdr_table.get(shdr_index).?;
2789 const section_offset = sym.value - self.phdrs.items[phdr_index].p_vaddr;
2790 const file_offset = self.shdrs.items[shdr_index].sh_offset + section_offset;
2791
2792 if (self.base.child_pid) |pid| {2866 if (self.base.child_pid) |pid| {
2793 switch (builtin.os.tag) {2867 switch (builtin.os.tag) {
2794 .linux => {2868 .linux => {
...@@ -2810,7 +2884,13 @@ fn updateDeclCode(...@@ -2810,7 +2884,13 @@ fn updateDeclCode(
2810 }2884 }
2811 }2885 }
28122886
2813 try self.base.file.?.pwriteAll(code, file_offset);2887 const shdr = self.shdrs.items[shdr_index];
2888 if (shdr.sh_type != elf.SHT_NOBITS) {
2889 const phdr_index = self.phdr_to_shdr_table.get(shdr_index).?;
2890 const section_offset = sym.value - self.phdrs.items[phdr_index].p_vaddr;
2891 const file_offset = shdr.sh_offset + section_offset;
2892 try self.base.file.?.pwriteAll(code, file_offset);
2893 }
2814}2894}
28152895
2816pub fn updateFunc(self: *Elf, mod: *Module, func_index: InternPool.Index, air: Air, liveness: Liveness) !void {2896pub fn updateFunc(self: *Elf, mod: *Module, func_index: InternPool.Index, air: Air, liveness: Liveness) !void {
...@@ -3358,9 +3438,14 @@ fn allocateLinkerDefinedSymbols(self: *Elf) void {...@@ -3358,9 +3438,14 @@ fn allocateLinkerDefinedSymbols(self: *Elf) void {
3358 // _end3438 // _end
3359 {3439 {
3360 const end_symbol = self.symbol(self.end_index.?);3440 const end_symbol = self.symbol(self.end_index.?);
3441 end_symbol.value = 0;
3361 for (self.shdrs.items, 0..) |*shdr, shndx| {3442 for (self.shdrs.items, 0..) |*shdr, shndx| {
3362 if (shdr.sh_flags & elf.SHF_ALLOC != 0) {3443 if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue;
3363 end_symbol.value = shdr.sh_addr + shdr.sh_size;3444 const phdr_index = self.phdr_to_shdr_table.get(@intCast(shndx)).?;
3445 const phdr = self.phdrs.items[phdr_index];
3446 const value = phdr.p_vaddr + phdr.p_memsz;
3447 if (end_symbol.value < value) {
3448 end_symbol.value = value;
3364 end_symbol.output_section_index = @intCast(shndx);3449 end_symbol.output_section_index = @intCast(shndx);
3365 }3450 }
3366 }3451 }
...@@ -3424,6 +3509,7 @@ fn updateSymtabSize(self: *Elf) !void {...@@ -3424,6 +3509,7 @@ fn updateSymtabSize(self: *Elf) !void {
3424 .p64 => @alignOf(elf.Elf64_Sym),3509 .p64 => @alignOf(elf.Elf64_Sym),
3425 };3510 };
3426 const needed_size = (sizes.nlocals + sizes.nglobals + 1) * sym_size;3511 const needed_size = (sizes.nlocals + sizes.nglobals + 1) * sym_size;
3512 shdr.sh_size = needed_size;
3427 try self.growNonAllocSection(self.symtab_section_index.?, needed_size, sym_align, true);3513 try self.growNonAllocSection(self.symtab_section_index.?, needed_size, sym_align, true);
3428}3514}
34293515
...@@ -3820,12 +3906,8 @@ pub fn calcImageBase(self: Elf) u64 {...@@ -3820,12 +3906,8 @@ pub fn calcImageBase(self: Elf) u64 {
3820 };3906 };
3821}3907}
38223908
3823pub fn defaultEntryAddress(self: Elf) u64 {3909pub fn isStatic(self: Elf) bool {
3824 if (self.entry_addr) |addr| return addr;3910 return self.base.options.link_mode == .Static;
3825 return switch (self.base.options.target.cpu.arch) {
3826 .spu_2 => 0,
3827 else => default_entry_addr,
3828 };
3829}3911}
38303912
3831pub fn isDynLib(self: Elf) bool {3913pub fn isDynLib(self: Elf) bool {
...@@ -4011,6 +4093,22 @@ pub fn comdatGroupOwner(self: *Elf, index: ComdatGroupOwner.Index) *ComdatGroupO...@@ -4011,6 +4093,22 @@ pub fn comdatGroupOwner(self: *Elf, index: ComdatGroupOwner.Index) *ComdatGroupO
4011 return &self.comdat_groups_owners.items[index];4093 return &self.comdat_groups_owners.items[index];
4012}4094}
40134095
4096pub fn tpAddress(self: *Elf) u64 {
4097 const index = self.phdr_tls_index orelse return 0;
4098 const phdr = self.phdrs.items[index];
4099 return mem.alignForward(u64, phdr.p_vaddr + phdr.p_memsz, phdr.p_align);
4100}
4101
4102pub fn dtpAddress(self: *Elf) u64 {
4103 return self.tlsAddress();
4104}
4105
4106pub fn tlsAddress(self: *Elf) u64 {
4107 const index = self.phdr_tls_index orelse return 0;
4108 const phdr = self.phdrs.items[index];
4109 return phdr.p_vaddr;
4110}
4111
4014const ErrorWithNotes = struct {4112const ErrorWithNotes = struct {
4015 /// Allocated index in misc_errors array.4113 /// Allocated index in misc_errors array.
4016 index: usize,4114 index: usize,
...@@ -4043,7 +4141,7 @@ const ErrorWithNotes = struct {...@@ -4043,7 +4141,7 @@ const ErrorWithNotes = struct {
4043 }4141 }
4044};4142};
40454143
4046fn addErrorWithNotes(self: *Elf, note_count: usize) error{OutOfMemory}!ErrorWithNotes {4144pub fn addErrorWithNotes(self: *Elf, note_count: usize) error{OutOfMemory}!ErrorWithNotes {
4047 try self.misc_errors.ensureUnusedCapacity(self.base.allocator, 1);4145 try self.misc_errors.ensureUnusedCapacity(self.base.allocator, 1);
4048 return self.addErrorWithNotesAssumeCapacity(note_count);4146 return self.addErrorWithNotesAssumeCapacity(note_count);
4049}4147}
src/link/Elf/Atom.zig+175-40
...@@ -59,38 +59,6 @@ pub fn outputShndx(self: Atom) ?u16 {...@@ -59,38 +59,6 @@ pub fn outputShndx(self: Atom) ?u16 {
59 return self.output_section_index;59 return self.output_section_index;
60}60}
6161
62pub fn codeInObject(self: Atom, elf_file: *Elf) error{Overflow}![]const u8 {
63 const object = self.file(elf_file).?.object;
64 return object.shdrContents(self.input_section_index);
65}
66
67/// Returns atom's code and optionally uncompresses data if required (for compressed sections).
68/// Caller owns the memory.
69pub fn codeInObjectUncompressAlloc(self: Atom, elf_file: *Elf) ![]u8 {
70 const gpa = elf_file.base.allocator;
71 const data = try self.codeInObject(elf_file);
72 const shdr = self.inputShdr(elf_file);
73 if (shdr.sh_flags & elf.SHF_COMPRESSED != 0) {
74 const chdr = @as(*align(1) const elf.Elf64_Chdr, @ptrCast(data.ptr)).*;
75 switch (chdr.ch_type) {
76 .ZLIB => {
77 var stream = std.io.fixedBufferStream(data[@sizeOf(elf.Elf64_Chdr)..]);
78 var zlib_stream = std.compress.zlib.decompressStream(gpa, stream.reader()) catch
79 return error.InputOutput;
80 defer zlib_stream.deinit();
81 const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow;
82 const decomp = try gpa.alloc(u8, size);
83 const nread = zlib_stream.reader().readAll(decomp) catch return error.InputOutput;
84 if (nread != decomp.len) {
85 return error.InputOutput;
86 }
87 return decomp;
88 },
89 else => @panic("TODO unhandled compression scheme"),
90 }
91 } else return gpa.dupe(u8, data);
92}
93
94pub fn priority(self: Atom, elf_file: *Elf) u64 {62pub fn priority(self: Atom, elf_file: *Elf) u64 {
95 const index = self.file(elf_file).?.index();63 const index = self.file(elf_file).?.index();
96 return (@as(u64, @intCast(index)) << 32) | @as(u64, @intCast(self.input_section_index));64 return (@as(u64, @intCast(index)) << 32) | @as(u64, @intCast(self.input_section_index));
...@@ -327,7 +295,15 @@ pub fn freeRelocs(self: Atom, elf_file: *Elf) void {...@@ -327,7 +295,15 @@ pub fn freeRelocs(self: Atom, elf_file: *Elf) void {
327 zig_module.relocs.items[self.relocs_section_index].clearRetainingCapacity();295 zig_module.relocs.items[self.relocs_section_index].clearRetainingCapacity();
328}296}
329297
330pub fn scanRelocs(self: Atom, elf_file: *Elf, undefs: anytype) !void {298pub fn scanRelocsRequiresCode(self: Atom, elf_file: *Elf) error{Overflow}!bool {
299 for (try self.relocs(elf_file)) |rel| {
300 if (rel.r_type() == elf.R_X86_64_GOTTPOFF) return true;
301 }
302 return false;
303}
304
305pub fn scanRelocs(self: Atom, elf_file: *Elf, code: ?[]const u8, undefs: anytype) !void {
306 const is_dyn_lib = elf_file.isDynLib();
331 const file_ptr = self.file(elf_file).?;307 const file_ptr = self.file(elf_file).?;
332 const rels = try self.relocs(elf_file);308 const rels = try self.relocs(elf_file);
333 var i: usize = 0;309 var i: usize = 0;
...@@ -336,6 +312,8 @@ pub fn scanRelocs(self: Atom, elf_file: *Elf, undefs: anytype) !void {...@@ -336,6 +312,8 @@ pub fn scanRelocs(self: Atom, elf_file: *Elf, undefs: anytype) !void {
336312
337 if (rel.r_type() == elf.R_X86_64_NONE) continue;313 if (rel.r_type() == elf.R_X86_64_NONE) continue;
338314
315 const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow;
316
339 const symbol_index = switch (file_ptr) {317 const symbol_index = switch (file_ptr) {
340 .zig_module => |x| x.symbol(rel.r_sym()),318 .zig_module => |x| x.symbol(rel.r_sym()),
341 .object => |x| x.symbols.items[rel.r_sym()],319 .object => |x| x.symbols.items[rel.r_sym()],
...@@ -388,7 +366,54 @@ pub fn scanRelocs(self: Atom, elf_file: *Elf, undefs: anytype) !void {...@@ -388,7 +366,54 @@ pub fn scanRelocs(self: Atom, elf_file: *Elf, undefs: anytype) !void {
388366
389 elf.R_X86_64_PC32 => {},367 elf.R_X86_64_PC32 => {},
390368
391 else => @panic("TODO"),369 elf.R_X86_64_TPOFF32,
370 elf.R_X86_64_TPOFF64,
371 => {
372 if (is_dyn_lib) {
373 // TODO
374 // self.picError(symbol, rel, elf_file);
375 }
376 },
377
378 elf.R_X86_64_TLSGD => {
379 // TODO verify followed by appropriate relocation such as PLT32 __tls_get_addr
380
381 if (elf_file.isStatic() or
382 (!symbol.flags.import and !is_dyn_lib))
383 {
384 // Relax if building with -static flag as __tls_get_addr() will not be present in libc.a
385 // We skip the next relocation.
386 i += 1;
387 } else if (!symbol.flags.import and is_dyn_lib) {
388 symbol.flags.needs_gottp = true;
389 i += 1;
390 } else {
391 symbol.flags.needs_tlsgd = true;
392 }
393 },
394
395 elf.R_X86_64_GOTTPOFF => {
396 const should_relax = blk: {
397 // if (!elf_file.options.relax or is_shared or symbol.flags.import) break :blk false;
398 if (!x86_64.canRelaxGotTpOff(code.?[r_offset - 3 ..])) break :blk false;
399 break :blk true;
400 };
401 if (!should_relax) {
402 symbol.flags.needs_gottp = true;
403 }
404 },
405
406 else => {
407 var err = try elf_file.addErrorWithNotes(1);
408 try err.addMsg(elf_file, "fatal linker error: unhandled relocation type {}", .{
409 fmtRelocType(rel.r_type()),
410 });
411 try err.addNote(elf_file, "in {}:{s} at offset 0x{x}", .{
412 self.file(elf_file).?.fmtPath(),
413 self.name(elf_file),
414 r_offset,
415 });
416 },
392 }417 }
393 }418 }
394}419}
...@@ -430,7 +455,10 @@ pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void {...@@ -430,7 +455,10 @@ pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void {
430 var stream = std.io.fixedBufferStream(code);455 var stream = std.io.fixedBufferStream(code);
431 const cwriter = stream.writer();456 const cwriter = stream.writer();
432457
433 for (try self.relocs(elf_file)) |rel| {458 const rels = try self.relocs(elf_file);
459 var i: usize = 0;
460 while (i < rels.len) : (i += 1) {
461 const rel = rels[i];
434 const r_type = rel.r_type();462 const r_type = rel.r_type();
435 if (r_type == elf.R_X86_64_NONE) continue;463 if (r_type == elf.R_X86_64_NONE) continue;
436464
...@@ -463,9 +491,9 @@ pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void {...@@ -463,9 +491,9 @@ pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void {
463 // Relative offset to the start of the global offset table.491 // Relative offset to the start of the global offset table.
464 const G = @as(i64, @intCast(target.gotAddress(elf_file))) - GOT;492 const G = @as(i64, @intCast(target.gotAddress(elf_file))) - GOT;
465 // // Address of the thread pointer.493 // // Address of the thread pointer.
466 // const TP = @as(i64, @intCast(elf_file.getTpAddress()));494 const TP = @as(i64, @intCast(elf_file.tpAddress()));
467 // // Address of the dynamic thread pointer.495 // // Address of the dynamic thread pointer.
468 // const DTP = @as(i64, @intCast(elf_file.getDtpAddress()));496 // const DTP = @as(i64, @intCast(elf_file.dtpAddress()));
469497
470 relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ({s})", .{498 relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ({s})", .{
471 fmtRelocType(r_type),499 fmtRelocType(r_type),
...@@ -512,10 +540,43 @@ pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void {...@@ -512,10 +540,43 @@ pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void {
512 try cwriter.writeIntLittle(i32, @as(i32, @intCast(G + GOT + A - P)));540 try cwriter.writeIntLittle(i32, @as(i32, @intCast(G + GOT + A - P)));
513 },541 },
514542
515 else => {543 elf.R_X86_64_TPOFF32 => try cwriter.writeIntLittle(i32, @as(i32, @truncate(S + A - TP))),
516 log.err("TODO: unhandled relocation type {}", .{fmtRelocType(rel.r_type())});544 elf.R_X86_64_TPOFF64 => try cwriter.writeIntLittle(i64, S + A - TP),
517 @panic("TODO unhandled relocation type");545
546 elf.R_X86_64_TLSGD => {
547 if (target.flags.has_tlsgd) {
548 // TODO
549 // const S_ = @as(i64, @intCast(target.tlsGdAddress(elf_file)));
550 // try cwriter.writeIntLittle(i32, @as(i32, @intCast(S_ + A - P)));
551 } else if (target.flags.has_gottp) {
552 // TODO
553 // const S_ = @as(i64, @intCast(target.getGotTpAddress(elf_file)));
554 // try relaxTlsGdToIe(relocs[i .. i + 2], @intCast(S_ - P), elf_file, &stream);
555 i += 1;
556 } else {
557 try x86_64.relaxTlsGdToLe(
558 self,
559 rels[i .. i + 2],
560 @as(i32, @intCast(S - TP)),
561 elf_file,
562 &stream,
563 );
564 i += 1;
565 }
566 },
567
568 elf.R_X86_64_GOTTPOFF => {
569 if (target.flags.has_gottp) {
570 // TODO
571 // const S_ = @as(i64, @intCast(target.gotTpAddress(elf_file)));
572 // try cwriter.writeIntLittle(i32, @as(i32, @intCast(S_ + A - P)));
573 } else {
574 x86_64.relaxGotTpOff(code[r_offset - 3 ..]) catch unreachable;
575 try cwriter.writeIntLittle(i32, @as(i32, @intCast(S - TP)));
576 }
518 },577 },
578
579 else => {},
519 }580 }
520 }581 }
521}582}
...@@ -681,6 +742,80 @@ const x86_64 = struct {...@@ -681,6 +742,80 @@ const x86_64 = struct {
681 }742 }
682 }743 }
683744
745 pub fn canRelaxGotTpOff(code: []const u8) bool {
746 const old_inst = disassemble(code) orelse return false;
747 switch (old_inst.encoding.mnemonic) {
748 .mov => if (Instruction.new(old_inst.prefix, .mov, &.{
749 old_inst.ops[0],
750 // TODO: hack to force imm32s in the assembler
751 .{ .imm = Immediate.s(-129) },
752 })) |inst| {
753 inst.encode(std.io.null_writer, .{}) catch return false;
754 return true;
755 } else |_| return false,
756 else => return false,
757 }
758 }
759
760 pub fn relaxGotTpOff(code: []u8) !void {
761 const old_inst = disassemble(code) orelse return error.RelaxFail;
762 switch (old_inst.encoding.mnemonic) {
763 .mov => {
764 const inst = try Instruction.new(old_inst.prefix, .mov, &.{
765 old_inst.ops[0],
766 // TODO: hack to force imm32s in the assembler
767 .{ .imm = Immediate.s(-129) },
768 });
769 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
770 encode(&.{inst}, code) catch return error.RelaxFail;
771 },
772 else => return error.RelaxFail,
773 }
774 }
775
776 pub fn relaxTlsGdToLe(
777 self: Atom,
778 rels: []align(1) const elf.Elf64_Rela,
779 value: i32,
780 elf_file: *Elf,
781 stream: anytype,
782 ) !void {
783 assert(rels.len == 2);
784 const writer = stream.writer();
785 switch (rels[1].r_type()) {
786 elf.R_X86_64_PC32,
787 elf.R_X86_64_PLT32,
788 elf.R_X86_64_GOTPCREL,
789 elf.R_X86_64_GOTPCRELX,
790 => {
791 var insts = [_]u8{
792 0x64, 0x48, 0x8b, 0x04, 0x25, 0, 0, 0, 0, // movq %fs:0,%rax
793 0x48, 0x81, 0xc0, 0, 0, 0, 0, // add $tp_offset, %rax
794 };
795 std.mem.writeIntLittle(i32, insts[12..][0..4], value);
796 try stream.seekBy(-4);
797 try writer.writeAll(&insts);
798 relocs_log.debug(" relaxing {} and {}", .{
799 fmtRelocType(rels[0].r_type()),
800 fmtRelocType(rels[1].r_type()),
801 });
802 },
803
804 else => {
805 var err = try elf_file.addErrorWithNotes(1);
806 try err.addMsg(elf_file, "fatal linker error: rewrite {} when followed by {}", .{
807 fmtRelocType(rels[0].r_type()),
808 fmtRelocType(rels[1].r_type()),
809 });
810 try err.addNote(elf_file, "in {}:{s} at offset 0x{x}", .{
811 self.file(elf_file).?.fmtPath(),
812 self.name(elf_file),
813 rels[0].r_offset,
814 });
815 },
816 }
817 }
818
684 fn disassemble(code: []const u8) ?Instruction {819 fn disassemble(code: []const u8) ?Instruction {
685 var disas = Disassembler.init(code);820 var disas = Disassembler.init(code);
686 const inst = disas.next() catch return null;821 const inst = disas.next() catch return null;
src/link/Elf/Object.zig+42-6
...@@ -208,6 +208,8 @@ fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) er...@@ -208,6 +208,8 @@ fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) er
208 break :blk prefix;208 break :blk prefix;
209 }209 }
210 }210 }
211 if (std.mem.eql(u8, name, ".tcommon")) break :blk ".tbss";
212 if (std.mem.eql(u8, name, ".common")) break :blk ".bss";
211 break :blk name;213 break :blk name;
212 };214 };
213 const @"type" = switch (shdr.sh_type) {215 const @"type" = switch (shdr.sh_type) {
...@@ -233,8 +235,7 @@ fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) er...@@ -233,8 +235,7 @@ fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) er
233 const is_alloc = flags & elf.SHF_ALLOC != 0;235 const is_alloc = flags & elf.SHF_ALLOC != 0;
234 const is_write = flags & elf.SHF_WRITE != 0;236 const is_write = flags & elf.SHF_WRITE != 0;
235 const is_exec = flags & elf.SHF_EXECINSTR != 0;237 const is_exec = flags & elf.SHF_EXECINSTR != 0;
236 const is_tls = flags & elf.SHF_TLS != 0;238 if (!is_alloc) {
237 if (!is_alloc or is_tls) {
238 log.err("{}: output section {s} not found", .{ self.fmtPath(), name });239 log.err("{}: output section {s} not found", .{ self.fmtPath(), name });
239 @panic("TODO: missing output section!");240 @panic("TODO: missing output section!");
240 }241 }
...@@ -243,7 +244,7 @@ fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) er...@@ -243,7 +244,7 @@ fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) er
243 if (is_exec) phdr_flags |= elf.PF_X;244 if (is_exec) phdr_flags |= elf.PF_X;
244 const phdr_index = try elf_file.allocateSegment(.{245 const phdr_index = try elf_file.allocateSegment(.{
245 .size = Elf.padToIdeal(shdr.sh_size),246 .size = Elf.padToIdeal(shdr.sh_size),
246 .alignment = if (is_tls) shdr.sh_addralign else elf_file.page_size,247 .alignment = elf_file.page_size,
247 .flags = phdr_flags,248 .flags = phdr_flags,
248 });249 });
249 const shndx = try elf_file.allocateAllocSection(.{250 const shndx = try elf_file.allocateAllocSection(.{
...@@ -428,7 +429,13 @@ pub fn scanRelocs(self: *Object, elf_file: *Elf, undefs: anytype) !void {...@@ -428,7 +429,13 @@ pub fn scanRelocs(self: *Object, elf_file: *Elf, undefs: anytype) !void {
428 const shdr = atom.inputShdr(elf_file);429 const shdr = atom.inputShdr(elf_file);
429 if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue;430 if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue;
430 if (shdr.sh_type == elf.SHT_NOBITS) continue;431 if (shdr.sh_type == elf.SHT_NOBITS) continue;
431 try atom.scanRelocs(elf_file, undefs);432 if (try atom.scanRelocsRequiresCode(elf_file)) {
433 // TODO ideally, we don't have to decompress at this stage (should already be done)
434 // and we just fetch the code slice.
435 const code = try self.codeDecompressAlloc(elf_file, atom_index);
436 defer elf_file.base.allocator.free(code);
437 try atom.scanRelocs(elf_file, code, undefs);
438 } else try atom.scanRelocs(elf_file, null, undefs);
432 }439 }
433440
434 for (self.cies.items) |cie| {441 for (self.cies.items) |cie| {
...@@ -591,7 +598,7 @@ pub fn convertCommonSymbols(self: *Object, elf_file: *Elf) !void {...@@ -591,7 +598,7 @@ pub fn convertCommonSymbols(self: *Object, elf_file: *Elf) !void {
591 try self.atoms.append(gpa, atom_index);598 try self.atoms.append(gpa, atom_index);
592599
593 const is_tls = global.getType(elf_file) == elf.STT_TLS;600 const is_tls = global.getType(elf_file) == elf.STT_TLS;
594 const name = if (is_tls) ".tls_common" else ".common";601 const name = if (is_tls) ".tbss" else ".bss";
595602
596 const atom = elf_file.atom(atom_index).?;603 const atom = elf_file.atom(atom_index).?;
597 atom.atom_index = atom_index;604 atom.atom_index = atom_index;
...@@ -685,7 +692,7 @@ pub fn globals(self: *Object) []const Symbol.Index {...@@ -685,7 +692,7 @@ pub fn globals(self: *Object) []const Symbol.Index {
685 return self.symbols.items[start..];692 return self.symbols.items[start..];
686}693}
687694
688pub fn shdrContents(self: *Object, index: u32) error{Overflow}![]const u8 {695fn shdrContents(self: Object, index: u32) error{Overflow}![]const u8 {
689 assert(index < self.shdrs.items.len);696 assert(index < self.shdrs.items.len);
690 const shdr = self.shdrs.items[index];697 const shdr = self.shdrs.items[index];
691 const offset = math.cast(usize, shdr.sh_offset) orelse return error.Overflow;698 const offset = math.cast(usize, shdr.sh_offset) orelse return error.Overflow;
...@@ -693,6 +700,35 @@ pub fn shdrContents(self: *Object, index: u32) error{Overflow}![]const u8 {...@@ -693,6 +700,35 @@ pub fn shdrContents(self: *Object, index: u32) error{Overflow}![]const u8 {
693 return self.data[offset..][0..size];700 return self.data[offset..][0..size];
694}701}
695702
703/// Returns atom's code and optionally uncompresses data if required (for compressed sections).
704/// Caller owns the memory.
705pub fn codeDecompressAlloc(self: Object, elf_file: *Elf, atom_index: Atom.Index) ![]u8 {
706 const gpa = elf_file.base.allocator;
707 const atom_ptr = elf_file.atom(atom_index).?;
708 assert(atom_ptr.file_index == self.index);
709 const data = try self.shdrContents(atom_ptr.input_section_index);
710 const shdr = atom_ptr.inputShdr(elf_file);
711 if (shdr.sh_flags & elf.SHF_COMPRESSED != 0) {
712 const chdr = @as(*align(1) const elf.Elf64_Chdr, @ptrCast(data.ptr)).*;
713 switch (chdr.ch_type) {
714 .ZLIB => {
715 var stream = std.io.fixedBufferStream(data[@sizeOf(elf.Elf64_Chdr)..]);
716 var zlib_stream = std.compress.zlib.decompressStream(gpa, stream.reader()) catch
717 return error.InputOutput;
718 defer zlib_stream.deinit();
719 const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow;
720 const decomp = try gpa.alloc(u8, size);
721 const nread = zlib_stream.reader().readAll(decomp) catch return error.InputOutput;
722 if (nread != decomp.len) {
723 return error.InputOutput;
724 }
725 return decomp;
726 },
727 else => @panic("TODO unhandled compression scheme"),
728 }
729 } else return gpa.dupe(u8, data);
730}
731
696fn getString(self: *Object, off: u32) [:0]const u8 {732fn getString(self: *Object, off: u32) [:0]const u8 {
697 assert(off < self.strtab.len);733 assert(off < self.strtab.len);
698 return mem.sliceTo(@as([*:0]const u8, @ptrCast(self.strtab.ptr + off)), 0);734 return mem.sliceTo(@as([*:0]const u8, @ptrCast(self.strtab.ptr + off)), 0);
src/link/Elf/Symbol.zig+8-5
...@@ -196,9 +196,10 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void {...@@ -196,9 +196,10 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void {
196 // if (symbol.flags.is_canonical) break :blk symbol.address(.{}, elf_file);196 // if (symbol.flags.is_canonical) break :blk symbol.address(.{}, elf_file);
197 // break :blk 0;197 // break :blk 0;
198 // }198 // }
199 // if (st_shndx == elf.SHN_ABS) break :blk symbol.value;199 if (st_shndx == elf.SHN_ABS) break :blk symbol.value;
200 // const shdr = &elf_file.sections.items(.shdr)[st_shndx];200 const shdr = &elf_file.shdrs.items[st_shndx];
201 // if (Elf.shdrIsTls(shdr)) break :blk symbol.value - elf_file.getTlsAddress();201 if (shdr.sh_flags & elf.SHF_TLS != 0 and file_ptr != .linker_defined)
202 break :blk symbol.value - elf_file.tlsAddress();
202 break :blk symbol.value;203 break :blk symbol.value;
203 };204 };
204 out.* = .{205 out.* = .{
...@@ -327,10 +328,12 @@ pub const Flags = packed struct {...@@ -327,10 +328,12 @@ pub const Flags = packed struct {
327 has_dynamic: bool = false,328 has_dynamic: bool = false,
328329
329 /// Whether the symbol contains TLSGD indirection.330 /// Whether the symbol contains TLSGD indirection.
330 tlsgd: bool = false,331 needs_tlsgd: bool = false,
332 has_tlsgd: bool = false,
331333
332 /// Whether the symbol contains GOTTP indirection.334 /// Whether the symbol contains GOTTP indirection.
333 gottp: bool = false,335 needs_gottp: bool = false,
336 has_gottp: bool = false,
334337
335 /// Whether the symbol contains TLSDESC indirection.338 /// Whether the symbol contains TLSDESC indirection.
336 tlsdesc: bool = false,339 tlsdesc: bool = false,
src/link/Elf/ZigModule.zig+24-1
...@@ -144,7 +144,14 @@ pub fn scanRelocs(self: *ZigModule, elf_file: *Elf, undefs: anytype) !void {...@@ -144,7 +144,14 @@ pub fn scanRelocs(self: *ZigModule, elf_file: *Elf, undefs: anytype) !void {
144 for (self.atoms.keys()) |atom_index| {144 for (self.atoms.keys()) |atom_index| {
145 const atom = elf_file.atom(atom_index) orelse continue;145 const atom = elf_file.atom(atom_index) orelse continue;
146 if (!atom.flags.alive) continue;146 if (!atom.flags.alive) continue;
147 try atom.scanRelocs(elf_file, undefs);147 if (try atom.scanRelocsRequiresCode(elf_file)) {
148 // TODO ideally we don't have to fetch the code here.
149 // Perhaps it would make sense to save the code until flushModule where we
150 // would free all of generated code?
151 const code = try self.codeAlloc(elf_file, atom_index);
152 defer elf_file.base.allocator.free(code);
153 try atom.scanRelocs(elf_file, code, undefs);
154 } else try atom.scanRelocs(elf_file, null, undefs);
148 }155 }
149}156}
150157
...@@ -253,6 +260,22 @@ pub fn asFile(self: *ZigModule) File {...@@ -253,6 +260,22 @@ pub fn asFile(self: *ZigModule) File {
253 return .{ .zig_module = self };260 return .{ .zig_module = self };
254}261}
255262
263/// Returns atom's code.
264/// Caller owns the memory.
265pub fn codeAlloc(self: ZigModule, elf_file: *Elf, atom_index: Atom.Index) ![]u8 {
266 const gpa = elf_file.base.allocator;
267 const atom = elf_file.atom(atom_index).?;
268 assert(atom.file_index == self.index);
269 const shdr = &elf_file.shdrs.items[atom.outputShndx().?];
270 const file_offset = shdr.sh_offset + atom.value - shdr.sh_addr;
271 const size = std.math.cast(usize, atom.size) orelse return error.Overflow;
272 const code = try gpa.alloc(u8, size);
273 errdefer gpa.free(code);
274 const amt = try elf_file.base.file.?.preadAll(code, file_offset);
275 if (amt != code.len) return error.InputOutput;
276 return code;
277}
278
256pub fn fmtSymtab(self: *ZigModule, elf_file: *Elf) std.fmt.Formatter(formatSymtab) {279pub fn fmtSymtab(self: *ZigModule, elf_file: *Elf) std.fmt.Formatter(formatSymtab) {
257 return .{ .data = .{280 return .{ .data = .{
258 .self = self,281 .self = self,
test/link/elf.zig+41-4
...@@ -18,7 +18,8 @@ pub fn build(b: *Build) void {...@@ -18,7 +18,8 @@ pub fn build(b: *Build) void {
18 // Exercise linker with LLVM backend18 // Exercise linker with LLVM backend
19 elf_step.dependOn(testEmptyObject(b, .{ .target = musl_target }));19 elf_step.dependOn(testEmptyObject(b, .{ .target = musl_target }));
20 elf_step.dependOn(testLinkingC(b, .{ .target = musl_target }));20 elf_step.dependOn(testLinkingC(b, .{ .target = musl_target }));
21 elf_step.dependOn(testLinkingZig(b, .{}));21 elf_step.dependOn(testLinkingZig(b, .{ .target = musl_target }));
22 elf_step.dependOn(testTlsStatic(b, .{ .target = musl_target }));
22}23}
2324
24fn testEmptyObject(b: *Build, opts: Options) *Step {25fn testEmptyObject(b: *Build, opts: Options) *Step {
...@@ -91,6 +92,37 @@ fn testLinkingZig(b: *Build, opts: Options) *Step {...@@ -91,6 +92,37 @@ fn testLinkingZig(b: *Build, opts: Options) *Step {
91 return test_step;92 return test_step;
92}93}
9394
95fn testTlsStatic(b: *Build, opts: Options) *Step {
96 const test_step = addTestStep(b, "tls-static", opts);
97
98 const exe = addExecutable(b, opts);
99 addCSourceBytes(exe,
100 \\#include <stdio.h>
101 \\_Thread_local int a = 10;
102 \\_Thread_local int b;
103 \\_Thread_local char c = 'a';
104 \\int main(int argc, char* argv[]) {
105 \\ printf("%d %d %c\n", a, b, c);
106 \\ a += 1;
107 \\ b += 1;
108 \\ c += 1;
109 \\ printf("%d %d %c\n", a, b, c);
110 \\ return 0;
111 \\}
112 );
113 exe.is_linking_libc = true;
114
115 const run = addRunArtifact(exe);
116 run.expectStdOutEqual(
117 \\10 0 a
118 \\11 1 b
119 \\
120 );
121 test_step.dependOn(&run.step);
122
123 return test_step;
124}
125
94const Options = struct {126const Options = struct {
95 target: CrossTarget = .{ .cpu_arch = .x86_64, .os_tag = .linux },127 target: CrossTarget = .{ .cpu_arch = .x86_64, .os_tag = .linux },
96 optimize: std.builtin.OptimizeMode = .Debug,128 optimize: std.builtin.OptimizeMode = .Debug,
...@@ -114,7 +146,6 @@ fn addExecutable(b: *Build, opts: Options) *Compile {...@@ -114,7 +146,6 @@ fn addExecutable(b: *Build, opts: Options) *Compile {
114 .name = "test",146 .name = "test",
115 .target = opts.target,147 .target = opts.target,
116 .optimize = opts.optimize,148 .optimize = opts.optimize,
117 .single_threaded = true, // TODO temp until we teach linker how to handle TLS
118 .use_llvm = opts.use_llvm,149 .use_llvm = opts.use_llvm,
119 .use_lld = false,150 .use_lld = false,
120 });151 });
...@@ -127,19 +158,25 @@ fn addRunArtifact(comp: *Compile) *Run {...@@ -127,19 +158,25 @@ fn addRunArtifact(comp: *Compile) *Run {
127 return run;158 return run;
128}159}
129160
130fn addZigSourceBytes(comp: *Compile, bytes: []const u8) void {161fn addZigSourceBytes(comp: *Compile, comptime bytes: []const u8) void {
131 const b = comp.step.owner;162 const b = comp.step.owner;
132 const file = WriteFile.create(b).add("a.zig", bytes);163 const file = WriteFile.create(b).add("a.zig", bytes);
133 file.addStepDependencies(&comp.step);164 file.addStepDependencies(&comp.step);
134 comp.root_src = file;165 comp.root_src = file;
135}166}
136167
137fn addCSourceBytes(comp: *Compile, bytes: []const u8) void {168fn addCSourceBytes(comp: *Compile, comptime bytes: []const u8) void {
138 const b = comp.step.owner;169 const b = comp.step.owner;
139 const file = WriteFile.create(b).add("a.c", bytes);170 const file = WriteFile.create(b).add("a.c", bytes);
140 comp.addCSourceFile(.{ .file = file, .flags = &.{} });171 comp.addCSourceFile(.{ .file = file, .flags = &.{} });
141}172}
142173
174fn addAsmSourceBytes(comp: *Compile, comptime bytes: []const u8) void {
175 const b = comp.step.owner;
176 const file = WriteFile.create(b).add("a.s", bytes ++ "\n");
177 comp.addAssemblyFile(file);
178}
179
143const std = @import("std");180const std = @import("std");
144181
145const Build = std.Build;182const Build = std.Build;
test/tests.zig+12-1
...@@ -196,6 +196,15 @@ const test_targets = blk: {...@@ -196,6 +196,15 @@ const test_targets = blk: {
196 },196 },
197 .link_libc = true,197 .link_libc = true,
198 },198 },
199 .{
200 .target = .{
201 .cpu_arch = .x86_64,
202 .os_tag = .linux,
203 .abi = .musl,
204 },
205 .link_libc = true,
206 .use_lld = false,
207 },
199208
200 .{209 .{
201 .target = .{210 .target = .{
...@@ -1031,6 +1040,7 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {...@@ -1031,6 +1040,7 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
1031 "-selfhosted"1040 "-selfhosted"
1032 else1041 else
1033 "";1042 "";
1043 const use_lld = if (test_target.use_lld == false) "-no-lld" else "";
10341044
1035 these_tests.addIncludePath(.{ .path = "test" });1045 these_tests.addIncludePath(.{ .path = "test" });
10361046
...@@ -1039,13 +1049,14 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {...@@ -1039,13 +1049,14 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
1039 these_tests.stack_size = 2 * 1024 * 1024;1049 these_tests.stack_size = 2 * 1024 * 1024;
1040 }1050 }
10411051
1042 const qualified_name = b.fmt("{s}-{s}-{s}{s}{s}{s}", .{1052 const qualified_name = b.fmt("{s}-{s}-{s}{s}{s}{s}{s}", .{
1043 options.name,1053 options.name,
1044 triple_txt,1054 triple_txt,
1045 @tagName(test_target.optimize_mode),1055 @tagName(test_target.optimize_mode),
1046 libc_suffix,1056 libc_suffix,
1047 single_threaded_suffix,1057 single_threaded_suffix,
1048 backend_suffix,1058 backend_suffix,
1059 use_lld,
1049 });1060 });
10501061
1051 if (test_target.target.ofmt == std.Target.ObjectFormat.c) {1062 if (test_target.target.ofmt == std.Target.ObjectFormat.c) {