| ... | ... | @@ -60,6 +60,7 @@ phdr_gnu_eh_frame_index: ?u16 = null, |
| 60 | 60 | /// PT_GNU_STACK |
| 61 | 61 | phdr_gnu_stack_index: ?u16 = null, |
| 62 | 62 | /// PT_TLS |
| 63 | /// TODO I think ELF permits multiple TLS segments but for now, assume one per file. |
| 63 | 64 | phdr_tls_index: ?u16 = null, |
| 64 | 65 | |
| 65 | 66 | entry_index: ?Symbol.Index = null, |
| ... | ... | @@ -1588,7 +1589,8 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node |
| 1588 | 1589 | try self.setVersionSymtab(); |
| 1589 | 1590 | try self.updateSectionSizes(); |
| 1590 | 1591 | |
| 1591 | | try self.allocateSections(); |
| 1592 | try self.initAndAllocateSegments(); |
| 1593 | self.allocateNonAllocSections(); |
| 1592 | 1594 | self.allocateSpecialPhdrs(); |
| 1593 | 1595 | self.allocateAtoms(); |
| 1594 | 1596 | self.allocateLinkerDefinedSymbols(); |
| ... | ... | @@ -3996,6 +3998,16 @@ fn initSpecialPhdrs(self: *Elf) !void { |
| 3996 | 3998 | .memsz = self.base.options.stack_size_override orelse 0, |
| 3997 | 3999 | .@"align" = 1, |
| 3998 | 4000 | }); |
| 4001 | |
| 4002 | const has_tls = for (self.shdrs.items) |shdr| { |
| 4003 | if (shdr.sh_flags & elf.SHF_TLS != 0) break true; |
| 4004 | } else false; |
| 4005 | if (has_tls) { |
| 4006 | self.phdr_tls_index = try self.addPhdr(.{ |
| 4007 | .type = elf.PT_TLS, |
| 4008 | .flags = elf.PF_R, |
| 4009 | }); |
| 4010 | } |
| 3999 | 4011 | } |
| 4000 | 4012 | |
| 4001 | 4013 | /// We need to sort constructors/destuctors in the following sections: |
| ... | ... | @@ -4402,6 +4414,7 @@ fn resetPhdrs(self: *Elf) !void { |
| 4402 | 4414 | const gpa = self.base.allocator; |
| 4403 | 4415 | const phdrs = try self.phdrs.toOwnedSlice(gpa); |
| 4404 | 4416 | try self.phdrs.ensureUnusedCapacity(gpa, phdrs.len); |
| 4417 | self.phdr_to_shdr_table.clearRetainingCapacity(); // TODO move this into Section structure |
| 4405 | 4418 | |
| 4406 | 4419 | for (&[_]?u16{ |
| 4407 | 4420 | self.phdr_table_index, |
| ... | ... | @@ -4410,6 +4423,7 @@ fn resetPhdrs(self: *Elf) !void { |
| 4410 | 4423 | self.phdr_dynamic_index, |
| 4411 | 4424 | self.phdr_gnu_eh_frame_index, |
| 4412 | 4425 | self.phdr_gnu_stack_index, |
| 4426 | self.phdr_tls_index, |
| 4413 | 4427 | }) |maybe_index| { |
| 4414 | 4428 | if (maybe_index) |index| { |
| 4415 | 4429 | self.phdrs.appendAssumeCapacity(phdrs[index]); |
| ... | ... | @@ -4420,69 +4434,43 @@ fn resetPhdrs(self: *Elf) !void { |
| 4420 | 4434 | fn initSegments(self: *Elf) !void { |
| 4421 | 4435 | // Add LOAD phdrs |
| 4422 | 4436 | const slice = self.shdrs.items; |
| 4423 | | { |
| 4424 | | var last_phdr: ?u16 = null; |
| 4425 | | var shndx: usize = 0; |
| 4426 | | while (shndx < slice.len) { |
| 4427 | | const shdr = &slice[shndx]; |
| 4428 | | if (!shdrIsAlloc(shdr) or shdrIsTbss(shdr)) { |
| 4429 | | shndx += 1; |
| 4430 | | continue; |
| 4431 | | } |
| 4432 | | last_phdr = try self.addPhdr(.{ |
| 4433 | | .type = elf.PT_LOAD, |
| 4434 | | .flags = shdrToPhdrFlags(shdr.sh_flags), |
| 4435 | | .@"align" = @max(self.page_size, shdr.sh_addralign), |
| 4436 | | .offset = if (last_phdr == null) 0 else shdr.sh_offset, |
| 4437 | | .addr = if (last_phdr == null) self.calcImageBase() else shdr.sh_addr, |
| 4438 | | }); |
| 4439 | | const p_flags = self.phdrs.items[last_phdr.?].p_flags; |
| 4440 | | try self.addShdrToPhdr(last_phdr.?, shdr); |
| 4437 | var last_phdr: ?u16 = null; |
| 4438 | var shndx: u16 = 0; |
| 4439 | while (shndx < slice.len) { |
| 4440 | const shdr = &slice[shndx]; |
| 4441 | if (!shdrIsAlloc(shdr) or shdrIsTbss(shdr)) { |
| 4441 | 4442 | shndx += 1; |
| 4442 | | |
| 4443 | | while (shndx < slice.len) : (shndx += 1) { |
| 4444 | | const next = &slice[shndx]; |
| 4445 | | if (shdrIsTbss(next)) continue; |
| 4446 | | if (p_flags == shdrToPhdrFlags(next.sh_flags)) { |
| 4447 | | if (shdrIsBss(next) or next.sh_offset - shdr.sh_offset == next.sh_addr - shdr.sh_addr) { |
| 4448 | | try self.addShdrToPhdr(last_phdr.?, next); |
| 4449 | | continue; |
| 4450 | | } |
| 4451 | | } |
| 4452 | | break; |
| 4453 | | } |
| 4443 | continue; |
| 4454 | 4444 | } |
| 4455 | | } |
| 4456 | | |
| 4457 | | // Add TLS phdr |
| 4458 | | { |
| 4459 | | var shndx: usize = 0; |
| 4460 | | outer: while (shndx < slice.len) { |
| 4461 | | const shdr = &slice[shndx]; |
| 4462 | | if (!shdrIsTls(shdr)) { |
| 4463 | | shndx += 1; |
| 4464 | | continue; |
| 4465 | | } |
| 4466 | | self.phdr_tls_index = try self.addPhdr(.{ |
| 4467 | | .type = elf.PT_TLS, |
| 4468 | | .flags = elf.PF_R, |
| 4469 | | .@"align" = shdr.sh_addralign, |
| 4470 | | .offset = shdr.sh_offset, |
| 4471 | | .addr = shdr.sh_addr, |
| 4472 | | }); |
| 4473 | | try self.addShdrToPhdr(self.phdr_tls_index.?, shdr); |
| 4474 | | shndx += 1; |
| 4475 | | |
| 4476 | | while (shndx < slice.len) : (shndx += 1) { |
| 4477 | | const next = &slice[shndx]; |
| 4478 | | if (!shdrIsTls(next)) continue :outer; |
| 4479 | | try self.addShdrToPhdr(self.phdr_tls_index.?, next); |
| 4445 | last_phdr = try self.addPhdr(.{ |
| 4446 | .type = elf.PT_LOAD, |
| 4447 | .flags = shdrToPhdrFlags(shdr.sh_flags), |
| 4448 | .@"align" = @max(self.page_size, shdr.sh_addralign), |
| 4449 | .offset = if (last_phdr == null) 0 else shdr.sh_offset, |
| 4450 | .addr = if (last_phdr == null) self.calcImageBase() else shdr.sh_addr, |
| 4451 | }); |
| 4452 | const p_flags = self.phdrs.items[last_phdr.?].p_flags; |
| 4453 | self.addShdrToPhdr(shndx, last_phdr.?); |
| 4454 | try self.phdr_to_shdr_table.putNoClobber(self.base.allocator, shndx, last_phdr.?); |
| 4455 | shndx += 1; |
| 4456 | |
| 4457 | while (shndx < slice.len) : (shndx += 1) { |
| 4458 | const next = &slice[shndx]; |
| 4459 | if (shdrIsTbss(next)) continue; |
| 4460 | if (p_flags == shdrToPhdrFlags(next.sh_flags)) { |
| 4461 | if (shdrIsBss(next) or next.sh_offset - shdr.sh_offset == next.sh_addr - shdr.sh_addr) { |
| 4462 | self.addShdrToPhdr(shndx, last_phdr.?); |
| 4463 | try self.phdr_to_shdr_table.putNoClobber(self.base.allocator, shndx, last_phdr.?); |
| 4464 | continue; |
| 4465 | } |
| 4480 | 4466 | } |
| 4467 | break; |
| 4481 | 4468 | } |
| 4482 | 4469 | } |
| 4483 | 4470 | } |
| 4484 | 4471 | |
| 4485 | | fn addShdrToPhdr(self: *Elf, phdr_index: u16, shdr: *const elf.Elf64_Shdr) !void { |
| 4472 | fn addShdrToPhdr(self: *Elf, shdr_index: u16, phdr_index: u16) void { |
| 4473 | const shdr = self.shdrs.items[shdr_index]; |
| 4486 | 4474 | const phdr = &self.phdrs.items[phdr_index]; |
| 4487 | 4475 | phdr.p_align = @max(phdr.p_align, shdr.sh_addralign); |
| 4488 | 4476 | if (shdr.sh_type != elf.SHT_NOBITS) { |
| ... | ... | @@ -4520,7 +4508,7 @@ pub inline fn shdrIsTls(shdr: *const elf.Elf64_Shdr) bool { |
| 4520 | 4508 | return shdr.sh_flags & elf.SHF_TLS != 0; |
| 4521 | 4509 | } |
| 4522 | 4510 | |
| 4523 | | fn allocateSectionsInMemory(self: *Elf, base_offset: u64) void { |
| 4511 | fn allocateAllocSectionsInMemory(self: *Elf, base_addr: u64) void { |
| 4524 | 4512 | // We use this struct to track maximum alignment of all TLS sections. |
| 4525 | 4513 | // According to https://github.com/rui314/mold/commit/bd46edf3f0fe9e1a787ea453c4657d535622e61f in mold, |
| 4526 | 4514 | // in-file offsets have to be aligned against the start of TLS program header. |
| ... | ... | @@ -4549,7 +4537,7 @@ fn allocateSectionsInMemory(self: *Elf, base_offset: u64) void { |
| 4549 | 4537 | alignment.tls_start_align = @max(alignment.tls_start_align, shdr.sh_addralign); |
| 4550 | 4538 | } |
| 4551 | 4539 | |
| 4552 | | var addr = self.calcImageBase() + base_offset; |
| 4540 | var addr = base_addr; |
| 4553 | 4541 | var i: usize = 0; |
| 4554 | 4542 | while (i < self.shdrs.items.len) : (i += 1) { |
| 4555 | 4543 | const shdr = &self.shdrs.items[i]; |
| ... | ... | @@ -4592,25 +4580,16 @@ fn allocateSectionsInMemory(self: *Elf, base_offset: u64) void { |
| 4592 | 4580 | } |
| 4593 | 4581 | } |
| 4594 | 4582 | |
| 4595 | | fn allocateSectionsInFile(self: *Elf, base_offset: u64) void { |
| 4583 | fn allocateAllocSectionsInFile(self: *Elf, base_offset: u64) void { |
| 4596 | 4584 | var offset = base_offset; |
| 4597 | 4585 | var i: usize = 0; |
| 4598 | 4586 | while (i < self.shdrs.items.len) { |
| 4599 | 4587 | const first = &self.shdrs.items[i]; |
| 4600 | | defer if (!shdrIsAlloc(first) or shdrIsZerofill(first)) { |
| 4588 | if (shdrIsZerofill(first) or first.sh_type == elf.SHT_NULL) { |
| 4601 | 4589 | i += 1; |
| 4602 | | }; |
| 4603 | | |
| 4604 | | if (first.sh_type == elf.SHT_NULL) continue; |
| 4605 | | |
| 4606 | | // Non-alloc sections don't need congruency with their allocated virtual memory addresses |
| 4607 | | if (!shdrIsAlloc(first)) { |
| 4608 | | first.sh_offset = mem.alignForward(u64, offset, first.sh_addralign); |
| 4609 | | offset = first.sh_offset + first.sh_size; |
| 4610 | 4590 | continue; |
| 4611 | 4591 | } |
| 4612 | | // Skip any zerofill section |
| 4613 | | if (shdrIsZerofill(first)) continue; |
| 4592 | if (!shdrIsAlloc(first)) break; |
| 4614 | 4593 | |
| 4615 | 4594 | // Set the offset to a value that is congruent with the section's allocated virtual memory address |
| 4616 | 4595 | if (first.sh_addralign > self.page_size) { |
| ... | ... | @@ -4637,11 +4616,26 @@ fn allocateSectionsInFile(self: *Elf, base_offset: u64) void { |
| 4637 | 4616 | offset = prev.sh_offset + prev.sh_size; |
| 4638 | 4617 | |
| 4639 | 4618 | // Skip any zerofill section |
| 4640 | | while (i < self.shdrs.items.len and shdrIsAlloc(&self.shdrs.items[i]) and shdrIsZerofill(&self.shdrs.items[i])) : (i += 1) {} |
| 4619 | while (i < self.shdrs.items.len and |
| 4620 | shdrIsAlloc(&self.shdrs.items[i]) and |
| 4621 | shdrIsZerofill(&self.shdrs.items[i])) : (i += 1) |
| 4622 | {} |
| 4623 | } |
| 4624 | } |
| 4625 | |
| 4626 | fn allocateNonAllocSectionsInFile(self: *Elf, base_offset: u64) void { |
| 4627 | var offset = base_offset; |
| 4628 | var i: usize = 0; |
| 4629 | while (i < self.shdrs.items.len) : (i += 1) { |
| 4630 | const first = &self.shdrs.items[i]; |
| 4631 | if (shdrIsAlloc(first) or first.sh_type == elf.SHT_NULL) continue; |
| 4632 | // Non-alloc sections don't need congruency with their allocated virtual memory addresses |
| 4633 | first.sh_offset = mem.alignForward(u64, offset, first.sh_addralign); |
| 4634 | offset = first.sh_offset + first.sh_size; |
| 4641 | 4635 | } |
| 4642 | 4636 | } |
| 4643 | 4637 | |
| 4644 | | fn allocateSections(self: *Elf) !void { |
| 4638 | fn initAndAllocateSegments(self: *Elf) !void { |
| 4645 | 4639 | const ehsize: u64 = switch (self.ptr_width) { |
| 4646 | 4640 | .p32 => @sizeOf(elf.Elf32_Ehdr), |
| 4647 | 4641 | .p64 => @sizeOf(elf.Elf64_Ehdr), |
| ... | ... | @@ -4650,17 +4644,29 @@ fn allocateSections(self: *Elf) !void { |
| 4650 | 4644 | .p32 => @sizeOf(elf.Elf32_Phdr), |
| 4651 | 4645 | .p64 => @sizeOf(elf.Elf64_Phdr), |
| 4652 | 4646 | }; |
| 4647 | const image_base = self.calcImageBase(); |
| 4653 | 4648 | while (true) { |
| 4654 | 4649 | const nphdrs = self.phdrs.items.len; |
| 4655 | | const base_offset: u64 = ehsize + nphdrs * phsize; |
| 4656 | | self.allocateSectionsInMemory(base_offset); |
| 4657 | | self.allocateSectionsInFile(base_offset); |
| 4650 | const off = ehsize + nphdrs * phsize; |
| 4651 | const addr = image_base + off; |
| 4652 | self.allocateAllocSectionsInMemory(addr); |
| 4653 | self.allocateAllocSectionsInFile(off); |
| 4658 | 4654 | try self.resetPhdrs(); |
| 4659 | 4655 | try self.initSegments(); |
| 4660 | 4656 | if (nphdrs == self.phdrs.items.len) break; |
| 4661 | 4657 | } |
| 4662 | 4658 | } |
| 4663 | 4659 | |
| 4660 | fn allocateNonAllocSections(self: *Elf) void { |
| 4661 | var off: u64 = 0; |
| 4662 | for (self.shdrs.items) |shdr| { |
| 4663 | if (shdr.sh_type == elf.SHT_NULL) continue; |
| 4664 | if (shdr.sh_flags & elf.SHF_ALLOC == 0) break; |
| 4665 | off = @max(off, shdr.sh_offset + shdr.sh_size); |
| 4666 | } |
| 4667 | self.allocateNonAllocSectionsInFile(off); |
| 4668 | } |
| 4669 | |
| 4664 | 4670 | fn allocateSpecialPhdrs(self: *Elf) void { |
| 4665 | 4671 | for (&[_]struct { ?u16, ?u16 }{ |
| 4666 | 4672 | .{ self.phdr_interp_index, self.interp_section_index }, |
| ... | ... | @@ -4677,6 +4683,32 @@ fn allocateSpecialPhdrs(self: *Elf) void { |
| 4677 | 4683 | phdr.p_memsz = shdr.sh_size; |
| 4678 | 4684 | } |
| 4679 | 4685 | } |
| 4686 | |
| 4687 | // Allocate TLS phdr |
| 4688 | if (self.phdr_tls_index) |index| { |
| 4689 | const slice = self.shdrs.items; |
| 4690 | const phdr = &self.phdrs.items[index]; |
| 4691 | var shndx: u16 = 0; |
| 4692 | outer: while (shndx < slice.len) { |
| 4693 | const shdr = slice[shndx]; |
| 4694 | if (shdr.sh_flags & elf.SHF_TLS == 0) { |
| 4695 | shndx += 1; |
| 4696 | continue; |
| 4697 | } |
| 4698 | phdr.p_offset = shdr.sh_offset; |
| 4699 | phdr.p_vaddr = shdr.sh_addr; |
| 4700 | phdr.p_paddr = shdr.sh_addr; |
| 4701 | phdr.p_align = shdr.sh_addralign; |
| 4702 | self.addShdrToPhdr(shndx, index); |
| 4703 | shndx += 1; |
| 4704 | |
| 4705 | while (shndx < slice.len) : (shndx += 1) { |
| 4706 | const next = slice[shndx]; |
| 4707 | if (next.sh_flags & elf.SHF_TLS == 0) continue :outer; |
| 4708 | self.addShdrToPhdr(shndx, index); |
| 4709 | } |
| 4710 | } |
| 4711 | } |
| 4680 | 4712 | } |
| 4681 | 4713 | |
| 4682 | 4714 | fn allocateAtoms(self: *Elf) void { |
| ... | ... | @@ -5807,8 +5839,21 @@ fn formatPhdrs( |
| 5807 | 5839 | if (exec) flags[0] = 'X'; |
| 5808 | 5840 | if (write) flags[1] = 'W'; |
| 5809 | 5841 | if (read) flags[2] = 'R'; |
| 5810 | | try writer.print("phdr({d}) : {s} : @{x} ({x}) : align({x}) : filesz({x}) : memsz({x})\n", .{ |
| 5811 | | i, flags, phdr.p_offset, phdr.p_vaddr, phdr.p_align, phdr.p_filesz, phdr.p_memsz, |
| 5842 | const p_type = switch (phdr.p_type) { |
| 5843 | elf.PT_LOAD => "LOAD", |
| 5844 | elf.PT_TLS => "TLS", |
| 5845 | elf.PT_GNU_EH_FRAME => "GNU_EH_FRAME", |
| 5846 | elf.PT_GNU_STACK => "GNU_STACK", |
| 5847 | elf.PT_DYNAMIC => "DYNAMIC", |
| 5848 | elf.PT_INTERP => "INTERP", |
| 5849 | elf.PT_NULL => "NULL", |
| 5850 | elf.PT_PHDR => "PHDR", |
| 5851 | elf.PT_NOTE => "NOTE", |
| 5852 | else => "UNKNOWN", |
| 5853 | }; |
| 5854 | try writer.print("phdr({d}) : {s} : {s} : @{x} ({x}) : align({x}) : filesz({x}) : memsz({x})\n", .{ |
| 5855 | i, p_type, flags, phdr.p_offset, phdr.p_vaddr, |
| 5856 | phdr.p_align, phdr.p_filesz, phdr.p_memsz, |
| 5812 | 5857 | }); |
| 5813 | 5858 | } |
| 5814 | 5859 | } |
Jakub Konka <kubkon@jakubkonka.com> 2023-10-11 12:03:27+02:00
src