authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-10-11 12:03:27+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-10-16 19:33:05+02:00
log8be71906d914563ea3e3e154ab948b4d78a1dcd3
tree47fc3995a3808b57749cb7d34cc1bde08806ddb4
parentbcce035636f14de6b2f39edea094d6eab3a321c9

elf: split allocating sections/segments into alloc and non-alloc


1 files changed, 123 insertions(+), 78 deletions(-)

src/link/Elf.zig+123-78
...@@ -60,6 +60,7 @@ phdr_gnu_eh_frame_index: ?u16 = null,...@@ -60,6 +60,7 @@ phdr_gnu_eh_frame_index: ?u16 = null,
60/// PT_GNU_STACK60/// PT_GNU_STACK
61phdr_gnu_stack_index: ?u16 = null,61phdr_gnu_stack_index: ?u16 = null,
62/// PT_TLS62/// PT_TLS
63/// TODO I think ELF permits multiple TLS segments but for now, assume one per file.
63phdr_tls_index: ?u16 = null,64phdr_tls_index: ?u16 = null,
6465
65entry_index: ?Symbol.Index = null,66entry_index: ?Symbol.Index = null,
...@@ -1588,7 +1589,8 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node...@@ -1588,7 +1589,8 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
1588 try self.setVersionSymtab();1589 try self.setVersionSymtab();
1589 try self.updateSectionSizes();1590 try self.updateSectionSizes();
15901591
1591 try self.allocateSections();1592 try self.initAndAllocateSegments();
1593 self.allocateNonAllocSections();
1592 self.allocateSpecialPhdrs();1594 self.allocateSpecialPhdrs();
1593 self.allocateAtoms();1595 self.allocateAtoms();
1594 self.allocateLinkerDefinedSymbols();1596 self.allocateLinkerDefinedSymbols();
...@@ -3996,6 +3998,16 @@ fn initSpecialPhdrs(self: *Elf) !void {...@@ -3996,6 +3998,16 @@ fn initSpecialPhdrs(self: *Elf) !void {
3996 .memsz = self.base.options.stack_size_override orelse 0,3998 .memsz = self.base.options.stack_size_override orelse 0,
3997 .@"align" = 1,3999 .@"align" = 1,
3998 });4000 });
4001
4002 const has_tls = for (self.shdrs.items) |shdr| {
4003 if (shdr.sh_flags & elf.SHF_TLS != 0) break true;
4004 } else false;
4005 if (has_tls) {
4006 self.phdr_tls_index = try self.addPhdr(.{
4007 .type = elf.PT_TLS,
4008 .flags = elf.PF_R,
4009 });
4010 }
3999}4011}
40004012
4001/// We need to sort constructors/destuctors in the following sections:4013/// We need to sort constructors/destuctors in the following sections:
...@@ -4402,6 +4414,7 @@ fn resetPhdrs(self: *Elf) !void {...@@ -4402,6 +4414,7 @@ fn resetPhdrs(self: *Elf) !void {
4402 const gpa = self.base.allocator;4414 const gpa = self.base.allocator;
4403 const phdrs = try self.phdrs.toOwnedSlice(gpa);4415 const phdrs = try self.phdrs.toOwnedSlice(gpa);
4404 try self.phdrs.ensureUnusedCapacity(gpa, phdrs.len);4416 try self.phdrs.ensureUnusedCapacity(gpa, phdrs.len);
4417 self.phdr_to_shdr_table.clearRetainingCapacity(); // TODO move this into Section structure
44054418
4406 for (&[_]?u16{4419 for (&[_]?u16{
4407 self.phdr_table_index,4420 self.phdr_table_index,
...@@ -4410,6 +4423,7 @@ fn resetPhdrs(self: *Elf) !void {...@@ -4410,6 +4423,7 @@ fn resetPhdrs(self: *Elf) !void {
4410 self.phdr_dynamic_index,4423 self.phdr_dynamic_index,
4411 self.phdr_gnu_eh_frame_index,4424 self.phdr_gnu_eh_frame_index,
4412 self.phdr_gnu_stack_index,4425 self.phdr_gnu_stack_index,
4426 self.phdr_tls_index,
4413 }) |maybe_index| {4427 }) |maybe_index| {
4414 if (maybe_index) |index| {4428 if (maybe_index) |index| {
4415 self.phdrs.appendAssumeCapacity(phdrs[index]);4429 self.phdrs.appendAssumeCapacity(phdrs[index]);
...@@ -4420,69 +4434,43 @@ fn resetPhdrs(self: *Elf) !void {...@@ -4420,69 +4434,43 @@ fn resetPhdrs(self: *Elf) !void {
4420fn initSegments(self: *Elf) !void {4434fn initSegments(self: *Elf) !void {
4421 // Add LOAD phdrs4435 // Add LOAD phdrs
4422 const slice = self.shdrs.items;4436 const slice = self.shdrs.items;
4423 {4437 var last_phdr: ?u16 = null;
4424 var last_phdr: ?u16 = null;4438 var shndx: u16 = 0;
4425 var shndx: usize = 0;4439 while (shndx < slice.len) {
4426 while (shndx < slice.len) {4440 const shdr = &slice[shndx];
4427 const shdr = &slice[shndx];4441 if (!shdrIsAlloc(shdr) or shdrIsTbss(shdr)) {
4428 if (!shdrIsAlloc(shdr) or shdrIsTbss(shdr)) {
4429 shndx += 1;
4430 continue;
4431 }
4432 last_phdr = try self.addPhdr(.{
4433 .type = elf.PT_LOAD,
4434 .flags = shdrToPhdrFlags(shdr.sh_flags),
4435 .@"align" = @max(self.page_size, shdr.sh_addralign),
4436 .offset = if (last_phdr == null) 0 else shdr.sh_offset,
4437 .addr = if (last_phdr == null) self.calcImageBase() else shdr.sh_addr,
4438 });
4439 const p_flags = self.phdrs.items[last_phdr.?].p_flags;
4440 try self.addShdrToPhdr(last_phdr.?, shdr);
4441 shndx += 1;4442 shndx += 1;
44424443 continue;
4443 while (shndx < slice.len) : (shndx += 1) {
4444 const next = &slice[shndx];
4445 if (shdrIsTbss(next)) continue;
4446 if (p_flags == shdrToPhdrFlags(next.sh_flags)) {
4447 if (shdrIsBss(next) or next.sh_offset - shdr.sh_offset == next.sh_addr - shdr.sh_addr) {
4448 try self.addShdrToPhdr(last_phdr.?, next);
4449 continue;
4450 }
4451 }
4452 break;
4453 }
4454 }4444 }
4455 }4445 last_phdr = try self.addPhdr(.{
44564446 .type = elf.PT_LOAD,
4457 // Add TLS phdr4447 .flags = shdrToPhdrFlags(shdr.sh_flags),
4458 {4448 .@"align" = @max(self.page_size, shdr.sh_addralign),
4459 var shndx: usize = 0;4449 .offset = if (last_phdr == null) 0 else shdr.sh_offset,
4460 outer: while (shndx < slice.len) {4450 .addr = if (last_phdr == null) self.calcImageBase() else shdr.sh_addr,
4461 const shdr = &slice[shndx];4451 });
4462 if (!shdrIsTls(shdr)) {4452 const p_flags = self.phdrs.items[last_phdr.?].p_flags;
4463 shndx += 1;4453 self.addShdrToPhdr(shndx, last_phdr.?);
4464 continue;4454 try self.phdr_to_shdr_table.putNoClobber(self.base.allocator, shndx, last_phdr.?);
4465 }4455 shndx += 1;
4466 self.phdr_tls_index = try self.addPhdr(.{4456
4467 .type = elf.PT_TLS,4457 while (shndx < slice.len) : (shndx += 1) {
4468 .flags = elf.PF_R,4458 const next = &slice[shndx];
4469 .@"align" = shdr.sh_addralign,4459 if (shdrIsTbss(next)) continue;
4470 .offset = shdr.sh_offset,4460 if (p_flags == shdrToPhdrFlags(next.sh_flags)) {
4471 .addr = shdr.sh_addr,4461 if (shdrIsBss(next) or next.sh_offset - shdr.sh_offset == next.sh_addr - shdr.sh_addr) {
4472 });4462 self.addShdrToPhdr(shndx, last_phdr.?);
4473 try self.addShdrToPhdr(self.phdr_tls_index.?, shdr);4463 try self.phdr_to_shdr_table.putNoClobber(self.base.allocator, shndx, last_phdr.?);
4474 shndx += 1;4464 continue;
44754465 }
4476 while (shndx < slice.len) : (shndx += 1) {
4477 const next = &slice[shndx];
4478 if (!shdrIsTls(next)) continue :outer;
4479 try self.addShdrToPhdr(self.phdr_tls_index.?, next);
4480 }4466 }
4467 break;
4481 }4468 }
4482 }4469 }
4483}4470}
44844471
4485fn addShdrToPhdr(self: *Elf, phdr_index: u16, shdr: *const elf.Elf64_Shdr) !void {4472fn addShdrToPhdr(self: *Elf, shdr_index: u16, phdr_index: u16) void {
4473 const shdr = self.shdrs.items[shdr_index];
4486 const phdr = &self.phdrs.items[phdr_index];4474 const phdr = &self.phdrs.items[phdr_index];
4487 phdr.p_align = @max(phdr.p_align, shdr.sh_addralign);4475 phdr.p_align = @max(phdr.p_align, shdr.sh_addralign);
4488 if (shdr.sh_type != elf.SHT_NOBITS) {4476 if (shdr.sh_type != elf.SHT_NOBITS) {
...@@ -4520,7 +4508,7 @@ pub inline fn shdrIsTls(shdr: *const elf.Elf64_Shdr) bool {...@@ -4520,7 +4508,7 @@ pub inline fn shdrIsTls(shdr: *const elf.Elf64_Shdr) bool {
4520 return shdr.sh_flags & elf.SHF_TLS != 0;4508 return shdr.sh_flags & elf.SHF_TLS != 0;
4521}4509}
45224510
4523fn allocateSectionsInMemory(self: *Elf, base_offset: u64) void {4511fn allocateAllocSectionsInMemory(self: *Elf, base_addr: u64) void {
4524 // We use this struct to track maximum alignment of all TLS sections.4512 // We use this struct to track maximum alignment of all TLS sections.
4525 // According to https://github.com/rui314/mold/commit/bd46edf3f0fe9e1a787ea453c4657d535622e61f in mold,4513 // According to https://github.com/rui314/mold/commit/bd46edf3f0fe9e1a787ea453c4657d535622e61f in mold,
4526 // in-file offsets have to be aligned against the start of TLS program header.4514 // in-file offsets have to be aligned against the start of TLS program header.
...@@ -4549,7 +4537,7 @@ fn allocateSectionsInMemory(self: *Elf, base_offset: u64) void {...@@ -4549,7 +4537,7 @@ fn allocateSectionsInMemory(self: *Elf, base_offset: u64) void {
4549 alignment.tls_start_align = @max(alignment.tls_start_align, shdr.sh_addralign);4537 alignment.tls_start_align = @max(alignment.tls_start_align, shdr.sh_addralign);
4550 }4538 }
45514539
4552 var addr = self.calcImageBase() + base_offset;4540 var addr = base_addr;
4553 var i: usize = 0;4541 var i: usize = 0;
4554 while (i < self.shdrs.items.len) : (i += 1) {4542 while (i < self.shdrs.items.len) : (i += 1) {
4555 const shdr = &self.shdrs.items[i];4543 const shdr = &self.shdrs.items[i];
...@@ -4592,25 +4580,16 @@ fn allocateSectionsInMemory(self: *Elf, base_offset: u64) void {...@@ -4592,25 +4580,16 @@ fn allocateSectionsInMemory(self: *Elf, base_offset: u64) void {
4592 }4580 }
4593}4581}
45944582
4595fn allocateSectionsInFile(self: *Elf, base_offset: u64) void {4583fn allocateAllocSectionsInFile(self: *Elf, base_offset: u64) void {
4596 var offset = base_offset;4584 var offset = base_offset;
4597 var i: usize = 0;4585 var i: usize = 0;
4598 while (i < self.shdrs.items.len) {4586 while (i < self.shdrs.items.len) {
4599 const first = &self.shdrs.items[i];4587 const first = &self.shdrs.items[i];
4600 defer if (!shdrIsAlloc(first) or shdrIsZerofill(first)) {4588 if (shdrIsZerofill(first) or first.sh_type == elf.SHT_NULL) {
4601 i += 1;4589 i += 1;
4602 };
4603
4604 if (first.sh_type == elf.SHT_NULL) continue;
4605
4606 // Non-alloc sections don't need congruency with their allocated virtual memory addresses
4607 if (!shdrIsAlloc(first)) {
4608 first.sh_offset = mem.alignForward(u64, offset, first.sh_addralign);
4609 offset = first.sh_offset + first.sh_size;
4610 continue;4590 continue;
4611 }4591 }
4612 // Skip any zerofill section4592 if (!shdrIsAlloc(first)) break;
4613 if (shdrIsZerofill(first)) continue;
46144593
4615 // Set the offset to a value that is congruent with the section's allocated virtual memory address4594 // Set the offset to a value that is congruent with the section's allocated virtual memory address
4616 if (first.sh_addralign > self.page_size) {4595 if (first.sh_addralign > self.page_size) {
...@@ -4637,11 +4616,26 @@ fn allocateSectionsInFile(self: *Elf, base_offset: u64) void {...@@ -4637,11 +4616,26 @@ fn allocateSectionsInFile(self: *Elf, base_offset: u64) void {
4637 offset = prev.sh_offset + prev.sh_size;4616 offset = prev.sh_offset + prev.sh_size;
46384617
4639 // Skip any zerofill section4618 // Skip any zerofill section
4640 while (i < self.shdrs.items.len and shdrIsAlloc(&self.shdrs.items[i]) and shdrIsZerofill(&self.shdrs.items[i])) : (i += 1) {}4619 while (i < self.shdrs.items.len and
4620 shdrIsAlloc(&self.shdrs.items[i]) and
4621 shdrIsZerofill(&self.shdrs.items[i])) : (i += 1)
4622 {}
4623 }
4624}
4625
4626fn allocateNonAllocSectionsInFile(self: *Elf, base_offset: u64) void {
4627 var offset = base_offset;
4628 var i: usize = 0;
4629 while (i < self.shdrs.items.len) : (i += 1) {
4630 const first = &self.shdrs.items[i];
4631 if (shdrIsAlloc(first) or first.sh_type == elf.SHT_NULL) continue;
4632 // Non-alloc sections don't need congruency with their allocated virtual memory addresses
4633 first.sh_offset = mem.alignForward(u64, offset, first.sh_addralign);
4634 offset = first.sh_offset + first.sh_size;
4641 }4635 }
4642}4636}
46434637
4644fn allocateSections(self: *Elf) !void {4638fn initAndAllocateSegments(self: *Elf) !void {
4645 const ehsize: u64 = switch (self.ptr_width) {4639 const ehsize: u64 = switch (self.ptr_width) {
4646 .p32 => @sizeOf(elf.Elf32_Ehdr),4640 .p32 => @sizeOf(elf.Elf32_Ehdr),
4647 .p64 => @sizeOf(elf.Elf64_Ehdr),4641 .p64 => @sizeOf(elf.Elf64_Ehdr),
...@@ -4650,17 +4644,29 @@ fn allocateSections(self: *Elf) !void {...@@ -4650,17 +4644,29 @@ fn allocateSections(self: *Elf) !void {
4650 .p32 => @sizeOf(elf.Elf32_Phdr),4644 .p32 => @sizeOf(elf.Elf32_Phdr),
4651 .p64 => @sizeOf(elf.Elf64_Phdr),4645 .p64 => @sizeOf(elf.Elf64_Phdr),
4652 };4646 };
4647 const image_base = self.calcImageBase();
4653 while (true) {4648 while (true) {
4654 const nphdrs = self.phdrs.items.len;4649 const nphdrs = self.phdrs.items.len;
4655 const base_offset: u64 = ehsize + nphdrs * phsize;4650 const off = ehsize + nphdrs * phsize;
4656 self.allocateSectionsInMemory(base_offset);4651 const addr = image_base + off;
4657 self.allocateSectionsInFile(base_offset);4652 self.allocateAllocSectionsInMemory(addr);
4653 self.allocateAllocSectionsInFile(off);
4658 try self.resetPhdrs();4654 try self.resetPhdrs();
4659 try self.initSegments();4655 try self.initSegments();
4660 if (nphdrs == self.phdrs.items.len) break;4656 if (nphdrs == self.phdrs.items.len) break;
4661 }4657 }
4662}4658}
46634659
4660fn allocateNonAllocSections(self: *Elf) void {
4661 var off: u64 = 0;
4662 for (self.shdrs.items) |shdr| {
4663 if (shdr.sh_type == elf.SHT_NULL) continue;
4664 if (shdr.sh_flags & elf.SHF_ALLOC == 0) break;
4665 off = @max(off, shdr.sh_offset + shdr.sh_size);
4666 }
4667 self.allocateNonAllocSectionsInFile(off);
4668}
4669
4664fn allocateSpecialPhdrs(self: *Elf) void {4670fn allocateSpecialPhdrs(self: *Elf) void {
4665 for (&[_]struct { ?u16, ?u16 }{4671 for (&[_]struct { ?u16, ?u16 }{
4666 .{ self.phdr_interp_index, self.interp_section_index },4672 .{ self.phdr_interp_index, self.interp_section_index },
...@@ -4677,6 +4683,32 @@ fn allocateSpecialPhdrs(self: *Elf) void {...@@ -4677,6 +4683,32 @@ fn allocateSpecialPhdrs(self: *Elf) void {
4677 phdr.p_memsz = shdr.sh_size;4683 phdr.p_memsz = shdr.sh_size;
4678 }4684 }
4679 }4685 }
4686
4687 // Allocate TLS phdr
4688 if (self.phdr_tls_index) |index| {
4689 const slice = self.shdrs.items;
4690 const phdr = &self.phdrs.items[index];
4691 var shndx: u16 = 0;
4692 outer: while (shndx < slice.len) {
4693 const shdr = slice[shndx];
4694 if (shdr.sh_flags & elf.SHF_TLS == 0) {
4695 shndx += 1;
4696 continue;
4697 }
4698 phdr.p_offset = shdr.sh_offset;
4699 phdr.p_vaddr = shdr.sh_addr;
4700 phdr.p_paddr = shdr.sh_addr;
4701 phdr.p_align = shdr.sh_addralign;
4702 self.addShdrToPhdr(shndx, index);
4703 shndx += 1;
4704
4705 while (shndx < slice.len) : (shndx += 1) {
4706 const next = slice[shndx];
4707 if (next.sh_flags & elf.SHF_TLS == 0) continue :outer;
4708 self.addShdrToPhdr(shndx, index);
4709 }
4710 }
4711 }
4680}4712}
46814713
4682fn allocateAtoms(self: *Elf) void {4714fn allocateAtoms(self: *Elf) void {
...@@ -5807,8 +5839,21 @@ fn formatPhdrs(...@@ -5807,8 +5839,21 @@ fn formatPhdrs(
5807 if (exec) flags[0] = 'X';5839 if (exec) flags[0] = 'X';
5808 if (write) flags[1] = 'W';5840 if (write) flags[1] = 'W';
5809 if (read) flags[2] = 'R';5841 if (read) flags[2] = 'R';
5810 try writer.print("phdr({d}) : {s} : @{x} ({x}) : align({x}) : filesz({x}) : memsz({x})\n", .{5842 const p_type = switch (phdr.p_type) {
5811 i, flags, phdr.p_offset, phdr.p_vaddr, phdr.p_align, phdr.p_filesz, phdr.p_memsz,5843 elf.PT_LOAD => "LOAD",
5844 elf.PT_TLS => "TLS",
5845 elf.PT_GNU_EH_FRAME => "GNU_EH_FRAME",
5846 elf.PT_GNU_STACK => "GNU_STACK",
5847 elf.PT_DYNAMIC => "DYNAMIC",
5848 elf.PT_INTERP => "INTERP",
5849 elf.PT_NULL => "NULL",
5850 elf.PT_PHDR => "PHDR",
5851 elf.PT_NOTE => "NOTE",
5852 else => "UNKNOWN",
5853 };
5854 try writer.print("phdr({d}) : {s} : {s} : @{x} ({x}) : align({x}) : filesz({x}) : memsz({x})\n", .{
5855 i, p_type, flags, phdr.p_offset, phdr.p_vaddr,
5856 phdr.p_align, phdr.p_filesz, phdr.p_memsz,
5812 });5857 });
5813 }5858 }
5814}5859}