| ... | @@ -1610,34 +1610,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node | ... | @@ -1610,34 +1610,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node |
| 1610 | try self.setVersionSymtab(); | 1610 | try self.setVersionSymtab(); |
| 1611 | try self.updateSectionSizes(); | 1611 | try self.updateSectionSizes(); |
| 1612 | | 1612 | |
| 1613 | // Allocate PHDR table. | 1613 | self.allocatePhdrTable(); |
| 1614 | { | | |
| 1615 | const new_load_segments = self.calcNumberOfSegments(); | | |
| 1616 | const phdr_table = &self.phdrs.items[self.phdr_table_index.?]; | | |
| 1617 | const phdr_table_load = &self.phdrs.items[self.phdr_table_load_index.?]; | | |
| 1618 | | | |
| 1619 | const phsize: u64 = switch (self.ptr_width) { | | |
| 1620 | .p32 => @sizeOf(elf.Elf32_Phdr), | | |
| 1621 | .p64 => @sizeOf(elf.Elf64_Phdr), | | |
| 1622 | }; | | |
| 1623 | const needed_size = (self.phdrs.items.len + new_load_segments) * phsize; | | |
| 1624 | | | |
| 1625 | if (needed_size > self.allocatedSize(phdr_table.p_offset)) { | | |
| 1626 | phdr_table.p_offset = 0; | | |
| 1627 | phdr_table.p_offset = self.findFreeSpace(needed_size, phdr_table.p_align); | | |
| 1628 | } | | |
| 1629 | | | |
| 1630 | phdr_table_load.p_offset = mem.alignBackward(u64, phdr_table.p_offset, phdr_table_load.p_align); | | |
| 1631 | const load_align_offset = phdr_table.p_offset - phdr_table_load.p_offset; | | |
| 1632 | phdr_table_load.p_filesz = load_align_offset + needed_size; | | |
| 1633 | phdr_table_load.p_memsz = load_align_offset + needed_size; | | |
| 1634 | | | |
| 1635 | phdr_table.p_filesz = needed_size; | | |
| 1636 | phdr_table.p_vaddr = phdr_table_load.p_vaddr + load_align_offset; | | |
| 1637 | phdr_table.p_paddr = phdr_table_load.p_paddr + load_align_offset; | | |
| 1638 | phdr_table.p_memsz = needed_size; | | |
| 1639 | } | | |
| 1640 | | | |
| 1641 | try self.allocateAllocSections(); | 1614 | try self.allocateAllocSections(); |
| 1642 | self.allocateNonAllocSections(); | 1615 | self.allocateNonAllocSections(); |
| 1643 | self.allocateSpecialPhdrs(); | 1616 | self.allocateSpecialPhdrs(); |
| ... | @@ -4441,84 +4414,6 @@ fn updateSectionSizes(self: *Elf) !void { | ... | @@ -4441,84 +4414,6 @@ fn updateSectionSizes(self: *Elf) !void { |
| 4441 | } | 4414 | } |
| 4442 | } | 4415 | } |
| 4443 | | 4416 | |
| 4444 | /// The assumed layout of PHDRs in file is as follows: | | |
| 4445 | /// PT_PHDR | | |
| 4446 | /// PT_LOAD for PT_PHDR | | |
| 4447 | /// PT_INTERP | | |
| 4448 | /// PT_DYNAMIC | | |
| 4449 | /// PT_GNU_EH_FRAME | | |
| 4450 | /// PT_GNU_STACK | | |
| 4451 | /// PT_LOAD... | | |
| 4452 | /// PT_TLS | | |
| 4453 | fn resetPhdrs(self: *Elf) !void { | | |
| 4454 | const gpa = self.base.allocator; | | |
| 4455 | const phdrs = try self.phdrs.toOwnedSlice(gpa); | | |
| 4456 | try self.phdrs.ensureUnusedCapacity(gpa, phdrs.len); | | |
| 4457 | self.phdr_to_shdr_table.clearRetainingCapacity(); // TODO move this into Section structure | | |
| 4458 | | | |
| 4459 | for (&[_]?u16{ | | |
| 4460 | self.phdr_table_index, | | |
| 4461 | self.phdr_table_load_index, | | |
| 4462 | self.phdr_interp_index, | | |
| 4463 | self.phdr_dynamic_index, | | |
| 4464 | self.phdr_gnu_eh_frame_index, | | |
| 4465 | self.phdr_gnu_stack_index, | | |
| 4466 | self.phdr_tls_index, | | |
| 4467 | }) |maybe_index| { | | |
| 4468 | if (maybe_index) |index| { | | |
| 4469 | self.phdrs.appendAssumeCapacity(phdrs[index]); | | |
| 4470 | } | | |
| 4471 | } | | |
| 4472 | } | | |
| 4473 | | | |
| 4474 | fn initSegments(self: *Elf) !void { | | |
| 4475 | // Add LOAD phdrs | | |
| 4476 | const gpa = self.base.allocator; | | |
| 4477 | const slice = self.shdrs.items; | | |
| 4478 | var shndx: u16 = 0; | | |
| 4479 | while (shndx < slice.len) { | | |
| 4480 | const shdr = &slice[shndx]; | | |
| 4481 | if (!shdrIsAlloc(shdr) or shdrIsTbss(shdr)) { | | |
| 4482 | shndx += 1; | | |
| 4483 | continue; | | |
| 4484 | } | | |
| 4485 | const phndx = try self.addPhdr(.{ | | |
| 4486 | .type = elf.PT_LOAD, | | |
| 4487 | .flags = shdrToPhdrFlags(shdr.sh_flags), | | |
| 4488 | .@"align" = @max(self.page_size, shdr.sh_addralign), | | |
| 4489 | .offset = shdr.sh_offset, | | |
| 4490 | .addr = shdr.sh_addr, | | |
| 4491 | }); | | |
| 4492 | const p_flags = self.phdrs.items[phndx].p_flags; | | |
| 4493 | self.addShdrToPhdr(shndx, phndx); | | |
| 4494 | try self.phdr_to_shdr_table.putNoClobber(gpa, shndx, phndx); | | |
| 4495 | shndx += 1; | | |
| 4496 | | | |
| 4497 | while (shndx < slice.len) : (shndx += 1) { | | |
| 4498 | const next = &slice[shndx]; | | |
| 4499 | if (shdrIsTbss(next)) continue; | | |
| 4500 | if (p_flags == shdrToPhdrFlags(next.sh_flags)) { | | |
| 4501 | if (shdrIsBss(next) or next.sh_offset - shdr.sh_offset == next.sh_addr - shdr.sh_addr) { | | |
| 4502 | self.addShdrToPhdr(shndx, phndx); | | |
| 4503 | try self.phdr_to_shdr_table.putNoClobber(self.base.allocator, shndx, phndx); | | |
| 4504 | continue; | | |
| 4505 | } | | |
| 4506 | } | | |
| 4507 | break; | | |
| 4508 | } | | |
| 4509 | } | | |
| 4510 | } | | |
| 4511 | | | |
| 4512 | fn addShdrToPhdr(self: *Elf, shdr_index: u16, phdr_index: u16) void { | | |
| 4513 | const shdr = self.shdrs.items[shdr_index]; | | |
| 4514 | const phdr = &self.phdrs.items[phdr_index]; | | |
| 4515 | phdr.p_align = @max(phdr.p_align, shdr.sh_addralign); | | |
| 4516 | if (shdr.sh_type != elf.SHT_NOBITS) { | | |
| 4517 | phdr.p_filesz = shdr.sh_offset + shdr.sh_size - phdr.p_offset; | | |
| 4518 | } | | |
| 4519 | phdr.p_memsz = shdr.sh_addr + shdr.sh_size - phdr.p_vaddr; | | |
| 4520 | } | | |
| 4521 | | | |
| 4522 | fn shdrToPhdrFlags(sh_flags: u64) u32 { | 4417 | fn shdrToPhdrFlags(sh_flags: u64) u32 { |
| 4523 | const write = sh_flags & elf.SHF_WRITE != 0; | 4418 | const write = sh_flags & elf.SHF_WRITE != 0; |
| 4524 | const exec = sh_flags & elf.SHF_EXECINSTR != 0; | 4419 | const exec = sh_flags & elf.SHF_EXECINSTR != 0; |
| ... | @@ -4528,26 +4423,8 @@ fn shdrToPhdrFlags(sh_flags: u64) u32 { | ... | @@ -4528,26 +4423,8 @@ fn shdrToPhdrFlags(sh_flags: u64) u32 { |
| 4528 | return out_flags; | 4423 | return out_flags; |
| 4529 | } | 4424 | } |
| 4530 | | 4425 | |
| 4531 | inline fn shdrIsAlloc(shdr: *const elf.Elf64_Shdr) bool { | 4426 | /// Calculates how many segments (PT_LOAD progam headers) are required |
| 4532 | return shdr.sh_flags & elf.SHF_ALLOC != 0; | 4427 | /// to cover the set of sections. |
| 4533 | } | | |
| 4534 | | | |
| 4535 | inline fn shdrIsBss(shdr: *const elf.Elf64_Shdr) bool { | | |
| 4536 | return shdrIsZerofill(shdr) and !shdrIsTls(shdr); | | |
| 4537 | } | | |
| 4538 | | | |
| 4539 | inline fn shdrIsTbss(shdr: *const elf.Elf64_Shdr) bool { | | |
| 4540 | return shdrIsZerofill(shdr) and shdrIsTls(shdr); | | |
| 4541 | } | | |
| 4542 | | | |
| 4543 | inline fn shdrIsZerofill(shdr: *const elf.Elf64_Shdr) bool { | | |
| 4544 | return shdr.sh_type == elf.SHT_NOBITS; | | |
| 4545 | } | | |
| 4546 | | | |
| 4547 | pub inline fn shdrIsTls(shdr: *const elf.Elf64_Shdr) bool { | | |
| 4548 | return shdr.sh_flags & elf.SHF_TLS != 0; | | |
| 4549 | } | | |
| 4550 | | | |
| 4551 | fn calcNumberOfSegments(self: *Elf) usize { | 4428 | fn calcNumberOfSegments(self: *Elf) usize { |
| 4552 | var count: usize = 0; | 4429 | var count: usize = 0; |
| 4553 | var flags: u64 = 0; | 4430 | var flags: u64 = 0; |
| ... | @@ -4560,137 +4437,36 @@ fn calcNumberOfSegments(self: *Elf) usize { | ... | @@ -4560,137 +4437,36 @@ fn calcNumberOfSegments(self: *Elf) usize { |
| 4560 | return count; | 4437 | return count; |
| 4561 | } | 4438 | } |
| 4562 | | 4439 | |
| 4563 | fn allocateAllocSectionsInMemory(self: *Elf, base_addr: u64) void { | 4440 | /// Allocates PHDR table in virtual memory and in file. |
| 4564 | // We use this struct to track maximum alignment of all TLS sections. | 4441 | fn allocatePhdrTable(self: *Elf) void { |
| 4565 | // According to https://github.com/rui314/mold/commit/bd46edf3f0fe9e1a787ea453c4657d535622e61f in mold, | 4442 | const new_load_segments = self.calcNumberOfSegments(); |
| 4566 | // in-file offsets have to be aligned against the start of TLS program header. | 4443 | const phdr_table = &self.phdrs.items[self.phdr_table_index.?]; |
| 4567 | // If that's not ensured, then in a multi-threaded context, TLS variables across a shared object | 4444 | const phdr_table_load = &self.phdrs.items[self.phdr_table_load_index.?]; |
| 4568 | // boundary may not get correctly loaded at an aligned address. | | |
| 4569 | const Align = struct { | | |
| 4570 | tls_start_align: u64 = 1, | | |
| 4571 | first_tls_index: ?usize = null, | | |
| 4572 | | | |
| 4573 | fn isFirstTlsShdr(this: @This(), other: usize) bool { | | |
| 4574 | if (this.first_tls_index) |index| return index == other; | | |
| 4575 | return false; | | |
| 4576 | } | | |
| 4577 | | 4445 | |
| 4578 | fn @"align"(this: @This(), index: usize, sh_addralign: u64, addr: u64) u64 { | 4446 | const phsize: u64 = switch (self.ptr_width) { |
| 4579 | const alignment = if (this.isFirstTlsShdr(index)) this.tls_start_align else sh_addralign; | 4447 | .p32 => @sizeOf(elf.Elf32_Phdr), |
| 4580 | return mem.alignForward(u64, addr, alignment); | 4448 | .p64 => @sizeOf(elf.Elf64_Phdr), |
| 4581 | } | | |
| 4582 | }; | 4449 | }; |
| | 4450 | const needed_size = (self.phdrs.items.len + new_load_segments) * phsize; |
| 4583 | | 4451 | |
| 4584 | var alignment = Align{}; | 4452 | if (needed_size > self.allocatedSize(phdr_table.p_offset)) { |
| 4585 | for (self.shdrs.items, 0..) |*shdr, i| { | 4453 | phdr_table.p_offset = 0; |
| 4586 | if (shdr.sh_type == elf.SHT_NULL) continue; | 4454 | phdr_table.p_offset = self.findFreeSpace(needed_size, phdr_table.p_align); |
| 4587 | if (!shdrIsTls(shdr)) continue; | | |
| 4588 | if (alignment.first_tls_index == null) alignment.first_tls_index = i; | | |
| 4589 | alignment.tls_start_align = @max(alignment.tls_start_align, shdr.sh_addralign); | | |
| 4590 | } | | |
| 4591 | | | |
| 4592 | var addr = base_addr; | | |
| 4593 | var i: usize = 0; | | |
| 4594 | while (i < self.shdrs.items.len) : (i += 1) { | | |
| 4595 | const shdr = &self.shdrs.items[i]; | | |
| 4596 | if (shdr.sh_type == elf.SHT_NULL) continue; | | |
| 4597 | if (!shdrIsAlloc(shdr)) continue; | | |
| 4598 | if (i > 0) { | | |
| 4599 | const prev_shdr = self.shdrs.items[i - 1]; | | |
| 4600 | if (shdrToPhdrFlags(shdr.sh_flags) != shdrToPhdrFlags(prev_shdr.sh_flags)) { | | |
| 4601 | // We need to advance by page size | | |
| 4602 | addr += self.page_size; | | |
| 4603 | } | | |
| 4604 | } | | |
| 4605 | if (shdrIsTbss(shdr)) { | | |
| 4606 | // .tbss is a little special as it's used only by the loader meaning it doesn't | | |
| 4607 | // need to be actually mmap'ed at runtime. We still need to correctly increment | | |
| 4608 | // the addresses of every TLS zerofill section tho. Thus, we hack it so that | | |
| 4609 | // we increment the start address like normal, however, after we are done, | | |
| 4610 | // the next ALLOC section will get its start address allocated within the same | | |
| 4611 | // range as the .tbss sections. We will get something like this: | | |
| 4612 | // | | |
| 4613 | // ... | | |
| 4614 | // .tbss 0x10 | | |
| 4615 | // .tcommon 0x20 | | |
| 4616 | // .data 0x10 | | |
| 4617 | // ... | | |
| 4618 | var tbss_addr = addr; | | |
| 4619 | while (i < self.shdrs.items.len and shdrIsTbss(&self.shdrs.items[i])) : (i += 1) { | | |
| 4620 | const tbss_shdr = &self.shdrs.items[i]; | | |
| 4621 | tbss_addr = alignment.@"align"(i, tbss_shdr.sh_addralign, tbss_addr); | | |
| 4622 | tbss_shdr.sh_addr = tbss_addr; | | |
| 4623 | tbss_addr += tbss_shdr.sh_size; | | |
| 4624 | } | | |
| 4625 | i -= 1; | | |
| 4626 | continue; | | |
| 4627 | } | | |
| 4628 | | | |
| 4629 | addr = alignment.@"align"(i, shdr.sh_addralign, addr); | | |
| 4630 | shdr.sh_addr = addr; | | |
| 4631 | addr += shdr.sh_size; | | |
| 4632 | } | 4455 | } |
| 4633 | } | | |
| 4634 | | | |
| 4635 | fn allocateAllocSectionsInFile(self: *Elf, base_offset: u64) void { | | |
| 4636 | var offset = base_offset; | | |
| 4637 | var i: usize = 0; | | |
| 4638 | while (i < self.shdrs.items.len) { | | |
| 4639 | const first = &self.shdrs.items[i]; | | |
| 4640 | if (shdrIsZerofill(first) or first.sh_type == elf.SHT_NULL) { | | |
| 4641 | i += 1; | | |
| 4642 | continue; | | |
| 4643 | } | | |
| 4644 | if (!shdrIsAlloc(first)) break; | | |
| 4645 | | | |
| 4646 | // Set the offset to a value that is congruent with the section's allocated virtual memory address | | |
| 4647 | if (first.sh_addralign > self.page_size) { | | |
| 4648 | offset = mem.alignForward(u64, offset, first.sh_addralign); | | |
| 4649 | } else { | | |
| 4650 | const val = mem.alignBackward(u64, offset, self.page_size) + @rem(first.sh_addr, self.page_size); | | |
| 4651 | offset = if (offset <= val) val else val + self.page_size; | | |
| 4652 | } | | |
| 4653 | | | |
| 4654 | while (true) { | | |
| 4655 | const prev = &self.shdrs.items[i]; | | |
| 4656 | prev.sh_offset = offset + prev.sh_addr - first.sh_addr; | | |
| 4657 | i += 1; | | |
| 4658 | | 4456 | |
| 4659 | const next = &self.shdrs.items[i]; | 4457 | phdr_table_load.p_offset = mem.alignBackward(u64, phdr_table.p_offset, phdr_table_load.p_align); |
| 4660 | if (i >= self.shdrs.items.len or !shdrIsAlloc(next) or shdrIsZerofill(next)) break; | 4458 | const load_align_offset = phdr_table.p_offset - phdr_table_load.p_offset; |
| 4661 | if (next.sh_addr < first.sh_addr) break; | 4459 | phdr_table_load.p_filesz = load_align_offset + needed_size; |
| | 4460 | phdr_table_load.p_memsz = load_align_offset + needed_size; |
| 4662 | | 4461 | |
| 4663 | const gap = next.sh_addr - prev.sh_addr - prev.sh_size; | 4462 | phdr_table.p_filesz = needed_size; |
| 4664 | if (gap >= self.page_size) break; | 4463 | phdr_table.p_vaddr = phdr_table_load.p_vaddr + load_align_offset; |
| 4665 | } | 4464 | phdr_table.p_paddr = phdr_table_load.p_paddr + load_align_offset; |
| 4666 | | 4465 | phdr_table.p_memsz = needed_size; |
| 4667 | const prev = &self.shdrs.items[i - 1]; | | |
| 4668 | offset = prev.sh_offset + prev.sh_size; | | |
| 4669 | | | |
| 4670 | // Skip any zerofill section | | |
| 4671 | while (i < self.shdrs.items.len and | | |
| 4672 | shdrIsAlloc(&self.shdrs.items[i]) and | | |
| 4673 | shdrIsZerofill(&self.shdrs.items[i])) : (i += 1) | | |
| 4674 | {} | | |
| 4675 | } | | |
| 4676 | } | | |
| 4677 | | | |
| 4678 | /// This function is really only responsible for allocating alloc sections, | | |
| 4679 | /// and creating load segments as-if incremental linking mode didn't exist. | | |
| 4680 | /// As a base address we take space immediately after the PHDR table, and | | |
| 4681 | /// aim for fitting between it and where the first incremental segment is | | |
| 4682 | /// allocated (see `initMetadata`). | | |
| 4683 | fn initAndAllocateSegments(self: *Elf, base_addr: u64, base_offset: u64) !void { | | |
| 4684 | while (true) { | | |
| 4685 | const nphdrs = self.phdrs.items.len; | | |
| 4686 | self.allocateAllocSectionsInMemory(base_addr); | | |
| 4687 | self.allocateAllocSectionsInFile(base_offset); | | |
| 4688 | try self.resetPhdrs(); | | |
| 4689 | try self.initSegments(); | | |
| 4690 | if (nphdrs == self.phdrs.items.len) break; | | |
| 4691 | } | | |
| 4692 | } | 4466 | } |
| 4693 | | 4467 | |
| | 4468 | /// Allocates alloc sections and creates load segments for sections |
| | 4469 | /// extracted from input object files. |
| 4694 | fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { | 4470 | fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { |
| 4695 | // We use this struct to track maximum alignment of all TLS sections. | 4471 | // We use this struct to track maximum alignment of all TLS sections. |
| 4696 | // According to https://github.com/rui314/mold/commit/bd46edf3f0fe9e1a787ea453c4657d535622e61f in mold, | 4472 | // According to https://github.com/rui314/mold/commit/bd46edf3f0fe9e1a787ea453c4657d535622e61f in mold, |
| ... | @@ -4713,17 +4489,25 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { | ... | @@ -4713,17 +4489,25 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { |
| 4713 | }; | 4489 | }; |
| 4714 | | 4490 | |
| 4715 | var alignment = Align{}; | 4491 | var alignment = Align{}; |
| 4716 | for (self.shdrs.items, 0..) |*shdr, i| { | 4492 | for (self.shdrs.items, 0..) |shdr, i| { |
| 4717 | if (shdr.sh_type == elf.SHT_NULL) continue; | 4493 | if (shdr.sh_type == elf.SHT_NULL) continue; |
| 4718 | if (!shdrIsTls(shdr)) continue; | 4494 | if (shdr.sh_flags & elf.SHF_TLS == 0) continue; |
| 4719 | if (alignment.first_tls_index == null) alignment.first_tls_index = i; | 4495 | if (alignment.first_tls_index == null) alignment.first_tls_index = i; |
| 4720 | alignment.tls_start_align = @max(alignment.tls_start_align, shdr.sh_addralign); | 4496 | alignment.tls_start_align = @max(alignment.tls_start_align, shdr.sh_addralign); |
| 4721 | } | 4497 | } |
| 4722 | | 4498 | |
| | 4499 | // Next, calculate segment covers by scanning all alloc sections. |
| | 4500 | // If a section matches segment flags with the preceeding section, |
| | 4501 | // we put it in the same segment. Otherwise, we create a new cover. |
| | 4502 | // This algorithm is simple but suboptimal in terms of space re-use: |
| | 4503 | // normally we would also take into account any gaps in allocated |
| | 4504 | // virtual and file offsets. However, the simple one will do for one |
| | 4505 | // as we are more interested in quick turnaround and compatibility |
| | 4506 | // with `findFreeSpace` mechanics than anything else. |
| 4723 | const gpa = self.base.allocator; | 4507 | const gpa = self.base.allocator; |
| 4724 | const nphdrs = self.calcNumberOfSegments(); | 4508 | const nphdrs = self.calcNumberOfSegments(); |
| 4725 | var cuts = try gpa.alloc(struct { start: u16, len: u16 }, nphdrs); | 4509 | var covers = try gpa.alloc(struct { start: u16, len: u16 }, nphdrs); |
| 4726 | defer gpa.free(cuts); | 4510 | defer gpa.free(covers); |
| 4727 | | 4511 | |
| 4728 | var shndx = for (self.shdrs.items, 0..) |shdr, in| { | 4512 | var shndx = for (self.shdrs.items, 0..) |shdr, in| { |
| 4729 | if (shdr.sh_type == elf.SHT_NULL) continue; | 4513 | if (shdr.sh_type == elf.SHT_NULL) continue; |
| ... | @@ -4731,31 +4515,35 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { | ... | @@ -4731,31 +4515,35 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { |
| 4731 | break @as(u16, @intCast(in)); | 4515 | break @as(u16, @intCast(in)); |
| 4732 | } else @as(u16, @intCast(self.shdrs.items.len)); | 4516 | } else @as(u16, @intCast(self.shdrs.items.len)); |
| 4733 | | 4517 | |
| 4734 | for (cuts) |*cut| { | 4518 | for (covers) |*cover| { |
| 4735 | cut.* = .{ .start = shndx, .len = 0 }; | 4519 | cover.* = .{ .start = shndx, .len = 0 }; |
| 4736 | var flags = shdrToPhdrFlags(self.shdrs.items[shndx].sh_flags); | 4520 | var flags = shdrToPhdrFlags(self.shdrs.items[shndx].sh_flags); |
| 4737 | | 4521 | |
| 4738 | while (shndx < self.shdrs.items.len) : (shndx += 1) { | 4522 | while (shndx < self.shdrs.items.len) : (shndx += 1) { |
| 4739 | const shdr = &self.shdrs.items[shndx]; | 4523 | const shdr = self.shdrs.items[shndx]; |
| 4740 | if (!shdrIsAlloc(shdr)) break; | 4524 | if (shdr.sh_flags & elf.SHF_ALLOC == 0) break; |
| 4741 | if (shdrToPhdrFlags(shdr.sh_flags) != flags) { | 4525 | if (shdrToPhdrFlags(shdr.sh_flags) != flags) { |
| 4742 | cut.len = shndx - cut.start; | 4526 | cover.len = shndx - cover.start; |
| 4743 | break; | 4527 | break; |
| 4744 | } | 4528 | } |
| 4745 | } | 4529 | } |
| 4746 | } | 4530 | } |
| 4747 | cuts[nphdrs - 1].len = shndx - cuts[nphdrs - 1].start; | 4531 | covers[nphdrs - 1].len = shndx - covers[nphdrs - 1].start; |
| 4748 | | 4532 | |
| 4749 | // Allocate in segments | 4533 | // Now we can proceed with allocating the sections in virtual memory. |
| | 4534 | // As the base address we take the end address of the PHDR table. |
| | 4535 | // When allocating we first find the largest required alignment |
| | 4536 | // of any section that is contained in a cover and use it to align |
| | 4537 | // the start address of the segement (and first section). |
| 4750 | const phdr_table = &self.phdrs.items[self.phdr_table_load_index.?]; | 4538 | const phdr_table = &self.phdrs.items[self.phdr_table_load_index.?]; |
| 4751 | var addr = phdr_table.p_vaddr + phdr_table.p_memsz; | 4539 | var addr = phdr_table.p_vaddr + phdr_table.p_memsz; |
| 4752 | | 4540 | |
| 4753 | for (cuts) |cut| { | 4541 | for (covers) |cover| { |
| 4754 | const slice = self.shdrs.items[cut.start..][0..cut.len]; | 4542 | const slice = self.shdrs.items[cover.start..][0..cover.len]; |
| 4755 | | 4543 | |
| 4756 | var @"align": u64 = self.page_size; | 4544 | var @"align": u64 = self.page_size; |
| 4757 | for (slice) |*shdr| { | 4545 | for (slice) |shdr| { |
| 4758 | if (shdrIsTbss(shdr)) continue; | 4546 | if (shdr.sh_type == elf.SHT_NOBITS and shdr.sh_flags & elf.SHF_TLS != 0) continue; |
| 4759 | @"align" = @max(@"align", shdr.sh_addralign); | 4547 | @"align" = @max(@"align", shdr.sh_addralign); |
| 4760 | } | 4548 | } |
| 4761 | | 4549 | |
| ... | @@ -4764,9 +4552,9 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { | ... | @@ -4764,9 +4552,9 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { |
| 4764 | var memsz: u64 = 0; | 4552 | var memsz: u64 = 0; |
| 4765 | var filesz: u64 = 0; | 4553 | var filesz: u64 = 0; |
| 4766 | var i: usize = 0; | 4554 | var i: usize = 0; |
| 4767 | while (i < cut.len) : (i += 1) { | 4555 | while (i < cover.len) : (i += 1) { |
| 4768 | const shdr = &slice[i]; | 4556 | const shdr = &slice[i]; |
| 4769 | if (shdrIsTbss(shdr)) { | 4557 | if (shdr.sh_type == elf.SHT_NOBITS and shdr.sh_flags & elf.SHF_TLS != 0) { |
| 4770 | // .tbss is a little special as it's used only by the loader meaning it doesn't | 4558 | // .tbss is a little special as it's used only by the loader meaning it doesn't |
| 4771 | // need to be actually mmap'ed at runtime. We still need to correctly increment | 4559 | // need to be actually mmap'ed at runtime. We still need to correctly increment |
| 4772 | // the addresses of every TLS zerofill section tho. Thus, we hack it so that | 4560 | // the addresses of every TLS zerofill section tho. Thus, we hack it so that |
| ... | @@ -4780,16 +4568,19 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { | ... | @@ -4780,16 +4568,19 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { |
| 4780 | // .data 0x10 | 4568 | // .data 0x10 |
| 4781 | // ... | 4569 | // ... |
| 4782 | var tbss_addr = addr; | 4570 | var tbss_addr = addr; |
| 4783 | while (i < cut.len and shdrIsTbss(&slice[i])) : (i += 1) { | 4571 | while (i < cover.len and |
| | 4572 | slice[i].sh_type == elf.SHT_NOBITS and |
| | 4573 | slice[i].sh_flags & elf.SHF_TLS != 0) : (i += 1) |
| | 4574 | { |
| 4784 | const tbss_shdr = &slice[i]; | 4575 | const tbss_shdr = &slice[i]; |
| 4785 | tbss_addr = alignment.@"align"(cut.start + i, tbss_shdr.sh_addralign, tbss_addr); | 4576 | tbss_addr = alignment.@"align"(cover.start + i, tbss_shdr.sh_addralign, tbss_addr); |
| 4786 | tbss_shdr.sh_addr = tbss_addr; | 4577 | tbss_shdr.sh_addr = tbss_addr; |
| 4787 | tbss_addr += tbss_shdr.sh_size; | 4578 | tbss_addr += tbss_shdr.sh_size; |
| 4788 | } | 4579 | } |
| 4789 | i -= 1; | 4580 | i -= 1; |
| 4790 | continue; | 4581 | continue; |
| 4791 | } | 4582 | } |
| 4792 | const next = alignment.@"align"(cut.start + i, shdr.sh_addralign, addr); | 4583 | const next = alignment.@"align"(cover.start + i, shdr.sh_addralign, addr); |
| 4793 | const padding = next - addr; | 4584 | const padding = next - addr; |
| 4794 | addr = next; | 4585 | addr = next; |
| 4795 | shdr.sh_addr = addr; | 4586 | shdr.sh_addr = addr; |
| ... | @@ -4812,18 +4603,19 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { | ... | @@ -4812,18 +4603,19 @@ fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { |
| 4812 | }); | 4603 | }); |
| 4813 | | 4604 | |
| 4814 | for (slice, 0..) |*shdr, ii| { | 4605 | for (slice, 0..) |*shdr, ii| { |
| 4815 | if (shdrIsZerofill(shdr)) continue; | 4606 | if (shdr.sh_type == elf.SHT_NOBITS) continue; |
| 4816 | off = alignment.@"align"(cut.start + ii, shdr.sh_addralign, off); | 4607 | off = alignment.@"align"(cover.start + ii, shdr.sh_addralign, off); |
| 4817 | // off = mem.alignForward(u64, off, shdr.sh_addralign); | 4608 | // off = mem.alignForward(u64, off, shdr.sh_addralign); |
| 4818 | shdr.sh_offset = off; | 4609 | shdr.sh_offset = off; |
| 4819 | off += shdr.sh_size; | 4610 | off += shdr.sh_size; |
| 4820 | try self.phdr_to_shdr_table.putNoClobber(gpa, @intCast(ii + cut.start), phndx); | 4611 | try self.phdr_to_shdr_table.putNoClobber(gpa, @intCast(ii + cover.start), phndx); |
| 4821 | } | 4612 | } |
| 4822 | | 4613 | |
| 4823 | addr += self.page_size; | 4614 | addr += self.page_size; |
| 4824 | } | 4615 | } |
| 4825 | } | 4616 | } |
| 4826 | | 4617 | |
| | 4618 | /// Allocates non-alloc sections (debug info, symtabs, etc.). |
| 4827 | fn allocateNonAllocSections(self: *Elf) void { | 4619 | fn allocateNonAllocSections(self: *Elf) void { |
| 4828 | for (self.shdrs.items) |*shdr| { | 4620 | for (self.shdrs.items) |*shdr| { |
| 4829 | if (shdr.sh_type == elf.SHT_NULL) continue; | 4621 | if (shdr.sh_type == elf.SHT_NULL) continue; |
| ... | @@ -4832,8 +4624,8 @@ fn allocateNonAllocSections(self: *Elf) void { | ... | @@ -4832,8 +4624,8 @@ fn allocateNonAllocSections(self: *Elf) void { |
| 4832 | if (needed_size > self.allocatedSize(shdr.sh_offset)) { | 4624 | if (needed_size > self.allocatedSize(shdr.sh_offset)) { |
| 4833 | shdr.sh_size = 0; | 4625 | shdr.sh_size = 0; |
| 4834 | shdr.sh_offset = self.findFreeSpace(needed_size, shdr.sh_addralign); | 4626 | shdr.sh_offset = self.findFreeSpace(needed_size, shdr.sh_addralign); |
| 4835 | shdr.sh_size = needed_size; | | |
| 4836 | } | 4627 | } |
| | 4628 | shdr.sh_size = needed_size; |
| 4837 | } | 4629 | } |
| 4838 | } | 4630 | } |
| 4839 | | 4631 | |
| ... | @@ -4854,7 +4646,7 @@ fn allocateSpecialPhdrs(self: *Elf) void { | ... | @@ -4854,7 +4646,7 @@ fn allocateSpecialPhdrs(self: *Elf) void { |
| 4854 | } | 4646 | } |
| 4855 | } | 4647 | } |
| 4856 | | 4648 | |
| 4857 | // Allocate TLS phdr | 4649 | // Set the TLS segment boundaries. |
| 4858 | // We assume TLS sections are laid out contiguously and that there is | 4650 | // We assume TLS sections are laid out contiguously and that there is |
| 4859 | // a single TLS segment. | 4651 | // a single TLS segment. |
| 4860 | if (self.phdr_tls_index) |index| { | 4652 | if (self.phdr_tls_index) |index| { |
| ... | @@ -4871,14 +4663,21 @@ fn allocateSpecialPhdrs(self: *Elf) void { | ... | @@ -4871,14 +4663,21 @@ fn allocateSpecialPhdrs(self: *Elf) void { |
| 4871 | phdr.p_vaddr = shdr.sh_addr; | 4663 | phdr.p_vaddr = shdr.sh_addr; |
| 4872 | phdr.p_paddr = shdr.sh_addr; | 4664 | phdr.p_paddr = shdr.sh_addr; |
| 4873 | phdr.p_align = shdr.sh_addralign; | 4665 | phdr.p_align = shdr.sh_addralign; |
| 4874 | self.addShdrToPhdr(shndx, index); | | |
| 4875 | shndx += 1; | 4666 | shndx += 1; |
| | 4667 | phdr.p_align = @max(phdr.p_align, shdr.sh_addralign); |
| | 4668 | if (shdr.sh_type != elf.SHT_NOBITS) { |
| | 4669 | phdr.p_filesz = shdr.sh_offset + shdr.sh_size - phdr.p_offset; |
| | 4670 | } |
| | 4671 | phdr.p_memsz = shdr.sh_addr + shdr.sh_size - phdr.p_vaddr; |
| 4876 | | 4672 | |
| 4877 | while (shndx < slice.len) : (shndx += 1) { | 4673 | while (shndx < slice.len) : (shndx += 1) { |
| 4878 | const next = slice[shndx]; | 4674 | const next = slice[shndx]; |
| 4879 | // if (next.sh_flags & elf.SHF_TLS == 0) continue :outer; // TODO uncomment if we permit more TLS segments | | |
| 4880 | if (next.sh_flags & elf.SHF_TLS == 0) break; | 4675 | if (next.sh_flags & elf.SHF_TLS == 0) break; |
| 4881 | self.addShdrToPhdr(shndx, index); | 4676 | phdr.p_align = @max(phdr.p_align, next.sh_addralign); |
| | 4677 | if (next.sh_type != elf.SHT_NOBITS) { |
| | 4678 | phdr.p_filesz = next.sh_offset + next.sh_size - phdr.p_offset; |
| | 4679 | } |
| | 4680 | phdr.p_memsz = next.sh_addr + next.sh_size - phdr.p_vaddr; |
| 4882 | } | 4681 | } |
| 4883 | } | 4682 | } |
| 4884 | } | 4683 | } |
Jakub Konka <kubkon@jakubkonka.com> 2023-10-13 11:50:01+02:00
src