authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-08-02 00:03:31+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-08-03 21:19:41+02:00
log1e710396d4489ce11e70820a95c33ec7463b1c12
tree9e0a75de3b1ab878eb8214f9582ed689bac6ffd3
parent7bba3d330ad90026e8b79cae9940a1e878628119

macho: fix linking in incremental context

Fix incorrect writing of symtab and strtab in dSYM bundle in incremental context. Fix incorrectly navigating unnamed consts (freeing) in incremental context. This is currently hard-coded to require all consts to land in `__TEXT,__const`, which is wrong and needs a rewrite.

2 files changed, 69 insertions(+), 50 deletions(-)

src/link/MachO.zig+29-28
...@@ -511,15 +511,14 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No...@@ -511,15 +511,14 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
511511
512 try self.createMhExecuteHeaderSymbol();512 try self.createMhExecuteHeaderSymbol();
513 try self.resolveDyldStubBinder();513 try self.resolveDyldStubBinder();
514 try self.createDyldPrivateAtom();
515 try self.createStubHelperPreambleAtom();
514 try self.resolveSymbolsInDylibs();516 try self.resolveSymbolsInDylibs();
515517
516 if (self.unresolved.count() > 0) {518 if (self.unresolved.count() > 0) {
517 return error.UndefinedSymbolReference;519 return error.UndefinedSymbolReference;
518 }520 }
519521
520 try self.createDyldPrivateAtom();
521 try self.createStubHelperPreambleAtom();
522
523 try self.allocateSpecialSymbols();522 try self.allocateSpecialSymbols();
524523
525 if (build_options.enable_logging) {524 if (build_options.enable_logging) {
...@@ -589,7 +588,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No...@@ -589,7 +588,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
589 } else null;588 } else null;
590589
591 var headers_buf = std.ArrayList(u8).init(arena);590 var headers_buf = std.ArrayList(u8).init(arena);
592 try self.writeSegmentHeaders(0, self.segments.items.len, &ncmds, headers_buf.writer());591 try self.writeSegmentHeaders(&ncmds, headers_buf.writer());
593592
594 try self.base.file.?.pwriteAll(headers_buf.items, @sizeOf(macho.mach_header_64));593 try self.base.file.?.pwriteAll(headers_buf.items, @sizeOf(macho.mach_header_64));
595 try self.base.file.?.pwriteAll(lc_buffer.items, @sizeOf(macho.mach_header_64) + headers_buf.items.len);594 try self.base.file.?.pwriteAll(lc_buffer.items, @sizeOf(macho.mach_header_64) + headers_buf.items.len);
...@@ -1203,7 +1202,7 @@ fn linkOneShot(self: *MachO, comp: *Compilation, prog_node: *std.Progress.Node)...@@ -1203,7 +1202,7 @@ fn linkOneShot(self: *MachO, comp: *Compilation, prog_node: *std.Progress.Node)
1203 } else null;1202 } else null;
12041203
1205 var headers_buf = std.ArrayList(u8).init(arena);1204 var headers_buf = std.ArrayList(u8).init(arena);
1206 try self.writeSegmentHeaders(0, self.segments.items.len, &ncmds, headers_buf.writer());1205 try self.writeSegmentHeaders(&ncmds, headers_buf.writer());
12071206
1208 try self.base.file.?.pwriteAll(headers_buf.items, @sizeOf(macho.mach_header_64));1207 try self.base.file.?.pwriteAll(headers_buf.items, @sizeOf(macho.mach_header_64));
1209 try self.base.file.?.pwriteAll(lc_buffer.items, @sizeOf(macho.mach_header_64) + headers_buf.items.len);1208 try self.base.file.?.pwriteAll(lc_buffer.items, @sizeOf(macho.mach_header_64) + headers_buf.items.len);
...@@ -3863,7 +3862,9 @@ pub fn deleteExport(self: *MachO, exp: Export) void {...@@ -3863,7 +3862,9 @@ pub fn deleteExport(self: *MachO, exp: Export) void {
3863fn freeUnnamedConsts(self: *MachO, decl_index: Module.Decl.Index) void {3862fn freeUnnamedConsts(self: *MachO, decl_index: Module.Decl.Index) void {
3864 const unnamed_consts = self.unnamed_const_atoms.getPtr(decl_index) orelse return;3863 const unnamed_consts = self.unnamed_const_atoms.getPtr(decl_index) orelse return;
3865 for (unnamed_consts.items) |atom| {3864 for (unnamed_consts.items) |atom| {
3866 const sect_id = atom.getSymbol(self).n_sect;3865 // TODO
3866 // const sect_id = atom.getSymbol(self).n_sect;
3867 const sect_id = self.getSectionByName("__TEXT", "__const").?;
3867 self.freeAtom(atom, sect_id, true);3868 self.freeAtom(atom, sect_id, true);
3868 self.locals_free_list.append(self.base.allocator, atom.sym_index) catch {};3869 self.locals_free_list.append(self.base.allocator, atom.sym_index) catch {};
3869 self.locals.items[atom.sym_index].n_type = 0;3870 self.locals.items[atom.sym_index].n_type = 0;
...@@ -4402,8 +4403,6 @@ fn initSection(...@@ -4402,8 +4403,6 @@ fn initSection(
4402 const index = try self.insertSection(segment_id, .{4403 const index = try self.insertSection(segment_id, .{
4403 .sectname = makeStaticString(sectname),4404 .sectname = makeStaticString(sectname),
4404 .segname = seg.segname,4405 .segname = seg.segname,
4405 .size = if (self.mode == .incremental) @intCast(u32, size) else 0,
4406 .@"align" = alignment,
4407 .flags = opts.flags,4406 .flags = opts.flags,
4408 .reserved1 = opts.reserved1,4407 .reserved1 = opts.reserved1,
4409 .reserved2 = opts.reserved2,4408 .reserved2 = opts.reserved2,
...@@ -4413,6 +4412,9 @@ fn initSection(...@@ -4413,6 +4412,9 @@ fn initSection(
44134412
4414 if (self.mode == .incremental) {4413 if (self.mode == .incremental) {
4415 const header = &self.sections.items(.header)[index];4414 const header = &self.sections.items(.header)[index];
4415 header.size = size;
4416 header.@"align" = alignment;
4417
4416 const prev_end_off = if (index > 0) blk: {4418 const prev_end_off = if (index > 0) blk: {
4417 const prev_section = self.sections.get(index - 1);4419 const prev_section = self.sections.get(index - 1);
4418 if (prev_section.segment_index == segment_id) {4420 if (prev_section.segment_index == segment_id) {
...@@ -4421,15 +4423,25 @@ fn initSection(...@@ -4421,15 +4423,25 @@ fn initSection(
4421 } else break :blk seg.fileoff;4423 } else break :blk seg.fileoff;
4422 } else 0;4424 } else 0;
4423 const alignment_pow_2 = try math.powi(u32, 2, alignment);4425 const alignment_pow_2 = try math.powi(u32, 2, alignment);
4424 const padding: u64 = if (index == 0) try self.calcMinHeaderPad() else 0;4426 // TODO better prealloc for __text section
4427 // const padding: u64 = if (index == 0) try self.calcMinHeaderPad() else 0;
4428 const padding: u64 = if (index == 0) 0x1000 else 0;
4425 const off = mem.alignForwardGeneric(u64, padding + prev_end_off, alignment_pow_2);4429 const off = mem.alignForwardGeneric(u64, padding + prev_end_off, alignment_pow_2);
4426 log.debug("allocating {s},{s} section at 0x{x}", .{ header.segName(), header.sectName(), off });
4427
4428 header.addr = seg.vmaddr + off - seg.fileoff;
44294430
4430 if (!header.isZerofill()) {4431 if (!header.isZerofill()) {
4431 header.offset = @intCast(u32, off);4432 header.offset = @intCast(u32, off);
4432 }4433 }
4434 header.addr = seg.vmaddr + off - seg.fileoff;
4435
4436 // TODO this will break if we are inserting section that is not the last section
4437 // in a segment.
4438 const max_size = self.allocatedSize(segment_id, off);
4439
4440 if (size > max_size) {
4441 try self.growSection(index, @intCast(u32, size));
4442 }
4443
4444 log.debug("allocating {s},{s} section at 0x{x}", .{ header.segName(), header.sectName(), off });
44334445
4434 self.updateSectionOrdinals(index + 1);4446 self.updateSectionOrdinals(index + 1);
4435 }4447 }
...@@ -4494,7 +4506,7 @@ fn updateSectionOrdinals(self: *MachO, start: u8) void {...@@ -4494,7 +4506,7 @@ fn updateSectionOrdinals(self: *MachO, start: u8) void {
44944506
4495 const slice = self.sections.slice();4507 const slice = self.sections.slice();
4496 for (slice.items(.last_atom)[start..]) |last_atom| {4508 for (slice.items(.last_atom)[start..]) |last_atom| {
4497 var atom = last_atom.?;4509 var atom = last_atom orelse continue;
44984510
4499 while (true) {4511 while (true) {
4500 const sym = atom.getSymbolPtr(self);4512 const sym = atom.getSymbolPtr(self);
...@@ -4536,17 +4548,6 @@ fn shiftLocalsByOffset(self: *MachO, sect_id: u8, offset: i64) !void {...@@ -4536,17 +4548,6 @@ fn shiftLocalsByOffset(self: *MachO, sect_id: u8, offset: i64) !void {
4536 }4548 }
4537}4549}
45384550
4539fn findFreeSpace(self: MachO, segment_id: u8, alignment: u64, start: ?u64) u64 {
4540 const seg = self.segments.items[segment_id];
4541 const indexes = self.getSectionIndexes(segment_id);
4542 if (indexes.end - indexes.start == 0) {
4543 return if (start) |v| v else seg.fileoff;
4544 }
4545 const last_sect = self.sections.items(.header)[indexes.end - 1];
4546 const final_off = last_sect.offset + padToIdeal(last_sect.size);
4547 return mem.alignForwardGeneric(u64, final_off, alignment);
4548}
4549
4550fn growSegment(self: *MachO, segment_index: u8, new_size: u64) !void {4551fn growSegment(self: *MachO, segment_index: u8, new_size: u64) !void {
4551 const segment = &self.segments.items[segment_index];4552 const segment = &self.segments.items[segment_index];
4552 const new_segment_size = mem.alignForwardGeneric(u64, new_size, self.page_size);4553 const new_segment_size = mem.alignForwardGeneric(u64, new_size, self.page_size);
...@@ -4885,14 +4886,14 @@ fn getSegmentAllocBase(self: MachO, indices: []const ?u8) struct { vmaddr: u64,...@@ -4885,14 +4886,14 @@ fn getSegmentAllocBase(self: MachO, indices: []const ?u8) struct { vmaddr: u64,
4885 return .{ .vmaddr = 0, .fileoff = 0 };4886 return .{ .vmaddr = 0, .fileoff = 0 };
4886}4887}
48874888
4888pub fn writeSegmentHeaders(self: *MachO, start: usize, end: usize, ncmds: *u32, writer: anytype) !void {4889fn writeSegmentHeaders(self: *MachO, ncmds: *u32, writer: anytype) !void {
4889 for (self.segments.items[start..end]) |seg, i| {4890 for (self.segments.items) |seg, i| {
4890 if (seg.nsects == 0 and4891 if (seg.nsects == 0 and
4891 (mem.eql(u8, seg.segName(), "__DATA_CONST") or4892 (mem.eql(u8, seg.segName(), "__DATA_CONST") or
4892 mem.eql(u8, seg.segName(), "__DATA"))) continue;4893 mem.eql(u8, seg.segName(), "__DATA"))) continue;
4893 try writer.writeStruct(seg);4894 try writer.writeStruct(seg);
48944895
4895 const indexes = self.getSectionIndexes(@intCast(u8, start + i));4896 const indexes = self.getSectionIndexes(@intCast(u8, i));
4896 for (self.sections.items(.header)[indexes.start..indexes.end]) |header| {4897 for (self.sections.items(.header)[indexes.start..indexes.end]) |header| {
4897 try writer.writeStruct(header);4898 try writer.writeStruct(header);
4898 }4899 }
...@@ -5718,7 +5719,7 @@ pub fn getSectionByName(self: MachO, segname: []const u8, sectname: []const u8)...@@ -5718,7 +5719,7 @@ pub fn getSectionByName(self: MachO, segname: []const u8, sectname: []const u8)
5718 } else return null;5719 } else return null;
5719}5720}
57205721
5721fn getSectionIndexes(self: MachO, segment_index: u8) struct { start: u8, end: u8 } {5722pub fn getSectionIndexes(self: MachO, segment_index: u8) struct { start: u8, end: u8 } {
5722 var start: u8 = 0;5723 var start: u8 = 0;
5723 const nsects = for (self.segments.items) |seg, i| {5724 const nsects = for (self.segments.items) |seg, i| {
5724 if (i == segment_index) break @intCast(u8, seg.nsects);5725 if (i == segment_index) break @intCast(u8, seg.nsects);
src/link/MachO/DebugSymbols.zig+40-22
...@@ -63,6 +63,10 @@ pub const Reloc = struct {...@@ -63,6 +63,10 @@ pub const Reloc = struct {
63pub fn populateMissingMetadata(self: *DebugSymbols, allocator: Allocator) !void {63pub fn populateMissingMetadata(self: *DebugSymbols, allocator: Allocator) !void {
64 if (self.linkedit_segment_cmd_index == null) {64 if (self.linkedit_segment_cmd_index == null) {
65 self.linkedit_segment_cmd_index = @intCast(u8, self.segments.items.len);65 self.linkedit_segment_cmd_index = @intCast(u8, self.segments.items.len);
66 log.debug("found __LINKEDIT segment free space 0x{x} to 0x{x}", .{
67 self.base.page_size,
68 self.base.page_size * 2,
69 });
66 // TODO this needs reworking70 // TODO this needs reworking
67 try self.segments.append(allocator, .{71 try self.segments.append(allocator, .{
68 .segname = makeStaticString("__LINKEDIT"),72 .segname = makeStaticString("__LINKEDIT"),
...@@ -79,7 +83,7 @@ pub fn populateMissingMetadata(self: *DebugSymbols, allocator: Allocator) !void...@@ -79,7 +83,7 @@ pub fn populateMissingMetadata(self: *DebugSymbols, allocator: Allocator) !void
79 if (self.dwarf_segment_cmd_index == null) {83 if (self.dwarf_segment_cmd_index == null) {
80 self.dwarf_segment_cmd_index = @intCast(u8, self.segments.items.len);84 self.dwarf_segment_cmd_index = @intCast(u8, self.segments.items.len);
8185
82 const linkedit = self.segments.items[self.base.linkedit_segment_cmd_index.?];86 const linkedit = self.segments.items[self.linkedit_segment_cmd_index.?];
83 const ideal_size: u16 = 200 + 128 + 160 + 250;87 const ideal_size: u16 = 200 + 128 + 160 + 250;
84 const needed_size = mem.alignForwardGeneric(u64, padToIdeal(ideal_size), self.base.page_size);88 const needed_size = mem.alignForwardGeneric(u64, padToIdeal(ideal_size), self.base.page_size);
85 const fileoff = linkedit.fileoff + linkedit.filesize;89 const fileoff = linkedit.fileoff + linkedit.filesize;
...@@ -290,20 +294,7 @@ pub fn flushModule(self: *DebugSymbols, allocator: Allocator, options: link.Opti...@@ -290,20 +294,7 @@ pub fn flushModule(self: *DebugSymbols, allocator: Allocator, options: link.Opti
290294
291 var headers_buf = std.ArrayList(u8).init(allocator);295 var headers_buf = std.ArrayList(u8).init(allocator);
292 defer headers_buf.deinit();296 defer headers_buf.deinit();
293 try self.base.writeSegmentHeaders(297 try self.writeSegmentHeaders(&ncmds, headers_buf.writer());
294 0,
295 self.base.linkedit_segment_cmd_index.?,
296 &ncmds,
297 headers_buf.writer(),
298 );
299
300 for (self.segments.items) |seg| {
301 try headers_buf.writer().writeStruct(seg);
302 ncmds += 2;
303 }
304 for (self.sections.items) |header| {
305 try headers_buf.writer().writeStruct(header);
306 }
307298
308 try self.file.pwriteAll(headers_buf.items, @sizeOf(macho.mach_header_64));299 try self.file.pwriteAll(headers_buf.items, @sizeOf(macho.mach_header_64));
309 try self.file.pwriteAll(lc_buffer.items, @sizeOf(macho.mach_header_64) + headers_buf.items.len);300 try self.file.pwriteAll(lc_buffer.items, @sizeOf(macho.mach_header_64) + headers_buf.items.len);
...@@ -349,6 +340,7 @@ fn updateDwarfSegment(self: *DebugSymbols) void {...@@ -349,6 +340,7 @@ fn updateDwarfSegment(self: *DebugSymbols) void {
349340
350 var max_offset: u64 = 0;341 var max_offset: u64 = 0;
351 for (self.sections.items) |*sect| {342 for (self.sections.items) |*sect| {
343 sect.addr += diff;
352 log.debug(" {s},{s} - 0x{x}-0x{x} - 0x{x}-0x{x}", .{344 log.debug(" {s},{s} - 0x{x}-0x{x} - 0x{x}-0x{x}", .{
353 sect.segName(),345 sect.segName(),
354 sect.sectName(),346 sect.sectName(),
...@@ -360,7 +352,6 @@ fn updateDwarfSegment(self: *DebugSymbols) void {...@@ -360,7 +352,6 @@ fn updateDwarfSegment(self: *DebugSymbols) void {
360 if (sect.offset + sect.size > max_offset) {352 if (sect.offset + sect.size > max_offset) {
361 max_offset = sect.offset + sect.size;353 max_offset = sect.offset + sect.size;
362 }354 }
363 sect.addr += diff;
364 }355 }
365356
366 const file_size = max_offset - dwarf_segment.fileoff;357 const file_size = max_offset - dwarf_segment.fileoff;
...@@ -372,6 +363,37 @@ fn updateDwarfSegment(self: *DebugSymbols) void {...@@ -372,6 +363,37 @@ fn updateDwarfSegment(self: *DebugSymbols) void {
372 }363 }
373}364}
374365
366fn writeSegmentHeaders(self: *DebugSymbols, ncmds: *u32, writer: anytype) !void {
367 // Write segment/section headers from the binary file first.
368 const end = self.base.linkedit_segment_cmd_index.?;
369 for (self.base.segments.items[0..end]) |seg, i| {
370 if (seg.nsects == 0 and
371 (mem.eql(u8, seg.segName(), "__DATA_CONST") or
372 mem.eql(u8, seg.segName(), "__DATA"))) continue;
373 var out_seg = seg;
374 out_seg.fileoff = 0;
375 out_seg.filesize = 0;
376 try writer.writeStruct(out_seg);
377
378 const indexes = self.base.getSectionIndexes(@intCast(u8, i));
379 for (self.base.sections.items(.header)[indexes.start..indexes.end]) |header| {
380 var out_header = header;
381 out_header.offset = 0;
382 try writer.writeStruct(out_header);
383 }
384
385 ncmds.* += 1;
386 }
387 // Next, commit DSYM's __LINKEDIT and __DWARF segments headers.
388 for (self.segments.items) |seg| {
389 try writer.writeStruct(seg);
390 ncmds.* += 1;
391 }
392 for (self.sections.items) |header| {
393 try writer.writeStruct(header);
394 }
395}
396
375fn writeHeader(self: *DebugSymbols, ncmds: u32, sizeofcmds: u32) !void {397fn writeHeader(self: *DebugSymbols, ncmds: u32, sizeofcmds: u32) !void {
376 var header: macho.mach_header_64 = .{};398 var header: macho.mach_header_64 = .{};
377 header.filetype = macho.MH_DSYM;399 header.filetype = macho.MH_DSYM;
...@@ -469,11 +491,7 @@ fn writeSymtab(self: *DebugSymbols, lc: *macho.symtab_command) !void {...@@ -469,11 +491,7 @@ fn writeSymtab(self: *DebugSymbols, lc: *macho.symtab_command) !void {
469 const nsyms = nlocals + nexports;491 const nsyms = nlocals + nexports;
470492
471 const seg = &self.segments.items[self.linkedit_segment_cmd_index.?];493 const seg = &self.segments.items[self.linkedit_segment_cmd_index.?];
472 const offset = mem.alignForwardGeneric(494 const offset = mem.alignForwardGeneric(u64, seg.fileoff, @alignOf(macho.nlist_64));
473 u64,
474 seg.fileoff + seg.filesize,
475 @alignOf(macho.nlist_64),
476 );
477 const needed_size = nsyms * @sizeOf(macho.nlist_64);495 const needed_size = nsyms * @sizeOf(macho.nlist_64);
478496
479 if (needed_size > seg.filesize) {497 if (needed_size > seg.filesize) {
...@@ -535,7 +553,7 @@ fn writeStrtab(self: *DebugSymbols, lc: *macho.symtab_command) !void {...@@ -535,7 +553,7 @@ fn writeStrtab(self: *DebugSymbols, lc: *macho.symtab_command) !void {
535 const needed_size = mem.alignForwardGeneric(u64, self.strtab.buffer.items.len, @alignOf(u64));553 const needed_size = mem.alignForwardGeneric(u64, self.strtab.buffer.items.len, @alignOf(u64));
536 lc.strsize = @intCast(u32, needed_size);554 lc.strsize = @intCast(u32, needed_size);
537555
538 if (offset + needed_size > seg.filesize) {556 if (symtab_size + needed_size > seg.filesize) {
539 const aligned_size = mem.alignForwardGeneric(u64, offset + needed_size, self.base.page_size);557 const aligned_size = mem.alignForwardGeneric(u64, offset + needed_size, self.base.page_size);
540 const diff = @intCast(u32, aligned_size - seg.filesize);558 const diff = @intCast(u32, aligned_size - seg.filesize);
541 const dwarf_seg = &self.segments.items[self.dwarf_segment_cmd_index.?];559 const dwarf_seg = &self.segments.items[self.dwarf_segment_cmd_index.?];