authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2020-10-02 21:24:38+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2020-10-04 15:31:47+02:00
logc4054f8e0a6a7dea8570507d2a38b38a392ddd0d
treed1ee1b48a87ab9aaded253a69c9201d1dfd44153
parent737a8bf2041158b3036bda9130f9ce3f6c1ad582

Refactor flushing of MachO exe

This commit refactors free space analysis (and VM analysis) somewhat. This is still far from perfect, but at least it's not using any hardcoded values. This commit also reorganizes different flushing bits here and there, so that bit common to Exe and Obj are put in the common path, while Exe-specific are behind an appropriate switch/if statement.

1 files changed, 139 insertions(+), 114 deletions(-)

src/link/MachO.zig+139-114
......@@ -247,14 +247,6 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void {
247247 const tracy = trace(@src());
248248 defer tracy.end();
249249
250 switch (self.base.options.output_mode) {
251 .Exe => {},
252 .Obj => return error.TODOImplementWritingObjFiles,
253 .Lib => return error.TODOImplementWritingLibFiles,
254 }
255
256 try self.writeExportTrie();
257
258250 // Unfortunately these have to be buffered and done at the end because MachO does not allow
259251 // mixing local, global and undefined symbols within a symbol table.
260252 try self.writeAllGlobalSymbols();
......@@ -262,79 +254,76 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void {
262254
263255 try self.writeStringTable();
264256
257 switch (self.base.options.output_mode) {
258 .Exe => {
259 if (self.entry_addr) |addr| {
260 // Write export trie.
261 try self.writeExportTrie();
262
263 // Update LC_MAIN with entry offset
264 const text_segment = self.load_commands.items[self.text_segment_cmd_index.?].Segment;
265 const main_cmd = &self.load_commands.items[self.main_cmd_index.?].EntryPoint;
266 main_cmd.entryoff = addr - text_segment.vmaddr;
267 }
268
269 {
270 // Update dynamic symbol table.
271 const nlocals = @intCast(u32, self.local_symbols.items.len);
272 const nglobals = @intCast(u32, self.global_symbols.items.len);
273 const nundefs = @intCast(u32, self.undef_symbols.items.len);
274 const dysymtab = &self.load_commands.items[self.dysymtab_cmd_index.?].Dysymtab;
275 dysymtab.nlocalsym = nlocals;
276 dysymtab.iextdefsym = nlocals;
277 dysymtab.nextdefsym = nglobals;
278 dysymtab.iundefsym = nlocals + nglobals;
279 dysymtab.nundefsym = nundefs;
280 }
281 {
282 // Write path to dyld loader.
283 var off: usize = @sizeOf(macho.mach_header_64);
284 for (self.load_commands.items) |cmd| {
285 if (cmd == .Dylinker) break;
286 off += cmd.cmdsize();
287 }
288 const cmd = &self.load_commands.items[self.dylinker_cmd_index.?].Dylinker;
289 off += cmd.name;
290 const padding = cmd.cmdsize - @sizeOf(macho.dylinker_command);
291 log.debug("writing LC_LOAD_DYLINKER padding of size {} at 0x{x}\n", .{ padding, off });
292 try self.addPadding(padding, off);
293 log.debug("writing LC_LOAD_DYLINKER path to dyld at 0x{x}\n", .{off});
294 try self.base.file.?.pwriteAll(mem.spanZ(DEFAULT_DYLD_PATH), off);
295 }
296 {
297 // Write path to libSystem.
298 var off: usize = @sizeOf(macho.mach_header_64);
299 for (self.load_commands.items) |cmd| {
300 if (cmd == .Dylib) break;
301 off += cmd.cmdsize();
302 }
303 const cmd = &self.load_commands.items[self.libsystem_cmd_index.?].Dylib;
304 off += cmd.dylib.name;
305 const padding = cmd.cmdsize - @sizeOf(macho.dylib_command);
306 log.debug("writing LC_LOAD_DYLIB padding of size {} at 0x{x}\n", .{ padding, off });
307 try self.addPadding(padding, off);
308 log.debug("writing LC_LOAD_DYLIB path to libSystem at 0x{x}\n", .{off});
309 try self.base.file.?.pwriteAll(mem.spanZ(LIB_SYSTEM_PATH), off);
310 }
311 },
312 .Obj => {},
313 .Lib => return error.TODOImplementWritingLibFiles,
314 }
315
316 if (self.cmd_table_dirty) try self.writeCmdHeaders();
317
265318 {
266 // update Symtab and Dysymtab commands with symbol counts
267 // TODO this could probably be done incrementally
319 // Update symbol table.
268320 const nlocals = @intCast(u32, self.local_symbols.items.len);
269321 const nglobals = @intCast(u32, self.global_symbols.items.len);
270322 const nundefs = @intCast(u32, self.undef_symbols.items.len);
271
272323 const symtab = &self.load_commands.items[self.symtab_cmd_index.?].Symtab;
273324 symtab.nsyms = nlocals + nglobals + nundefs;
274
275 const dysymtab = &self.load_commands.items[self.dysymtab_cmd_index.?].Dysymtab;
276 dysymtab.nlocalsym = nlocals;
277 dysymtab.iextdefsym = nlocals;
278 dysymtab.nextdefsym = nglobals;
279 dysymtab.iundefsym = nlocals + nglobals;
280 dysymtab.nundefsym = nundefs;
281 }
282 if (self.entry_addr) |addr| {
283 // update LC_MAIN with entry offset
284 const text_segment = self.load_commands.items[self.text_segment_cmd_index.?].Segment;
285 const main_cmd = &self.load_commands.items[self.main_cmd_index.?].EntryPoint;
286 main_cmd.entryoff = addr - text_segment.vmaddr;
287 }
288 {
289 var last_cmd_offset: usize = @sizeOf(macho.mach_header_64);
290 for (self.load_commands.items) |cmd| {
291 try cmd.write(&self.base.file.?, last_cmd_offset);
292 last_cmd_offset += cmd.cmdsize();
293 }
294 }
295 {
296 // write __text section
297 const off = @sizeOf(macho.mach_header_64) + @sizeOf(macho.segment_command_64) * 2;
298 log.debug("writing text section {} at 0x{x}\n", .{ self.sections.items[0..1], off });
299 try self.base.file.?.pwriteAll(mem.sliceAsBytes(self.sections.items[0..1]), off);
300 }
301 {
302 // write __got section
303 const text = &self.load_commands.items[self.text_segment_cmd_index.?];
304 const off = @sizeOf(macho.mach_header_64) + @sizeOf(macho.segment_command_64) * 2 + text.cmdsize();
305 log.debug("writing got section {} at 0x{x}\n", .{ self.sections.items[1..2], off });
306 try self.base.file.?.pwriteAll(mem.sliceAsBytes(self.sections.items[1..2]), off);
307 }
308 {
309 // write path to dyld
310 var off: usize = @sizeOf(macho.mach_header_64);
311 for (self.load_commands.items) |cmd| {
312 if (cmd == .Dylinker) break;
313 off += cmd.cmdsize();
314 }
315 const cmd = &self.load_commands.items[self.dylinker_cmd_index.?].Dylinker;
316 off += cmd.name;
317 const padding = cmd.cmdsize - @sizeOf(macho.dylinker_command);
318 log.debug("writing LC_LOAD_DYLINKER padding of size {} at 0x{x}\n", .{ padding, off });
319 try self.addPadding(padding, off);
320 log.debug("writing LC_LOAD_DYLINKER path to dyld at 0x{x}\n", .{off});
321 try self.base.file.?.pwriteAll(mem.spanZ(DEFAULT_DYLD_PATH), off);
322 }
323 {
324 // write path to libSystem
325 var off: usize = @sizeOf(macho.mach_header_64);
326 for (self.load_commands.items) |cmd| {
327 if (cmd == .Dylib) break;
328 off += cmd.cmdsize();
329 }
330 const cmd = &self.load_commands.items[self.libsystem_cmd_index.?].Dylib;
331 off += cmd.dylib.name;
332 const padding = cmd.cmdsize - @sizeOf(macho.dylib_command);
333 log.debug("writing LC_LOAD_DYLIB padding of size {} at 0x{x}\n", .{ padding, off });
334 try self.addPadding(padding, off);
335 log.debug("writing LC_LOAD_DYLIB path to libSystem at 0x{x}\n", .{off});
336 try self.base.file.?.pwriteAll(mem.spanZ(LIB_SYSTEM_PATH), off);
337325 }
326
338327 if (self.entry_addr == null and self.base.options.output_mode == .Exe) {
339328 log.debug("flushing. no_entry_point_found = true\n", .{});
340329 self.error_flags.no_entry_point_found = true;
......@@ -775,7 +764,6 @@ pub fn updateDecl(self: *MachO, module: *Module, decl: *Module.Decl) !void {
775764 return;
776765 },
777766 };
778 log.debug("generated code {}\n", .{code});
779767
780768 const required_alignment = typed_value.ty.abiAlignment(self.base.options.target);
781769 const symbol = &self.local_symbols.items[decl.link.macho.local_sym_index];
......@@ -784,7 +772,6 @@ pub fn updateDecl(self: *MachO, module: *Module, decl: *Module.Decl) !void {
784772 const name_str_index = try self.makeString(decl_name);
785773 const addr = try self.allocateTextBlock(&decl.link.macho, code.len, required_alignment);
786774 log.debug("allocated text block for {} at 0x{x}\n", .{ decl_name, addr });
787 log.debug("updated text section {}\n", .{self.sections.items[self.text_section_index.?]});
788775
789776 symbol.* = .{
790777 .n_strx = name_str_index,
......@@ -801,7 +788,6 @@ pub fn updateDecl(self: *MachO, module: *Module, decl: *Module.Decl) !void {
801788 const text_section = self.sections.items[self.text_section_index.?];
802789 const section_offset = symbol.n_value - text_section.addr;
803790 const file_offset = text_section.offset + section_offset;
804 log.debug("file_offset 0x{x}\n", .{file_offset});
805791
806792 try self.base.file.?.pwriteAll(code, file_offset);
807793
......@@ -889,6 +875,12 @@ pub fn getDeclVAddr(self: *MachO, decl: *const Module.Decl) u64 {
889875}
890876
891877pub fn populateMissingMetadata(self: *MachO) !void {
878 switch (self.base.options.output_mode) {
879 .Exe => {},
880 .Obj => return error.TODOImplementWritingObjFiles,
881 .Lib => return error.TODOImplementWritingLibFiles,
882 }
883
892884 if (self.pagezero_segment_cmd_index == null) {
893885 self.pagezero_segment_cmd_index = @intCast(u16, self.load_commands.items.len);
894886 try self.load_commands.append(self.base.allocator, .{
......@@ -934,7 +926,7 @@ pub fn populateMissingMetadata(self: *MachO) !void {
934926 text_segment.cmdsize += @sizeOf(macho.section_64);
935927 text_segment.nsects += 1;
936928
937 const file_size = self.base.options.program_code_size_hint;
929 const file_size = mem.alignForwardGeneric(u64, self.base.options.program_code_size_hint, 0x1000);
938930 const off = @intCast(u32, self.findFreeSpace(file_size, 0x1000)); // TODO maybe findFreeSpace should return u32 directly?
939931 const flags = macho.S_REGULAR | macho.S_ATTR_PURE_INSTRUCTIONS | macho.S_ATTR_SOME_INSTRUCTIONS;
940932
......@@ -946,7 +938,7 @@ pub fn populateMissingMetadata(self: *MachO) !void {
946938 .addr = text_segment.vmaddr + off,
947939 .size = file_size,
948940 .offset = off,
949 .@"align" = 12,
941 .@"align" = 12, // 2^12 = 4096
950942 .reloff = 0,
951943 .nreloc = 0,
952944 .flags = flags,
......@@ -955,11 +947,8 @@ pub fn populateMissingMetadata(self: *MachO) !void {
955947 .reserved3 = 0,
956948 });
957949
958 text_segment.vmsize = file_size + off;
950 text_segment.vmsize = file_size + off; // We add off here since __TEXT segment includes everything prior to __text section.
959951 text_segment.filesize = file_size + off;
960
961 log.debug("initial text section {}\n", .{self.sections.items[self.text_section_index.?]});
962 log.debug("updated text segment {}\n", .{text_segment});
963952 }
964953 if (self.data_segment_cmd_index == null) {
965954 self.data_segment_cmd_index = @intCast(u16, self.load_commands.items.len);
......@@ -970,7 +959,7 @@ pub fn populateMissingMetadata(self: *MachO) !void {
970959 .cmd = macho.LC_SEGMENT_64,
971960 .cmdsize = @sizeOf(macho.segment_command_64),
972961 .segname = makeStaticString("__DATA"),
973 .vmaddr = text_segment.vmaddr + text_segment.vmsize, // TODO this should be found when running findFreeSpace
962 .vmaddr = text_segment.vmaddr + text_segment.vmsize,
974963 .vmsize = 0,
975964 .fileoff = 0,
976965 .filesize = 0,
......@@ -999,7 +988,7 @@ pub fn populateMissingMetadata(self: *MachO) !void {
999988 .addr = data_segment.vmaddr,
1000989 .size = file_size,
1001990 .offset = off,
1002 .@"align" = 3,
991 .@"align" = 3, // 2^3 = 8
1003992 .reloff = 0,
1004993 .nreloc = 0,
1005994 .flags = macho.S_REGULAR,
......@@ -1012,9 +1001,6 @@ pub fn populateMissingMetadata(self: *MachO) !void {
10121001 data_segment.vmsize = segment_size;
10131002 data_segment.filesize = segment_size;
10141003 data_segment.fileoff = off;
1015
1016 log.debug("initial got section {}\n", .{self.sections.items[self.got_section_index.?]});
1017 log.debug("updated data segment {}\n", .{data_segment});
10181004 }
10191005 if (self.linkedit_segment_cmd_index == null) {
10201006 self.linkedit_segment_cmd_index = @intCast(u16, self.load_commands.items.len);
......@@ -1025,7 +1011,7 @@ pub fn populateMissingMetadata(self: *MachO) !void {
10251011 .cmd = macho.LC_SEGMENT_64,
10261012 .cmdsize = @sizeOf(macho.segment_command_64),
10271013 .segname = makeStaticString("__LINKEDIT"),
1028 .vmaddr = data_segment.vmaddr + data_segment.vmsize, // TODO this should be found when running findFreeSpace
1014 .vmaddr = data_segment.vmaddr + data_segment.vmsize,
10291015 .vmsize = 0,
10301016 .fileoff = 0,
10311017 .filesize = 0,
......@@ -1101,11 +1087,11 @@ pub fn populateMissingMetadata(self: *MachO) !void {
11011087 }
11021088 if (self.dylinker_cmd_index == null) {
11031089 self.dylinker_cmd_index = @intCast(u16, self.load_commands.items.len);
1104 const cmdsize = commandSize(@sizeOf(macho.dylinker_command) + mem.lenZ(DEFAULT_DYLD_PATH));
1090 const cmdsize = mem.alignForwardGeneric(u64, @sizeOf(macho.dylinker_command) + mem.lenZ(DEFAULT_DYLD_PATH), @sizeOf(u64));
11051091 try self.load_commands.append(self.base.allocator, .{
11061092 .Dylinker = .{
11071093 .cmd = macho.LC_LOAD_DYLINKER,
1108 .cmdsize = cmdsize,
1094 .cmdsize = @intCast(u32, cmdsize),
11091095 .name = @sizeOf(macho.dylinker_command),
11101096 },
11111097 });
......@@ -1113,7 +1099,7 @@ pub fn populateMissingMetadata(self: *MachO) !void {
11131099 }
11141100 if (self.libsystem_cmd_index == null) {
11151101 self.libsystem_cmd_index = @intCast(u16, self.load_commands.items.len);
1116 const cmdsize = commandSize(@sizeOf(macho.dylib_command) + mem.lenZ(LIB_SYSTEM_PATH));
1102 const cmdsize = mem.alignForwardGeneric(u64, @sizeOf(macho.dylib_command) + mem.lenZ(LIB_SYSTEM_PATH), @sizeOf(u64));
11171103 // TODO Find a way to work out runtime version from the OS version triple stored in std.Target.
11181104 // In the meantime, we're gonna hardcode to the minimum compatibility version of 1.0.0.
11191105 const min_version = 0x10000;
......@@ -1126,7 +1112,7 @@ pub fn populateMissingMetadata(self: *MachO) !void {
11261112 try self.load_commands.append(self.base.allocator, .{
11271113 .Dylib = .{
11281114 .cmd = macho.LC_LOAD_DYLIB,
1129 .cmdsize = cmdsize,
1115 .cmdsize = @intCast(u32, cmdsize),
11301116 .dylib = dylib,
11311117 },
11321118 });
......@@ -1156,10 +1142,8 @@ pub fn populateMissingMetadata(self: *MachO) !void {
11561142 dyld_info.export_size = @intCast(u32, file_size);
11571143
11581144 const segment_size = mem.alignForwardGeneric(u64, file_size, 0x1000);
1159 linkedit.vmsize = 4 * segment_size;
1145 linkedit.vmsize += segment_size;
11601146 linkedit.fileoff = off;
1161
1162 log.debug("updated linkedit segment {}\n", .{linkedit});
11631147 }
11641148 }
11651149 {
......@@ -1172,6 +1156,9 @@ pub fn populateMissingMetadata(self: *MachO) !void {
11721156 log.debug("found symbol table free space 0x{x} to 0x{x}\n", .{ off, off + file_size });
11731157 symtab.symoff = off;
11741158 symtab.nsyms = @intCast(u32, nsyms);
1159
1160 const segment_size = mem.alignForwardGeneric(u64, file_size, 0x1000);
1161 linkedit.vmsize += segment_size;
11751162 }
11761163 if (symtab.stroff == 0) {
11771164 try self.string_table.append(self.base.allocator, 0);
......@@ -1180,6 +1167,9 @@ pub fn populateMissingMetadata(self: *MachO) !void {
11801167 log.debug("found string table free space 0x{x} to 0x{x}\n", .{ off, off + file_size });
11811168 symtab.stroff = off;
11821169 symtab.strsize = file_size;
1170
1171 const segment_size = mem.alignForwardGeneric(u64, file_size, 0x1000);
1172 linkedit.vmsize += segment_size;
11831173 }
11841174 }
11851175 if (self.dyld_stub_binder_index == null) {
......@@ -1215,13 +1205,11 @@ fn allocateTextBlock(self: *MachO, text_block: *TextBlock, new_block_size: u64,
12151205 break :blk text_section.addr;
12161206 }
12171207 };
1218 log.debug("computed symbol address 0x{x}\n", .{addr});
12191208
12201209 const expand_text_section = block_placement == null or block_placement.?.next == null;
12211210 if (expand_text_section) {
12221211 const text_capacity = self.allocatedSize(text_section.offset);
12231212 const needed_size = (addr + new_block_size) - text_section.addr;
1224 log.debug("text capacity 0x{x}, needed size 0x{x}\n", .{ text_capacity, needed_size });
12251213 assert(needed_size <= text_capacity); // TODO handle growth
12261214
12271215 self.last_text_block = text_block;
......@@ -1276,18 +1264,6 @@ fn updateString(self: *MachO, old_str_off: u32, new_name: []const u8) !u32 {
12761264 return self.makeString(new_name);
12771265}
12781266
1279fn alignSize(comptime Int: type, min_size: anytype, alignment: Int) Int {
1280 const size = @intCast(Int, min_size);
1281 if (size % alignment == 0) return size;
1282
1283 const div = size / alignment;
1284 return (div + 1) * alignment;
1285}
1286
1287fn commandSize(min_size: anytype) u32 {
1288 return alignSize(u32, min_size, @sizeOf(u64));
1289}
1290
12911267fn addPadding(self: *MachO, size: u64, file_offset: u64) !void {
12921268 if (size == 0) return;
12931269
......@@ -1424,7 +1400,7 @@ fn writeAllUndefSymbols(self: *MachO) !void {
14241400}
14251401
14261402fn writeExportTrie(self: *MachO) !void {
1427 if (self.entry_addr == null) return;
1403 assert(self.entry_addr != null);
14281404
14291405 // TODO implement mechanism for generating a prefix tree of the exported symbols
14301406 // single branch export trie
......@@ -1440,7 +1416,6 @@ fn writeExportTrie(self: *MachO) !void {
14401416 const written = try std.debug.leb.writeULEB128Mem(buf[12..], addr);
14411417 buf[10] = @intCast(u8, written) + 1;
14421418 buf[11] = 0;
1443 log.debug("WAT = {}, {x}\n", .{ written, buf[0..] });
14441419
14451420 const dyld_info = &self.load_commands.items[self.dyld_info_cmd_index.?].DyldInfo;
14461421 try self.base.file.?.pwriteAll(buf[0..], dyld_info.export_off);
......@@ -1450,7 +1425,6 @@ fn writeStringTable(self: *MachO) !void {
14501425 const symtab = &self.load_commands.items[self.symtab_cmd_index.?].Symtab;
14511426 const allocated_size = self.allocatedSize(symtab.stroff);
14521427 const needed_size = self.string_table.items.len;
1453 log.debug("allocated_size = 0x{x}, needed_size = 0x{x}\n", .{ allocated_size, needed_size });
14541428
14551429 if (needed_size > allocated_size) {
14561430 symtab.strsize = 0;
......@@ -1462,11 +1436,62 @@ fn writeStringTable(self: *MachO) !void {
14621436
14631437 try self.base.file.?.pwriteAll(self.string_table.items, symtab.stroff);
14641438
1465 // FIXME
1439 // TODO rework how we preallocate space for the entire __LINKEDIT segment instead of
1440 // doing dynamic updates like this.
14661441 const linkedit = &self.load_commands.items[self.linkedit_segment_cmd_index.?].Segment;
14671442 linkedit.filesize = symtab.stroff + symtab.strsize - linkedit.fileoff;
14681443}
14691444
1445fn writeCmdHeaders(self: *MachO) !void {
1446 assert(self.cmd_table_dirty);
1447
1448 // Write all load command headers first.
1449 // Since command sizes are up-to-date and accurate, we will correctly
1450 // leave space for any section headers that any of the segment load
1451 // commands might consist of.
1452 var last_cmd_offset: usize = @sizeOf(macho.mach_header_64);
1453 for (self.load_commands.items) |cmd| {
1454 try cmd.write(&self.base.file.?, last_cmd_offset);
1455 last_cmd_offset += cmd.cmdsize();
1456 }
1457 {
1458 // write __text section header
1459 const off = if (self.text_segment_cmd_index) |text_segment_index| blk: {
1460 var i: usize = 0;
1461 var cmdsize: usize = @sizeOf(macho.mach_header_64) + @sizeOf(macho.segment_command_64);
1462 while (i < text_segment_index) : (i += 1) {
1463 cmdsize += self.load_commands.items[i].cmdsize();
1464 }
1465 break :blk cmdsize;
1466 } else {
1467 // If we've landed in here, we are building a MachO object file, so we have
1468 // only one, noname segment to append this section header to.
1469 return error.TODOImplementWritingObjFiles;
1470 };
1471 const idx = self.text_section_index.?;
1472 log.debug("writing text section {} at 0x{x}\n", .{ self.sections.items[idx .. idx + 1], off });
1473 try self.base.file.?.pwriteAll(mem.sliceAsBytes(self.sections.items[idx .. idx + 1]), off);
1474 }
1475 {
1476 // write __got section header
1477 const off = if (self.data_segment_cmd_index) |data_segment_index| blk: {
1478 var i: usize = 0;
1479 var cmdsize: usize = @sizeOf(macho.mach_header_64) + @sizeOf(macho.segment_command_64);
1480 while (i < data_segment_index) : (i += 1) {
1481 cmdsize += self.load_commands.items[i].cmdsize();
1482 }
1483 break :blk cmdsize;
1484 } else {
1485 // If we've landed in here, we are building a MachO object file, so we have
1486 // only one, noname segment to append this section header to.
1487 return error.TODOImplementWritingObjFiles;
1488 };
1489 const idx = self.got_section_index.?;
1490 log.debug("writing got section {} at 0x{x}\n", .{ self.sections.items[idx .. idx + 1], off });
1491 try self.base.file.?.pwriteAll(mem.sliceAsBytes(self.sections.items[idx .. idx + 1]), off);
1492 }
1493}
1494
14701495/// Writes Mach-O file header.
14711496/// Should be invoked last as it needs up-to-date values of ncmds and sizeof_cmds bookkeeping
14721497/// variables.