authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2021-09-13 23:02:21+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2021-09-13 23:02:21+02:00
log760241ce50eaa9031339f6b591358b53f5797486
tree713b4ad62085eba2c29390d8003866f3290e16ce
parent46a10401f035b50122af9a91348edeb3f57e864e

macho: use the cache system to know if need to relink objects

This applies to stage2 where we make use of the cache system to work out if we need to relink objects when performing incremental updates. When the process is restarted however, while in principle the idea is to carry on where we left off by reparsing the prelinked binary from file, the required machinery is not there yet, and therefore we always fully relink upon restart.

2 files changed, 51 insertions(+), 66 deletions(-)

src/link/MachO.zig+40-55
......@@ -166,11 +166,13 @@ error_flags: File.ErrorFlags = File.ErrorFlags{},
166166
167167load_commands_dirty: bool = false,
168168sections_order_dirty: bool = false,
169
170169has_dices: bool = false,
171170has_stabs: bool = false,
172
173args_digest: [Cache.hex_digest_len]u8 = undefined,
171/// A helper var to indicate if we are at the start of the incremental updates, or
172/// already somewhere further along the update-and-run chain.
173/// TODO once we add opening a prelinked output binary from file, this will become
174/// obsolete as we will carry on where we left off.
175cold_start: bool = false,
174176
175177section_ordinals: std.AutoArrayHashMapUnmanaged(MatchingSection, void) = .{},
176178
......@@ -336,6 +338,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
336338 return self;
337339 }
338340
341 // TODO Migrate DebugSymbols to the merged linker codepaths
339342 // if (!options.strip and options.module != null) {
340343 // // Create dSYM bundle.
341344 // const dir = options.module.?.zig_cache_artifact_directory;
......@@ -456,8 +459,11 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
456459 defer if (!self.base.options.disable_lld_caching) man.deinit();
457460
458461 var digest: [Cache.hex_digest_len]u8 = undefined;
462 var needs_full_relink = true;
463
464 cache: {
465 if (use_stage1 and self.base.options.disable_lld_caching) break :cache;
459466
460 if (!self.base.options.disable_lld_caching) {
461467 man = comp.cache_parent.obtain();
462468
463469 // We are about to obtain this lock, so here we give other processes a chance first.
......@@ -491,17 +497,36 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
491497 id_symlink_basename,
492498 &prev_digest_buf,
493499 ) catch |err| blk: {
494 log.debug("MachO Zld new_digest={s} error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) });
500 log.debug("MachO Zld new_digest={s} error: {s}", .{
501 std.fmt.fmtSliceHexLower(&digest),
502 @errorName(err),
503 });
495504 // Handle this as a cache miss.
496505 break :blk prev_digest_buf[0..0];
497506 };
498507 if (mem.eql(u8, prev_digest, &digest)) {
499 log.debug("MachO Zld digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)});
500508 // Hot diggity dog! The output binary is already there.
501 self.base.lock = man.toOwnedLock();
502 return;
509
510 if (use_stage1) {
511 log.debug("MachO Zld digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)});
512 self.base.lock = man.toOwnedLock();
513 return;
514 } else {
515 log.debug("MachO Zld digest={s} match", .{std.fmt.fmtSliceHexLower(&digest)});
516 if (!self.cold_start) {
517 log.debug(" no need to relink objects", .{});
518 needs_full_relink = false;
519 } else {
520 log.debug(" TODO parse prelinked binary and continue linking where we left off", .{});
521 // TODO until such time however, perform a full relink of objects.
522 needs_full_relink = true;
523 }
524 }
503525 }
504 log.debug("MachO Zld prev_digest={s} new_digest={s}", .{ std.fmt.fmtSliceHexLower(prev_digest), std.fmt.fmtSliceHexLower(&digest) });
526 log.debug("MachO Zld prev_digest={s} new_digest={s}", .{
527 std.fmt.fmtSliceHexLower(prev_digest),
528 std.fmt.fmtSliceHexLower(&digest),
529 });
505530
506531 // We are about to change the output file to be different, so we invalidate the build hash now.
507532 directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
......@@ -509,7 +534,6 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
509534 else => |e| return e,
510535 };
511536 }
512
513537 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
514538
515539 if (self.base.options.output_mode == .Obj) {
......@@ -557,34 +581,6 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
557581 try self.strtab.append(self.base.allocator, 0);
558582 }
559583
560 const needs_full_relink = blk: {
561 if (use_stage1) break :blk true;
562
563 var hh: Cache.HashHelper = .{};
564 hh.addListOfBytes(self.base.options.objects);
565 for (comp.c_object_table.keys()) |key| {
566 hh.addBytes(key.status.success.object_path);
567 }
568 hh.addOptionalBytes(module_obj_path);
569 if (comp.compiler_rt_static_lib) |lib| {
570 hh.addBytes(lib.full_object_path);
571 }
572 if (self.base.options.link_libcpp) {
573 hh.addBytes(comp.libcxxabi_static_lib.?.full_object_path);
574 hh.addBytes(comp.libcxx_static_lib.?.full_object_path);
575 }
576 hh.addListOfBytes(self.base.options.lib_dirs);
577 hh.addListOfBytes(self.base.options.framework_dirs);
578 hh.addListOfBytes(self.base.options.frameworks);
579 hh.addListOfBytes(self.base.options.rpath_list);
580 hh.addStringSet(self.base.options.system_libs);
581 hh.addOptionalBytes(self.base.options.sysroot);
582 const new_digest = hh.final();
583 const needs_full_relink = !mem.eql(u8, &new_digest, &self.args_digest);
584 mem.copy(u8, &self.args_digest, &new_digest);
585 break :blk needs_full_relink;
586 };
587
588584 if (needs_full_relink) {
589585 self.objects.clearRetainingCapacity();
590586 self.archives.clearRetainingCapacity();
......@@ -848,22 +844,6 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
848844 try self.allocateGlobalSymbols();
849845 try self.writeAtoms();
850846
851 // log.warn("Locals:", .{});
852 // for (self.locals.items) |sym, i| {
853 // log.warn(" => {d}: {s}, {}", .{ i, self.getString(sym.n_strx), sym });
854 // }
855 // log.warn("Globals:", .{});
856 // for (self.globals.items) |sym, i| {
857 // log.warn(" => {d}: {s} {}", .{ i, self.getString(sym.n_strx), sym });
858 // }
859 // {
860 // log.warn("Resolver:", .{});
861 // var it = self.symbol_resolver.iterator();
862 // while (it.next()) |entry| {
863 // log.warn(" => {s}: {}", .{ self.getString(entry.key_ptr.*), entry.value_ptr.* });
864 // }
865 // }
866
867847 if (self.bss_section_index) |idx| {
868848 const seg = &self.load_commands.items[self.data_segment_cmd_index.?].Segment;
869849 const sect = &seg.sections.items[idx];
......@@ -880,7 +860,8 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
880860 try self.flushModule(comp);
881861 }
882862
883 if (!self.base.options.disable_lld_caching) {
863 cache: {
864 if (use_stage1 and self.base.options.disable_lld_caching) break :cache;
884865 // Update the file with the digest. If it fails we can continue; it only
885866 // means that the next invocation will have an unnecessary cache miss.
886867 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
......@@ -894,6 +875,8 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
894875 // other processes clobbering it.
895876 self.base.lock = man.toOwnedLock();
896877 }
878
879 self.cold_start = false;
897880}
898881
899882pub fn flushModule(self: *MachO, comp: *Compilation) !void {
......@@ -3929,6 +3912,8 @@ pub fn populateMissingMetadata(self: *MachO) !void {
39293912 });
39303913 self.load_commands_dirty = true;
39313914 }
3915
3916 self.cold_start = true;
39323917}
39333918
39343919const AllocateSectionOpts = struct {
test/stage2/darwin.zig+11-11
......@@ -27,8 +27,8 @@ pub fn addCases(ctx: *TestContext) !void {
2727
2828 // Regular old hello world
2929 case.addCompareOutput(
30 \\extern "c" fn write(usize, usize, usize) usize;
31 \\extern "c" fn exit(usize) noreturn;
30 \\extern fn write(usize, usize, usize) usize;
31 \\extern fn exit(usize) noreturn;
3232 \\
3333 \\pub export fn main() noreturn {
3434 \\ print();
......@@ -47,8 +47,8 @@ pub fn addCases(ctx: *TestContext) !void {
4747
4848 // Print it 4 times and force growth and realloc.
4949 case.addCompareOutput(
50 \\extern "c" fn write(usize, usize, usize) usize;
51 \\extern "c" fn exit(usize) noreturn;
50 \\extern fn write(usize, usize, usize) usize;
51 \\extern fn exit(usize) noreturn;
5252 \\
5353 \\pub export fn main() noreturn {
5454 \\ print();
......@@ -74,8 +74,8 @@ pub fn addCases(ctx: *TestContext) !void {
7474
7575 // Print it once, and change the message.
7676 case.addCompareOutput(
77 \\extern "c" fn write(usize, usize, usize) usize;
78 \\extern "c" fn exit(usize) noreturn;
77 \\extern fn write(usize, usize, usize) usize;
78 \\extern fn exit(usize) noreturn;
7979 \\
8080 \\pub export fn main() noreturn {
8181 \\ print();
......@@ -94,8 +94,8 @@ pub fn addCases(ctx: *TestContext) !void {
9494
9595 // Now we print it twice.
9696 case.addCompareOutput(
97 \\extern "c" fn write(usize, usize, usize) usize;
98 \\extern "c" fn exit(usize) noreturn;
97 \\extern fn write(usize, usize, usize) usize;
98 \\extern fn exit(usize) noreturn;
9999 \\
100100 \\pub export fn main() noreturn {
101101 \\ print();
......@@ -121,7 +121,7 @@ pub fn addCases(ctx: *TestContext) !void {
121121 // This test case also covers an infrequent scenarion where the string table *may* be relocated
122122 // into the position preceeding the symbol table which results in a dyld error.
123123 case.addCompareOutput(
124 \\extern "c" fn exit(usize) noreturn;
124 \\extern fn exit(usize) noreturn;
125125 \\
126126 \\pub export fn main() noreturn {
127127 \\ exit(0);
......@@ -131,8 +131,8 @@ pub fn addCases(ctx: *TestContext) !void {
131131 );
132132
133133 case.addCompareOutput(
134 \\extern "c" fn exit(usize) noreturn;
135 \\extern "c" fn write(usize, usize, usize) usize;
134 \\extern fn exit(usize) noreturn;
135 \\extern fn write(usize, usize, usize) usize;
136136 \\
137137 \\pub export fn main() noreturn {
138138 \\ _ = write(1, @ptrToInt("Hey!\n"), 5);