authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-02 20:57:04-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-03 14:49:35-07:00
logd94303be2bcee33e7efba22a186fd06eaa809707
tree883b8f12094809995e885658652e9c43334def23
parent663ffa5a7a1d53d1adf9c1ea6991aad8bf82aadc

stage2: introduce renameTmpIntoCache into the linker API

Doc comments reproduced here: This function is called by the frontend before flush(). It communicates that `options.bin_file.emit` directory needs to be renamed from `[zig-cache]/tmp/[random]` to `[zig-cache]/o/[digest]`. The frontend would like to simply perform a file system rename, however, some linker backends care about the file paths of the objects they are linking. So this function call tells linker backends to rename the paths of object files to observe the new directory path. Linker backends which do not have this requirement can fall back to the simple implementation at the bottom of this function. This function is only called when CacheMode is `whole`. This solves stack trace regressions on Windows and macOS because the linker backends do not observe object file paths until flush().

7 files changed, 166 insertions(+), 79 deletions(-)

src/Compilation.zig+50-34
......@@ -2093,39 +2093,15 @@ pub fn update(comp: *Compilation) !void {
20932093 }
20942094
20952095 if (comp.totalErrorCount() != 0) {
2096 // Skip flushing.
2096 // Skip flushing and keep source files loaded for error reporting.
20972097 comp.link_error_flags = .{};
20982098 return;
20992099 }
21002100
2101 // This is needed before reading the error flags.
2102 try comp.bin_file.flush(comp);
2103 comp.link_error_flags = comp.bin_file.errorFlags();
2104
2105 if (!use_stage1) {
2106 if (comp.bin_file.options.module) |module| {
2107 try link.File.C.flushEmitH(module);
2108 }
2109 }
2110
21112101 // Flush takes care of -femit-bin, but we still have -femit-llvm-ir, -femit-llvm-bc, and
21122102 // -femit-asm to handle, in the case of C objects.
21132103 comp.emitOthers();
21142104
2115 // If there are any errors, we anticipate the source files being loaded
2116 // to report error messages. Otherwise we unload all source files to save memory.
2117 // The ZIR needs to stay loaded in memory because (1) Decl objects contain references
2118 // to it, and (2) generic instantiations, comptime calls, inline calls will need
2119 // to reference the ZIR.
2120 if (!comp.keep_source_files_loaded) {
2121 if (comp.bin_file.options.module) |module| {
2122 for (module.import_table.values()) |file| {
2123 file.unloadTree(comp.gpa);
2124 file.unloadSource(comp.gpa);
2125 }
2126 }
2127 }
2128
21292105 if (comp.whole_cache_manifest != null) {
21302106 const digest = man.final();
21312107
......@@ -2139,23 +2115,63 @@ pub fn update(comp: *Compilation) !void {
21392115 const o_sub_path = try std.fs.path.join(comp.gpa, &[_][]const u8{ "o", &digest });
21402116 defer comp.gpa.free(o_sub_path);
21412117
2142 try std.fs.rename(
2143 comp.local_cache_directory.handle,
2144 tmp_dir_sub_path,
2145 comp.local_cache_directory.handle,
2146 o_sub_path,
2147 );
2118 try comp.bin_file.renameTmpIntoCache(comp.local_cache_directory, tmp_dir_sub_path, o_sub_path);
2119 comp.wholeCacheModeSetBinFilePath(&digest);
2120
2121 // This is intentionally sandwiched between renameTmpIntoCache() and writeManifest().
2122 if (comp.bin_file.options.module) |module| {
2123 // We need to set the zig_cache_artifact_directory for -femit-asm, -femit-llvm-ir,
2124 // etc to know where to output to.
2125 var artifact_dir = try comp.local_cache_directory.handle.openDir(o_sub_path, .{});
2126 defer artifact_dir.close();
2127
2128 var dir_path = try comp.local_cache_directory.join(comp.gpa, &.{o_sub_path});
2129 defer comp.gpa.free(dir_path);
2130
2131 module.zig_cache_artifact_directory = .{
2132 .handle = artifact_dir,
2133 .path = dir_path,
2134 };
2135
2136 try comp.flush();
2137 } else {
2138 try comp.flush();
2139 }
21482140
21492141 // Failure here only means an unnecessary cache miss.
21502142 man.writeManifest() catch |err| {
21512143 log.warn("failed to write cache manifest: {s}", .{@errorName(err)});
21522144 };
21532145
2154 comp.wholeCacheModeSetBinFilePath(&digest);
2155
21562146 assert(comp.bin_file.lock == null);
21572147 comp.bin_file.lock = man.toOwnedLock();
2158 return;
2148 } else {
2149 try comp.flush();
2150 }
2151
2152 // Unload all source files to save memory.
2153 // The ZIR needs to stay loaded in memory because (1) Decl objects contain references
2154 // to it, and (2) generic instantiations, comptime calls, inline calls will need
2155 // to reference the ZIR.
2156 if (!comp.keep_source_files_loaded) {
2157 if (comp.bin_file.options.module) |module| {
2158 for (module.import_table.values()) |file| {
2159 file.unloadTree(comp.gpa);
2160 file.unloadSource(comp.gpa);
2161 }
2162 }
2163 }
2164}
2165
2166fn flush(comp: *Compilation) !void {
2167 try comp.bin_file.flush(comp); // This is needed before reading the error flags.
2168 comp.link_error_flags = comp.bin_file.errorFlags();
2169
2170 const use_stage1 = build_options.is_stage1 and comp.bin_file.options.use_stage1;
2171 if (!use_stage1) {
2172 if (comp.bin_file.options.module) |module| {
2173 try link.File.C.flushEmitH(module);
2174 }
21592175 }
21602176}
21612177
src/codegen/llvm.zig+6-4
......@@ -324,10 +324,12 @@ pub const Object = struct {
324324 const mod = comp.bin_file.options.module.?;
325325 const cache_dir = mod.zig_cache_artifact_directory;
326326
327 const emit_bin_path: ?[*:0]const u8 = if (comp.bin_file.options.emit) |emit|
328 try emit.directory.joinZ(arena, &[_][]const u8{self.sub_path})
329 else
330 null;
327 const emit_bin_path: ?[*:0]const u8 = if (comp.bin_file.options.emit) |emit| blk: {
328 const full_out_path = try emit.directory.join(arena, &[_][]const u8{emit.sub_path});
329 break :blk try std.fs.path.joinZ(arena, &.{
330 std.fs.path.dirname(full_out_path).?, self.sub_path,
331 });
332 } else null;
331333
332334 const emit_asm_path = try locPath(arena, comp.emit_asm, cache_dir);
333335 const emit_llvm_ir_path = try locPath(arena, comp.emit_llvm_ir, cache_dir);
src/link.zig+48-11
......@@ -636,6 +636,36 @@ pub const File = struct {
636636 }
637637 }
638638
639 /// This function is called by the frontend before flush(). It communicates that
640 /// `options.bin_file.emit` directory needs to be renamed from
641 /// `[zig-cache]/tmp/[random]` to `[zig-cache]/o/[digest]`.
642 /// The frontend would like to simply perform a file system rename, however,
643 /// some linker backends care about the file paths of the objects they are linking.
644 /// So this function call tells linker backends to rename the paths of object files
645 /// to observe the new directory path.
646 /// Linker backends which do not have this requirement can fall back to the simple
647 /// implementation at the bottom of this function.
648 /// This function is only called when CacheMode is `whole`.
649 pub fn renameTmpIntoCache(
650 base: *File,
651 cache_directory: Compilation.Directory,
652 tmp_dir_sub_path: []const u8,
653 o_sub_path: []const u8,
654 ) !void {
655 // So far, none of the linker backends need to respond to this event, however,
656 // it makes sense that they might want to. So we leave this mechanism here
657 // for now. Once the linker backends get more mature, if it turns out this
658 // is not needed we can refactor this into having the frontend do the rename
659 // directly, and remove this function from link.zig.
660 _ = base;
661 try std.fs.rename(
662 cache_directory.handle,
663 tmp_dir_sub_path,
664 cache_directory.handle,
665 o_sub_path,
666 );
667 }
668
639669 pub fn linkAsArchive(base: *File, comp: *Compilation) !void {
640670 const tracy = trace(@src());
641671 defer tracy.end();
......@@ -645,9 +675,11 @@ pub const File = struct {
645675 const arena = arena_allocator.allocator();
646676
647677 const directory = base.options.emit.?.directory; // Just an alias to make it shorter to type.
678 const full_out_path = try directory.join(arena, &[_][]const u8{base.options.emit.?.sub_path});
679 const full_out_path_z = try arena.dupeZ(u8, full_out_path);
648680
649 // If there is no Zig code to compile, then we should skip flushing the output file because it
650 // will not be part of the linker line anyway.
681 // If there is no Zig code to compile, then we should skip flushing the output file
682 // because it will not be part of the linker line anyway.
651683 const module_obj_path: ?[]const u8 = if (base.options.module) |module| blk: {
652684 const use_stage1 = build_options.is_stage1 and base.options.use_stage1;
653685 if (use_stage1) {
......@@ -656,20 +688,28 @@ pub const File = struct {
656688 .target = base.options.target,
657689 .output_mode = .Obj,
658690 });
659 const o_directory = module.zig_cache_artifact_directory;
660 const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
661 break :blk full_obj_path;
691 switch (base.options.cache_mode) {
692 .incremental => break :blk try module.zig_cache_artifact_directory.join(
693 arena,
694 &[_][]const u8{obj_basename},
695 ),
696 .whole => break :blk try fs.path.join(arena, &.{
697 fs.path.dirname(full_out_path_z).?, obj_basename,
698 }),
699 }
662700 }
663701 if (base.options.object_format == .macho) {
664702 try base.cast(MachO).?.flushObject(comp);
665703 } else {
666704 try base.flushModule(comp);
667705 }
668 const obj_basename = base.intermediary_basename.?;
669 const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
670 break :blk full_obj_path;
706 break :blk try fs.path.join(arena, &.{
707 fs.path.dirname(full_out_path_z).?, base.intermediary_basename.?,
708 });
671709 } else null;
672710
711 log.debug("module_obj_path={s}", .{if (module_obj_path) |s| s else "(null)"});
712
673713 const compiler_rt_path: ?[]const u8 = if (base.options.include_compiler_rt)
674714 comp.compiler_rt_obj.?.full_object_path
675715 else
......@@ -742,9 +782,6 @@ pub const File = struct {
742782 object_files.appendAssumeCapacity(try arena.dupeZ(u8, p));
743783 }
744784
745 const full_out_path = try directory.join(arena, &[_][]const u8{base.options.emit.?.sub_path});
746 const full_out_path_z = try arena.dupeZ(u8, full_out_path);
747
748785 if (base.options.verbose_link) {
749786 std.debug.print("ar rcs {s}", .{full_out_path_z});
750787 for (object_files.items) |arg| {
src/link/Coff.zig+14-7
......@@ -880,6 +880,7 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
880880 const arena = arena_allocator.allocator();
881881
882882 const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
883 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
883884
884885 // If there is no Zig code to compile, then we should skip flushing the output file because it
885886 // will not be part of the linker line anyway.
......@@ -891,15 +892,22 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
891892 .target = self.base.options.target,
892893 .output_mode = .Obj,
893894 });
894 const o_directory = module.zig_cache_artifact_directory;
895 const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
896 break :blk full_obj_path;
895 switch (self.base.options.cache_mode) {
896 .incremental => break :blk try module.zig_cache_artifact_directory.join(
897 arena,
898 &[_][]const u8{obj_basename},
899 ),
900 .whole => break :blk try fs.path.join(arena, &.{
901 fs.path.dirname(full_out_path).?, obj_basename,
902 }),
903 }
897904 }
898905
899906 try self.flushModule(comp);
900 const obj_basename = self.base.intermediary_basename.?;
901 const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
902 break :blk full_obj_path;
907
908 break :blk try fs.path.join(arena, &.{
909 fs.path.dirname(full_out_path).?, self.base.intermediary_basename.?,
910 });
903911 } else null;
904912
905913 const is_lib = self.base.options.output_mode == .Lib;
......@@ -978,7 +986,6 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
978986 };
979987 }
980988
981 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
982989 if (self.base.options.output_mode == .Obj) {
983990 // LLD's COFF driver does not support the equivalent of `-r` so we do a simple file copy
984991 // here. TODO: think carefully about how we can avoid this redundant operation when doing
src/link/Elf.zig+18-8
......@@ -297,6 +297,7 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Elf {
297297 else => return error.UnsupportedELFArchitecture,
298298 };
299299 const self = try gpa.create(Elf);
300 errdefer gpa.destroy(self);
300301 self.* = .{
301302 .base = .{
302303 .tag = .elf,
......@@ -306,6 +307,9 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Elf {
306307 },
307308 .ptr_width = ptr_width,
308309 };
310 // TODO get rid of the sub_path parameter to LlvmObject.create
311 // and create the llvm_object here. Also openPath needs to
312 // not override this field or there will be a memory leak.
309313 return self;
310314}
311315
......@@ -1298,6 +1302,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
12981302 const arena = arena_allocator.allocator();
12991303
13001304 const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
1305 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
13011306
13021307 // If there is no Zig code to compile, then we should skip flushing the output file because it
13031308 // will not be part of the linker line anyway.
......@@ -1309,15 +1314,22 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
13091314 .target = self.base.options.target,
13101315 .output_mode = .Obj,
13111316 });
1312 const o_directory = module.zig_cache_artifact_directory;
1313 const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
1314 break :blk full_obj_path;
1317 switch (self.base.options.cache_mode) {
1318 .incremental => break :blk try module.zig_cache_artifact_directory.join(
1319 arena,
1320 &[_][]const u8{obj_basename},
1321 ),
1322 .whole => break :blk try fs.path.join(arena, &.{
1323 fs.path.dirname(full_out_path).?, obj_basename,
1324 }),
1325 }
13151326 }
13161327
13171328 try self.flushModule(comp);
1318 const obj_basename = self.base.intermediary_basename.?;
1319 const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
1320 break :blk full_obj_path;
1329
1330 break :blk try fs.path.join(arena, &.{
1331 fs.path.dirname(full_out_path).?, self.base.intermediary_basename.?,
1332 });
13211333 } else null;
13221334
13231335 const is_obj = self.base.options.output_mode == .Obj;
......@@ -1434,8 +1446,6 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
14341446 };
14351447 }
14361448
1437 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
1438
14391449 // Due to a deficiency in LLD, we need to special-case BPF to a simple file copy when generating
14401450 // relocatables. Normally, we would expect `lld -r` to work. However, because LLD wants to resolve
14411451 // BPF relocations which it shouldn't, it fails before even generating the relocatable.
src/link/MachO.zig+16-7
......@@ -423,6 +423,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void {
423423 const arena = arena_allocator.allocator();
424424
425425 const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
426 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
426427
427428 // If there is no Zig code to compile, then we should skip flushing the output file because it
428429 // will not be part of the linker line anyway.
......@@ -433,15 +434,24 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void {
433434 .target = self.base.options.target,
434435 .output_mode = .Obj,
435436 });
436 const o_directory = module.zig_cache_artifact_directory;
437 const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
438 break :blk full_obj_path;
437 switch (self.base.options.cache_mode) {
438 .incremental => break :blk try module.zig_cache_artifact_directory.join(
439 arena,
440 &[_][]const u8{obj_basename},
441 ),
442 .whole => break :blk try fs.path.join(arena, &.{
443 fs.path.dirname(full_out_path).?, obj_basename,
444 }),
445 }
439446 }
440447
441448 const obj_basename = self.base.intermediary_basename orelse break :blk null;
449
442450 try self.flushObject(comp);
443 const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
444 break :blk full_obj_path;
451
452 break :blk try fs.path.join(arena, &.{
453 fs.path.dirname(full_out_path).?, obj_basename,
454 });
445455 } else null;
446456
447457 const is_lib = self.base.options.output_mode == .Lib;
......@@ -534,7 +544,6 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void {
534544 else => |e| return e,
535545 };
536546 }
537 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
538547
539548 if (self.base.options.output_mode == .Obj) {
540549 // LLD's MachO driver does not support the equivalent of `-r` so we do a simple file copy
......@@ -1269,7 +1278,7 @@ fn parseInputFiles(self: *MachO, files: []const []const u8, syslibroot: ?[]const
12691278 for (files) |file_name| {
12701279 const full_path = full_path: {
12711280 var buffer: [fs.MAX_PATH_BYTES]u8 = undefined;
1272 const path = try std.fs.realpath(file_name, &buffer);
1281 const path = try fs.realpath(file_name, &buffer);
12731282 break :full_path try self.base.allocator.dupe(u8, path);
12741283 };
12751284 defer self.base.allocator.free(full_path);
src/link/Wasm.zig+14-8
......@@ -1050,6 +1050,7 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
10501050 const arena = arena_allocator.allocator();
10511051
10521052 const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
1053 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
10531054
10541055 // If there is no Zig code to compile, then we should skip flushing the output file because it
10551056 // will not be part of the linker line anyway.
......@@ -1061,15 +1062,22 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
10611062 .target = self.base.options.target,
10621063 .output_mode = .Obj,
10631064 });
1064 const o_directory = module.zig_cache_artifact_directory;
1065 const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
1066 break :blk full_obj_path;
1065 switch (self.base.options.cache_mode) {
1066 .incremental => break :blk try module.zig_cache_artifact_directory.join(
1067 arena,
1068 &[_][]const u8{obj_basename},
1069 ),
1070 .whole => break :blk try fs.path.join(arena, &.{
1071 fs.path.dirname(full_out_path).?, obj_basename,
1072 }),
1073 }
10671074 }
10681075
10691076 try self.flushModule(comp);
1070 const obj_basename = self.base.intermediary_basename.?;
1071 const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
1072 break :blk full_obj_path;
1077
1078 break :blk try fs.path.join(arena, &.{
1079 fs.path.dirname(full_out_path).?, self.base.intermediary_basename.?,
1080 });
10731081 } else null;
10741082
10751083 const is_obj = self.base.options.output_mode == .Obj;
......@@ -1143,8 +1151,6 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
11431151 };
11441152 }
11451153
1146 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
1147
11481154 if (self.base.options.output_mode == .Obj) {
11491155 // LLD's WASM driver does not support the equivalent of `-r` so we do a simple file copy
11501156 // here. TODO: think carefully about how we can avoid this redundant operation when doing