authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-12-26 23:45:44-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-01-01 17:51:21-07:00
log524dc756b30c19db070362c8094459c7c8e4fd8a
treef0fc599599b9c8bfcdbfa0e6eed737ea21311fc2
parent57562c8d507667b6fefcb7fbc7a305fbd610b5dd

Compilation: several branch regression fixes

* move wasi_emulated_libs into Compilation - It needs to be accessed from Compilation, which needs to potentially build those artifacts. * Compilation: improve error reporting for two cases - the setMiscFailure mechanism is handy - let's use it! * fix one instance of incorrectly checking for emit_bin via `comp.bin_file != null`. There are more instances of this that need to be fixed in a future commit. * fix renameTmpIntoCache not handling the case where it needs to make the "o" directory in the zig-cache directory. - while I'm at it, simplify the logic for handling the fact that Windows returns error.AccessDenied rather than error.PathAlreadyExists for failure to rename a directory over another one. * fix missing cache hash additions - there are still more to add in a future commit - addNonIncrementalStuffToCacheManifest is called when bin_file is always null, and then it incorrectly checks if bin_file is non-null and only then adds a bunch of stuff to the cache hash. It needs to instead add to the cache hash based on lf_open_opts.

3 files changed, 102 insertions(+), 72 deletions(-)

src/Compilation.zig+91-57
......@@ -190,6 +190,7 @@ compiler_rt_lib: ?CRTFile = null,
190190compiler_rt_obj: ?CRTFile = null,
191191
192192glibc_so_files: ?glibc.BuiltSharedObjects = null,
193wasi_emulated_libs: []const wasi_libc.CRTFile,
193194
194195/// For example `Scrt1.o` and `libc_nonshared.a`. These are populated after building libc from source,
195196/// The set of needed CRT (C runtime) files differs depending on the target and compilation settings.
......@@ -707,6 +708,8 @@ pub const Win32Resource = struct {
707708
708709pub const MiscTask = enum {
709710 write_builtin_zig,
711 rename_results,
712 check_whole_cache,
710713 glibc_crt_file,
711714 glibc_shared_objects,
712715 musl_crt_file,
......@@ -1500,6 +1503,7 @@ pub fn create(gpa: Allocator, options: CreateOptions) !*Compilation {
15001503 .function_sections = options.function_sections,
15011504 .data_sections = options.data_sections,
15021505 .native_system_include_paths = options.native_system_include_paths,
1506 .wasi_emulated_libs = options.wasi_emulated_libs,
15031507 };
15041508
15051509 // Prevent some footguns by making the "any" fields of config reflect
......@@ -1521,7 +1525,6 @@ pub fn create(gpa: Allocator, options: CreateOptions) !*Compilation {
15211525 .z_max_page_size = options.linker_z_max_page_size,
15221526 .darwin_sdk_layout = libc_dirs.darwin_sdk_layout,
15231527 .frameworks = options.frameworks,
1524 .wasi_emulated_libs = options.wasi_emulated_libs,
15251528 .lib_dirs = options.lib_dirs,
15261529 .rpath_list = options.rpath_list,
15271530 .symbol_wrap_set = options.symbol_wrap_set,
......@@ -1672,10 +1675,10 @@ pub fn create(gpa: Allocator, options: CreateOptions) !*Compilation {
16721675 };
16731676 errdefer comp.destroy();
16741677
1675 const target = options.root_mod.resolved_target.result;
1678 const target = comp.root_mod.resolved_target.result;
16761679
1677 const capable_of_building_compiler_rt = canBuildLibCompilerRt(target, options.config.use_llvm);
1678 const capable_of_building_zig_libc = canBuildZigLibC(target, options.config.use_llvm);
1680 const capable_of_building_compiler_rt = canBuildLibCompilerRt(target, comp.config.use_llvm);
1681 const capable_of_building_zig_libc = canBuildZigLibC(target, comp.config.use_llvm);
16791682
16801683 // Add a `CObject` for each `c_source_files`.
16811684 try comp.c_object_table.ensureTotalCapacity(gpa, options.c_source_files.len);
......@@ -1768,25 +1771,21 @@ pub fn create(gpa: Allocator, options: CreateOptions) !*Compilation {
17681771 });
17691772 }
17701773
1771 if (comp.bin_file) |lf| {
1772 if (lf.cast(link.File.Wasm)) |wasm| {
1773 if (comp.wantBuildWasiLibcFromSource()) {
1774 if (!target_util.canBuildLibC(target)) return error.LibCUnavailable;
1774 if (comp.wantBuildWasiLibcFromSource()) {
1775 if (!target_util.canBuildLibC(target)) return error.LibCUnavailable;
17751776
1776 // worst-case we need all components
1777 try comp.work_queue.ensureUnusedCapacity(wasm.wasi_emulated_libs.len + 2);
1777 // worst-case we need all components
1778 try comp.work_queue.ensureUnusedCapacity(comp.wasi_emulated_libs.len + 2);
17781779
1779 for (wasm.wasi_emulated_libs) |crt_file| {
1780 comp.work_queue.writeItemAssumeCapacity(.{
1781 .wasi_libc_crt_file = crt_file,
1782 });
1783 }
1784 comp.work_queue.writeAssumeCapacity(&[_]Job{
1785 .{ .wasi_libc_crt_file = wasi_libc.execModelCrtFile(options.config.wasi_exec_model) },
1786 .{ .wasi_libc_crt_file = .libc_a },
1787 });
1788 }
1780 for (comp.wasi_emulated_libs) |crt_file| {
1781 comp.work_queue.writeItemAssumeCapacity(.{
1782 .wasi_libc_crt_file = crt_file,
1783 });
17891784 }
1785 comp.work_queue.writeAssumeCapacity(&[_]Job{
1786 .{ .wasi_libc_crt_file = wasi_libc.execModelCrtFile(comp.config.wasi_exec_model) },
1787 .{ .wasi_libc_crt_file = .libc_a },
1788 });
17901789 }
17911790
17921791 if (comp.wantBuildMinGWFromSource()) {
......@@ -1832,11 +1831,11 @@ pub fn create(gpa: Allocator, options: CreateOptions) !*Compilation {
18321831 }
18331832
18341833 if (comp.bin_file) |lf| {
1835 if (comp.getTarget().isMinGW() and comp.config.any_non_single_threaded) {
1834 if (target.isMinGW() and comp.config.any_non_single_threaded) {
18361835 // LLD might drop some symbols as unused during LTO and GCing, therefore,
18371836 // we force mark them for resolution here.
18381837
1839 const tls_index_sym = switch (comp.getTarget().cpu.arch) {
1838 const tls_index_sym = switch (target.cpu.arch) {
18401839 .x86 => "__tls_index",
18411840 else => "_tls_index",
18421841 };
......@@ -2024,12 +2023,14 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
20242023 try comp.addNonIncrementalStuffToCacheManifest(&man);
20252024
20262025 const is_hit = man.hit() catch |err| {
2027 // TODO properly bubble these up instead of emitting a warning
20282026 const i = man.failed_file_index orelse return err;
20292027 const pp = man.files.items[i].prefixed_path orelse return err;
20302028 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
2031 std.log.warn("{s}: {s}{s}", .{ @errorName(err), prefix, pp.sub_path });
2032 return err;
2029 return comp.setMiscFailure(
2030 .check_whole_cache,
2031 "unable to check cache: stat file '{}{s}{s}' failed: {s}",
2032 .{ comp.local_cache_directory, prefix, pp.sub_path, @errorName(err) },
2033 );
20332034 };
20342035 if (is_hit) {
20352036 comp.last_update_was_cache_hit = true;
......@@ -2221,7 +2222,17 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
22212222 const tmp_dir_sub_path = "tmp" ++ s ++ Package.Manifest.hex64(tmp_dir_rand_int);
22222223 const o_sub_path = "o" ++ s ++ digest;
22232224
2224 try renameTmpIntoCache(comp.local_cache_directory, tmp_dir_sub_path, o_sub_path);
2225 renameTmpIntoCache(comp.local_cache_directory, tmp_dir_sub_path, o_sub_path) catch |err| {
2226 return comp.setMiscFailure(
2227 .rename_results,
2228 "failed to rename compilation results ('{}{s}') into local cache ('{}{s}'): {s}",
2229 .{
2230 comp.local_cache_directory, tmp_dir_sub_path,
2231 comp.local_cache_directory, o_sub_path,
2232 @errorName(err),
2233 },
2234 );
2235 };
22252236 comp.wholeCacheModeSetBinFilePath(whole, &digest);
22262237
22272238 // Failure here only means an unnecessary cache miss.
......@@ -2251,42 +2262,36 @@ fn renameTmpIntoCache(
22512262 tmp_dir_sub_path: []const u8,
22522263 o_sub_path: []const u8,
22532264) !void {
2265 var seen_eaccess = false;
22542266 while (true) {
2255 if (builtin.os.tag == .windows) {
2256 // Work around windows `renameW` can't fail with `PathAlreadyExists`
2267 std.fs.rename(
2268 cache_directory.handle,
2269 tmp_dir_sub_path,
2270 cache_directory.handle,
2271 o_sub_path,
2272 ) catch |err| switch (err) {
2273 // On Windows, rename fails with `AccessDenied` rather than `PathAlreadyExists`.
22572274 // See https://github.com/ziglang/zig/issues/8362
2258 if (cache_directory.handle.access(o_sub_path, .{})) |_| {
2259 try cache_directory.handle.deleteTree(o_sub_path);
2260 continue;
2261 } else |err| switch (err) {
2262 error.FileNotFound => {},
2263 else => |e| return e,
2264 }
2265 std.fs.rename(
2266 cache_directory.handle,
2267 tmp_dir_sub_path,
2268 cache_directory.handle,
2269 o_sub_path,
2270 ) catch |err| {
2271 log.err("unable to rename cache dir {s} to {s}: {s}", .{ tmp_dir_sub_path, o_sub_path, @errorName(err) });
2272 return err;
2273 };
2274 break;
2275 } else {
2276 std.fs.rename(
2277 cache_directory.handle,
2278 tmp_dir_sub_path,
2279 cache_directory.handle,
2280 o_sub_path,
2281 ) catch |err| switch (err) {
2282 error.PathAlreadyExists => {
2275 error.AccessDenied => switch (builtin.os.tag) {
2276 .windows => {
2277 if (!seen_eaccess) return error.AccessDenied;
2278 seen_eaccess = true;
22832279 try cache_directory.handle.deleteTree(o_sub_path);
22842280 continue;
22852281 },
2286 else => |e| return e,
2287 };
2288 break;
2289 }
2282 else => return error.AccessDenied,
2283 },
2284 error.PathAlreadyExists => {
2285 try cache_directory.handle.deleteTree(o_sub_path);
2286 continue;
2287 },
2288 error.FileNotFound => {
2289 try cache_directory.handle.makePath("o");
2290 continue;
2291 },
2292 else => |e| return e,
2293 };
2294 break;
22902295 }
22912296}
22922297
......@@ -2386,6 +2391,10 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
23862391 try addModuleTableToCacheHash(gpa, arena, &man.hash, mod.main_mod, .{ .files = man });
23872392
23882393 // Synchronize with other matching comments: ZigOnlyHashStuff
2394 man.hash.add(comp.config.use_llvm);
2395 man.hash.add(comp.config.use_lib_llvm);
2396 man.hash.add(comp.config.dll_export_fns);
2397 man.hash.add(comp.config.is_test);
23892398 man.hash.add(comp.config.test_evented_io);
23902399 man.hash.addOptionalBytes(comp.test_filter);
23912400 man.hash.addOptionalBytes(comp.test_name_prefix);
......@@ -2393,6 +2402,20 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
23932402 man.hash.add(comp.formatted_panics);
23942403 man.hash.add(mod.emit_h != null);
23952404 man.hash.add(mod.error_limit);
2405 } else {
2406 cache_helpers.addResolvedTarget(&man.hash, comp.root_mod.resolved_target);
2407 man.hash.add(comp.root_mod.optimize_mode);
2408 man.hash.add(comp.root_mod.code_model);
2409 man.hash.add(comp.root_mod.single_threaded);
2410 man.hash.add(comp.root_mod.error_tracing);
2411 man.hash.add(comp.root_mod.pic);
2412 man.hash.add(comp.root_mod.omit_frame_pointer);
2413 man.hash.add(comp.root_mod.stack_check);
2414 man.hash.add(comp.root_mod.red_zone);
2415 man.hash.add(comp.root_mod.sanitize_c);
2416 man.hash.add(comp.root_mod.sanitize_thread);
2417 man.hash.add(comp.root_mod.unwind_tables);
2418 man.hash.add(comp.root_mod.structured_cfg);
23962419 }
23972420
23982421 for (comp.objects) |obj| {
......@@ -3829,8 +3852,19 @@ pub fn obtainCObjectCacheManifest(
38293852 // Only things that need to be added on top of the base hash, and only things
38303853 // that apply both to @cImport and compiling C objects. No linking stuff here!
38313854 // Also nothing that applies only to compiling .zig code.
3855 cache_helpers.addResolvedTarget(&man.hash, owner_mod.resolved_target);
3856 man.hash.add(owner_mod.optimize_mode);
3857 man.hash.add(owner_mod.code_model);
3858 man.hash.add(owner_mod.single_threaded);
3859 man.hash.add(owner_mod.error_tracing);
3860 man.hash.add(owner_mod.pic);
3861 man.hash.add(owner_mod.omit_frame_pointer);
3862 man.hash.add(owner_mod.stack_check);
3863 man.hash.add(owner_mod.red_zone);
38323864 man.hash.add(owner_mod.sanitize_c);
38333865 man.hash.add(owner_mod.sanitize_thread);
3866 man.hash.add(owner_mod.unwind_tables);
3867 man.hash.add(owner_mod.structured_cfg);
38343868 man.hash.addListOfBytes(owner_mod.cc_argv);
38353869 man.hash.add(comp.config.link_libcpp);
38363870
src/link.zig-2
......@@ -170,8 +170,6 @@ pub const File = struct {
170170 /// (Windows) .def file to specify when linking
171171 module_definition_file: ?[]const u8,
172172
173 wasi_emulated_libs: []const wasi_libc.CRTFile,
174
175173 pub const Entry = union(enum) {
176174 default,
177175 disabled,
src/link/Wasm.zig+11-13
......@@ -43,7 +43,6 @@ export_symbol_names: []const []const u8,
4343global_base: ?u64,
4444initial_memory: ?u64,
4545max_memory: ?u64,
46wasi_emulated_libs: []const wasi_libc.CRTFile,
4746/// Output name of the file
4847name: []const u8,
4948/// If this is not null, an object file is created by LLVM and linked with LLD afterwards.
......@@ -435,7 +434,6 @@ pub fn createEmpty(
435434 .global_base = options.global_base,
436435 .initial_memory = options.initial_memory,
437436 .max_memory = options.max_memory,
438 .wasi_emulated_libs = options.wasi_emulated_libs,
439437
440438 .entry_name = switch (options.entry) {
441439 .disabled => null,
......@@ -3626,7 +3624,7 @@ fn linkWithZld(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) l
36263624 const is_exe_or_dyn_lib = output_mode == .Exe or
36273625 (output_mode == .Lib and link_mode == .Dynamic);
36283626 if (is_exe_or_dyn_lib) {
3629 for (wasm.wasi_emulated_libs) |crt_file| {
3627 for (comp.wasi_emulated_libs) |crt_file| {
36303628 try positionals.append(try comp.get_libc_crt_file(
36313629 arena,
36323630 wasi_libc.emulatedLibCRFileLibName(crt_file),
......@@ -4601,7 +4599,7 @@ fn linkWithLLD(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) !
46014599 const import_memory = comp.config.import_memory;
46024600 const target = comp.root_mod.resolved_target.result;
46034601
4604 const gpa = wasm.base.comp.gpa;
4602 const gpa = comp.gpa;
46054603 var arena_allocator = std.heap.ArenaAllocator.init(gpa);
46064604 defer arena_allocator.deinit();
46074605 const arena = arena_allocator.allocator();
......@@ -4611,7 +4609,7 @@ fn linkWithLLD(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) !
46114609
46124610 // If there is no Zig code to compile, then we should skip flushing the output file because it
46134611 // will not be part of the linker line anyway.
4614 const module_obj_path: ?[]const u8 = if (wasm.base.comp.module != null) blk: {
4612 const module_obj_path: ?[]const u8 = if (comp.module != null) blk: {
46154613 try wasm.flushModule(comp, prog_node);
46164614
46174615 if (fs.path.dirname(full_out_path)) |dirname| {
......@@ -4626,7 +4624,7 @@ fn linkWithLLD(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) !
46264624 sub_prog_node.context.refresh();
46274625 defer sub_prog_node.end();
46284626
4629 const is_obj = wasm.base.comp.config.output_mode == .Obj;
4627 const is_obj = comp.config.output_mode == .Obj;
46304628 const compiler_rt_path: ?[]const u8 = blk: {
46314629 if (comp.compiler_rt_lib) |lib| break :blk lib.full_object_path;
46324630 if (comp.compiler_rt_obj) |obj| break :blk obj.full_object_path;
......@@ -4823,7 +4821,7 @@ fn linkWithLLD(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) !
48234821 try argv.append("--allow-undefined");
48244822 }
48254823
4826 if (wasm.base.comp.config.output_mode == .Lib and wasm.base.comp.config.link_mode == .Dynamic) {
4824 if (comp.config.output_mode == .Lib and comp.config.link_mode == .Dynamic) {
48274825 try argv.append("--shared");
48284826 }
48294827 if (comp.config.pie) {
......@@ -4842,10 +4840,10 @@ fn linkWithLLD(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) !
48424840 }
48434841
48444842 if (target.os.tag == .wasi) {
4845 const is_exe_or_dyn_lib = wasm.base.comp.config.output_mode == .Exe or
4846 (wasm.base.comp.config.output_mode == .Lib and wasm.base.comp.config.link_mode == .Dynamic);
4843 const is_exe_or_dyn_lib = comp.config.output_mode == .Exe or
4844 (comp.config.output_mode == .Lib and comp.config.link_mode == .Dynamic);
48474845 if (is_exe_or_dyn_lib) {
4848 for (wasm.wasi_emulated_libs) |crt_file| {
4846 for (comp.wasi_emulated_libs) |crt_file| {
48494847 try argv.append(try comp.get_libc_crt_file(
48504848 arena,
48514849 wasi_libc.emulatedLibCRFileLibName(crt_file),
......@@ -4891,7 +4889,7 @@ fn linkWithLLD(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) !
48914889 try argv.append(p);
48924890 }
48934891
4894 if (wasm.base.comp.config.output_mode != .Obj and
4892 if (comp.config.output_mode != .Obj and
48954893 !comp.skip_linker_dependencies and
48964894 !comp.config.link_libc)
48974895 {
......@@ -4902,7 +4900,7 @@ fn linkWithLLD(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) !
49024900 try argv.append(p);
49034901 }
49044902
4905 if (wasm.base.comp.verbose_link) {
4903 if (comp.verbose_link) {
49064904 // Skip over our own name so that the LLD linker name is the first argv item.
49074905 Compilation.dump_argv(argv.items[1..]);
49084906 }
......@@ -4977,7 +4975,7 @@ fn linkWithLLD(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) !
49774975 // it, and then can react to that in the same way as trying to run an ELF file
49784976 // from a foreign CPU architecture.
49794977 if (fs.has_executable_bit and target.os.tag == .wasi and
4980 wasm.base.comp.config.output_mode == .Exe)
4978 comp.config.output_mode == .Exe)
49814979 {
49824980 // TODO: what's our strategy for reporting linker errors from this function?
49834981 // report a nice error here with the file path if it fails instead of