From a08cc7d2ae3bd6e90f8d26ac13f2e0652687dc30 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 15 Jun 2023 17:10:42 -0700 Subject: [PATCH] compiler: resolve library paths in the frontend search_strategy is no longer passed to Compilation at all; instead it is used in the CLI code only. When using Zig CLI mode, `-l` no longer has the ability to link statically; use positional arguments for this. The CLI has a small abstraction around library resolution handling which is used to remove some code duplication regarding static libraries, as well as handle the difference between zig cc CLI mode and zig CLI mode. Thanks to this, system libraries are now included in the cache hash, and thus changes to them will correctly cause cache misses. In the future, lib_dirs should no longer be passed to Compilation at all, because it is a frontend-only concept. Previously, -search_paths_first and -search_dylibs_first were Darwin-only arguments; they now work the same for all targets. Same thing with --sysroot. Improved the error reporting for failure to find a system library. An example error now looks like this: ``` $ zig build-exe test.zig -lfoo -L. -L/a -target x86_64-macos --sysroot /home/andy/local error: unable to find Dynamic system library 'foo' using strategy 'no_fallback'. search paths: ./libfoo.tbd ./libfoo.dylib ./libfoo.so /home/andy/local/a/libfoo.tbd /home/andy/local/a/libfoo.dylib /home/andy/local/a/libfoo.so /a/libfoo.tbd /a/libfoo.dylib /a/libfoo.so ``` closes #14963 --- src/Compilation.zig | 20 +- src/link.zig | 33 ++- src/link/Coff/lld.zig | 2 +- src/link/Elf.zig | 10 +- src/link/MachO.zig | 29 ++- src/link/MachO/zld.zig | 91 +------ src/main.zig | 521 ++++++++++++++++++++++++++++++----------- 7 files changed, 457 insertions(+), 249 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 2e269a7e80dfcc49df2a1922003e4db8ba4ccbbb..f966f1e17f891656f9d5431212b2d5b3064d4c7b 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -448,6 +448,7 @@ pub const ClangPreprocessorMode = enum { stdout, }; +pub const Framework = link.Framework; pub const SystemLib = link.SystemLib; pub const CacheMode = link.CacheMode; @@ -505,7 +506,7 @@ pub const InitOptions = struct { c_source_files: []const CSourceFile = &[0]CSourceFile{}, link_objects: []LinkObject = &[0]LinkObject{}, framework_dirs: []const []const u8 = &[0][]const u8{}, - frameworks: std.StringArrayHashMapUnmanaged(SystemLib) = .{}, + frameworks: std.StringArrayHashMapUnmanaged(Framework) = .{}, system_lib_names: []const []const u8 = &.{}, system_lib_infos: []const SystemLib = &.{}, /// These correspond to the WASI libc emulated subcomponents including: @@ -644,8 +645,6 @@ pub const InitOptions = struct { entitlements: ?[]const u8 = null, /// (Darwin) size of the __PAGEZERO segment pagezero_size: ?u64 = null, - /// (Darwin) search strategy for system libraries - search_strategy: ?link.File.MachO.SearchStrategy = null, /// (Darwin) set minimum space for future expansion of the load commands headerpad_size: ?u32 = null, /// (Darwin) set enough space as if all paths were MATPATHLEN @@ -1567,7 +1566,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .install_name = options.install_name, .entitlements = options.entitlements, .pagezero_size = options.pagezero_size, - .search_strategy = options.search_strategy, .headerpad_size = options.headerpad_size, .headerpad_max_install_names = options.headerpad_max_install_names, .dead_strip_dylibs = options.dead_strip_dylibs, @@ -1727,15 +1725,18 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { // When linking mingw-w64 there are some import libs we always need. for (mingw.always_link_libs) |name| { - try comp.bin_file.options.system_libs.put(comp.gpa, name, .{}); + try comp.bin_file.options.system_libs.put(comp.gpa, name, .{ + .needed = false, + .weak = false, + .path = name, + }); } } // Generate Windows import libs. if (target.os.tag == .windows) { const count = comp.bin_file.options.system_libs.count(); try comp.work_queue.ensureUnusedCapacity(count); - var i: usize = 0; - while (i < count) : (i += 1) { + for (0..count) |i| { comp.work_queue.writeItemAssumeCapacity(.{ .windows_import_lib = i }); } } @@ -2377,7 +2378,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes } man.hash.addOptionalBytes(comp.bin_file.options.soname); man.hash.addOptional(comp.bin_file.options.version); - link.hashAddSystemLibs(&man.hash, comp.bin_file.options.system_libs); + try link.hashAddSystemLibs(man, comp.bin_file.options.system_libs); man.hash.addListOfBytes(comp.bin_file.options.force_undefined_symbols.keys()); man.hash.addOptional(comp.bin_file.options.allow_shlib_undefined); man.hash.add(comp.bin_file.options.bind_global_refs_locally); @@ -2395,10 +2396,9 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes // Mach-O specific stuff man.hash.addListOfBytes(comp.bin_file.options.framework_dirs); - link.hashAddSystemLibs(&man.hash, comp.bin_file.options.frameworks); + link.hashAddFrameworks(&man.hash, comp.bin_file.options.frameworks); try man.addOptionalFile(comp.bin_file.options.entitlements); man.hash.addOptional(comp.bin_file.options.pagezero_size); - man.hash.addOptional(comp.bin_file.options.search_strategy); man.hash.addOptional(comp.bin_file.options.headerpad_size); man.hash.add(comp.bin_file.options.headerpad_max_install_names); man.hash.add(comp.bin_file.options.dead_strip_dylibs); diff --git a/src/link.zig b/src/link.zig index 42322cbda88221d8af195b6eb310201c5e82825f..9edba501235983cc50c7dc7de0c83683d35717ae 100644 --- a/src/link.zig +++ b/src/link.zig @@ -21,7 +21,16 @@ const Type = @import("type.zig").Type; const TypedValue = @import("TypedValue.zig"); /// When adding a new field, remember to update `hashAddSystemLibs`. +/// These are *always* dynamically linked. Static libraries will be +/// provided as positional arguments. pub const SystemLib = struct { + needed: bool, + weak: bool, + path: []const u8, +}; + +/// When adding a new field, remember to update `hashAddFrameworks`. +pub const Framework = struct { needed: bool = false, weak: bool = false, }; @@ -31,11 +40,23 @@ pub const SortSection = enum { name, alignment }; pub const CacheMode = enum { incremental, whole }; pub fn hashAddSystemLibs( - hh: *Cache.HashHelper, + man: *Cache.Manifest, hm: std.StringArrayHashMapUnmanaged(SystemLib), +) !void { + const keys = hm.keys(); + man.hash.addListOfBytes(keys); + for (hm.values()) |value| { + man.hash.add(value.needed); + man.hash.add(value.weak); + _ = try man.addFile(value.path, null); + } +} + +pub fn hashAddFrameworks( + hh: *Cache.HashHelper, + hm: std.StringArrayHashMapUnmanaged(Framework), ) void { const keys = hm.keys(); - hh.add(keys.len); hh.addListOfBytes(keys); for (hm.values()) |value| { hh.add(value.needed); @@ -183,9 +204,12 @@ pub const Options = struct { objects: []Compilation.LinkObject, framework_dirs: []const []const u8, - frameworks: std.StringArrayHashMapUnmanaged(SystemLib), + frameworks: std.StringArrayHashMapUnmanaged(Framework), + /// These are *always* dynamically linked. Static libraries will be + /// provided as positional arguments. system_libs: std.StringArrayHashMapUnmanaged(SystemLib), wasi_emulated_libs: []const wasi_libc.CRTFile, + // TODO: remove this. libraries are resolved by the frontend. lib_dirs: []const []const u8, rpath_list: []const []const u8, @@ -225,9 +249,6 @@ pub const Options = struct { /// (Darwin) size of the __PAGEZERO segment pagezero_size: ?u64 = null, - /// (Darwin) search strategy for system libraries - search_strategy: ?File.MachO.SearchStrategy = null, - /// (Darwin) set minimum space for future expansion of the load commands headerpad_size: ?u32 = null, diff --git a/src/link/Coff/lld.zig b/src/link/Coff/lld.zig index 800b128081339f9ad99dd30e1d075c2404e976c3..fe2055027f8b8064deee59dcc3db34765dfaa165 100644 --- a/src/link/Coff/lld.zig +++ b/src/link/Coff/lld.zig @@ -88,7 +88,7 @@ pub fn linkWithLLD(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod } } } - link.hashAddSystemLibs(&man.hash, self.base.options.system_libs); + try link.hashAddSystemLibs(&man, self.base.options.system_libs); man.hash.addListOfBytes(self.base.options.force_undefined_symbols.keys()); man.hash.addOptional(self.base.options.subsystem); man.hash.add(self.base.options.is_test); diff --git a/src/link/Elf.zig b/src/link/Elf.zig index c4371fbcbff68d005d5e7a8f940fa44d318f7798..3292168bce8493cd73cdd5f51c0c85a966b69189 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1428,7 +1428,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v } man.hash.addOptionalBytes(self.base.options.soname); man.hash.addOptional(self.base.options.version); - link.hashAddSystemLibs(&man.hash, self.base.options.system_libs); + try link.hashAddSystemLibs(&man, self.base.options.system_libs); man.hash.addListOfBytes(self.base.options.force_undefined_symbols.keys()); man.hash.add(allow_shlib_undefined); man.hash.add(self.base.options.bind_global_refs_locally); @@ -1824,8 +1824,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v argv.appendAssumeCapacity("--as-needed"); var as_needed = true; - for (system_libs, 0..) |link_lib, i| { - const lib_as_needed = !system_libs_values[i].needed; + for (system_libs_values) |lib_info| { + const lib_as_needed = !lib_info.needed; switch ((@as(u2, @intFromBool(lib_as_needed)) << 1) | @intFromBool(as_needed)) { 0b00, 0b11 => {}, 0b01 => { @@ -1842,9 +1842,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v // libraries and not static libraries (the check for that needs to be earlier), // but they could be full paths to .so files, in which case we // want to avoid prepending "-l". - const ext = Compilation.classifyFileExt(link_lib); - const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{s}", .{link_lib}); - argv.appendAssumeCapacity(arg); + argv.appendAssumeCapacity(lib_info.path); } if (!as_needed) { diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 44e6acc3976d5abfa4c8405001db1ea53fe64cf8..389d275fd57e07acb2d2f1ad84e44204c6d7a902 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -58,11 +58,6 @@ const Rebase = @import("MachO/dyld_info/Rebase.zig"); pub const base_tag: File.Tag = File.Tag.macho; -pub const SearchStrategy = enum { - paths_first, - dylibs_first, -}; - /// Mode of operation of the linker. pub const Mode = enum { /// Incremental mode will preallocate segments/sections and is compatible with @@ -840,7 +835,11 @@ pub fn resolveLibSystem( // re-exports every single symbol definition. for (search_dirs) |dir| { if (try resolveLib(arena, dir, "System", ".tbd")) |full_path| { - try out_libs.put(full_path, .{ .needed = true }); + try out_libs.put(full_path, .{ + .needed = true, + .weak = false, + .path = full_path, + }); libsystem_available = true; break :blk; } @@ -850,8 +849,16 @@ pub fn resolveLibSystem( for (search_dirs) |dir| { if (try resolveLib(arena, dir, "System", ".dylib")) |libsystem_path| { if (try resolveLib(arena, dir, "c", ".dylib")) |libc_path| { - try out_libs.put(libsystem_path, .{ .needed = true }); - try out_libs.put(libc_path, .{ .needed = true }); + try out_libs.put(libsystem_path, .{ + .needed = true, + .weak = false, + .path = libsystem_path, + }); + try out_libs.put(libc_path, .{ + .needed = true, + .weak = false, + .path = libc_path, + }); libsystem_available = true; break :blk; } @@ -865,7 +872,11 @@ pub fn resolveLibSystem( const full_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "darwin", libsystem_name, }); - try out_libs.put(full_path, .{ .needed = true }); + try out_libs.put(full_path, .{ + .needed = true, + .weak = false, + .path = full_path, + }); } } diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index 39b64fe1787d39d3cb1ca2ae970aae0ead00e913..c2a814b319db3cd53dcf71d73d5a556ab5225d37 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -3410,7 +3410,6 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr // installation sources because they are always a product of the compiler version + target information. man.hash.add(stack_size); man.hash.addOptional(options.pagezero_size); - man.hash.addOptional(options.search_strategy); man.hash.addOptional(options.headerpad_size); man.hash.add(options.headerpad_max_install_names); man.hash.add(gc_sections); @@ -3418,13 +3417,13 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr man.hash.add(options.strip); man.hash.addListOfBytes(options.lib_dirs); man.hash.addListOfBytes(options.framework_dirs); - link.hashAddSystemLibs(&man.hash, options.frameworks); + link.hashAddFrameworks(&man.hash, options.frameworks); man.hash.addListOfBytes(options.rpath_list); if (is_dyn_lib) { man.hash.addOptionalBytes(options.install_name); man.hash.addOptional(options.version); } - link.hashAddSystemLibs(&man.hash, options.system_libs); + try link.hashAddSystemLibs(&man, options.system_libs); man.hash.addOptionalBytes(options.sysroot); man.hash.addListOfBytes(options.force_undefined_symbols.keys()); try man.addOptionalFile(options.entitlements); @@ -3550,84 +3549,20 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try positionals.append(comp.libcxx_static_lib.?.full_object_path); } - // Shared and static libraries passed via `-l` flag. - var candidate_libs = std.StringArrayHashMap(link.SystemLib).init(arena); - - const system_lib_names = options.system_libs.keys(); - for (system_lib_names) |system_lib_name| { - // By this time, we depend on these libs being dynamically linked libraries and not static libraries - // (the check for that needs to be earlier), but they could be full paths to .dylib files, in which - // case we want to avoid prepending "-l". - if (Compilation.classifyFileExt(system_lib_name) == .shared_library) { - try positionals.append(system_lib_name); - continue; - } - - const system_lib_info = options.system_libs.get(system_lib_name).?; - try candidate_libs.put(system_lib_name, .{ - .needed = system_lib_info.needed, - .weak = system_lib_info.weak, - }); - } - - var lib_dirs = std.ArrayList([]const u8).init(arena); - for (options.lib_dirs) |dir| { - if (try MachO.resolveSearchDir(arena, dir, options.sysroot)) |search_dir| { - try lib_dirs.append(search_dir); - } else { - log.warn("directory not found for '-L{s}'", .{dir}); - } + { + // Add all system library paths to positionals. + const vals = options.system_libs.values(); + try positionals.ensureUnusedCapacity(vals.len); + for (vals) |info| positionals.appendAssumeCapacity(info.path); } var libs = std.StringArrayHashMap(link.SystemLib).init(arena); - // Assume ld64 default -search_paths_first if no strategy specified. - const search_strategy = options.search_strategy orelse .paths_first; - outer: for (candidate_libs.keys()) |lib_name| { - switch (search_strategy) { - .paths_first => { - // Look in each directory for a dylib (stub first), and then for archive - for (lib_dirs.items) |dir| { - for (&[_][]const u8{ ".tbd", ".dylib", ".a" }) |ext| { - if (try MachO.resolveLib(arena, dir, lib_name, ext)) |full_path| { - try libs.put(full_path, candidate_libs.get(lib_name).?); - continue :outer; - } - } - } else { - log.warn("library not found for '-l{s}'", .{lib_name}); - lib_not_found = true; - } - }, - .dylibs_first => { - // First, look for a dylib in each search dir - for (lib_dirs.items) |dir| { - for (&[_][]const u8{ ".tbd", ".dylib" }) |ext| { - if (try MachO.resolveLib(arena, dir, lib_name, ext)) |full_path| { - try libs.put(full_path, candidate_libs.get(lib_name).?); - continue :outer; - } - } - } else for (lib_dirs.items) |dir| { - if (try MachO.resolveLib(arena, dir, lib_name, ".a")) |full_path| { - try libs.put(full_path, candidate_libs.get(lib_name).?); - } else { - log.warn("library not found for '-l{s}'", .{lib_name}); - lib_not_found = true; - } - } - }, - } + for (options.system_libs.values()) |v| { + try libs.put(v.path, v); } - if (lib_not_found) { - log.warn("Library search paths:", .{}); - for (lib_dirs.items) |dir| { - log.warn(" {s}", .{dir}); - } - } - - try MachO.resolveLibSystem(arena, comp, options.sysroot, target, lib_dirs.items, &libs); + try MachO.resolveLibSystem(arena, comp, options.sysroot, target, options.lib_dirs, &libs); // frameworks var framework_dirs = std.ArrayList([]const u8).init(arena); @@ -3647,6 +3582,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try libs.put(full_path, .{ .needed = info.needed, .weak = info.weak, + .path = full_path, }); continue :outer; } @@ -3698,11 +3634,6 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try argv.append(try std.fmt.allocPrint(arena, "0x{x}", .{pagezero_size})); } - if (options.search_strategy) |strat| switch (strat) { - .paths_first => try argv.append("-search_paths_first"), - .dylibs_first => try argv.append("-search_dylibs_first"), - }; - if (options.headerpad_size) |headerpad_size| { try argv.append("-headerpad_size"); try argv.append(try std.fmt.allocPrint(arena, "0x{x}", .{headerpad_size})); diff --git a/src/main.zig b/src/main.zig index c29c35ede49d14e4a97ebeed30f54dfe8c175fda..83e1e2bc387845a31fdce292c91cc076b4c16ced 100644 --- a/src/main.zig +++ b/src/main.zig @@ -527,11 +527,11 @@ const usage_build_generic = \\ --subsystem [subsystem] (Windows) /SUBSYSTEM: to the linker \\ --stack [size] Override default stack size \\ --image-base [addr] Set base address for executable image - \\ -weak-l[lib] (Darwin) link against system library and mark it and all referenced symbols as weak + \\ -weak-l[lib] link against system library and mark it and all referenced symbols as weak \\ -weak_library [lib] \\ -framework [name] (Darwin) link against framework \\ -needed_framework [name] (Darwin) link against framework (even if unused) - \\ -needed_library [lib] (Darwin) link against system library (even if unused) + \\ -needed_library [lib] link against system library (even if unused) \\ -weak_framework [name] (Darwin) link against framework and mark it and all referenced symbols as weak \\ -F[dir] (Darwin) add search path for frameworks \\ -install_name=[value] (Darwin) add dylib's install name @@ -716,6 +716,27 @@ const ArgsIterator = struct { } }; +/// In contrast to `link.SystemLib`, this stores arguments that may need to be +/// resolved into static libraries so that we can pass only dynamic libraries +/// as system libs to `Compilation`. +const SystemLib = struct { + needed: bool, + weak: bool, + + preferred_mode: std.builtin.LinkMode, + search_strategy: SearchStrategy, + + const SearchStrategy = enum { paths_first, mode_first, no_fallback }; + + fn fallbackMode(this: SystemLib) std.builtin.LinkMode { + assert(this.search_strategy != .no_fallback); + return switch (this.preferred_mode) { + .Dynamic => .Static, + .Static => .Dynamic, + }; + } +}; + fn buildOutputType( gpa: Allocator, arena: Allocator, @@ -854,7 +875,8 @@ fn buildOutputType( var hash_style: link.HashStyle = .both; var entitlements: ?[]const u8 = null; var pagezero_size: ?u64 = null; - var search_strategy: ?link.File.MachO.SearchStrategy = null; + var lib_search_strategy: ?SystemLib.SearchStrategy = null; + var lib_preferred_mode: ?std.builtin.LinkMode = null; var headerpad_size: ?u32 = null; var headerpad_max_install_names: bool = false; var dead_strip_dylibs: bool = false; @@ -869,11 +891,7 @@ fn buildOutputType( var llvm_m_args = std.ArrayList([]const u8).init(gpa); defer llvm_m_args.deinit(); - var system_libs = std.StringArrayHashMap(Compilation.SystemLib).init(gpa); - defer system_libs.deinit(); - - var static_libs = std.ArrayList([]const u8).init(gpa); - defer static_libs.deinit(); + var system_libs = std.StringArrayHashMap(SystemLib).init(arena); var wasi_emulated_libs = std.ArrayList(wasi_libc.CRTFile).init(gpa); defer wasi_emulated_libs.deinit(); @@ -884,8 +902,8 @@ fn buildOutputType( var extra_cflags = std.ArrayList([]const u8).init(gpa); defer extra_cflags.deinit(); - var lib_dirs = std.ArrayList([]const u8).init(gpa); - defer lib_dirs.deinit(); + // These are before resolving sysroot. + var lib_dir_args = std.ArrayList([]const u8).init(arena); var rpath_list = std.ArrayList([]const u8).init(gpa); defer rpath_list.deinit(); @@ -901,7 +919,7 @@ fn buildOutputType( var framework_dirs = std.ArrayList([]const u8).init(gpa); defer framework_dirs.deinit(); - var frameworks: std.StringArrayHashMapUnmanaged(Compilation.SystemLib) = .{}; + var frameworks: std.StringArrayHashMapUnmanaged(Compilation.Framework) = .{}; // null means replace with the test executable binary var test_exec_args = std.ArrayList(?[]const u8).init(gpa); @@ -1061,7 +1079,7 @@ fn buildOutputType( } else if (mem.eql(u8, arg, "-rpath")) { try rpath_list.append(args_iter.nextOrFatal()); } else if (mem.eql(u8, arg, "--library-directory") or mem.eql(u8, arg, "-L")) { - try lib_dirs.append(args_iter.nextOrFatal()); + try lib_dir_args.append(args_iter.nextOrFatal()); } else if (mem.eql(u8, arg, "-F")) { try framework_dirs.append(args_iter.nextOrFatal()); } else if (mem.eql(u8, arg, "-framework")) { @@ -1085,9 +1103,11 @@ fn buildOutputType( fatal("unable to parse pagezero size'{s}': {s}", .{ next_arg, @errorName(err) }); }; } else if (mem.eql(u8, arg, "-search_paths_first")) { - search_strategy = .paths_first; + lib_search_strategy = .paths_first; + lib_preferred_mode = .Dynamic; } else if (mem.eql(u8, arg, "-search_dylibs_first")) { - search_strategy = .dylibs_first; + lib_search_strategy = .mode_first; + lib_preferred_mode = .Dynamic; } else if (mem.eql(u8, arg, "-headerpad")) { const next_arg = args_iter.nextOrFatal(); headerpad_size = std.fmt.parseUnsigned(u32, eatIntPrefix(next_arg, 16), 16) catch |err| { @@ -1104,17 +1124,36 @@ fn buildOutputType( } else if (mem.eql(u8, arg, "-version-script") or mem.eql(u8, arg, "--version-script")) { version_script = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "--library") or mem.eql(u8, arg, "-l")) { - // We don't know whether this library is part of libc or libc++ until - // we resolve the target, so we simply append to the list for now. - try system_libs.put(args_iter.nextOrFatal(), .{}); + // We don't know whether this library is part of libc + // or libc++ until we resolve the target, so we append + // to the list for now. + try system_libs.put(args_iter.nextOrFatal(), .{ + .needed = false, + .weak = false, + // -l always dynamic links. For static libraries, + // users are expected to use positional arguments + // which are always unambiguous. + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.eql(u8, arg, "--needed-library") or mem.eql(u8, arg, "-needed-l") or mem.eql(u8, arg, "-needed_library")) { const next_arg = args_iter.nextOrFatal(); - try system_libs.put(next_arg, .{ .needed = true }); + try system_libs.put(next_arg, .{ + .needed = true, + .weak = false, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.eql(u8, arg, "-weak_library") or mem.eql(u8, arg, "-weak-l")) { - try system_libs.put(args_iter.nextOrFatal(), .{ .weak = true }); + try system_libs.put(args_iter.nextOrFatal(), .{ + .needed = false, + .weak = true, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.eql(u8, arg, "-D")) { try clang_argv.append(arg); try clang_argv.append(args_iter.nextOrFatal()); @@ -1486,17 +1525,36 @@ fn buildOutputType( } else if (mem.startsWith(u8, arg, "-T")) { linker_script = arg[2..]; } else if (mem.startsWith(u8, arg, "-L")) { - try lib_dirs.append(arg[2..]); + try lib_dir_args.append(arg[2..]); } else if (mem.startsWith(u8, arg, "-F")) { try framework_dirs.append(arg[2..]); } else if (mem.startsWith(u8, arg, "-l")) { - // We don't know whether this library is part of libc or libc++ until - // we resolve the target, so we simply append to the list for now. - try system_libs.put(arg["-l".len..], .{}); + // We don't know whether this library is part of libc + // or libc++ until we resolve the target, so we append + // to the list for now. + try system_libs.put(arg["-l".len..], .{ + .needed = false, + .weak = false, + // -l always dynamic links. For static libraries, + // users are expected to use positional arguments + // which are always unambiguous. + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.startsWith(u8, arg, "-needed-l")) { - try system_libs.put(arg["-needed-l".len..], .{ .needed = true }); + try system_libs.put(arg["-needed-l".len..], .{ + .needed = true, + .weak = false, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.startsWith(u8, arg, "-weak-l")) { - try system_libs.put(arg["-weak-l".len..], .{ .weak = true }); + try system_libs.put(arg["-weak-l".len..], .{ + .needed = false, + .weak = true, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.startsWith(u8, arg, "-D")) { try clang_argv.append(arg); } else if (mem.startsWith(u8, arg, "-I")) { @@ -1642,9 +1700,22 @@ fn buildOutputType( .loption = true, }); } else if (force_static_libs) { - try static_libs.append(it.only_arg); + try system_libs.put(it.only_arg, .{ + .needed = false, + .weak = false, + .preferred_mode = .Static, + .search_strategy = .no_fallback, + }); } else { - try system_libs.put(it.only_arg, .{ .needed = needed }); + // C compilers are traditionally expected to look + // first for dynamic libraries and then fall back + // to static libraries. + try system_libs.put(it.only_arg, .{ + .needed = needed, + .weak = false, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .paths_first, + }); } }, .ignore => {}, @@ -1748,9 +1819,11 @@ fn buildOutputType( { force_static_libs = true; } else if (mem.eql(u8, linker_arg, "-search_paths_first")) { - search_strategy = .paths_first; + lib_search_strategy = .paths_first; + lib_preferred_mode = .Dynamic; } else if (mem.eql(u8, linker_arg, "-search_dylibs_first")) { - search_strategy = .dylibs_first; + lib_search_strategy = .mode_first; + lib_preferred_mode = .Dynamic; } else { try linker_args.append(linker_arg); } @@ -1828,7 +1901,7 @@ fn buildOutputType( try linker_args.append("-z"); try linker_args.append(it.only_arg); }, - .lib_dir => try lib_dirs.append(it.only_arg), + .lib_dir => try lib_dir_args.append(it.only_arg), .mcpu => target_mcpu = it.only_arg, .m => try llvm_m_args.append(it.only_arg), .dep_file => { @@ -1860,7 +1933,12 @@ fn buildOutputType( .force_undefined_symbol => { try force_undefined_symbols.put(gpa, it.only_arg, {}); }, - .weak_library => try system_libs.put(it.only_arg, .{ .weak = true }), + .weak_library => try system_libs.put(it.only_arg, .{ + .needed = false, + .weak = true, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .paths_first, + }), .weak_framework => try frameworks.put(gpa, it.only_arg, .{ .weak = true }), .headerpad_max_install_names => headerpad_max_install_names = true, .compress_debug_sections => { @@ -2156,11 +2234,26 @@ fn buildOutputType( } else if (mem.eql(u8, arg, "-needed_framework")) { try frameworks.put(gpa, linker_args_it.nextOrFatal(), .{ .needed = true }); } else if (mem.eql(u8, arg, "-needed_library")) { - try system_libs.put(linker_args_it.nextOrFatal(), .{ .needed = true }); + try system_libs.put(linker_args_it.nextOrFatal(), .{ + .weak = false, + .needed = true, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .paths_first, + }); } else if (mem.startsWith(u8, arg, "-weak-l")) { - try system_libs.put(arg["-weak-l".len..], .{ .weak = true }); + try system_libs.put(arg["-weak-l".len..], .{ + .weak = true, + .needed = false, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .paths_first, + }); } else if (mem.eql(u8, arg, "-weak_library")) { - try system_libs.put(linker_args_it.nextOrFatal(), .{ .weak = true }); + try system_libs.put(linker_args_it.nextOrFatal(), .{ + .weak = true, + .needed = false, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .paths_first, + }); } else if (mem.eql(u8, arg, "-compatibility_version")) { const compat_version = linker_args_it.nextOrFatal(); compatibility_version = std.SemanticVersion.parse(compat_version) catch |err| { @@ -2458,40 +2551,63 @@ fn buildOutputType( } } + // Resolve the library path arguments with respect to sysroot. + var lib_dirs = std.ArrayList([]const u8).init(arena); + if (sysroot) |root| { + for (lib_dir_args.items) |dir| { + if (fs.path.isAbsolute(dir)) { + const stripped_dir = dir[fs.path.diskDesignator(dir).len..]; + const full_path = try fs.path.join(arena, &[_][]const u8{ root, stripped_dir }); + try lib_dirs.append(full_path); + } + try lib_dirs.append(dir); + } + } else { + lib_dirs = lib_dir_args; + } + lib_dir_args = undefined; // From here we use lib_dirs instead. + // Now that we have target info, we can find out if any of the system libraries // are part of libc or libc++. We remove them from the list and communicate their // existence via flags instead. + // Similarly, if any libs in this list are statically provided, we omit + // them from the resolved list and populate the link_objects array instead. + var resolved_system_libs: std.MultiArrayList(struct { + name: []const u8, + lib: Compilation.SystemLib, + }) = .{}; + { - // Similarly, if any libs in this list are statically provided, we remove - // them from this list and populate the link_objects array instead. - const sep = fs.path.sep_str; var test_path = std.ArrayList(u8).init(gpa); defer test_path.deinit(); - var i: usize = 0; - syslib: while (i < system_libs.count()) { - const lib_name = system_libs.keys()[i]; + var checked_paths = std.ArrayList(u8).init(gpa); + defer checked_paths.deinit(); + var failed_libs = std.ArrayList(struct { + name: []const u8, + strategy: SystemLib.SearchStrategy, + checked_paths: []const u8, + preferred_mode: std.builtin.LinkMode, + }).init(arena); + + syslib: for (system_libs.keys(), system_libs.values()) |lib_name, info| { if (target_util.is_libc_lib_name(target_info.target, lib_name)) { link_libc = true; - system_libs.orderedRemoveAt(i); continue; } if (target_util.is_libcpp_lib_name(target_info.target, lib_name)) { link_libcpp = true; - system_libs.orderedRemoveAt(i); continue; } switch (target_util.classifyCompilerRtLibName(target_info.target, lib_name)) { .none => {}, .only_libunwind, .both => { link_libunwind = true; - system_libs.orderedRemoveAt(i); continue; }, .only_compiler_rt => { std.log.warn("ignoring superfluous library '{s}': this dependency is fulfilled instead by compiler-rt which zig unconditionally provides", .{lib_name}); - system_libs.orderedRemoveAt(i); continue; }, } @@ -2503,53 +2619,150 @@ fn buildOutputType( if (target_info.target.os.tag == .wasi) { if (wasi_libc.getEmulatedLibCRTFile(lib_name)) |crt_file| { try wasi_emulated_libs.append(crt_file); - system_libs.orderedRemoveAt(i); continue; } } - for (lib_dirs.items) |lib_dir_path| { - if (cross_target.isDarwin()) break; // Targeting Darwin we let the linker resolve the libraries in the correct order - test_path.clearRetainingCapacity(); - try test_path.writer().print("{s}" ++ sep ++ "{s}{s}{s}", .{ - lib_dir_path, - target_info.target.libPrefix(), - lib_name, - target_info.target.staticLibSuffix(), + checked_paths.clearRetainingCapacity(); + + switch (info.search_strategy) { + .mode_first, .no_fallback => { + // check for preferred mode + for (lib_dirs.items) |lib_dir_path| { + if (try accessLibPath( + &test_path, + &checked_paths, + lib_dir_path, + lib_name, + target_info.target, + info.preferred_mode, + )) { + const path = try arena.dupe(u8, test_path.items); + switch (info.preferred_mode) { + .Static => try link_objects.append(.{ .path = path }), + .Dynamic => try resolved_system_libs.append(arena, .{ + .name = lib_name, + .lib = .{ + .needed = info.needed, + .weak = info.weak, + .path = path, + }, + }), + } + continue :syslib; + } + } + // check for fallback mode + if (info.search_strategy == .no_fallback) { + try failed_libs.append(.{ + .name = lib_name, + .strategy = info.search_strategy, + .checked_paths = try arena.dupe(u8, checked_paths.items), + .preferred_mode = info.preferred_mode, + }); + continue :syslib; + } + for (lib_dirs.items) |lib_dir_path| { + if (try accessLibPath( + &test_path, + &checked_paths, + lib_dir_path, + lib_name, + target_info.target, + info.fallbackMode(), + )) { + const path = try arena.dupe(u8, test_path.items); + switch (info.preferred_mode) { + .Static => try link_objects.append(.{ .path = path }), + .Dynamic => try resolved_system_libs.append(arena, .{ + .name = lib_name, + .lib = .{ + .needed = info.needed, + .weak = info.weak, + .path = path, + }, + }), + } + continue :syslib; + } + } + try failed_libs.append(.{ + .name = lib_name, + .strategy = info.search_strategy, + .checked_paths = try arena.dupe(u8, checked_paths.items), + .preferred_mode = info.preferred_mode, + }); + continue :syslib; + }, + .paths_first => { + for (lib_dirs.items) |lib_dir_path| { + // check for preferred mode + if (try accessLibPath( + &test_path, + &checked_paths, + lib_dir_path, + lib_name, + target_info.target, + info.preferred_mode, + )) { + const path = try arena.dupe(u8, test_path.items); + switch (info.preferred_mode) { + .Static => try link_objects.append(.{ .path = path }), + .Dynamic => try resolved_system_libs.append(arena, .{ + .name = lib_name, + .lib = .{ + .needed = info.needed, + .weak = info.weak, + .path = path, + }, + }), + } + continue :syslib; + } + + // check for fallback mode + if (try accessLibPath( + &test_path, + &checked_paths, + lib_dir_path, + lib_name, + target_info.target, + info.fallbackMode(), + )) { + const path = try arena.dupe(u8, test_path.items); + switch (info.preferred_mode) { + .Static => try link_objects.append(.{ .path = path }), + .Dynamic => try resolved_system_libs.append(arena, .{ + .name = lib_name, + .lib = .{ + .needed = info.needed, + .weak = info.weak, + .path = path, + }, + }), + } + continue :syslib; + } + } + try failed_libs.append(.{ + .name = lib_name, + .strategy = info.search_strategy, + .checked_paths = try arena.dupe(u8, checked_paths.items), + .preferred_mode = info.preferred_mode, + }); + continue :syslib; + }, + } + @compileError("unreachable"); + } + + if (failed_libs.items.len > 0) { + for (failed_libs.items) |f| { + std.log.err("unable to find {s} system library '{s}' using strategy '{s}'. searched paths:{s}", .{ + @tagName(f.preferred_mode), f.name, @tagName(f.strategy), f.checked_paths, }); - fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { - error.FileNotFound => continue, - else => |e| fatal("unable to search for static library '{s}': {s}", .{ - test_path.items, @errorName(e), - }), - }; - try link_objects.append(.{ .path = try arena.dupe(u8, test_path.items) }); - system_libs.orderedRemoveAt(i); - continue :syslib; } - - // Unfortunately, in the case of MinGW we also need to look for `libfoo.a`. - if (target_info.target.isMinGW()) { - for (lib_dirs.items) |lib_dir_path| { - test_path.clearRetainingCapacity(); - try test_path.writer().print("{s}" ++ sep ++ "lib{s}.a", .{ - lib_dir_path, lib_name, - }); - fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { - error.FileNotFound => continue, - else => |e| fatal("unable to search for static library '{s}': {s}", .{ - test_path.items, @errorName(e), - }), - }; - try link_objects.append(.{ .path = try arena.dupe(u8, test_path.items) }); - system_libs.orderedRemoveAt(i); - continue :syslib; - } - } - - std.log.scoped(.cli).debug("depending on system for -l{s}", .{lib_name}); - - i += 1; + process.exit(1); } } // libc++ depends on libc @@ -2576,7 +2789,7 @@ fn buildOutputType( } if (sysroot == null and cross_target.isNativeOs() and - (system_libs.count() != 0 or want_native_include_dirs)) + (resolved_system_libs.len != 0 or want_native_include_dirs)) { const paths = std.zig.system.NativePaths.detect(arena, target_info) catch |err| { fatal("unable to detect native system paths: {s}", .{@errorName(err)}); @@ -2613,54 +2826,8 @@ fn buildOutputType( framework_dirs.appendAssumeCapacity(framework_dir); } - for (paths.lib_dirs.items) |lib_dir| { - try lib_dirs.append(lib_dir); - } - for (paths.rpaths.items) |rpath| { - try rpath_list.append(rpath); - } - } - - { - // Resolve static libraries into full paths. - const sep = fs.path.sep_str; - - var test_path = std.ArrayList(u8).init(gpa); - defer test_path.deinit(); - - for (static_libs.items) |static_lib| { - for (lib_dirs.items) |lib_dir_path| { - test_path.clearRetainingCapacity(); - try test_path.writer().print("{s}" ++ sep ++ "{s}{s}{s}", .{ - lib_dir_path, - target_info.target.libPrefix(), - static_lib, - target_info.target.staticLibSuffix(), - }); - fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { - error.FileNotFound => continue, - else => |e| fatal("unable to search for static library '{s}': {s}", .{ - test_path.items, @errorName(e), - }), - }; - try link_objects.append(.{ .path = try arena.dupe(u8, test_path.items) }); - break; - } else { - var search_paths = std.ArrayList(u8).init(arena); - for (lib_dirs.items) |lib_dir_path| { - try search_paths.writer().print("\n {s}" ++ sep ++ "{s}{s}{s}", .{ - lib_dir_path, - target_info.target.libPrefix(), - static_lib, - target_info.target.staticLibSuffix(), - }); - } - try search_paths.appendSlice("\n suggestion: use full paths to static libraries on the command line rather than using -l and -L arguments"); - fatal("static library '{s}' not found. search paths: {s}", .{ - static_lib, search_paths.items, - }); - } - } + try lib_dirs.appendSlice(paths.lib_dirs.items); + try rpath_list.appendSlice(paths.rpaths.items); } const object_format = target_info.target.ofmt; @@ -3086,8 +3253,8 @@ fn buildOutputType( .link_objects = link_objects.items, .framework_dirs = framework_dirs.items, .frameworks = frameworks, - .system_lib_names = system_libs.keys(), - .system_lib_infos = system_libs.values(), + .system_lib_names = resolved_system_libs.items(.name), + .system_lib_infos = resolved_system_libs.items(.lib), .wasi_emulated_libs = wasi_emulated_libs.items, .link_libc = link_libc, .link_libcpp = link_libcpp, @@ -3196,7 +3363,6 @@ fn buildOutputType( .install_name = install_name, .entitlements = entitlements, .pagezero_size = pagezero_size, - .search_strategy = search_strategy, .headerpad_size = headerpad_size, .headerpad_max_install_names = headerpad_max_install_names, .dead_strip_dylibs = dead_strip_dylibs, @@ -6068,3 +6234,84 @@ fn renderOptions(color: Color) std.zig.ErrorBundle.RenderOptions { .include_reference_trace = ttyconf != .no_color, }; } + +fn accessLibPath( + test_path: *std.ArrayList(u8), + checked_paths: *std.ArrayList(u8), + lib_dir_path: []const u8, + lib_name: []const u8, + target: std.Target, + link_mode: std.builtin.LinkMode, +) !bool { + const sep = fs.path.sep_str; + + if (target.isDarwin() and link_mode == .Dynamic) tbd: { + // Prefer .tbd over .dylib. + test_path.clearRetainingCapacity(); + try test_path.writer().print("{s}" ++ sep ++ "lib{s}.tbd", .{ lib_dir_path, lib_name }); + try checked_paths.writer().print("\n {s}", .{test_path.items}); + fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { + error.FileNotFound => break :tbd, + else => |e| fatal("unable to search for tbd library '{s}': {s}", .{ + test_path.items, @errorName(e), + }), + }; + return true; + } + + main_check: { + test_path.clearRetainingCapacity(); + try test_path.writer().print("{s}" ++ sep ++ "{s}{s}{s}", .{ + lib_dir_path, + target.libPrefix(), + lib_name, + switch (link_mode) { + .Static => target.staticLibSuffix(), + .Dynamic => target.dynamicLibSuffix(), + }, + }); + try checked_paths.writer().print("\n {s}", .{test_path.items}); + fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { + error.FileNotFound => break :main_check, + else => |e| fatal("unable to search for {s} library '{s}': {s}", .{ + @tagName(link_mode), test_path.items, @errorName(e), + }), + }; + return true; + } + + // In the case of Darwin, the main check will be .dylib, so here we + // additionally check for .so files. + if (target.isDarwin() and link_mode == .Dynamic) so: { + // Prefer .tbd over .dylib. + test_path.clearRetainingCapacity(); + try test_path.writer().print("{s}" ++ sep ++ "lib{s}.so", .{ lib_dir_path, lib_name }); + try checked_paths.writer().print("\n {s}", .{test_path.items}); + fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { + error.FileNotFound => break :so, + else => |e| fatal("unable to search for tbd library '{s}': {s}", .{ + test_path.items, @errorName(e), + }), + }; + return true; + } + + // In the case of MinGW, the main check will be .lib but we also need to + // look for `libfoo.a`. + if (target.isMinGW() and link_mode == .Static) mingw: { + test_path.clearRetainingCapacity(); + try test_path.writer().print("{s}" ++ sep ++ "lib{s}.a", .{ + lib_dir_path, lib_name, + }); + try checked_paths.writer().print("\n {s}", .{test_path.items}); + fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { + error.FileNotFound => break :mingw, + else => |e| fatal("unable to search for static library '{s}': {s}", .{ + test_path.items, @errorName(e), + }), + }; + return true; + } + + return false; +} -- 2.54.0