authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2020-11-05 11:29:58+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2020-11-05 18:35:51+01:00
loge0e3ceac19c0de0f8b3e408f9d3728496bc051f7
tree676bca514c95c540837e42106cd8b9e9bb998531
parent17837affd22a6055c65a14252fa38610fdeabc3a

Re-enable system linker hack

It is now possible to force linking with system linker `ld` instead of the LLVM `lld` linker when building natively on the target. This can be done at each stage by specifying `--system-linker-hack` flag, and can be useful on platforms where `lld` fails to operate properly such as macOS 11 Big Sur on ARM64 where every binary/dylib is expected to be codesigned. Some example invocations for each stage of compilation of Zig toolchain: ``` cmake .. -DCMAKE_PREFIX_PATH=/path/to/llvm -DSYSTEM_LINKER_HACK=1 ``` ``` build/zig build test --system-linker-hack ``` ``` build/zig build --prefix $(pwd)/stage2 -Denable-llvm --system-linker-hack ``` ``` build/zig build-exe hello.zig --system-linker-hack ```

8 files changed, 173 insertions(+), 90 deletions(-)

CMakeLists.txt+18-7
......@@ -505,13 +505,24 @@ add_dependencies(zig zig_build_zig1)
505505
506506install(TARGETS zig DESTINATION bin)
507507
508set(ZIG_INSTALL_ARGS "build"
509 --override-lib-dir "${CMAKE_SOURCE_DIR}/lib"
510 "-Dlib-files-only"
511 --prefix "${CMAKE_INSTALL_PREFIX}"
512 "-Dconfig_h=${ZIG_CONFIG_H_OUT}"
513 install
514)
508if(NOT SYSTEM_LINKER_HACK)
509 set(ZIG_INSTALL_ARGS "build"
510 --override-lib-dir "${CMAKE_SOURCE_DIR}/lib"
511 "-Dlib-files-only"
512 --prefix "${CMAKE_INSTALL_PREFIX}"
513 "-Dconfig_h=${ZIG_CONFIG_H_OUT}"
514 install
515 )
516else()
517 set(ZIG_INSTALL_ARGS "build"
518 --override-lib-dir "${CMAKE_SOURCE_DIR}/lib"
519 "-Dlib-files-only"
520 --prefix "${CMAKE_INSTALL_PREFIX}"
521 "-Dconfig_h=${ZIG_CONFIG_H_OUT}"
522 --system-linker-hack
523 install
524 )
525endif()
515526
516527# CODE has no effect with Visual Studio build system generator, therefore
517528# when using Visual Studio build system generator we resort to running
lib/std/build.zig+3-4
......@@ -67,6 +67,7 @@ pub const Builder = struct {
6767 vcpkg_root: VcpkgRoot,
6868 pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null,
6969 args: ?[][]const u8 = null,
70 system_linker_hack: bool,
7071
7172 const PkgConfigError = error{
7273 PkgConfigCrashed,
......@@ -173,6 +174,7 @@ pub const Builder = struct {
173174 .install_path = undefined,
174175 .vcpkg_root = VcpkgRoot{ .Unattempted = {} },
175176 .args = null,
177 .system_linker_hack = false,
176178 };
177179 try self.top_level_steps.append(&self.install_tls);
178180 try self.top_level_steps.append(&self.uninstall_tls);
......@@ -2074,6 +2076,7 @@ pub const LibExeObjStep = struct {
20742076 if (builder.verbose_link or self.verbose_link) zig_args.append("--verbose-link") catch unreachable;
20752077 if (builder.verbose_cc or self.verbose_cc) zig_args.append("--verbose-cc") catch unreachable;
20762078 if (builder.verbose_llvm_cpu_features) zig_args.append("--verbose-llvm-cpu-features") catch unreachable;
2079 if (builder.system_linker_hack or self.system_linker_hack) zig_args.append("--system-linker-hack") catch unreachable;
20772080
20782081 if (self.emit_llvm_ir) try zig_args.append("-femit-llvm-ir");
20792082 if (self.emit_asm) try zig_args.append("-femit-asm");
......@@ -2283,10 +2286,6 @@ pub const LibExeObjStep = struct {
22832286 }
22842287 }
22852288
2286 if (self.system_linker_hack) {
2287 try zig_args.append("--system-linker-hack");
2288 }
2289
22902289 if (self.valgrind_support) |valgrind_support| {
22912290 if (valgrind_support) {
22922291 try zig_args.append("-fvalgrind");
lib/std/special/build_runner.zig+3
......@@ -112,6 +112,8 @@ pub fn main() !void {
112112 builder.verbose_cc = true;
113113 } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) {
114114 builder.verbose_llvm_cpu_features = true;
115 } else if (mem.eql(u8, arg, "--system-linker-hack")) {
116 builder.system_linker_hack = true;
115117 } else if (mem.eql(u8, arg, "--")) {
116118 builder.args = argsRest(args, arg_idx);
117119 break;
......@@ -213,6 +215,7 @@ fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void
213215 \\ --verbose-cimport Enable compiler debug output for C imports
214216 \\ --verbose-cc Enable compiler debug output for C compilation
215217 \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features
218 \\ --system-linker-hack Use system linker LD instead of LLD (requires LLVM enabled)
216219 \\
217220 );
218221}
src/Compilation.zig+3
......@@ -348,6 +348,8 @@ pub const InitOptions = struct {
348348 want_valgrind: ?bool = null,
349349 use_llvm: ?bool = null,
350350 use_lld: ?bool = null,
351 /// When set, this option will overwrite LLD with the system linker LD.
352 system_linker_hack: bool = false,
351353 use_clang: ?bool = null,
352354 rdynamic: bool = false,
353355 strip: bool = false,
......@@ -775,6 +777,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
775777 .optimize_mode = options.optimize_mode,
776778 .use_lld = use_lld,
777779 .use_llvm = use_llvm,
780 .system_linker_hack = options.system_linker_hack,
778781 .link_libc = link_libc,
779782 .link_libcpp = options.link_libcpp,
780783 .objects = options.link_objects,
src/link.zig+3
......@@ -57,6 +57,9 @@ pub const Options = struct {
5757 /// other objects.
5858 /// Otherwise (depending on `use_lld`) this link code directly outputs and updates the final binary.
5959 use_llvm: bool,
60 /// If this is true and `use_llvm` is true, this link code will use system linker `ld` instead of
61 /// the LLD.
62 system_linker_hack: bool,
6063 link_libc: bool,
6164 link_libcpp: bool,
6265 function_sections: bool,
src/link/Elf.zig+66-40
......@@ -1353,14 +1353,20 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
13531353 // Create an LLD command line and invoke it.
13541354 var argv = std.ArrayList([]const u8).init(self.base.allocator);
13551355 defer argv.deinit();
1356 // Even though we're calling LLD as a library it thinks the first argument is its own exe name.
1357 try argv.append("lld");
1356
1357 if (self.base.options.is_native_os and self.base.options.system_linker_hack) {
1358 try argv.append("ld");
1359 } else {
1360 // Even though we're calling LLD as a library it thinks the first argument is its own exe name.
1361 try argv.append("lld");
1362
1363 try argv.append("-error-limit=0");
1364 }
1365
13581366 if (is_obj) {
13591367 try argv.append("-r");
13601368 }
13611369
1362 try argv.append("-error-limit=0");
1363
13641370 if (self.base.options.output_mode == .Exe) {
13651371 try argv.append("-z");
13661372 try argv.append(try std.fmt.allocPrint(arena, "stack-size={}", .{stack_size}));
......@@ -1616,43 +1622,63 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
16161622 Compilation.dump_argv(argv.items);
16171623 }
16181624
1619 // Oh, snapplesauce! We need null terminated argv.
1620 const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
1621 for (argv.items) |arg, i| {
1622 new_argv[i] = try arena.dupeZ(u8, arg);
1623 }
1625 if (self.base.options.is_native_os and self.base.options.system_linker_hack) {
1626 const result = try std.ChildProcess.exec(.{ .allocator = self.base.allocator, .argv = argv.items });
1627 defer {
1628 self.base.allocator.free(result.stdout);
1629 self.base.allocator.free(result.stderr);
1630 }
1631 if (result.stdout.len != 0) {
1632 std.log.warn("unexpected LD stdout: {}", .{result.stdout});
1633 }
1634 if (result.stderr.len != 0) {
1635 std.log.warn("unexpected LD stderr: {}", .{result.stderr});
1636 }
1637 if (result.term.Exited != 0) {
1638 // TODO parse this output and surface with the Compilation API rather than
1639 // directly outputting to stderr here.
1640 std.debug.print("{}", .{result.stderr});
1641 return error.LDReportedFailure;
1642 }
1643 } else {
1644 // Oh, snapplesauce! We need null terminated argv.
1645 const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
1646 for (argv.items) |arg, i| {
1647 new_argv[i] = try arena.dupeZ(u8, arg);
1648 }
16241649
1625 var stderr_context: LLDContext = .{
1626 .elf = self,
1627 .data = std.ArrayList(u8).init(self.base.allocator),
1628 };
1629 defer stderr_context.data.deinit();
1630 var stdout_context: LLDContext = .{
1631 .elf = self,
1632 .data = std.ArrayList(u8).init(self.base.allocator),
1633 };
1634 defer stdout_context.data.deinit();
1635 const llvm = @import("../llvm.zig");
1636 const ok = llvm.Link(
1637 .ELF,
1638 new_argv.ptr,
1639 new_argv.len,
1640 append_diagnostic,
1641 @ptrToInt(&stdout_context),
1642 @ptrToInt(&stderr_context),
1643 );
1644 if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory;
1645 if (stdout_context.data.items.len != 0) {
1646 std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items});
1647 }
1648 if (!ok) {
1649 // TODO parse this output and surface with the Compilation API rather than
1650 // directly outputting to stderr here.
1651 std.debug.print("{}", .{stderr_context.data.items});
1652 return error.LLDReportedFailure;
1653 }
1654 if (stderr_context.data.items.len != 0) {
1655 std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items});
1650 var stderr_context: LLDContext = .{
1651 .elf = self,
1652 .data = std.ArrayList(u8).init(self.base.allocator),
1653 };
1654 defer stderr_context.data.deinit();
1655 var stdout_context: LLDContext = .{
1656 .elf = self,
1657 .data = std.ArrayList(u8).init(self.base.allocator),
1658 };
1659 defer stdout_context.data.deinit();
1660 const llvm = @import("../llvm.zig");
1661 const ok = llvm.Link(
1662 .ELF,
1663 new_argv.ptr,
1664 new_argv.len,
1665 append_diagnostic,
1666 @ptrToInt(&stdout_context),
1667 @ptrToInt(&stderr_context),
1668 );
1669 if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory;
1670 if (stdout_context.data.items.len != 0) {
1671 std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items});
1672 }
1673 if (!ok) {
1674 // TODO parse this output and surface with the Compilation API rather than
1675 // directly outputting to stderr here.
1676 std.debug.print("{}", .{stderr_context.data.items});
1677 return error.LLDReportedFailure;
1678 }
1679 if (stderr_context.data.items.len != 0) {
1680 std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items});
1681 }
16561682 }
16571683
16581684 if (!self.base.options.disable_lld_caching) {
src/link/MachO.zig+66-39
......@@ -532,11 +532,17 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
532532 // Create an LLD command line and invoke it.
533533 var argv = std.ArrayList([]const u8).init(self.base.allocator);
534534 defer argv.deinit();
535 // Even though we're calling LLD as a library it thinks the first argument is its own exe name.
536 try argv.append("lld");
537535
538 try argv.append("-error-limit");
539 try argv.append("0");
536 // TODO https://github.com/ziglang/zig/issues/6971
537 if (self.base.options.is_native_os and self.base.options.system_linker_hack) {
538 try argv.append("ld");
539 } else {
540 // Even though we're calling LLD as a library it thinks the first argument is its own exe name.
541 try argv.append("lld");
542
543 try argv.append("-error-limit");
544 try argv.append("0");
545 }
540546
541547 try argv.append("-demangle");
542548
......@@ -703,42 +709,63 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
703709 Compilation.dump_argv(argv.items);
704710 }
705711
706 const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
707 for (argv.items) |arg, i| {
708 new_argv[i] = try arena.dupeZ(u8, arg);
709 }
712 // TODO https://github.com/ziglang/zig/issues/6971
713 if (self.base.options.is_native_os and self.base.options.system_linker_hack) {
714 const result = try std.ChildProcess.exec(.{ .allocator = self.base.allocator, .argv = argv.items });
715 defer {
716 self.base.allocator.free(result.stdout);
717 self.base.allocator.free(result.stderr);
718 }
719 if (result.stdout.len != 0) {
720 std.log.warn("unexpected LD stdout: {}", .{result.stdout});
721 }
722 if (result.stderr.len != 0) {
723 std.log.warn("unexpected LD stderr: {}", .{result.stderr});
724 }
725 if (result.term.Exited != 0) {
726 // TODO parse this output and surface with the Compilation API rather than
727 // directly outputting to stderr here.
728 std.debug.print("{}", .{result.stderr});
729 return error.LDReportedFailure;
730 }
731 } else {
732 const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
733 for (argv.items) |arg, i| {
734 new_argv[i] = try arena.dupeZ(u8, arg);
735 }
710736
711 var stderr_context: LLDContext = .{
712 .macho = self,
713 .data = std.ArrayList(u8).init(self.base.allocator),
714 };
715 defer stderr_context.data.deinit();
716 var stdout_context: LLDContext = .{
717 .macho = self,
718 .data = std.ArrayList(u8).init(self.base.allocator),
719 };
720 defer stdout_context.data.deinit();
721 const llvm = @import("../llvm.zig");
722 const ok = llvm.Link(
723 .MachO,
724 new_argv.ptr,
725 new_argv.len,
726 append_diagnostic,
727 @ptrToInt(&stdout_context),
728 @ptrToInt(&stderr_context),
729 );
730 if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory;
731 if (stdout_context.data.items.len != 0) {
732 std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items});
733 }
734 if (!ok) {
735 // TODO parse this output and surface with the Compilation API rather than
736 // directly outputting to stderr here.
737 std.debug.print("{}", .{stderr_context.data.items});
738 return error.LLDReportedFailure;
739 }
740 if (stderr_context.data.items.len != 0) {
741 std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items});
737 var stderr_context: LLDContext = .{
738 .macho = self,
739 .data = std.ArrayList(u8).init(self.base.allocator),
740 };
741 defer stderr_context.data.deinit();
742 var stdout_context: LLDContext = .{
743 .macho = self,
744 .data = std.ArrayList(u8).init(self.base.allocator),
745 };
746 defer stdout_context.data.deinit();
747 const llvm = @import("../llvm.zig");
748 const ok = llvm.Link(
749 .MachO,
750 new_argv.ptr,
751 new_argv.len,
752 append_diagnostic,
753 @ptrToInt(&stdout_context),
754 @ptrToInt(&stderr_context),
755 );
756 if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory;
757 if (stdout_context.data.items.len != 0) {
758 std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items});
759 }
760 if (!ok) {
761 // TODO parse this output and surface with the Compilation API rather than
762 // directly outputting to stderr here.
763 std.debug.print("{}", .{stderr_context.data.items});
764 return error.LLDReportedFailure;
765 }
766 if (stderr_context.data.items.len != 0) {
767 std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items});
768 }
742769 }
743770 }
744771
src/main.zig+11
......@@ -317,6 +317,7 @@ const usage_build_generic =
317317 \\ --image-base [addr] Set base address for executable image
318318 \\ -framework [name] (darwin) link against framework
319319 \\ -F[dir] (darwin) add search path for frameworks
320 \\ --system-linker-hack Use system linker LD instead of LLD (requires LLVM enabled)
320321 \\
321322 \\Test Options:
322323 \\ --test-filter [text] Skip tests that do not match filter
......@@ -474,6 +475,7 @@ fn buildOutputType(
474475 var image_base_override: ?u64 = null;
475476 var use_llvm: ?bool = null;
476477 var use_lld: ?bool = null;
478 var system_linker_hack = false;
477479 var use_clang: ?bool = null;
478480 var link_eh_frame_hdr = false;
479481 var link_emit_relocs = false;
......@@ -915,6 +917,8 @@ fn buildOutputType(
915917 mem.startsWith(u8, arg, "-I"))
916918 {
917919 try clang_argv.append(arg);
920 } else if (mem.startsWith(u8, arg, "--system-linker-hack")) {
921 system_linker_hack = true;
918922 } else {
919923 fatal("unrecognized parameter: '{}'", .{arg});
920924 }
......@@ -1639,6 +1643,7 @@ fn buildOutputType(
16391643 .want_valgrind = want_valgrind,
16401644 .use_llvm = use_llvm,
16411645 .use_lld = use_lld,
1646 .system_linker_hack = system_linker_hack,
16421647 .use_clang = use_clang,
16431648 .rdynamic = rdynamic,
16441649 .linker_script = linker_script,
......@@ -2160,6 +2165,7 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
21602165 var override_global_cache_dir: ?[]const u8 = null;
21612166 var override_local_cache_dir: ?[]const u8 = null;
21622167 var child_argv = std.ArrayList([]const u8).init(arena);
2168 var system_linker_hack = false;
21632169
21642170 const argv_index_exe = child_argv.items.len;
21652171 _ = try child_argv.addOne();
......@@ -2200,6 +2206,10 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
22002206 override_global_cache_dir = args[i];
22012207 try child_argv.appendSlice(&[_][]const u8{ arg, args[i] });
22022208 continue;
2209 } else if (mem.eql(u8, arg, "--system-linker-hack")) {
2210 system_linker_hack = true;
2211 try child_argv.append(arg);
2212 continue;
22032213 }
22042214 }
22052215 try child_argv.append(arg);
......@@ -2335,6 +2345,7 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
23352345 .optimize_mode = .Debug,
23362346 .self_exe_path = self_exe_path,
23372347 .rand = &default_prng.random,
2348 .system_linker_hack = system_linker_hack,
23382349 }) catch |err| {
23392350 fatal("unable to create compilation: {}", .{@errorName(err)});
23402351 };