From 4e4722a65e3bc3d6bb4518df3082ffea7c2a7c5d Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 7 Jun 2021 11:02:11 +0200 Subject: [PATCH] cc,wasi: build referenced-only emulated components Move parsing of system libs into `main.zig` next to where we decide if we should link libC, and, if targeting WASI, if the specified libname equals one of the emulated components, save it on the side and remove it from the system libs. Then, build *only* those parts of WASI libc that were preserved in the previous step. This also fixes building of different crt1 bits needed to support reactors and commands. --- src/Compilation.zig | 43 ++++-- src/link.zig | 1 + src/link/Wasm.zig | 15 ++- src/main.zig | 12 ++ src/wasi_libc.zig | 310 +++++++++++++++++++++++++------------------- 5 files changed, 227 insertions(+), 154 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 4c015cb58fbade0650f0d7dc43f3ff92493ac2c5..444e69c557e7609480da9dbabd4a20a86a37c0a3 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -203,8 +203,8 @@ const Job = union(enum) { /// needed when not linking libc and using LLVM for code generation because it generates /// calls to, for example, memcpy and memset. zig_libc: void, - /// WASI libc sysroot - wasi_libc_sysroot: void, + /// one of WASI libc static objects + wasi_libc_crt_file: wasi_libc.CRTFile, /// Use stage1 C++ code to compile zig code into an object file. stage1_module: void, @@ -279,7 +279,7 @@ pub const MiscTask = enum { libcxx, libcxxabi, libtsan, - wasi_libc_sysroot, + wasi_libc_crt_file, compiler_rt, libssp, zig_libc, @@ -646,6 +646,12 @@ pub const InitOptions = struct { framework_dirs: []const []const u8 = &[0][]const u8{}, frameworks: []const []const u8 = &[0][]const u8{}, system_libs: []const []const u8 = &[0][]const u8{}, + /// These correspond to the WASI libc emulated subcomponents including: + /// * process clocks + /// * getpid + /// * mman + /// * signal + wasi_emulated_libs: []const []const u8 = &[0][]const u8{}, link_libc: bool = false, link_libcpp: bool = false, link_libunwind: bool = false, @@ -1286,6 +1292,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .framework_dirs = options.framework_dirs, .system_libs = system_libs, .syslibroot = darwin_options.syslibroot, + .wasi_emulated_libs = options.wasi_emulated_libs, .lib_dirs = options.lib_dirs, .rpath_list = options.rpath_list, .strip = strip, @@ -1426,8 +1433,19 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { }, }); } - if (comp.wantBuildWasiLibcSysrootFromSource()) { - try comp.work_queue.write(&[_]Job{.{ .wasi_libc_sysroot = {} }}); + if (comp.wantBuildWasiLibcFromSource()) { + try comp.work_queue.ensureUnusedCapacity(6); // worst-case we need all components + const wasi_emulated_libs = comp.bin_file.options.wasi_emulated_libs; + for (wasi_emulated_libs) |lib_name| { + comp.work_queue.writeItemAssumeCapacity(.{ + .wasi_libc_crt_file = wasi_libc.getEmulatedLibCRTFile(lib_name).?, + }); + } + // TODO add logic deciding which crt1 we want here. + comp.work_queue.writeAssumeCapacity(&[_]Job{ + .{ .wasi_libc_crt_file = .crt1_o }, + .{ .wasi_libc_crt_file = .libc_a }, + }); } if (comp.wantBuildMinGWFromSource()) { const static_lib_jobs = [_]Job{ @@ -2169,12 +2187,12 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor ); }; }, - .wasi_libc_sysroot => { - wasi_libc.buildWasiLibcSysroot(self) catch |err| { + .wasi_libc_crt_file => |crt_file| { + wasi_libc.buildCRTFile(self, crt_file) catch |err| { // TODO Surface more error details. try self.setMiscFailure( - .wasi_libc_sysroot, - "unable to build WASI libc sysroot: {s}", + .wasi_libc_crt_file, + "unable to build WASI libc CRT file: {s}", .{@errorName(err)}, ); }; @@ -3303,7 +3321,7 @@ pub fn get_libc_crt_file(comp: *Compilation, arena: *Allocator, basename: []cons if (comp.wantBuildGLibCFromSource() or comp.wantBuildMuslFromSource() or comp.wantBuildMinGWFromSource() or - comp.wantBuildWasiLibcSysrootFromSource()) + comp.wantBuildWasiLibcFromSource()) { return comp.crt_files.get(basename).?.full_object_path; } @@ -3343,8 +3361,9 @@ fn wantBuildMuslFromSource(comp: Compilation) bool { !comp.getTarget().isWasm(); } -fn wantBuildWasiLibcSysrootFromSource(comp: Compilation) bool { - return comp.wantBuildLibCFromSource() and comp.getTarget().isWasm(); +fn wantBuildWasiLibcFromSource(comp: Compilation) bool { + return comp.wantBuildLibCFromSource() and comp.getTarget().isWasm() and + comp.getTarget().os.tag == .wasi; } fn wantBuildMinGWFromSource(comp: Compilation) bool { diff --git a/src/link.zig b/src/link.zig index fcb263f03e3a5a8abaa97549afa6a02090db2c77..c013db4947b0a82038bee787acef3c1d0e62ad64 100644 --- a/src/link.zig +++ b/src/link.zig @@ -110,6 +110,7 @@ pub const Options = struct { framework_dirs: []const []const u8, frameworks: []const []const u8, system_libs: std.StringArrayHashMapUnmanaged(void), + wasi_emulated_libs: []const []const u8, lib_dirs: []const []const u8, rpath_list: []const []const u8, diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index 443ace855ab1028e11dc9ef9d8c16f398fd74042..0168028354860b144defeefaec7cce48ffc99a83 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -699,18 +699,19 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { if (link_in_crt) { // TODO work out if we want standard crt, a reactor or a command - try argv.append(try comp.get_libc_crt_file(arena, "crt.o")); + try argv.append(try comp.get_libc_crt_file(arena, "crt1.o")); } if (!is_obj) { const system_libs = self.base.options.system_libs.keys(); for (system_libs) |link_lib| { - const full_name = try std.fmt.allocPrint(arena, "lib{s}.a", .{link_lib}); - if (comp.crt_files.get(full_name)) |crt| { - try argv.append(crt.full_object_path); - } else { - try argv.append(try std.fmt.allocPrint(arena, "-l{s}", .{link_lib})); - } + try argv.append(try std.fmt.allocPrint(arena, "-l{s}", .{link_lib})); + } + + const wasi_emulated_libs = self.base.options.wasi_emulated_libs; + for (wasi_emulated_libs) |lib_name| { + const full_lib_name = try std.fmt.allocPrint(arena, "lib{s}.a", .{lib_name}); + try argv.append(try comp.get_libc_crt_file(arena, full_lib_name)); } if (self.base.options.link_libc) { diff --git a/src/main.zig b/src/main.zig index 9245f5fd8a09d6ff286a2de307f7d2808a17d6fc..1e4ec0183f272885919e1eefea5565dd51b50456 100644 --- a/src/main.zig +++ b/src/main.zig @@ -15,6 +15,7 @@ const Package = @import("Package.zig"); const build_options = @import("build_options"); const introspect = @import("introspect.zig"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; +const wasi_libc = @import("wasi_libc.zig"); const translate_c = @import("translate_c.zig"); const Cache = @import("Cache.zig"); const target_util = @import("target.zig"); @@ -616,6 +617,9 @@ fn buildOutputType( var system_libs = std.ArrayList([]const u8).init(gpa); defer system_libs.deinit(); + var wasi_emulated_libs = std.ArrayList([]const u8).init(gpa); + defer wasi_emulated_libs.deinit(); + var clang_argv = std.ArrayList([]const u8).init(gpa); defer clang_argv.deinit(); @@ -1586,6 +1590,13 @@ fn buildOutputType( if (std.fs.path.isAbsolute(lib_name)) { fatal("cannot use absolute path as a system library: {s}", .{lib_name}); } + if (target_info.target.os.tag == .wasi) { + if (wasi_libc.getEmulatedLibCRTFile(lib_name)) |_| { + try wasi_emulated_libs.append(lib_name); + _ = system_libs.orderedRemove(i); + continue; + } + } i += 1; } } @@ -1895,6 +1906,7 @@ fn buildOutputType( .framework_dirs = framework_dirs.items, .frameworks = frameworks.items, .system_libs = system_libs.items, + .wasi_emulated_libs = wasi_emulated_libs.items, .link_libc = link_libc, .link_libcpp = link_libcpp, .link_libunwind = link_libunwind, diff --git a/src/wasi_libc.zig b/src/wasi_libc.zig index 37e7f4448811e99d9f44ae6513a42e1f63dd530c..0c178891c74fd9a0f1d218ddd70e5a214e195c99 100644 --- a/src/wasi_libc.zig +++ b/src/wasi_libc.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const mem = std.mem; const path = std.fs.path; const Allocator = std.mem.Allocator; @@ -7,7 +8,34 @@ const build_options = @import("build_options"); const target_util = @import("target.zig"); const musl = @import("musl.zig"); -pub fn buildWasiLibcSysroot(comp: *Compilation) !void { +pub const CRTFile = enum { + crt1_o, + crt1_reactor_o, + crt1_command_o, + libc_a, + libwasi_emulated_process_clocks_a, + libwasi_emulated_getpid_a, + libwasi_emulated_mman_a, + libwasi_emulated_signal_a, +}; + +pub fn getEmulatedLibCRTFile(lib_name: []const u8) ?CRTFile { + if (mem.eql(u8, lib_name, "wasi-emulated-process-clocks")) { + return .libwasi_emulated_process_clocks_a; + } + if (mem.eql(u8, lib_name, "wasi-emulated-getpid")) { + return .libwasi_emulated_getpid_a; + } + if (mem.eql(u8, lib_name, "wasi-emulated-mman")) { + return .libwasi_emulated_mman_a; + } + if (mem.eql(u8, lib_name, "wasi-emulated-signal")) { + return .libwasi_emulated_signal_a; + } + return null; +} + +pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { if (!build_options.have_llvm) { return error.ZigCompilerNotBuiltWithLLVMExtensions; } @@ -17,176 +45,190 @@ pub fn buildWasiLibcSysroot(comp: *Compilation) !void { defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; - { - // Compile crt sources. - var args = std.ArrayList([]const u8).init(arena); - try addCCArgs(comp, arena, &args, false); - try addLibcBottomHalfIncludes(comp, arena, &args); - - var crt_sources = std.ArrayList(Compilation.CSourceFile).init(arena); - for (crt_src_files) |file_path| { - try crt_sources.append(.{ - .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ - "libc", try sanitize(arena, file_path), - }), - .extra_flags = args.items, - }); - } - try comp.build_crt_file("crt", .Obj, crt_sources.items); - } - - { - // Compile WASI libc (sysroot). - var libc_sources = std.ArrayList(Compilation.CSourceFile).init(arena); - - { - // Compile dlmalloc. + switch (crt_file) { + .crt1_o => { var args = std.ArrayList([]const u8).init(arena); - try addCCArgs(comp, arena, &args, true); - try args.appendSlice(&[_][]const u8{ - "-I", - try comp.zig_lib_directory.join(arena, &[_][]const u8{ - "libc", - "wasi", - "dlmalloc", - "include", - }), + try addCCArgs(comp, arena, &args, false); + try addLibcBottomHalfIncludes(comp, arena, &args); + return comp.build_crt_file("crt1", .Obj, &[1]Compilation.CSourceFile{ + .{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ + "libc", try sanitize(arena, crt1_src_file), + }), + .extra_flags = args.items, + }, }); - - for (dlmalloc_src_files) |file_path| { - try libc_sources.append(.{ + }, + .crt1_reactor_o => { + var args = std.ArrayList([]const u8).init(arena); + try addCCArgs(comp, arena, &args, false); + try addLibcBottomHalfIncludes(comp, arena, &args); + return comp.build_crt_file("crt1-reactor", .Obj, &[1]Compilation.CSourceFile{ + .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ - "libc", try sanitize(arena, file_path), + "libc", try sanitize(arena, crt1_reactor_src_file), }), .extra_flags = args.items, - }); - } - } - - { - // Compile libc-bottom-half. + }, + }); + }, + .crt1_command_o => { var args = std.ArrayList([]const u8).init(arena); - try addCCArgs(comp, arena, &args, true); + try addCCArgs(comp, arena, &args, false); try addLibcBottomHalfIncludes(comp, arena, &args); - - for (libc_bottom_half_src_files) |file_path| { - try libc_sources.append(.{ + return comp.build_crt_file("crt1-command", .Obj, &[1]Compilation.CSourceFile{ + .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ - "libc", try sanitize(arena, file_path), + "libc", try sanitize(arena, crt1_command_src_file), }), .extra_flags = args.items, + }, + }); + }, + .libc_a => { + var libc_sources = std.ArrayList(Compilation.CSourceFile).init(arena); + + { + // Compile dlmalloc. + var args = std.ArrayList([]const u8).init(arena); + try addCCArgs(comp, arena, &args, true); + try args.appendSlice(&[_][]const u8{ + "-I", + try comp.zig_lib_directory.join(arena, &[_][]const u8{ + "libc", + "wasi", + "dlmalloc", + "include", + }), }); + + for (dlmalloc_src_files) |file_path| { + try libc_sources.append(.{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ + "libc", try sanitize(arena, file_path), + }), + .extra_flags = args.items, + }); + } } - } - { - // Compile libc-top-half. + { + // Compile libc-bottom-half. + var args = std.ArrayList([]const u8).init(arena); + try addCCArgs(comp, arena, &args, true); + try addLibcBottomHalfIncludes(comp, arena, &args); + + for (libc_bottom_half_src_files) |file_path| { + try libc_sources.append(.{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ + "libc", try sanitize(arena, file_path), + }), + .extra_flags = args.items, + }); + } + } + + { + // Compile libc-top-half. + var args = std.ArrayList([]const u8).init(arena); + try addCCArgs(comp, arena, &args, true); + try addLibcTopHalfIncludes(comp, arena, &args); + + for (libc_top_half_src_files) |file_path| { + try libc_sources.append(.{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ + "libc", try sanitize(arena, file_path), + }), + .extra_flags = args.items, + }); + } + } + + try comp.build_crt_file("c", .Lib, libc_sources.items); + }, + .libwasi_emulated_process_clocks_a => { var args = std.ArrayList([]const u8).init(arena); try addCCArgs(comp, arena, &args, true); - try addLibcTopHalfIncludes(comp, arena, &args); + try addLibcBottomHalfIncludes(comp, arena, &args); - for (libc_top_half_src_files) |file_path| { - try libc_sources.append(.{ + var emu_clocks_sources = std.ArrayList(Compilation.CSourceFile).init(arena); + for (emulated_process_clocks_src_files) |file_path| { + try emu_clocks_sources.append(.{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", try sanitize(arena, file_path), }), .extra_flags = args.items, }); } - } - - try comp.build_crt_file("c", .Lib, libc_sources.items); - } - - { - // Compile emulated process clocks. - var args = std.ArrayList([]const u8).init(arena); - try addCCArgs(comp, arena, &args, true); - try addLibcBottomHalfIncludes(comp, arena, &args); - - var emu_clocks_sources = std.ArrayList(Compilation.CSourceFile).init(arena); - for (emulated_process_clocks_src_files) |file_path| { - try emu_clocks_sources.append(.{ - .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ - "libc", try sanitize(arena, file_path), - }), - .extra_flags = args.items, - }); - } - try comp.build_crt_file("wasi-emulated-process-clocks", .Lib, emu_clocks_sources.items); - } - - { - // Compile emulated getpid. - var args = std.ArrayList([]const u8).init(arena); - try addCCArgs(comp, arena, &args, true); - try addLibcBottomHalfIncludes(comp, arena, &args); - - var emu_getpid_sources = std.ArrayList(Compilation.CSourceFile).init(arena); - for (emulated_getpid_src_files) |file_path| { - try emu_getpid_sources.append(.{ - .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ - "libc", try sanitize(arena, file_path), - }), - .extra_flags = args.items, - }); - } - try comp.build_crt_file("wasi-emulated-getpid", .Lib, emu_getpid_sources.items); - } - - { - // Compile emulated mman. - var args = std.ArrayList([]const u8).init(arena); - try addCCArgs(comp, arena, &args, true); - try addLibcBottomHalfIncludes(comp, arena, &args); - - var emu_mman_sources = std.ArrayList(Compilation.CSourceFile).init(arena); - for (emulated_mman_src_files) |file_path| { - try emu_mman_sources.append(.{ - .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ - "libc", try sanitize(arena, file_path), - }), - .extra_flags = args.items, - }); - } - try comp.build_crt_file("wasi-emulated-mman", .Lib, emu_mman_sources.items); - } - - { - // Compile emulated signals. - var emu_signal_sources = std.ArrayList(Compilation.CSourceFile).init(arena); - - { + try comp.build_crt_file("wasi-emulated-process-clocks", .Lib, emu_clocks_sources.items); + }, + .libwasi_emulated_getpid_a => { var args = std.ArrayList([]const u8).init(arena); try addCCArgs(comp, arena, &args, true); + try addLibcBottomHalfIncludes(comp, arena, &args); - for (emulated_signal_bottom_half_src_files) |file_path| { - try emu_signal_sources.append(.{ + var emu_getpid_sources = std.ArrayList(Compilation.CSourceFile).init(arena); + for (emulated_getpid_src_files) |file_path| { + try emu_getpid_sources.append(.{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", try sanitize(arena, file_path), }), .extra_flags = args.items, }); } - } - - { + try comp.build_crt_file("wasi-emulated-getpid", .Lib, emu_getpid_sources.items); + }, + .libwasi_emulated_mman_a => { var args = std.ArrayList([]const u8).init(arena); try addCCArgs(comp, arena, &args, true); - try addLibcTopHalfIncludes(comp, arena, &args); - try args.append("-D_WASI_EMULATED_SIGNAL"); + try addLibcBottomHalfIncludes(comp, arena, &args); - for (emulated_signal_top_half_src_files) |file_path| { - try emu_signal_sources.append(.{ + var emu_mman_sources = std.ArrayList(Compilation.CSourceFile).init(arena); + for (emulated_mman_src_files) |file_path| { + try emu_mman_sources.append(.{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", try sanitize(arena, file_path), }), .extra_flags = args.items, }); } - } + try comp.build_crt_file("wasi-emulated-mman", .Lib, emu_mman_sources.items); + }, + .libwasi_emulated_signal_a => { + var emu_signal_sources = std.ArrayList(Compilation.CSourceFile).init(arena); + + { + var args = std.ArrayList([]const u8).init(arena); + try addCCArgs(comp, arena, &args, true); + + for (emulated_signal_bottom_half_src_files) |file_path| { + try emu_signal_sources.append(.{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ + "libc", try sanitize(arena, file_path), + }), + .extra_flags = args.items, + }); + } + } + + { + var args = std.ArrayList([]const u8).init(arena); + try addCCArgs(comp, arena, &args, true); + try addLibcTopHalfIncludes(comp, arena, &args); + try args.append("-D_WASI_EMULATED_SIGNAL"); + + for (emulated_signal_top_half_src_files) |file_path| { + try emu_signal_sources.append(.{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ + "libc", try sanitize(arena, file_path), + }), + .extra_flags = args.items, + }); + } + } - try comp.build_crt_file("wasi-emulated-signal", .Lib, emu_signal_sources.items); + try comp.build_crt_file("wasi-emulated-signal", .Lib, emu_signal_sources.items); + }, } } @@ -1084,11 +1126,9 @@ const libc_top_half_src_files = [_][]const u8{ "wasi/libc-top-half/sources/arc4random.c", }; -const crt_src_files = &[_][]const u8{ - "wasi/libc-bottom-half/crt/crt1.c", - "wasi/libc-bottom-half/crt/crt1-command.c", - "wasi/libc-bottom-half/crt/crt1-reactor.c", -}; +const crt1_src_file = "wasi/libc-bottom-half/crt/crt1.c"; +const crt1_command_src_file = "wasi/libc-bottom-half/crt/crt1-command.c"; +const crt1_reactor_src_file = "wasi/libc-bottom-half/crt/crt1-reactor.c"; const emulated_process_clocks_src_files = &[_][]const u8{ "wasi/libc-bottom-half/clocks/clock.c", -- 2.54.0