authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2021-06-07 11:02:11+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2021-06-09 01:25:38+02:00
log4e4722a65e3bc3d6bb4518df3082ffea7c2a7c5d
tree02bf3ff87229ec8df06bd8296050e318e1cb90cc
parent93a4403271ef62d8ae2cd10f58cc23d4b7bbdf4d

cc,wasi: build referenced-only emulated components

Move parsing of system libs into `main.zig` next to where we decide if we should link libC, and, if targeting WASI, if the specified libname equals one of the emulated components, save it on the side and remove it from the system libs. Then, build *only* those parts of WASI libc that were preserved in the previous step. This also fixes building of different crt1 bits needed to support reactors and commands.

5 files changed, 227 insertions(+), 154 deletions(-)

src/Compilation.zig+31-12
......@@ -203,8 +203,8 @@ const Job = union(enum) {
203203 /// needed when not linking libc and using LLVM for code generation because it generates
204204 /// calls to, for example, memcpy and memset.
205205 zig_libc: void,
206 /// WASI libc sysroot
207 wasi_libc_sysroot: void,
206 /// one of WASI libc static objects
207 wasi_libc_crt_file: wasi_libc.CRTFile,
208208
209209 /// Use stage1 C++ code to compile zig code into an object file.
210210 stage1_module: void,
......@@ -279,7 +279,7 @@ pub const MiscTask = enum {
279279 libcxx,
280280 libcxxabi,
281281 libtsan,
282 wasi_libc_sysroot,
282 wasi_libc_crt_file,
283283 compiler_rt,
284284 libssp,
285285 zig_libc,
......@@ -646,6 +646,12 @@ pub const InitOptions = struct {
646646 framework_dirs: []const []const u8 = &[0][]const u8{},
647647 frameworks: []const []const u8 = &[0][]const u8{},
648648 system_libs: []const []const u8 = &[0][]const u8{},
649 /// These correspond to the WASI libc emulated subcomponents including:
650 /// * process clocks
651 /// * getpid
652 /// * mman
653 /// * signal
654 wasi_emulated_libs: []const []const u8 = &[0][]const u8{},
649655 link_libc: bool = false,
650656 link_libcpp: bool = false,
651657 link_libunwind: bool = false,
......@@ -1286,6 +1292,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
12861292 .framework_dirs = options.framework_dirs,
12871293 .system_libs = system_libs,
12881294 .syslibroot = darwin_options.syslibroot,
1295 .wasi_emulated_libs = options.wasi_emulated_libs,
12891296 .lib_dirs = options.lib_dirs,
12901297 .rpath_list = options.rpath_list,
12911298 .strip = strip,
......@@ -1426,8 +1433,19 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
14261433 },
14271434 });
14281435 }
1429 if (comp.wantBuildWasiLibcSysrootFromSource()) {
1430 try comp.work_queue.write(&[_]Job{.{ .wasi_libc_sysroot = {} }});
1436 if (comp.wantBuildWasiLibcFromSource()) {
1437 try comp.work_queue.ensureUnusedCapacity(6); // worst-case we need all components
1438 const wasi_emulated_libs = comp.bin_file.options.wasi_emulated_libs;
1439 for (wasi_emulated_libs) |lib_name| {
1440 comp.work_queue.writeItemAssumeCapacity(.{
1441 .wasi_libc_crt_file = wasi_libc.getEmulatedLibCRTFile(lib_name).?,
1442 });
1443 }
1444 // TODO add logic deciding which crt1 we want here.
1445 comp.work_queue.writeAssumeCapacity(&[_]Job{
1446 .{ .wasi_libc_crt_file = .crt1_o },
1447 .{ .wasi_libc_crt_file = .libc_a },
1448 });
14311449 }
14321450 if (comp.wantBuildMinGWFromSource()) {
14331451 const static_lib_jobs = [_]Job{
......@@ -2169,12 +2187,12 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
21692187 );
21702188 };
21712189 },
2172 .wasi_libc_sysroot => {
2173 wasi_libc.buildWasiLibcSysroot(self) catch |err| {
2190 .wasi_libc_crt_file => |crt_file| {
2191 wasi_libc.buildCRTFile(self, crt_file) catch |err| {
21742192 // TODO Surface more error details.
21752193 try self.setMiscFailure(
2176 .wasi_libc_sysroot,
2177 "unable to build WASI libc sysroot: {s}",
2194 .wasi_libc_crt_file,
2195 "unable to build WASI libc CRT file: {s}",
21782196 .{@errorName(err)},
21792197 );
21802198 };
......@@ -3303,7 +3321,7 @@ pub fn get_libc_crt_file(comp: *Compilation, arena: *Allocator, basename: []cons
33033321 if (comp.wantBuildGLibCFromSource() or
33043322 comp.wantBuildMuslFromSource() or
33053323 comp.wantBuildMinGWFromSource() or
3306 comp.wantBuildWasiLibcSysrootFromSource())
3324 comp.wantBuildWasiLibcFromSource())
33073325 {
33083326 return comp.crt_files.get(basename).?.full_object_path;
33093327 }
......@@ -3343,8 +3361,9 @@ fn wantBuildMuslFromSource(comp: Compilation) bool {
33433361 !comp.getTarget().isWasm();
33443362}
33453363
3346fn wantBuildWasiLibcSysrootFromSource(comp: Compilation) bool {
3347 return comp.wantBuildLibCFromSource() and comp.getTarget().isWasm();
3364fn wantBuildWasiLibcFromSource(comp: Compilation) bool {
3365 return comp.wantBuildLibCFromSource() and comp.getTarget().isWasm() and
3366 comp.getTarget().os.tag == .wasi;
33483367}
33493368
33503369fn wantBuildMinGWFromSource(comp: Compilation) bool {
src/link.zig+1
......@@ -110,6 +110,7 @@ pub const Options = struct {
110110 framework_dirs: []const []const u8,
111111 frameworks: []const []const u8,
112112 system_libs: std.StringArrayHashMapUnmanaged(void),
113 wasi_emulated_libs: []const []const u8,
113114 lib_dirs: []const []const u8,
114115 rpath_list: []const []const u8,
115116
src/link/Wasm.zig+8-7
......@@ -699,18 +699,19 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
699699
700700 if (link_in_crt) {
701701 // TODO work out if we want standard crt, a reactor or a command
702 try argv.append(try comp.get_libc_crt_file(arena, "crt.o"));
702 try argv.append(try comp.get_libc_crt_file(arena, "crt1.o"));
703703 }
704704
705705 if (!is_obj) {
706706 const system_libs = self.base.options.system_libs.keys();
707707 for (system_libs) |link_lib| {
708 const full_name = try std.fmt.allocPrint(arena, "lib{s}.a", .{link_lib});
709 if (comp.crt_files.get(full_name)) |crt| {
710 try argv.append(crt.full_object_path);
711 } else {
712 try argv.append(try std.fmt.allocPrint(arena, "-l{s}", .{link_lib}));
713 }
708 try argv.append(try std.fmt.allocPrint(arena, "-l{s}", .{link_lib}));
709 }
710
711 const wasi_emulated_libs = self.base.options.wasi_emulated_libs;
712 for (wasi_emulated_libs) |lib_name| {
713 const full_lib_name = try std.fmt.allocPrint(arena, "lib{s}.a", .{lib_name});
714 try argv.append(try comp.get_libc_crt_file(arena, full_lib_name));
714715 }
715716
716717 if (self.base.options.link_libc) {
src/main.zig+12
......@@ -15,6 +15,7 @@ const Package = @import("Package.zig");
1515const build_options = @import("build_options");
1616const introspect = @import("introspect.zig");
1717const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
18const wasi_libc = @import("wasi_libc.zig");
1819const translate_c = @import("translate_c.zig");
1920const Cache = @import("Cache.zig");
2021const target_util = @import("target.zig");
......@@ -616,6 +617,9 @@ fn buildOutputType(
616617 var system_libs = std.ArrayList([]const u8).init(gpa);
617618 defer system_libs.deinit();
618619
620 var wasi_emulated_libs = std.ArrayList([]const u8).init(gpa);
621 defer wasi_emulated_libs.deinit();
622
619623 var clang_argv = std.ArrayList([]const u8).init(gpa);
620624 defer clang_argv.deinit();
621625
......@@ -1586,6 +1590,13 @@ fn buildOutputType(
15861590 if (std.fs.path.isAbsolute(lib_name)) {
15871591 fatal("cannot use absolute path as a system library: {s}", .{lib_name});
15881592 }
1593 if (target_info.target.os.tag == .wasi) {
1594 if (wasi_libc.getEmulatedLibCRTFile(lib_name)) |_| {
1595 try wasi_emulated_libs.append(lib_name);
1596 _ = system_libs.orderedRemove(i);
1597 continue;
1598 }
1599 }
15891600 i += 1;
15901601 }
15911602 }
......@@ -1895,6 +1906,7 @@ fn buildOutputType(
18951906 .framework_dirs = framework_dirs.items,
18961907 .frameworks = frameworks.items,
18971908 .system_libs = system_libs.items,
1909 .wasi_emulated_libs = wasi_emulated_libs.items,
18981910 .link_libc = link_libc,
18991911 .link_libcpp = link_libcpp,
19001912 .link_libunwind = link_libunwind,
src/wasi_libc.zig+175-135
......@@ -1,4 +1,5 @@
11const std = @import("std");
2const mem = std.mem;
23const path = std.fs.path;
34
45const Allocator = std.mem.Allocator;
......@@ -7,7 +8,34 @@ const build_options = @import("build_options");
78const target_util = @import("target.zig");
89const musl = @import("musl.zig");
910
10pub fn buildWasiLibcSysroot(comp: *Compilation) !void {
11pub const CRTFile = enum {
12 crt1_o,
13 crt1_reactor_o,
14 crt1_command_o,
15 libc_a,
16 libwasi_emulated_process_clocks_a,
17 libwasi_emulated_getpid_a,
18 libwasi_emulated_mman_a,
19 libwasi_emulated_signal_a,
20};
21
22pub fn getEmulatedLibCRTFile(lib_name: []const u8) ?CRTFile {
23 if (mem.eql(u8, lib_name, "wasi-emulated-process-clocks")) {
24 return .libwasi_emulated_process_clocks_a;
25 }
26 if (mem.eql(u8, lib_name, "wasi-emulated-getpid")) {
27 return .libwasi_emulated_getpid_a;
28 }
29 if (mem.eql(u8, lib_name, "wasi-emulated-mman")) {
30 return .libwasi_emulated_mman_a;
31 }
32 if (mem.eql(u8, lib_name, "wasi-emulated-signal")) {
33 return .libwasi_emulated_signal_a;
34 }
35 return null;
36}
37
38pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
1139 if (!build_options.have_llvm) {
1240 return error.ZigCompilerNotBuiltWithLLVMExtensions;
1341 }
......@@ -17,176 +45,190 @@ pub fn buildWasiLibcSysroot(comp: *Compilation) !void {
1745 defer arena_allocator.deinit();
1846 const arena = &arena_allocator.allocator;
1947
20 {
21 // Compile crt sources.
22 var args = std.ArrayList([]const u8).init(arena);
23 try addCCArgs(comp, arena, &args, false);
24 try addLibcBottomHalfIncludes(comp, arena, &args);
25
26 var crt_sources = std.ArrayList(Compilation.CSourceFile).init(arena);
27 for (crt_src_files) |file_path| {
28 try crt_sources.append(.{
29 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
30 "libc", try sanitize(arena, file_path),
31 }),
32 .extra_flags = args.items,
33 });
34 }
35 try comp.build_crt_file("crt", .Obj, crt_sources.items);
36 }
37
38 {
39 // Compile WASI libc (sysroot).
40 var libc_sources = std.ArrayList(Compilation.CSourceFile).init(arena);
41
42 {
43 // Compile dlmalloc.
48 switch (crt_file) {
49 .crt1_o => {
4450 var args = std.ArrayList([]const u8).init(arena);
45 try addCCArgs(comp, arena, &args, true);
46 try args.appendSlice(&[_][]const u8{
47 "-I",
48 try comp.zig_lib_directory.join(arena, &[_][]const u8{
49 "libc",
50 "wasi",
51 "dlmalloc",
52 "include",
53 }),
51 try addCCArgs(comp, arena, &args, false);
52 try addLibcBottomHalfIncludes(comp, arena, &args);
53 return comp.build_crt_file("crt1", .Obj, &[1]Compilation.CSourceFile{
54 .{
55 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
56 "libc", try sanitize(arena, crt1_src_file),
57 }),
58 .extra_flags = args.items,
59 },
5460 });
55
56 for (dlmalloc_src_files) |file_path| {
57 try libc_sources.append(.{
61 },
62 .crt1_reactor_o => {
63 var args = std.ArrayList([]const u8).init(arena);
64 try addCCArgs(comp, arena, &args, false);
65 try addLibcBottomHalfIncludes(comp, arena, &args);
66 return comp.build_crt_file("crt1-reactor", .Obj, &[1]Compilation.CSourceFile{
67 .{
5868 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
59 "libc", try sanitize(arena, file_path),
69 "libc", try sanitize(arena, crt1_reactor_src_file),
6070 }),
6171 .extra_flags = args.items,
62 });
63 }
64 }
65
66 {
67 // Compile libc-bottom-half.
72 },
73 });
74 },
75 .crt1_command_o => {
6876 var args = std.ArrayList([]const u8).init(arena);
69 try addCCArgs(comp, arena, &args, true);
77 try addCCArgs(comp, arena, &args, false);
7078 try addLibcBottomHalfIncludes(comp, arena, &args);
71
72 for (libc_bottom_half_src_files) |file_path| {
73 try libc_sources.append(.{
79 return comp.build_crt_file("crt1-command", .Obj, &[1]Compilation.CSourceFile{
80 .{
7481 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
75 "libc", try sanitize(arena, file_path),
82 "libc", try sanitize(arena, crt1_command_src_file),
7683 }),
7784 .extra_flags = args.items,
85 },
86 });
87 },
88 .libc_a => {
89 var libc_sources = std.ArrayList(Compilation.CSourceFile).init(arena);
90
91 {
92 // Compile dlmalloc.
93 var args = std.ArrayList([]const u8).init(arena);
94 try addCCArgs(comp, arena, &args, true);
95 try args.appendSlice(&[_][]const u8{
96 "-I",
97 try comp.zig_lib_directory.join(arena, &[_][]const u8{
98 "libc",
99 "wasi",
100 "dlmalloc",
101 "include",
102 }),
78103 });
104
105 for (dlmalloc_src_files) |file_path| {
106 try libc_sources.append(.{
107 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
108 "libc", try sanitize(arena, file_path),
109 }),
110 .extra_flags = args.items,
111 });
112 }
79113 }
80 }
81114
82 {
83 // Compile libc-top-half.
115 {
116 // Compile libc-bottom-half.
117 var args = std.ArrayList([]const u8).init(arena);
118 try addCCArgs(comp, arena, &args, true);
119 try addLibcBottomHalfIncludes(comp, arena, &args);
120
121 for (libc_bottom_half_src_files) |file_path| {
122 try libc_sources.append(.{
123 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
124 "libc", try sanitize(arena, file_path),
125 }),
126 .extra_flags = args.items,
127 });
128 }
129 }
130
131 {
132 // Compile libc-top-half.
133 var args = std.ArrayList([]const u8).init(arena);
134 try addCCArgs(comp, arena, &args, true);
135 try addLibcTopHalfIncludes(comp, arena, &args);
136
137 for (libc_top_half_src_files) |file_path| {
138 try libc_sources.append(.{
139 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
140 "libc", try sanitize(arena, file_path),
141 }),
142 .extra_flags = args.items,
143 });
144 }
145 }
146
147 try comp.build_crt_file("c", .Lib, libc_sources.items);
148 },
149 .libwasi_emulated_process_clocks_a => {
84150 var args = std.ArrayList([]const u8).init(arena);
85151 try addCCArgs(comp, arena, &args, true);
86 try addLibcTopHalfIncludes(comp, arena, &args);
152 try addLibcBottomHalfIncludes(comp, arena, &args);
87153
88 for (libc_top_half_src_files) |file_path| {
89 try libc_sources.append(.{
154 var emu_clocks_sources = std.ArrayList(Compilation.CSourceFile).init(arena);
155 for (emulated_process_clocks_src_files) |file_path| {
156 try emu_clocks_sources.append(.{
90157 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
91158 "libc", try sanitize(arena, file_path),
92159 }),
93160 .extra_flags = args.items,
94161 });
95162 }
96 }
97
98 try comp.build_crt_file("c", .Lib, libc_sources.items);
99 }
100
101 {
102 // Compile emulated process clocks.
103 var args = std.ArrayList([]const u8).init(arena);
104 try addCCArgs(comp, arena, &args, true);
105 try addLibcBottomHalfIncludes(comp, arena, &args);
106
107 var emu_clocks_sources = std.ArrayList(Compilation.CSourceFile).init(arena);
108 for (emulated_process_clocks_src_files) |file_path| {
109 try emu_clocks_sources.append(.{
110 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
111 "libc", try sanitize(arena, file_path),
112 }),
113 .extra_flags = args.items,
114 });
115 }
116 try comp.build_crt_file("wasi-emulated-process-clocks", .Lib, emu_clocks_sources.items);
117 }
118
119 {
120 // Compile emulated getpid.
121 var args = std.ArrayList([]const u8).init(arena);
122 try addCCArgs(comp, arena, &args, true);
123 try addLibcBottomHalfIncludes(comp, arena, &args);
124
125 var emu_getpid_sources = std.ArrayList(Compilation.CSourceFile).init(arena);
126 for (emulated_getpid_src_files) |file_path| {
127 try emu_getpid_sources.append(.{
128 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
129 "libc", try sanitize(arena, file_path),
130 }),
131 .extra_flags = args.items,
132 });
133 }
134 try comp.build_crt_file("wasi-emulated-getpid", .Lib, emu_getpid_sources.items);
135 }
136
137 {
138 // Compile emulated mman.
139 var args = std.ArrayList([]const u8).init(arena);
140 try addCCArgs(comp, arena, &args, true);
141 try addLibcBottomHalfIncludes(comp, arena, &args);
142
143 var emu_mman_sources = std.ArrayList(Compilation.CSourceFile).init(arena);
144 for (emulated_mman_src_files) |file_path| {
145 try emu_mman_sources.append(.{
146 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
147 "libc", try sanitize(arena, file_path),
148 }),
149 .extra_flags = args.items,
150 });
151 }
152 try comp.build_crt_file("wasi-emulated-mman", .Lib, emu_mman_sources.items);
153 }
154
155 {
156 // Compile emulated signals.
157 var emu_signal_sources = std.ArrayList(Compilation.CSourceFile).init(arena);
158
159 {
163 try comp.build_crt_file("wasi-emulated-process-clocks", .Lib, emu_clocks_sources.items);
164 },
165 .libwasi_emulated_getpid_a => {
160166 var args = std.ArrayList([]const u8).init(arena);
161167 try addCCArgs(comp, arena, &args, true);
168 try addLibcBottomHalfIncludes(comp, arena, &args);
162169
163 for (emulated_signal_bottom_half_src_files) |file_path| {
164 try emu_signal_sources.append(.{
170 var emu_getpid_sources = std.ArrayList(Compilation.CSourceFile).init(arena);
171 for (emulated_getpid_src_files) |file_path| {
172 try emu_getpid_sources.append(.{
165173 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
166174 "libc", try sanitize(arena, file_path),
167175 }),
168176 .extra_flags = args.items,
169177 });
170178 }
171 }
172
173 {
179 try comp.build_crt_file("wasi-emulated-getpid", .Lib, emu_getpid_sources.items);
180 },
181 .libwasi_emulated_mman_a => {
174182 var args = std.ArrayList([]const u8).init(arena);
175183 try addCCArgs(comp, arena, &args, true);
176 try addLibcTopHalfIncludes(comp, arena, &args);
177 try args.append("-D_WASI_EMULATED_SIGNAL");
184 try addLibcBottomHalfIncludes(comp, arena, &args);
178185
179 for (emulated_signal_top_half_src_files) |file_path| {
180 try emu_signal_sources.append(.{
186 var emu_mman_sources = std.ArrayList(Compilation.CSourceFile).init(arena);
187 for (emulated_mman_src_files) |file_path| {
188 try emu_mman_sources.append(.{
181189 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
182190 "libc", try sanitize(arena, file_path),
183191 }),
184192 .extra_flags = args.items,
185193 });
186194 }
187 }
195 try comp.build_crt_file("wasi-emulated-mman", .Lib, emu_mman_sources.items);
196 },
197 .libwasi_emulated_signal_a => {
198 var emu_signal_sources = std.ArrayList(Compilation.CSourceFile).init(arena);
199
200 {
201 var args = std.ArrayList([]const u8).init(arena);
202 try addCCArgs(comp, arena, &args, true);
203
204 for (emulated_signal_bottom_half_src_files) |file_path| {
205 try emu_signal_sources.append(.{
206 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
207 "libc", try sanitize(arena, file_path),
208 }),
209 .extra_flags = args.items,
210 });
211 }
212 }
188213
189 try comp.build_crt_file("wasi-emulated-signal", .Lib, emu_signal_sources.items);
214 {
215 var args = std.ArrayList([]const u8).init(arena);
216 try addCCArgs(comp, arena, &args, true);
217 try addLibcTopHalfIncludes(comp, arena, &args);
218 try args.append("-D_WASI_EMULATED_SIGNAL");
219
220 for (emulated_signal_top_half_src_files) |file_path| {
221 try emu_signal_sources.append(.{
222 .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
223 "libc", try sanitize(arena, file_path),
224 }),
225 .extra_flags = args.items,
226 });
227 }
228 }
229
230 try comp.build_crt_file("wasi-emulated-signal", .Lib, emu_signal_sources.items);
231 },
190232 }
191233}
192234
......@@ -1084,11 +1126,9 @@ const libc_top_half_src_files = [_][]const u8{
10841126 "wasi/libc-top-half/sources/arc4random.c",
10851127};
10861128
1087const crt_src_files = &[_][]const u8{
1088 "wasi/libc-bottom-half/crt/crt1.c",
1089 "wasi/libc-bottom-half/crt/crt1-command.c",
1090 "wasi/libc-bottom-half/crt/crt1-reactor.c",
1091};
1129const crt1_src_file = "wasi/libc-bottom-half/crt/crt1.c";
1130const crt1_command_src_file = "wasi/libc-bottom-half/crt/crt1-command.c";
1131const crt1_reactor_src_file = "wasi/libc-bottom-half/crt/crt1-reactor.c";
10921132
10931133const emulated_process_clocks_src_files = &[_][]const u8{
10941134 "wasi/libc-bottom-half/clocks/clock.c",