authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-18 14:09:55-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-05-25 18:54:34-07:00
logb3d162d6bfe82d84d55edd58016347a420732fe2
treef4b64b73de31b3d662d99d1f466444f4d7c9242f
parentef050483dff9f4fe57107b4c3ddba9a2692bdef4

build maker: rename files to match type

23 files changed, 9347 insertions(+), 9347 deletions(-)

lib/compiler/Maker.zig created+1848
...@@ -0,0 +1,1848 @@
1const Maker = @This();
2const builtin = @import("builtin");
3
4const std = @import("std");
5const Allocator = std.mem.Allocator;
6const Cache = std.Build.Cache;
7const Configuration = std.Build.Configuration;
8const File = std.Io.File;
9const Io = std.Io;
10const Path = std.Build.Cache.Path;
11const Writer = std.Io.Writer;
12const assert = std.debug.assert;
13const fatal = std.process.fatal;
14const fmt = std.fmt;
15const log = std.log;
16const mem = std.mem;
17const process = std.process;
18
19const Fuzz = @import("Maker/Fuzz.zig");
20const Graph = @import("Maker/Graph.zig");
21const Step = @import("Maker/Step.zig");
22const Watch = @import("Maker/Watch.zig");
23const WebServer = @import("Maker/WebServer.zig");
24
25pub const std_options: std.Options = .{
26 .side_channels_mitigations = .none,
27 .http_disable_tls = true,
28};
29
30gpa: Allocator,
31graph: *Graph,
32install_paths: InstallPaths,
33scanned_config: *const ScannedConfig,
34steps: []Step,
35
36available_rss: usize,
37max_rss_is_default: bool,
38max_rss_mutex: Io.Mutex,
39skip_oom_steps: bool,
40unit_test_timeout_ns: ?u64,
41watch: bool,
42web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn,
43/// Allocated into `gpa`.
44memory_blocked_steps: std.ArrayList(Configuration.Step.Index),
45/// Allocated into `gpa`.
46step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void),
47
48error_style: ErrorStyle,
49multiline_errors: MultilineErrors,
50summary: Summary,
51
52pub fn main(init: process.Init.Minimal) !void {
53 // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not
54 // always the case. So, we do need a true gpa for some things.
55 var safe_gpa_state: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{});
56 defer _ = safe_gpa_state.deinit();
57 const gpa = safe_gpa_state.allocator();
58
59 var threaded: std.Io.Threaded = .init(gpa, .{
60 .environ = init.environ,
61 .argv0 = .init(init.args),
62 });
63 defer threaded.deinit();
64 const io = threaded.io();
65
66 // ...but we'll back our arena by `std.heap.page_allocator` for efficiency.
67 var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
68 defer arena_instance.deinit();
69 const arena = arena_instance.allocator();
70
71 const args = try init.args.toSlice(arena);
72
73 // skip my own exe name
74 var arg_idx: usize = 1;
75
76 const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig");
77 const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir");
78 const build_root = expectArgOrFatal(args, &arg_idx, "--build-root");
79 const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache");
80 const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache");
81 const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration");
82
83 const cwd: Io.Dir = .cwd();
84
85 const zig_lib_directory: Cache.Directory = .{
86 .path = zig_lib_dir,
87 .handle = try cwd.openDir(io, zig_lib_dir, .{}),
88 };
89
90 const build_root_directory: Cache.Directory = .{
91 .path = build_root,
92 .handle = try cwd.openDir(io, build_root, .{}),
93 };
94
95 const local_cache_directory: Cache.Directory = .{
96 .path = local_cache_root,
97 .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}),
98 };
99
100 const global_cache_directory: Cache.Directory = .{
101 .path = global_cache_root,
102 .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}),
103 };
104
105 var graph: Graph = .{
106 .io = io,
107 .arena = arena,
108 .cache = .{
109 .io = io,
110 .gpa = gpa,
111 .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}),
112 .cwd = try process.currentPathAlloc(io, arena),
113 },
114 .zig_exe = zig_exe,
115 .environ_map = try init.environ.createMap(arena),
116 .global_cache_root = global_cache_directory,
117 .zig_lib_directory = zig_lib_directory,
118 };
119
120 graph.cache.addPrefix(.{ .path = null, .handle = cwd });
121 graph.cache.addPrefix(build_root_directory);
122 graph.cache.addPrefix(local_cache_directory);
123 graph.cache.addPrefix(global_cache_directory);
124 graph.cache.hash.addBytes(builtin.zig_version_string);
125
126 var step_names: std.ArrayList([]const u8) = .empty;
127 var debug_log_scopes: std.ArrayList([]const u8) = .empty;
128 var help_menu = false;
129 var steps_menu = false;
130 var print_configuration = false;
131 var override_install_prefix: ?[]const u8 = null;
132 var override_lib_dir: ?[]const u8 = null;
133 var override_bin_dir: ?[]const u8 = null;
134 var override_include_dir: ?[]const u8 = null;
135 var error_style: ErrorStyle = .verbose;
136 var multiline_errors: MultilineErrors = .indent;
137 var summary: ?Summary = null;
138 var max_rss: u64 = 0;
139 var skip_oom_steps = false;
140 var test_timeout_ns: ?u64 = null;
141 var color: Color = .auto;
142 var watch = false;
143 var fuzz: ?Fuzz.Mode = null;
144 var debounce_interval_ms: u16 = 50;
145 var webui_listen: ?Io.net.IpAddress = null;
146 var verbose = false;
147 var sysroot: ?[]const u8 = null;
148 var search_prefixes: std.ArrayList([]const u8) = .empty;
149 var libc_file: ?[]const u8 = null;
150 var debug_pkg_config: bool = false;
151 // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md,
152 // this will be the directory $glibc-build-dir/install/glibcs
153 // Given the example of the aarch64 target, this is the directory
154 // that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`.
155 // Also works for dynamic musl.
156 var libc_runtimes_dir: ?[]const u8 = null;
157 var enable_wine = false;
158 var enable_qemu = false;
159 var enable_wasmtime = false;
160 var enable_darling = false;
161 var enable_rosetta = false;
162 var reference_trace: ?u32 = null;
163 var run_args: ?[]const []const u8 = null;
164
165 if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| {
166 if (std.meta.stringToEnum(ErrorStyle, str)) |style| {
167 error_style = style;
168 }
169 }
170
171 if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| {
172 if (std.meta.stringToEnum(MultilineErrors, str)) |style| {
173 multiline_errors = style;
174 }
175 }
176
177 while (nextArg(args, &arg_idx)) |arg| {
178 if (mem.startsWith(u8, arg, "-")) {
179 if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
180 help_menu = true;
181 } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) {
182 steps_menu = true;
183 } else if (mem.eql(u8, arg, "--print-configuration")) {
184 print_configuration = true;
185 } else if (mem.eql(u8, arg, "--verbose")) {
186 verbose = true;
187 } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) {
188 override_install_prefix = nextArgOrFatal(args, &arg_idx);
189 } else if (mem.eql(u8, arg, "--prefix-lib-dir")) {
190 override_lib_dir = nextArgOrFatal(args, &arg_idx);
191 } else if (mem.eql(u8, arg, "--prefix-exe-dir")) {
192 override_bin_dir = nextArgOrFatal(args, &arg_idx);
193 } else if (mem.eql(u8, arg, "--prefix-include-dir")) {
194 override_include_dir = nextArgOrFatal(args, &arg_idx);
195 } else if (mem.eql(u8, arg, "--sysroot")) {
196 sysroot = nextArgOrFatal(args, &arg_idx);
197 } else if (mem.eql(u8, arg, "--maxrss")) {
198 const max_rss_text = nextArgOrFatal(args, &arg_idx);
199 max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err|
200 fatal("invalid byte size: '{s}': {t}", .{ max_rss_text, err });
201 } else if (mem.eql(u8, arg, "--skip-oom-steps")) {
202 skip_oom_steps = true;
203 } else if (mem.eql(u8, arg, "--test-timeout")) {
204 const units: []const struct { []const u8, u64 } = &.{
205 .{ "ns", 1 },
206 .{ "nanosecond", 1 },
207 .{ "us", std.time.ns_per_us },
208 .{ "microsecond", std.time.ns_per_us },
209 .{ "ms", std.time.ns_per_ms },
210 .{ "millisecond", std.time.ns_per_ms },
211 .{ "s", std.time.ns_per_s },
212 .{ "second", std.time.ns_per_s },
213 .{ "m", std.time.ns_per_min },
214 .{ "minute", std.time.ns_per_min },
215 .{ "h", std.time.ns_per_hour },
216 .{ "hour", std.time.ns_per_hour },
217 };
218 const timeout_str = nextArgOrFatal(args, &arg_idx);
219 const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal(
220 "invalid timeout '{s}': expected unit (ns, us, ms, s, m, h)",
221 .{timeout_str},
222 );
223 const num_str = timeout_str[0 .. num_end_idx + 1];
224 const unit_str = timeout_str[num_end_idx + 1 ..];
225 const unit_factor: f64 = for (units) |unit_and_factor| {
226 if (std.mem.eql(u8, unit_str, unit_and_factor[0])) {
227 break @floatFromInt(unit_and_factor[1]);
228 }
229 } else fatal(
230 "invalid timeout '{s}': invalid unit '{s}' (expected ns, us, ms, s, m, h)",
231 .{ timeout_str, unit_str },
232 );
233 const num_parsed = std.fmt.parseFloat(f64, num_str) catch |err| fatal(
234 "invalid timeout '{s}': invalid number '{s}' ({t})",
235 .{ timeout_str, num_str, err },
236 );
237 test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed);
238 } else if (mem.eql(u8, arg, "--search-prefix")) {
239 try search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx));
240 } else if (mem.eql(u8, arg, "--libc")) {
241 libc_file = nextArgOrFatal(args, &arg_idx);
242 } else if (mem.eql(u8, arg, "--color")) {
243 const next_arg = nextArg(args, &arg_idx) orelse
244 fatalWithHint("expected [auto|on|off] after '{s}'", .{arg});
245 color = std.meta.stringToEnum(Color, next_arg) orelse {
246 fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{
247 arg, next_arg,
248 });
249 };
250 } else if (mem.eql(u8, arg, "--error-style")) {
251 const next_arg = nextArg(args, &arg_idx) orelse
252 fatalWithHint("expected style after '{s}'", .{arg});
253 error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse {
254 fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg });
255 };
256 } else if (mem.eql(u8, arg, "--multiline-errors")) {
257 const next_arg = nextArg(args, &arg_idx) orelse
258 fatalWithHint("expected style after '{s}'", .{arg});
259 multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse {
260 fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg });
261 };
262 } else if (mem.eql(u8, arg, "--summary")) {
263 const next_arg = nextArg(args, &arg_idx) orelse
264 fatalWithHint("expected [all|new|failures|line|none] after '{s}'", .{arg});
265 summary = std.meta.stringToEnum(Summary, next_arg) orelse {
266 fatalWithHint("expected [all|new|failures|line|none] after '{s}', found '{s}'", .{
267 arg, next_arg,
268 });
269 };
270 } else if (mem.eql(u8, arg, "--seed")) {
271 const next_arg = nextArg(args, &arg_idx) orelse
272 fatalWithHint("expected u32 after '{s}'", .{arg});
273 graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| {
274 fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {t}", .{ next_arg, err });
275 };
276 } else if (mem.eql(u8, arg, "--debounce")) {
277 const next_arg = nextArg(args, &arg_idx) orelse
278 fatalWithHint("expected u16 after '{s}'", .{arg});
279 debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| {
280 fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {t}\n", .{
281 next_arg, err,
282 });
283 };
284 } else if (mem.eql(u8, arg, "--webui")) {
285 if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) };
286 } else if (mem.startsWith(u8, arg, "--webui=")) {
287 const addr_str = arg["--webui=".len..];
288 if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{});
289 webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| {
290 fatal("invalid web UI address '{s}': {t}", .{ addr_str, err });
291 };
292 } else if (mem.eql(u8, arg, "--debug-log")) {
293 const next_arg = nextArgOrFatal(args, &arg_idx);
294 try debug_log_scopes.append(arena, next_arg);
295 } else if (mem.eql(u8, arg, "--debug-pkg-config")) {
296 debug_pkg_config = true;
297 } else if (mem.eql(u8, arg, "--debug-rt")) {
298 graph.debug_compiler_runtime_libs = .Debug;
299 } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| {
300 graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse
301 fatal("unrecognized optimization mode: {s}", .{rest});
302 } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) {
303 // --glibc-runtimes was the old name of the flag; kept for compatibility for now.
304 libc_runtimes_dir = nextArgOrFatal(args, &arg_idx);
305 } else if (mem.eql(u8, arg, "--watch")) {
306 watch = true;
307 } else if (mem.eql(u8, arg, "--time-report")) {
308 graph.time_report = true;
309 if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) };
310 } else if (mem.eql(u8, arg, "--fuzz")) {
311 fuzz = .{ .forever = undefined };
312 if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) };
313 } else if (mem.startsWith(u8, arg, "--fuzz=")) {
314 const value = arg["--fuzz=".len..];
315 if (value.len == 0) fatal("missing argument to --fuzz", .{});
316
317 const unit: u8 = value[value.len - 1];
318 const digits = switch (unit) {
319 '0'...'9' => value,
320 'K', 'M', 'G' => value[0 .. value.len - 1],
321 else => fatal(
322 "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
323 .{},
324 ),
325 };
326
327 const amount = std.fmt.parseInt(u64, digits, 10) catch {
328 fatal(
329 "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
330 .{},
331 );
332 };
333
334 const normalized_amount = std.math.mul(u64, amount, switch (unit) {
335 else => unreachable,
336 '0'...'9' => 1,
337 'K' => 1000,
338 'M' => 1_000_000,
339 'G' => 1_000_000_000,
340 }) catch fatal("fuzzing limit amount overflows u64", .{});
341
342 fuzz = .{
343 .limit = .{
344 .amount = normalized_amount,
345 },
346 };
347 } else if (mem.eql(u8, arg, "-fincremental")) {
348 graph.incremental = true;
349 } else if (mem.eql(u8, arg, "-fno-incremental")) {
350 graph.incremental = false;
351 } else if (mem.eql(u8, arg, "-fwine")) {
352 enable_wine = true;
353 } else if (mem.eql(u8, arg, "-fno-wine")) {
354 enable_wine = false;
355 } else if (mem.eql(u8, arg, "-fqemu")) {
356 enable_qemu = true;
357 } else if (mem.eql(u8, arg, "-fno-qemu")) {
358 enable_qemu = false;
359 } else if (mem.eql(u8, arg, "-fwasmtime")) {
360 enable_wasmtime = true;
361 } else if (mem.eql(u8, arg, "-fno-wasmtime")) {
362 enable_wasmtime = false;
363 } else if (mem.eql(u8, arg, "-frosetta")) {
364 enable_rosetta = true;
365 } else if (mem.eql(u8, arg, "-fno-rosetta")) {
366 enable_rosetta = false;
367 } else if (mem.eql(u8, arg, "-fdarling")) {
368 enable_darling = true;
369 } else if (mem.eql(u8, arg, "-fno-darling")) {
370 enable_darling = false;
371 } else if (mem.eql(u8, arg, "-fallow-so-scripts")) {
372 graph.allow_so_scripts = true;
373 } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) {
374 graph.allow_so_scripts = false;
375 } else if (mem.eql(u8, arg, "-freference-trace")) {
376 reference_trace = 256;
377 } else if (mem.startsWith(u8, arg, "-freference-trace=")) {
378 const num = arg["-freference-trace=".len..];
379 reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| {
380 std.debug.print("unable to parse reference_trace count '{s}': {t}", .{ num, err });
381 process.exit(1);
382 };
383 } else if (mem.eql(u8, arg, "-fno-reference-trace")) {
384 reference_trace = null;
385 } else if (mem.cutPrefix(u8, arg, "-j")) |text| {
386 const n = std.fmt.parseUnsigned(u32, text, 10) catch |err|
387 fatal("unable to parse jobs count '{s}': {t}", .{ text, err });
388 if (n < 1) fatal("number of jobs must be at least 1", .{});
389 threaded.setAsyncLimit(.limited(n));
390 graph.max_jobs = n;
391 } else if (mem.eql(u8, arg, "--")) {
392 run_args = argsRest(args, arg_idx);
393 break;
394 } else {
395 fatalWithHint("unrecognized argument: '{s}'", .{arg});
396 }
397 } else {
398 try step_names.append(arena, arg);
399 }
400 }
401
402 const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map);
403 const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map);
404
405 graph.stderr_mode = switch (color) {
406 .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE),
407 .on => .escape_codes,
408 .off => .no_color,
409 };
410
411 const scanned_config: ScannedConfig = sc: {
412 const configuration = c: {
413 var file = cwd.openFile(io, configure_path, .{}) catch |err|
414 fatal("failed to open configuration file {s}: {t}", .{ configure_path, err });
415 defer file.close(io);
416 break :c Configuration.loadFile(arena, io, file) catch |err|
417 fatal("failed to load configuration file {s}: {t}", .{ configure_path, err });
418 };
419 var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty;
420 for (configuration.steps, 0..) |*conf_step, step_index_usize| {
421 const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize);
422 const flags: Configuration.Step.Flags = @bitCast(configuration.extra[conf_step.extra_index]);
423 if (flags.tag == .top_level) {
424 const name = step_index.ptr(&configuration).name.slice(&configuration);
425 try top_level_steps.put(arena, name, step_index);
426 }
427 }
428 break :sc .{
429 .configuration = configuration,
430 .top_level_steps = top_level_steps,
431 };
432 };
433
434 if (help_menu) {
435 var w = initStdoutWriter(io);
436 scanned_config.printUsage(&graph, w) catch |err| switch (err) {
437 error.WriteFailed => return stdout_writer_allocation.err.?,
438 else => |e| return e,
439 };
440 w.flush() catch return stdout_writer_allocation.err.?;
441 return;
442 } else if (steps_menu) {
443 var w = initStdoutWriter(io);
444 scanned_config.printSteps(&graph, w) catch |err| switch (err) {
445 error.WriteFailed => return stdout_writer_allocation.err.?,
446 else => |e| return e,
447 };
448 w.flush() catch return stdout_writer_allocation.err.?;
449 return;
450 } else if (print_configuration) {
451 var w = initStdoutWriter(io);
452 scanned_config.print(w) catch return stdout_writer_allocation.err.?;
453 w.flush() catch return stdout_writer_allocation.err.?;
454 return;
455 }
456
457 if (webui_listen != null) {
458 if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{});
459 if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{});
460 }
461
462 const main_progress_node = std.Progress.start(io, .{
463 .disable_printing = (color == .off),
464 });
465 defer main_progress_node.end();
466
467 const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{
468 .root_dir = .cwd(),
469 .sub_path = try Io.Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }),
470 } else if (override_install_prefix) |cwd_relative| .{
471 .root_dir = .cwd(),
472 .sub_path = cwd_relative,
473 } else .{
474 .root_dir = build_root_directory,
475 .sub_path = "zig-out",
476 };
477
478 const install_lib_path: Path = if (override_lib_dir) |cwd_relative| .{
479 .root_dir = .cwd(),
480 .sub_path = cwd_relative,
481 } else try install_prefix_path.join(arena, "lib");
482
483 const install_bin_path: Path = if (override_bin_dir) |cwd_relative| .{
484 .root_dir = .cwd(),
485 .sub_path = cwd_relative,
486 } else try install_prefix_path.join(arena, "bin");
487
488 const install_include_path: Path = if (override_include_dir) |cwd_relative| .{
489 .root_dir = .cwd(),
490 .sub_path = cwd_relative,
491 } else try install_prefix_path.join(arena, "include");
492
493 var maker: Maker = .{
494 .gpa = gpa,
495 .graph = &graph,
496 .scanned_config = &scanned_config,
497 .install_paths = .{
498 .prefix = install_prefix_path,
499 .lib = install_lib_path,
500 .bin = install_bin_path,
501 .include = install_include_path,
502 },
503 .steps = try arena.alloc(Step, scanned_config.configuration.steps.len),
504
505 .available_rss = max_rss,
506 .max_rss_is_default = false,
507 .max_rss_mutex = .init,
508 .skip_oom_steps = skip_oom_steps,
509 .unit_test_timeout_ns = test_timeout_ns,
510
511 .watch = watch,
512 .web_server = undefined, // set after `prepare`
513 .memory_blocked_steps = .empty,
514 .step_stack = .empty,
515
516 .error_style = error_style,
517 .multiline_errors = multiline_errors,
518 .summary = summary orelse if (watch or webui_listen != null) .line else .failures,
519 };
520 defer {
521 maker.memory_blocked_steps.deinit(gpa);
522 maker.step_stack.deinit(gpa);
523 }
524
525 if (maker.available_rss == 0) {
526 maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64);
527 maker.max_rss_is_default = true;
528 }
529
530 maker.prepare(step_names.items) catch |err| switch (err) {
531 error.DependencyLoopDetected, error.InsufficientMemory => {
532 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
533 process.exit(1);
534 },
535 else => |e| return e,
536 };
537
538 var w: Watch = w: {
539 if (!watch) break :w undefined;
540 if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag});
541 break :w try .init(graph.cache.cwd, &scanned_config.configuration, maker.steps);
542 };
543
544 const now = Io.Clock.Timestamp.now(io, .awake);
545
546 maker.web_server = if (webui_listen) |listen_address| ws: {
547 if (builtin.single_threaded) unreachable; // `fatal` above
548 break :ws .init(.{
549 .gpa = gpa,
550 .graph = &graph,
551 .all_steps = maker.step_stack.keys(),
552 .root_prog_node = main_progress_node,
553 .watch = watch,
554 .listen_address = listen_address,
555 .base_timestamp = now,
556 .configuration = &scanned_config.configuration,
557 });
558 } else null;
559
560 if (maker.web_server) |*ws| {
561 ws.start() catch |err| fatal("failed to start web server: {t}", .{err});
562 }
563
564 rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) {
565 const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
566 defer io.unlockStderr();
567 try stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H");
568 }) {
569 if (maker.web_server) |*ws| ws.startBuild();
570
571 try maker.makeStepNames(step_names.items, main_progress_node, fuzz);
572
573 if (maker.web_server) |*web_server| {
574 if (fuzz) |mode| if (mode != .forever) fatal(
575 "error: limited fuzzing is not implemented yet for --webui",
576 .{},
577 );
578
579 web_server.finishBuild(.{ .fuzz = fuzz != null });
580 }
581
582 if (maker.web_server) |*ws| {
583 const c = &scanned_config.configuration;
584 assert(!watch); // fatal error after CLI parsing
585 while (true) switch (try ws.wait()) {
586 .rebuild => {
587 for (maker.step_stack.keys()) |step_index| {
588 const step = maker.stepByIndex(step_index);
589 step.state = .precheck_done;
590 const deps = step_index.ptr(c).deps.slice(c);
591 step.pending_deps = @intCast(deps.len);
592 step.reset(gpa);
593 }
594 continue :rebuild;
595 },
596 };
597 }
598
599 // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`.
600 if (!Watch.have_impl) unreachable;
601
602 try w.update(gpa, maker.step_stack.keys());
603
604 // Wait until a file system notification arrives. Read all such events
605 // until the buffer is empty. Then wait for a debounce interval, resetting
606 // if any more events come in. After the debounce interval has passed,
607 // trigger a rebuild on all steps with modified inputs, as well as their
608 // recursive dependants.
609 var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined;
610 const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{
611 w.dir_count, countSubProcesses(maker.steps, maker.step_stack.keys()),
612 }) catch &caption_buf;
613 var debouncing_node = main_progress_node.start(caption, 0);
614 var in_debounce = false;
615 while (true) switch (try w.wait(gpa, io, if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) {
616 .timeout => {
617 assert(in_debounce);
618 debouncing_node.end();
619 markFailedStepsDirty(gpa, maker.steps, maker.step_stack.keys());
620 continue :rebuild;
621 },
622 .dirty => if (!in_debounce) {
623 in_debounce = true;
624 debouncing_node.end();
625 debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0);
626 },
627 .clean => {},
628 };
629 }
630}
631
632fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const Configuration.Step.Index) void {
633 for (all_steps) |step_index| {
634 const step = &make_steps[@intFromEnum(step_index)];
635 switch (step.state) {
636 .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa),
637 else => continue,
638 }
639 }
640 // Now that all dirty steps have been found, the remaining steps that
641 // succeeded from last run shall be marked "cached".
642 for (all_steps) |step_index| {
643 const step = &make_steps[@intFromEnum(step_index)];
644 switch (step.state) {
645 .success => step.result_cached = true,
646 else => continue,
647 }
648 }
649}
650
651fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.Index) usize {
652 var count: usize = 0;
653 for (all_steps) |step_index| {
654 const s = &make_steps[@intFromEnum(step_index)];
655 count += @intFromBool(s.getZigProcess() != null);
656 }
657 return count;
658}
659
660const InstallPaths = struct {
661 prefix: Path,
662 lib: Path,
663 bin: Path,
664 include: Path,
665};
666
667fn stepByIndex(maker: *const Maker, i: Configuration.Step.Index) *Step {
668 return &maker.steps[@intFromEnum(i)];
669}
670
671fn prepare(maker: *Maker, step_names: []const []const u8) !void {
672 const gpa = maker.gpa;
673 const graph = maker.graph;
674 const arena = graph.arena;
675 const seed: u32 = graph.random_seed;
676 const step_stack = &maker.step_stack;
677 const c = &maker.scanned_config.configuration;
678
679 @memset(maker.steps, .{});
680
681 if (step_names.len == 0) {
682 try step_stack.put(gpa, c.default_step, {});
683 } else {
684 try step_stack.ensureUnusedCapacity(gpa, step_names.len);
685 for (0..step_names.len) |i| {
686 const step_name = step_names[step_names.len - i - 1];
687 const s = maker.scanned_config.top_level_steps.get(step_name) orelse {
688 log.info("to list available steps: zig build -l", .{});
689 fatal("no such step: {s}", .{step_name});
690 };
691 step_stack.putAssumeCapacity(s, {});
692 }
693 }
694
695 const starting_steps = try arena.dupe(Configuration.Step.Index, step_stack.keys());
696
697 var rng = std.Random.DefaultPrng.init(seed);
698 const rand = rng.random();
699 rand.shuffle(Configuration.Step.Index, starting_steps);
700
701 for (starting_steps) |s| {
702 try constructGraphAndCheckForDependencyLoop(gpa, c, maker.steps, s, &maker.step_stack, rand);
703 }
704
705 {
706 // Check that we have enough memory to complete the build.
707 var any_problems = false;
708 var max_needed: usize = 0;
709 for (step_stack.keys()) |step_index| {
710 const make_step = maker.stepByIndex(step_index);
711 const conf_step = step_index.ptr(c);
712 const max_rss = conf_step.max_rss.toBytes();
713 if (max_rss == 0) continue;
714 max_needed = @max(max_needed, max_rss);
715 if (max_rss > maker.available_rss) {
716 if (maker.skip_oom_steps) {
717 make_step.state = .skipped_oom;
718 for (make_step.dependants.items) |dependant| {
719 maker.stepByIndex(dependant).pending_deps -= 1;
720 }
721 } else {
722 log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{
723 conf_step.owner.depPrefixSlice(c),
724 conf_step.name.slice(c),
725 max_rss,
726 maker.available_rss,
727 });
728 any_problems = true;
729 }
730 }
731 }
732 if (any_problems) {
733 if (maker.max_rss_is_default) {
734 std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{
735 max_needed,
736 });
737 }
738 return error.InsufficientMemory;
739 }
740 }
741}
742
743fn makeStepNames(
744 maker: *Maker,
745 step_names: []const []const u8,
746 parent_prog_node: std.Progress.Node,
747 fuzz: ?Fuzz.Mode,
748) !void {
749 const graph = maker.graph;
750 const gpa = maker.gpa;
751 const io = graph.io;
752 const step_stack = &maker.step_stack;
753 const top_level_steps = &maker.scanned_config.top_level_steps;
754 const c = &maker.scanned_config.configuration;
755
756 {
757 // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer,
758 // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking
759 // a step is initial when it actually became ready due to an earlier initial step.
760 var initial_set: std.ArrayList(Configuration.Step.Index) = .empty;
761 defer initial_set.deinit(gpa);
762 try initial_set.ensureUnusedCapacity(gpa, step_stack.count());
763 for (step_stack.keys()) |step_index| {
764 const s = maker.stepByIndex(step_index);
765 if (s.state == .precheck_done and s.pending_deps == 0) {
766 initial_set.appendAssumeCapacity(step_index);
767 }
768 }
769
770 const step_prog = parent_prog_node.start("steps", step_stack.count());
771 defer step_prog.end();
772
773 var group: Io.Group = .init;
774 defer group.cancel(io);
775 // Start working on all of the initial steps...
776 for (initial_set.items) |step_index| try stepReady(maker, &group, step_index, step_prog);
777 // ...and `makeStep` will trigger every other step when their last dependency finishes.
778 try group.await(io);
779 }
780
781 assert(maker.memory_blocked_steps.items.len == 0);
782
783 var test_pass_count: usize = 0;
784 var test_skip_count: usize = 0;
785 var test_fail_count: usize = 0;
786 var test_crash_count: usize = 0;
787 var test_timeout_count: usize = 0;
788
789 var test_count: usize = 0;
790
791 var success_count: usize = 0;
792 var skipped_count: usize = 0;
793 var failure_count: usize = 0;
794 var pending_count: usize = 0;
795 var total_compile_errors: usize = 0;
796
797 var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() });
798 defer cleanup_task.await(io);
799
800 for (step_stack.keys()) |step_index| {
801 const make_step = maker.stepByIndex(step_index);
802 test_pass_count += make_step.test_results.passCount();
803 test_skip_count += make_step.test_results.skip_count;
804 test_fail_count += make_step.test_results.fail_count;
805 test_crash_count += make_step.test_results.crash_count;
806 test_timeout_count += make_step.test_results.timeout_count;
807
808 test_count += make_step.test_results.test_count;
809
810 switch (make_step.state) {
811 .precheck_unstarted => unreachable,
812 .precheck_started => unreachable,
813 .precheck_done => unreachable,
814 .dependency_failure => pending_count += 1,
815 .success => success_count += 1,
816 .skipped, .skipped_oom => skipped_count += 1,
817 .failure => {
818 failure_count += 1;
819 const compile_errors_len = make_step.result_error_bundle.errorMessageCount();
820 if (compile_errors_len > 0) {
821 total_compile_errors += compile_errors_len;
822 }
823 },
824 }
825 }
826
827 if (fuzz) |mode| blk: {
828 switch (builtin.os.tag) {
829 // Current implementation depends on two things that need to be ported to Windows:
830 // * Memory-mapping to share data between the fuzzer and build runner.
831 // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
832 // many addresses to source locations).
833 .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}),
834 else => {},
835 }
836 if (@bitSizeOf(usize) != 64) {
837 // Current implementation depends on posix.mmap()'s second parameter, `length: usize`,
838 // being compatible with file system's u64 return value. This is not the case
839 // on 32-bit platforms.
840 // Affects or affected by issues #5185, #22523, and #22464.
841 fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
842 }
843
844 switch (mode) {
845 .forever => break :blk,
846 .limit => {},
847 }
848
849 assert(mode == .limit);
850 var f = Fuzz.init(
851 gpa,
852 io,
853 step_stack.keys(),
854 parent_prog_node,
855 mode,
856 ) catch |err| fatal("failed to start fuzzer: {t}", .{err});
857 defer f.deinit();
858
859 f.start();
860 try f.waitAndPrintReport();
861 }
862
863 // Every test has a state
864 assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count);
865
866 if (failure_count == 0) {
867 std.Progress.setStatus(.success);
868 } else {
869 std.Progress.setStatus(.failure);
870 }
871
872 summary: {
873 switch (maker.summary) {
874 .all, .new, .line => {},
875 .failures => if (failure_count == 0) break :summary,
876 .none => break :summary,
877 }
878
879 const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
880 defer io.unlockStderr();
881 const t = stderr.terminal();
882 const w = &stderr.file_writer.interface;
883
884 const total_count = success_count + failure_count + pending_count + skipped_count;
885 t.setColor(.cyan) catch {};
886 t.setColor(.bold) catch {};
887 w.writeAll("Build Summary: ") catch {};
888 t.setColor(.reset) catch {};
889 w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {};
890 {
891 t.setColor(.dim) catch {};
892 var first = true;
893 if (skipped_count > 0) {
894 w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {};
895 first = false;
896 }
897 if (failure_count > 0) {
898 w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {};
899 first = false;
900 }
901 if (!first) w.writeByte(')') catch {};
902 t.setColor(.reset) catch {};
903 }
904
905 if (test_count > 0) {
906 w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {};
907 t.setColor(.dim) catch {};
908 var first = true;
909 if (test_skip_count > 0) {
910 w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {};
911 first = false;
912 }
913 if (test_fail_count > 0) {
914 w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {};
915 first = false;
916 }
917 if (test_crash_count > 0) {
918 w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {};
919 first = false;
920 }
921 if (test_timeout_count > 0) {
922 w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {};
923 first = false;
924 }
925 if (!first) w.writeByte(')') catch {};
926 t.setColor(.reset) catch {};
927 }
928
929 w.writeAll("\n") catch {};
930
931 if (maker.summary == .line) break :summary;
932
933 // Print a fancy tree with build results.
934 var step_stack_copy = try step_stack.clone(gpa);
935 defer step_stack_copy.deinit(gpa);
936
937 var print_node: PrintNode = .{ .parent = null };
938 if (step_names.len == 0) {
939 print_node.last = true;
940 printTreeStep(maker, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) {
941 error.Canceled => |e| return e,
942 else => {},
943 };
944 } else {
945 const last_index = if (maker.summary == .all) top_level_steps.count() else blk: {
946 var i: usize = step_names.len;
947 while (i > 0) {
948 i -= 1;
949 const step_index = top_level_steps.get(step_names[i]).?;
950 const step = maker.stepByIndex(step_index);
951 const found = switch (maker.summary) {
952 .all, .line, .none => unreachable,
953 .failures => step.state != .success,
954 .new => !step.result_cached,
955 };
956 if (found) break :blk i;
957 }
958 break :blk top_level_steps.count();
959 };
960 for (step_names, 0..) |step_name, i| {
961 const step_index = top_level_steps.get(step_name).?;
962 print_node.last = i + 1 == last_index;
963 printTreeStep(maker, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) {
964 error.Canceled => |e| return e,
965 else => {},
966 };
967 }
968 }
969 w.writeByte('\n') catch {};
970 }
971
972 if (maker.watch or maker.web_server != null) return;
973
974 // Perhaps in the future there could be an Advanced Options flag such as
975 // --debug-build-runner-leaks which would make this code return instead of
976 // calling exit.
977
978 const code: u8 = code: {
979 if (failure_count == 0) break :code 0; // success
980 if (maker.error_style.verboseContext()) break :code 1; // failure; print build command
981 break :code 2; // failure; do not print build command
982 };
983 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
984 process.exit(code);
985}
986
987fn stepReady(
988 maker: *Maker,
989 group: *Io.Group,
990 step_index: Configuration.Step.Index,
991 root_prog_node: std.Progress.Node,
992) Io.Cancelable!void {
993 const graph = maker.graph;
994 const io = graph.io;
995 const c = &maker.scanned_config.configuration;
996 const max_rss = step_index.ptr(c).max_rss.toBytes();
997 if (max_rss != 0) {
998 try maker.max_rss_mutex.lock(io);
999 defer maker.max_rss_mutex.unlock(io);
1000 if (maker.available_rss < max_rss) {
1001 // Running this step right now could possibly exceed the allotted RSS.
1002 maker.memory_blocked_steps.append(maker.gpa, step_index) catch
1003 @panic("TODO eliminate memory allocation here");
1004 return;
1005 }
1006 maker.available_rss -= max_rss;
1007 }
1008 group.async(io, makeStep, .{ maker, group, step_index, root_prog_node });
1009}
1010
1011/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready
1012/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must
1013/// have already subtracted this value from `maker.available_rss`. This function will release the RSS
1014/// claim (i.e. add `s.max_rss` back into `maker.available_rss`) and queue any viable memory-blocked
1015/// steps after "make" completes for `s`.
1016fn makeStep(
1017 maker: *Maker,
1018 group: *Io.Group,
1019 step_index: Configuration.Step.Index,
1020 root_prog_node: std.Progress.Node,
1021) Io.Cancelable!void {
1022 const graph = maker.graph;
1023 const io = graph.io;
1024 const gpa = maker.gpa;
1025 const c = &maker.scanned_config.configuration;
1026 const conf_step = step_index.ptr(c);
1027 const step_name = conf_step.name.slice(c);
1028 const deps = conf_step.deps.slice(c);
1029 const make_step = maker.stepByIndex(step_index);
1030
1031 {
1032 const step_prog_node = root_prog_node.start(step_name, 0);
1033 defer step_prog_node.end();
1034
1035 if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .wip);
1036
1037 const new_state: Step.State = for (deps) |dep_index| {
1038 const dep_make_step = maker.stepByIndex(dep_index);
1039 switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) {
1040 .precheck_unstarted => unreachable,
1041 .precheck_started => unreachable,
1042 .precheck_done => unreachable,
1043
1044 .failure,
1045 .dependency_failure,
1046 .skipped_oom,
1047 => break .dependency_failure,
1048
1049 .success, .skipped => {},
1050 }
1051 } else if (make_step.make(.{
1052 .progress_node = step_prog_node,
1053 .watch = maker.watch,
1054 .web_server = if (maker.web_server) |*ws| ws else null,
1055 .unit_test_timeout_ns = maker.unit_test_timeout_ns,
1056 .gpa = gpa,
1057 })) state: {
1058 break :state .success;
1059 } else |err| switch (err) {
1060 error.MakeFailed => .failure,
1061 error.MakeSkipped => .skipped,
1062 };
1063
1064 @atomicStore(Step.State, &make_step.state, new_state, .monotonic);
1065
1066 switch (new_state) {
1067 .precheck_unstarted => unreachable,
1068 .precheck_started => unreachable,
1069 .precheck_done => unreachable,
1070
1071 .failure,
1072 .dependency_failure,
1073 .skipped_oom,
1074 => {
1075 if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .failure);
1076 std.Progress.setStatus(.failure_working);
1077 },
1078
1079 .success,
1080 .skipped,
1081 => {
1082 if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .success);
1083 },
1084 }
1085 }
1086
1087 // No matter the result, we want to display error/warning messages.
1088 if (make_step.result_error_bundle.errorMessageCount() > 0 or
1089 make_step.result_error_msgs.items.len > 0 or
1090 make_step.result_stderr.len > 0)
1091 {
1092 const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
1093 defer io.unlockStderr();
1094 printErrorMessages(gpa, c, maker.steps, step_index, .{}, stderr.terminal(), maker.error_style, maker.multiline_errors) catch |err| switch (err) {
1095 error.Canceled => |e| return e,
1096 error.WriteFailed => switch (stderr.file_writer.err.?) {
1097 error.Canceled => |e| return e,
1098 else => {},
1099 },
1100 else => {},
1101 };
1102 }
1103
1104 const max_rss = conf_step.max_rss.toBytes();
1105 if (max_rss != 0) {
1106 var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty;
1107 defer dispatch_set.deinit(gpa);
1108
1109 // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set`
1110 // as a staging buffer to avoid recursing into `makeStep` while `maker.max_rss_mutex` is held.
1111 {
1112 try maker.max_rss_mutex.lock(io);
1113 defer maker.max_rss_mutex.unlock(io);
1114 maker.available_rss += max_rss;
1115 dispatch_set.ensureUnusedCapacity(gpa, maker.memory_blocked_steps.items.len) catch
1116 @panic("TODO eliminate memory allocation here");
1117 while (maker.memory_blocked_steps.getLast()) |candidate_index| {
1118 const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes();
1119 if (maker.available_rss < candidate_max_rss) break;
1120 assert(maker.memory_blocked_steps.pop() == candidate_index);
1121 dispatch_set.appendAssumeCapacity(candidate_index);
1122 }
1123 }
1124 for (dispatch_set.items) |candidate| {
1125 group.async(io, makeStep, .{ maker, group, candidate, root_prog_node });
1126 }
1127 }
1128
1129 for (make_step.dependants.items) |dependant_index| {
1130 const dependant = maker.stepByIndex(dependant_index);
1131 // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0.
1132 if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) {
1133 try stepReady(maker, group, dependant_index, root_prog_node);
1134 }
1135 }
1136}
1137
1138fn printTreeStep(
1139 maker: *const Maker,
1140 step_index: Configuration.Step.Index,
1141 stderr: Io.Terminal,
1142 parent_node: *PrintNode,
1143 step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void),
1144) !void {
1145 const writer = stderr.writer;
1146 const first = step_stack.swapRemove(step_index);
1147 const summary = maker.summary;
1148 const c = &maker.scanned_config.configuration;
1149 const conf_step = step_index.ptr(c);
1150 const make_step = maker.stepByIndex(step_index);
1151 const skip = switch (summary) {
1152 .none, .line => unreachable,
1153 .all => false,
1154 .new => make_step.result_cached,
1155 .failures => make_step.state == .success,
1156 };
1157 if (skip) return;
1158 try printPrefix(parent_node, stderr);
1159
1160 if (parent_node.parent != null) {
1161 if (parent_node.last) {
1162 try printChildNodePrefix(stderr);
1163 } else {
1164 try writer.writeAll(switch (stderr.mode) {
1165 .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─
1166 else => "+- ",
1167 });
1168 }
1169 }
1170
1171 if (!first) try stderr.setColor(.dim);
1172
1173 // dep_prefix omitted here because it is redundant with the tree.
1174 try writer.writeAll(conf_step.name.slice(c));
1175
1176 const deps = conf_step.deps.slice(c);
1177
1178 if (first) {
1179 try printStepStatus(maker, step_index, stderr);
1180
1181 const last_index = if (summary == .all) deps.len -| 1 else blk: {
1182 var i: usize = deps.len;
1183 while (i > 0) {
1184 i -= 1;
1185
1186 const dep_index = deps[i];
1187 const dep = maker.stepByIndex(dep_index);
1188 const found = switch (summary) {
1189 .all, .line, .none => unreachable,
1190 .failures => dep.state != .success,
1191 .new => !dep.result_cached,
1192 };
1193 if (found) break :blk i;
1194 }
1195 break :blk deps.len -| 1;
1196 };
1197 for (deps, 0..) |dep, i| {
1198 var print_node: PrintNode = .{
1199 .parent = parent_node,
1200 .last = i == last_index,
1201 };
1202 try printTreeStep(maker, dep, stderr, &print_node, step_stack);
1203 }
1204 } else {
1205 if (deps.len == 0) {
1206 try writer.writeAll(" (reused)\n");
1207 } else {
1208 try writer.print(" (+{d} more reused dependencies)\n", .{deps.len});
1209 }
1210 try stderr.setColor(.reset);
1211 }
1212}
1213
1214fn printStepStatus(maker: *const Maker, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void {
1215 const s = maker.stepByIndex(step_index);
1216 const writer = stderr.writer;
1217 switch (s.state) {
1218 .precheck_unstarted => unreachable,
1219 .precheck_started => unreachable,
1220 .precheck_done => unreachable,
1221
1222 .dependency_failure => {
1223 try stderr.setColor(.dim);
1224 try writer.writeAll(" transitive failure\n");
1225 try stderr.setColor(.reset);
1226 },
1227
1228 .success => {
1229 try stderr.setColor(.green);
1230 if (s.result_cached) {
1231 try writer.writeAll(" cached");
1232 } else if (s.test_results.test_count > 0) {
1233 const pass_count = s.test_results.passCount();
1234 assert(s.test_results.test_count == pass_count + s.test_results.skip_count);
1235 try writer.print(" {d} pass", .{pass_count});
1236 if (s.test_results.skip_count > 0) {
1237 try stderr.setColor(.reset);
1238 try writer.writeAll(", ");
1239 try stderr.setColor(.yellow);
1240 try writer.print("{d} skip", .{s.test_results.skip_count});
1241 }
1242 try stderr.setColor(.reset);
1243 try writer.print(" ({d} total)", .{s.test_results.test_count});
1244 } else {
1245 try writer.writeAll(" success");
1246 }
1247 try stderr.setColor(.reset);
1248 if (s.result_duration_ns) |ns| {
1249 try stderr.setColor(.dim);
1250 if (ns >= std.time.ns_per_min) {
1251 try writer.print(" {d}m", .{ns / std.time.ns_per_min});
1252 } else if (ns >= std.time.ns_per_s) {
1253 try writer.print(" {d}s", .{ns / std.time.ns_per_s});
1254 } else if (ns >= std.time.ns_per_ms) {
1255 try writer.print(" {d}ms", .{ns / std.time.ns_per_ms});
1256 } else if (ns >= std.time.ns_per_us) {
1257 try writer.print(" {d}us", .{ns / std.time.ns_per_us});
1258 } else {
1259 try writer.print(" {d}ns", .{ns});
1260 }
1261 try stderr.setColor(.reset);
1262 }
1263 if (s.result_peak_rss != 0) {
1264 const rss = s.result_peak_rss;
1265 try stderr.setColor(.dim);
1266 if (rss >= 1000_000_000) {
1267 try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000});
1268 } else if (rss >= 1000_000) {
1269 try writer.print(" MaxRSS:{d}M", .{rss / 1000_000});
1270 } else if (rss >= 1000) {
1271 try writer.print(" MaxRSS:{d}K", .{rss / 1000});
1272 } else {
1273 try writer.print(" MaxRSS:{d}B", .{rss});
1274 }
1275 try stderr.setColor(.reset);
1276 }
1277 try writer.writeAll("\n");
1278 },
1279 .skipped => {
1280 try stderr.setColor(.yellow);
1281 try writer.writeAll(" skipped\n");
1282 try stderr.setColor(.reset);
1283 },
1284 .skipped_oom => {
1285 const c = &maker.scanned_config.configuration;
1286 const max_rss = step_index.ptr(c).max_rss.toBytes();
1287 try stderr.setColor(.yellow);
1288 try writer.writeAll(" skipped (not enough memory)");
1289 try stderr.setColor(.dim);
1290 try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{
1291 max_rss, maker.available_rss,
1292 });
1293 try stderr.setColor(.reset);
1294 },
1295 .failure => {
1296 try printStepFailure(maker.steps, step_index, stderr, false);
1297 try stderr.setColor(.reset);
1298 },
1299 }
1300}
1301
1302fn printStepFailure(
1303 make_steps: []Step,
1304 step_index: Configuration.Step.Index,
1305 stderr: Io.Terminal,
1306 dim: bool,
1307) !void {
1308 const w = stderr.writer;
1309 const s = &make_steps[@intFromEnum(step_index)];
1310 if (s.result_error_bundle.errorMessageCount() > 0) {
1311 try stderr.setColor(.red);
1312 try w.print(" {d} errors\n", .{
1313 s.result_error_bundle.errorMessageCount(),
1314 });
1315 } else if (!s.test_results.isSuccess()) {
1316 // These first values include all of the test "statuses". Every test is either passsed,
1317 // skipped, failed, crashed, or timed out.
1318 try stderr.setColor(.green);
1319 try w.print(" {d} pass", .{s.test_results.passCount()});
1320 try stderr.setColor(.reset);
1321 if (dim) try stderr.setColor(.dim);
1322 if (s.test_results.skip_count > 0) {
1323 try w.writeAll(", ");
1324 try stderr.setColor(.yellow);
1325 try w.print("{d} skip", .{s.test_results.skip_count});
1326 try stderr.setColor(.reset);
1327 if (dim) try stderr.setColor(.dim);
1328 }
1329 if (s.test_results.fail_count > 0) {
1330 try w.writeAll(", ");
1331 try stderr.setColor(.red);
1332 try w.print("{d} fail", .{s.test_results.fail_count});
1333 try stderr.setColor(.reset);
1334 if (dim) try stderr.setColor(.dim);
1335 }
1336 if (s.test_results.crash_count > 0) {
1337 try w.writeAll(", ");
1338 try stderr.setColor(.red);
1339 try w.print("{d} crash", .{s.test_results.crash_count});
1340 try stderr.setColor(.reset);
1341 if (dim) try stderr.setColor(.dim);
1342 }
1343 if (s.test_results.timeout_count > 0) {
1344 try w.writeAll(", ");
1345 try stderr.setColor(.red);
1346 try w.print("{d} timeout", .{s.test_results.timeout_count});
1347 try stderr.setColor(.reset);
1348 if (dim) try stderr.setColor(.dim);
1349 }
1350 try w.print(" ({d} total)", .{s.test_results.test_count});
1351
1352 // Memory leaks are intentionally written after the total, because is isn't a test *status*,
1353 // but just a flag that any tests -- even passed ones -- can have. We also use a different
1354 // separator, so it looks like:
1355 // 2 pass, 1 skip, 2 fail (5 total); 2 leaks
1356 if (s.test_results.leak_count > 0) {
1357 try w.writeAll("; ");
1358 try stderr.setColor(.red);
1359 try w.print("{d} leaks", .{s.test_results.leak_count});
1360 try stderr.setColor(.reset);
1361 if (dim) try stderr.setColor(.dim);
1362 }
1363
1364 // It's usually not helpful to know how many error logs there were because they tend to
1365 // just come with other errors (e.g. crashes and leaks print stack traces, and clean
1366 // failures print error traces). So only mention them if they're the only thing causing
1367 // the failure.
1368 const show_err_logs: bool = show: {
1369 var alt_results = s.test_results;
1370 alt_results.log_err_count = 0;
1371 break :show alt_results.isSuccess();
1372 };
1373 if (show_err_logs) {
1374 try w.writeAll("; ");
1375 try stderr.setColor(.red);
1376 try w.print("{d} error logs", .{s.test_results.log_err_count});
1377 try stderr.setColor(.reset);
1378 if (dim) try stderr.setColor(.dim);
1379 }
1380
1381 try w.writeAll("\n");
1382 } else if (s.result_error_msgs.items.len > 0) {
1383 try stderr.setColor(.red);
1384 try w.writeAll(" failure\n");
1385 } else {
1386 assert(s.result_stderr.len > 0);
1387 try stderr.setColor(.red);
1388 try w.writeAll(" w\n");
1389 }
1390}
1391
1392const PrintNode = struct {
1393 parent: ?*PrintNode,
1394 last: bool = false,
1395};
1396
1397fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void {
1398 const parent = node.parent orelse return;
1399 const writer = stderr.writer;
1400 if (parent.parent == null) return;
1401 try printPrefix(parent, stderr);
1402 if (parent.last) {
1403 try writer.writeAll(" ");
1404 } else {
1405 try writer.writeAll(switch (stderr.mode) {
1406 .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │
1407 else => "| ",
1408 });
1409 }
1410}
1411
1412fn printChildNodePrefix(stderr: Io.Terminal) !void {
1413 try stderr.writer.writeAll(switch (stderr.mode) {
1414 .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─
1415 else => "+- ",
1416 });
1417}
1418
1419/// Traverse the dependency graph depth-first and make it undirected by having
1420/// steps know their dependants (they only know dependencies at start).
1421/// Along the way, check that there is no dependency loop, and record the steps
1422/// in traversal order in `step_stack`.
1423/// Each step has its dependencies traversed in random order, this accomplishes
1424/// two things:
1425/// - `step_stack` will be in randomized-depth-first order, so the build runner
1426/// spawns initial steps in a random order
1427/// - each step's `dependants` list is also filled in a random order, so that
1428/// when it finishes executing in `makeStep`, it spawns next steps to run in
1429/// random order
1430fn constructGraphAndCheckForDependencyLoop(
1431 gpa: Allocator,
1432 c: *const Configuration,
1433 steps: []Step,
1434 step_index: Configuration.Step.Index,
1435 step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void),
1436 rand: std.Random,
1437) error{ DependencyLoopDetected, OutOfMemory }!void {
1438 const make_step: *Step = &steps[@intFromEnum(step_index)];
1439 switch (make_step.state) {
1440 .precheck_started => {
1441 log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)});
1442 return error.DependencyLoopDetected;
1443 },
1444 .precheck_unstarted => {
1445 make_step.state = .precheck_started;
1446
1447 const step = step_index.ptr(c);
1448 const dependencies = step.deps.slice(c);
1449 try step_stack.ensureUnusedCapacity(gpa, dependencies.len);
1450
1451 // We dupe to avoid shuffling the steps in the summary, it depends
1452 // on dependencies' order.
1453 const deps = try gpa.dupe(Configuration.Step.Index, dependencies);
1454 defer gpa.free(deps);
1455
1456 rand.shuffle(Configuration.Step.Index, deps);
1457
1458 for (deps) |dep| {
1459 const dep_step: *Step = &steps[@intFromEnum(dep)];
1460 try step_stack.put(gpa, dep, {});
1461 try dep_step.dependants.append(gpa, step_index);
1462 constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) {
1463 error.DependencyLoopDetected => {
1464 log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)});
1465 return err;
1466 },
1467 else => return err,
1468 };
1469 }
1470
1471 make_step.state = .precheck_done;
1472 make_step.pending_deps = @intCast(dependencies.len);
1473 },
1474 .precheck_done => {},
1475
1476 // These don't happen until we actually run the step graph.
1477 .dependency_failure => unreachable,
1478 .success => unreachable,
1479 .failure => unreachable,
1480 .skipped => unreachable,
1481 .skipped_oom => unreachable,
1482 }
1483}
1484
1485pub fn printErrorMessages(
1486 gpa: Allocator,
1487 c: *const Configuration,
1488 make_steps: []Step,
1489 failing_step_index: Configuration.Step.Index,
1490 options: std.zig.ErrorBundle.RenderOptions,
1491 stderr: Io.Terminal,
1492 error_style: ErrorStyle,
1493 multiline_errors: MultilineErrors,
1494) !void {
1495 const writer = stderr.writer;
1496 if (error_style.verboseContext()) {
1497 // Provide context for where these error messages are coming from by
1498 // printing the corresponding Step subtree.
1499 var step_stack: std.ArrayList(Configuration.Step.Index) = .empty;
1500 defer step_stack.deinit(gpa);
1501 try step_stack.append(gpa, failing_step_index);
1502 while (true) {
1503 const last_step = &make_steps[@intFromEnum(step_stack.items[step_stack.items.len - 1])];
1504 if (last_step.dependants.items.len == 0) break;
1505 try step_stack.append(gpa, last_step.dependants.items[0]);
1506 }
1507
1508 // Now, `step_stack` has the subtree that we want to print, in reverse order.
1509 try stderr.setColor(.dim);
1510 var indent: usize = 0;
1511 while (step_stack.pop()) |step_index| : (indent += 1) {
1512 if (indent > 0) {
1513 try writer.splatByteAll(' ', (indent - 1) * 3);
1514 try printChildNodePrefix(stderr);
1515 }
1516
1517 try writer.writeAll(step_index.ptr(c).name.slice(c));
1518
1519 if (step_index == failing_step_index) {
1520 try printStepFailure(make_steps, step_index, stderr, true);
1521 } else {
1522 try writer.writeAll("\n");
1523 }
1524 }
1525 try stderr.setColor(.reset);
1526 } else {
1527 // Just print the failing step itself.
1528 try stderr.setColor(.dim);
1529 try writer.writeAll(failing_step_index.ptr(c).name.slice(c));
1530 try printStepFailure(make_steps, failing_step_index, stderr, true);
1531 try stderr.setColor(.reset);
1532 }
1533
1534 const failing_step = &make_steps[@intFromEnum(failing_step_index)];
1535
1536 if (failing_step.result_stderr.len > 0) {
1537 try writer.writeAll(failing_step.result_stderr);
1538 if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) {
1539 try writer.writeAll("\n");
1540 }
1541 }
1542
1543 try failing_step.result_error_bundle.renderToTerminal(options, stderr);
1544
1545 for (failing_step.result_error_msgs.items) |msg| {
1546 try stderr.setColor(.red);
1547 try writer.writeAll("error:");
1548 try stderr.setColor(.reset);
1549 if (std.mem.indexOfScalar(u8, msg, '\n') == null) {
1550 try writer.print(" {s}\n", .{msg});
1551 } else switch (multiline_errors) {
1552 .indent => {
1553 var it = std.mem.splitScalar(u8, msg, '\n');
1554 try writer.print(" {s}\n", .{it.first()});
1555 while (it.next()) |line| {
1556 try writer.print(" {s}\n", .{line});
1557 }
1558 },
1559 .newline => try writer.print("\n{s}\n", .{msg}),
1560 .none => try writer.print(" {s}\n", .{msg}),
1561 }
1562 }
1563
1564 if (error_style.verboseContext()) {
1565 if (failing_step.result_failed_command) |cmd_str| {
1566 try stderr.setColor(.red);
1567 try writer.writeAll("failed command: ");
1568 try stderr.setColor(.reset);
1569 try writer.writeAll(cmd_str);
1570 try writer.writeByte('\n');
1571 }
1572 }
1573
1574 try writer.writeByte('\n');
1575}
1576
1577fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 {
1578 if (idx.* >= args.len) return null;
1579 defer idx.* += 1;
1580 return args[idx.*];
1581}
1582
1583fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 {
1584 return nextArg(args, idx) orelse {
1585 fatalWithHint("expected argument after {q}", .{args[idx.* - 1]});
1586 };
1587}
1588
1589fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 {
1590 const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first});
1591 if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg });
1592 const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first});
1593 return arg;
1594}
1595
1596fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 {
1597 if (idx >= args.len) return null;
1598 return args[idx..];
1599}
1600
1601const Color = std.zig.Color;
1602const ErrorStyle = enum {
1603 verbose,
1604 minimal,
1605 verbose_clear,
1606 minimal_clear,
1607 fn verboseContext(s: ErrorStyle) bool {
1608 return switch (s) {
1609 .verbose, .verbose_clear => true,
1610 .minimal, .minimal_clear => false,
1611 };
1612 }
1613 fn clearOnUpdate(s: ErrorStyle) bool {
1614 return switch (s) {
1615 .verbose, .minimal => false,
1616 .verbose_clear, .minimal_clear => true,
1617 };
1618 }
1619};
1620const MultilineErrors = enum { indent, newline, none };
1621const Summary = enum { all, new, failures, line, none };
1622
1623fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn {
1624 log.info("to access the help menu: zig build -h", .{});
1625 fatal(f, args);
1626}
1627
1628fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void {
1629 for (steps) |step_index| {
1630 if (true) @panic("TODO");
1631 const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue;
1632 if (wf.mode != .tmp) continue;
1633 const path = wf.generated_directory.path orelse continue;
1634 Io.Dir.cwd().deleteTree(io, path) catch |err| {
1635 log.warn("failed to delete {s}: {t}", .{ path, err });
1636 };
1637 }
1638}
1639
1640var stdio_buffer_allocation: [256]u8 = undefined;
1641var stdout_writer_allocation: Io.File.Writer = undefined;
1642
1643fn initStdoutWriter(io: Io) *Writer {
1644 stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation);
1645 return &stdout_writer_allocation.interface;
1646}
1647
1648const ScannedConfig = struct {
1649 configuration: Configuration,
1650 top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index),
1651
1652 fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void {
1653 const c = &sc.configuration;
1654 var serializer: std.zon.Serializer = .{ .writer = w };
1655 var s = try serializer.beginStruct(.{});
1656
1657 try s.field("default_step", @intFromEnum(c.default_step), .{});
1658 {
1659 var ss = try s.beginStructField("top_level_steps", .{});
1660 for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step| {
1661 try ss.field(name, @intFromEnum(step), .{});
1662 }
1663 try ss.end();
1664 }
1665
1666 try s.end();
1667 }
1668
1669 fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void {
1670 const arena = graph.arena;
1671 const c = &sc.configuration;
1672 for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step_index| {
1673 const step = step_index.ptr(c);
1674 const decorated_name = if (step_index == c.default_step)
1675 try fmt.allocPrint(arena, "{s} (default)", .{name})
1676 else
1677 name;
1678 const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index);
1679 const description = top_level.description.slice(c);
1680 try w.print(" {s:<28} {s}\n", .{ decorated_name, description });
1681 }
1682 }
1683
1684 fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void {
1685 const arena = graph.arena;
1686
1687 try w.print(
1688 \\Usage: {s} build [steps] [options]
1689 \\
1690 \\Steps:
1691 \\
1692 , .{graph.zig_exe});
1693 try printSteps(sc, graph, w);
1694 try w.writeAll(
1695 \\
1696 \\Project-Specific Options:
1697 \\
1698 );
1699
1700 const available_options = sc.configuration.available_options;
1701 if (available_options.len == 0) {
1702 try w.print(" (none)\n", .{});
1703 } else {
1704 for (available_options) |option| {
1705 const name = option.name.slice(&sc.configuration);
1706 const description = option.description.slice(&sc.configuration);
1707 const help = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ name, option.type });
1708 try w.print("{s:<30} {s}\n", .{ help, description });
1709 if (option.enum_options.slice(&sc.configuration)) |enum_options| {
1710 const padding: [33]u8 = @splat(' ');
1711 try w.writeAll(padding ++ "Supported Values:\n");
1712 for (enum_options) |enum_option_index| {
1713 const enum_option = enum_option_index.slice(&sc.configuration);
1714 try w.print(padding ++ " {s}\n", .{enum_option});
1715 }
1716 }
1717 }
1718 }
1719
1720 try w.writeAll(
1721 \\
1722 \\System Integration Options:
1723 \\ --search-prefix [path] Add a path to look for binaries, libraries, headers
1724 \\ --sysroot [path] Set the system root directory (usually /)
1725 \\ --libc [file] Provide a file which specifies libc paths
1726 \\
1727 \\ --system [pkgdir] Disable package fetching; enable all integrations
1728 \\ -fsys=[name] Enable a system integration
1729 \\ -fno-sys=[name] Disable a system integration
1730 \\
1731 \\ -fdarling, -fno-darling Integration with system-installed Darling to
1732 \\ execute macOS programs on Linux hosts
1733 \\ (default: no)
1734 \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute
1735 \\ foreign-architecture programs on Linux hosts
1736 \\ (default: no)
1737 \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc
1738 \\ (e.g. glibc or musl) built for multiple foreign
1739 \\ architectures, allowing execution of non-native
1740 \\ programs that link with libc.
1741 \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on
1742 \\ ARM64 macOS hosts. (default: no)
1743 \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to
1744 \\ execute WASI binaries. (default: no)
1745 \\ -fwine, -fno-wine Integration with system-installed Wine to execute
1746 \\ Windows programs on Linux hosts. (default: no)
1747 \\
1748 \\ Available System Integrations: Enabled:
1749 \\
1750 );
1751 if (sc.configuration.system_integrations.len == 0) {
1752 try w.writeAll(" (none) -\n");
1753 } else {
1754 for (sc.configuration.system_integrations) |system_integration| {
1755 const name = system_integration.name.slice(&sc.configuration);
1756 const status = switch (system_integration.status) {
1757 .disabled => "no",
1758 .enabled => "yes",
1759 };
1760 try w.print(" {s:<43} {s}\n", .{ name, status });
1761 }
1762 }
1763
1764 try w.writeAll(
1765 \\
1766 \\General Options:
1767 \\ -h, --help Print this help to stdout and exit
1768 \\ -l, --list-steps Print available steps to stdout and exit
1769 \\
1770 \\ -p, --prefix [path] Where to install files (default: zig-out)
1771 \\ --prefix-lib-dir [path] Where to install libraries
1772 \\ --prefix-exe-dir [path] Where to install executables
1773 \\ --prefix-include-dir [path] Where to install C header files
1774 \\ --release[=mode] Request release mode, optionally specifying a
1775 \\ preferred optimization mode: fast, safe, small
1776 \\
1777 \\ --verbose Print commands before executing them
1778 \\ --color [auto|off|on] Enable or disable colored error messages
1779 \\ --error-style [style] Control how build errors are printed
1780 \\ verbose (Default) Report errors with full context
1781 \\ minimal Report errors after summary, excluding context like command lines
1782 \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update
1783 \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update
1784 \\ --multiline-errors [style] Control how multi-line error messages are printed
1785 \\ indent (Default) Indent non-initial lines to align with initial line
1786 \\ newline Include a leading newline so that the error message is on its own lines
1787 \\ none Print as usual so the first line is misaligned
1788 \\ --summary [mode] Control the printing of the build summary
1789 \\ all Print the build summary in its entirety
1790 \\ new Omit cached steps
1791 \\ failures (Default if short-lived) Only print failed steps
1792 \\ line (Default if long-lived) Only print the single-line summary
1793 \\ none Do not print the build summary
1794 \\ -j<N> Limit concurrent jobs (default is to use all CPU cores)
1795 \\ --maxrss <bytes> Limit memory usage (default is to use available memory)
1796 \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss
1797 \\ --test-timeout <timeout> Limit execution time of unit tests, terminating if exceeded.
1798 \\ The timeout must include a unit: ns, us, ms, s, m, h
1799 \\ --watch Continuously rebuild when source files are modified
1800 \\ --debounce <ms> Delay before rebuilding after changed file detected
1801 \\ --webui[=ip] Enable the web interface on the given IP address
1802 \\ --fuzz[=limit] Continuously search for unit test failures with an optional
1803 \\ limit to the max number of iterations. The argument supports
1804 \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies
1805 \\ '--webui' when no limit is specified.
1806 \\ --time-report Force full rebuild and provide detailed information on
1807 \\ compilation time of Zig source code (implies '--webui')
1808 \\ -fincremental Enable incremental compilation
1809 \\ -fno-incremental Disable incremental compilation
1810 \\
1811 \\Package Management Options:
1812 \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit
1813 \\ needed (Default) Lazy dependencies are fetched as needed
1814 \\ all Lazy dependencies are always fetched
1815 \\ --fork=[path] Override one or more projects from dependency tree
1816 \\
1817 \\Advanced Options:
1818 \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error
1819 \\ -fno-reference-trace Disable reference trace
1820 \\ -fallow-so-scripts Allows .so files to be GNU ld scripts
1821 \\ -fno-allow-so-scripts (default) .so files must be ELF files
1822 \\ --build-file [file] Override path to build.zig
1823 \\ --cache-dir [path] Override path to local Zig cache directory
1824 \\ --global-cache-dir [path] Override path to global Zig cache directory
1825 \\ --zig-lib-dir [arg] Override path to Zig lib directory
1826 \\ --build-runner [file] Override path to build runner
1827 \\ --seed [integer] For shuffling dependency traversal order (default: random)
1828 \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries
1829 \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM)
1830 \\ sha1, tree 20-byte cryptographic hash (ELF, WASM)
1831 \\ md5 16-byte cryptographic hash (ELF)
1832 \\ uuid 16-byte random UUID (ELF, WASM)
1833 \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM)
1834 \\ none (default) No build ID
1835 \\ --debug-log [scope] Enable debugging the compiler
1836 \\ --debug-pkg-config Fail if unknown pkg-config flags encountered
1837 \\ --debug-rt Debug compiler runtime libraries
1838 \\ --verbose-link Enable compiler debug output for linking
1839 \\ --verbose-air Enable compiler debug output for Zig AIR
1840 \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR
1841 \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC
1842 \\ --verbose-cimport Enable compiler debug output for C imports
1843 \\ --verbose-cc Enable compiler debug output for C compilation
1844 \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features
1845 \\
1846 );
1847 }
1848};
lib/compiler/Maker/Fuzz.zig created+606
...@@ -0,0 +1,606 @@
1const Fuzz = @This();
2
3const std = @import("std");
4const Allocator = std.mem.Allocator;
5const Build = std.Build;
6const Cache = std.Build.Cache;
7const Coverage = std.debug.Coverage;
8const Configuration = std.Build.Configuration;
9const Io = std.Io;
10const abi = std.Build.abi.fuzz;
11const assert = std.debug.assert;
12const fatal = std.process.fatal;
13const log = std.log;
14
15const Maker = @import("../Maker.zig");
16const WebServer = @import("WebServer.zig");
17
18gpa: Allocator,
19io: Io,
20mode: Mode,
21
22/// Allocated into `gpa`.
23run_steps: []const Configuration.Step.Index,
24
25group: Io.Group,
26root_prog_node: std.Progress.Node,
27prog_node: std.Progress.Node,
28
29/// Protects `coverage_files`.
30coverage_mutex: Io.Mutex,
31coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap),
32
33queue_mutex: Io.Mutex,
34queue_cond: Io.Condition,
35msg_queue: std.ArrayList(Msg),
36
37pub const Mode = union(enum) {
38 forever: struct { ws: *WebServer },
39 limit: Limited,
40
41 pub const Limited = struct {
42 amount: u64,
43 };
44};
45
46const Msg = union(enum) {
47 coverage: struct {
48 id: u64,
49 cumulative: struct {
50 runs: u64,
51 unique: u64,
52 coverage: u64,
53 },
54 run: Configuration.Step.Index,
55 },
56 entry_point: struct {
57 coverage_id: u64,
58 addr: u64,
59 },
60};
61
62const CoverageMap = struct {
63 mapped_memory: []align(std.heap.page_size_min) const u8,
64 coverage: Coverage,
65 source_locations: []Coverage.SourceLocation,
66 /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested.
67 entry_points: std.ArrayList(u32),
68 start_timestamp: i64,
69 start_n_runs: u64,
70
71 fn deinit(cm: *CoverageMap, gpa: Allocator) void {
72 std.posix.munmap(cm.mapped_memory);
73 cm.coverage.deinit(gpa);
74 cm.* = undefined;
75 }
76};
77
78pub fn init(
79 gpa: Allocator,
80 io: Io,
81 all_steps: []const Configuration.Step.Index,
82 root_prog_node: std.Progress.Node,
83 mode: Mode,
84) error{ OutOfMemory, Canceled }!Fuzz {
85 const run_steps: []const Configuration.Step.Index = steps: {
86 var steps: std.ArrayList(Configuration.Step.Index) = .empty;
87 defer steps.deinit(gpa);
88 const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);
89 defer rebuild_node.end();
90 var rebuild_group: Io.Group = .init;
91 defer rebuild_group.cancel(io);
92
93 for (all_steps) |step| {
94 if (true) @panic("TODO");
95 const run = step.cast(std.Build.Step.Run) orelse continue;
96 if (run.producer == null) continue;
97 if (run.fuzz_tests.items.len == 0) continue;
98 try steps.append(gpa, run);
99 rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, rebuild_node });
100 }
101
102 if (steps.items.len == 0) fatal("no fuzz tests found", .{});
103 rebuild_node.setEstimatedTotalItems(steps.items.len);
104 const run_steps = try gpa.dupe(Configuration.Step.Index, steps.items);
105 try rebuild_group.await(io);
106 break :steps run_steps;
107 };
108 errdefer gpa.free(run_steps);
109
110 for (run_steps) |run_step_index| {
111 if (true) @panic("TODO");
112 assert(run_step_index.fuzz_tests.items.len > 0);
113 if (run_step_index.rebuilt_executable == null)
114 fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{});
115 }
116
117 return .{
118 .gpa = gpa,
119 .io = io,
120 .mode = mode,
121 .run_steps = run_steps,
122 .group = .init,
123 .root_prog_node = root_prog_node,
124 .prog_node = .none,
125 .coverage_files = .empty,
126 .coverage_mutex = .init,
127 .queue_mutex = .init,
128 .queue_cond = .init,
129 .msg_queue = .empty,
130 };
131}
132
133pub fn start(fuzz: *Fuzz) void {
134 const io = fuzz.io;
135 fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", 0);
136
137 if (fuzz.mode == .forever) {
138 // For polling messages and sending updates to subscribers.
139 fuzz.group.concurrent(io, coverageRun, .{fuzz}) catch |err|
140 fatal("unable to spawn coverage task: {t}", .{err});
141 }
142
143 if (true) @panic("TODO");
144
145 for (fuzz.run_steps) |run| {
146 assert(run.rebuilt_executable != null);
147 fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run });
148 }
149}
150
151pub fn deinit(fuzz: *Fuzz) void {
152 const io = fuzz.io;
153 fuzz.group.cancel(io);
154 fuzz.prog_node.end();
155 fuzz.gpa.free(fuzz.run_steps);
156}
157
158fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void {
159 rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| {
160 const compile = run.producer.?;
161 log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err });
162 };
163}
164
165fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) !void {
166 const graph = run.step.owner.graph;
167 const io = graph.io;
168 const compile = run.producer.?;
169 const prog_node = parent_prog_node.start(compile.step.name, 0);
170 defer prog_node.end();
171
172 const result = compile.rebuildInFuzzMode(gpa, prog_node);
173
174 const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0;
175 const show_error_msgs = compile.step.result_error_msgs.items.len > 0;
176 const show_stderr = compile.step.result_stderr.len > 0;
177
178 if (show_error_msgs or show_compile_errors or show_stderr) {
179 var buf: [256]u8 = undefined;
180 const stderr = try io.lockStderr(&buf, graph.stderr_mode);
181 defer io.unlockStderr();
182 Maker.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {};
183 }
184
185 const rebuilt_bin_path = result catch |err| switch (err) {
186 error.MakeFailed => return,
187 else => |other| return other,
188 };
189 run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);
190}
191
192fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void {
193 const owner = run.step.owner;
194 const gpa = owner.allocator;
195 const graph = owner.graph;
196 const io = graph.io;
197
198 run.rerunInFuzzMode(fuzz, fuzz.prog_node) catch |err| switch (err) {
199 error.MakeFailed => {
200 var buf: [256]u8 = undefined;
201 const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) {
202 error.Canceled => return,
203 };
204 defer io.unlockStderr();
205 Maker.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {};
206 return;
207 },
208 else => {
209 log.err("step '{s}': failed to rerun in fuzz mode: {t}", .{ run.step.name, err });
210 return;
211 },
212 };
213}
214
215pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
216 if (true) @panic("TODO");
217 assert(fuzz.mode == .forever);
218
219 var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa);
220 defer arena_state.deinit();
221 const arena = arena_state.allocator();
222
223 const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
224 var dedup_table: DedupTable = .empty;
225 defer dedup_table.deinit(fuzz.gpa);
226
227 for (fuzz.run_steps) |run_step| {
228 const compile_inputs = run_step.producer.?.step.inputs.table;
229 for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| {
230 try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len);
231 for (file_list.items) |sub_path| {
232 if (!std.mem.endsWith(u8, sub_path, ".zig")) continue;
233 const joined_path = try dir_path.join(arena, sub_path);
234 dedup_table.putAssumeCapacity(joined_path, {});
235 }
236 }
237 }
238
239 const deduped_paths = dedup_table.keys();
240 const SortContext = struct {
241 pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool {
242 _ = this;
243 return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) {
244 .lt => true,
245 .gt => false,
246 .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path),
247 };
248 }
249 };
250 std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
251 return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths);
252}
253
254pub const Previous = struct {
255 unique_runs: usize,
256 entry_points: usize,
257 sent_source_index: bool,
258 pub const init: Previous = .{
259 .unique_runs = 0,
260 .entry_points = 0,
261 .sent_source_index = false,
262 };
263};
264pub fn sendUpdate(
265 fuzz: *Fuzz,
266 socket: *std.http.Server.WebSocket,
267 prev: *Previous,
268) !void {
269 const io = fuzz.io;
270
271 try fuzz.coverage_mutex.lock(io);
272 defer fuzz.coverage_mutex.unlock(io);
273
274 const coverage_maps = fuzz.coverage_files.values();
275 if (coverage_maps.len == 0) return;
276 // TODO: handle multiple fuzz steps in the WebSocket packets
277 const coverage_map = &coverage_maps[0];
278 const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
279 // TODO: this isn't sound! We need to do volatile reads of these bits rather than handing the
280 // buffer off to the kernel, because we might race with the fuzzer process[es]. This brings the
281 // whole mmap strategy into question. Incidentally, I wonder if post-writergate we could pass
282 // this data straight to the socket with sendfile...
283 const seen_pcs = cov_header.seenBits();
284 const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic);
285 const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic);
286 {
287 if (!prev.sent_source_index) {
288 prev.sent_source_index = true;
289 // We need to send initial context.
290 const header: abi.SourceIndexHeader = .{
291 .directories_len = @intCast(coverage_map.coverage.directories.entries.len),
292 .files_len = @intCast(coverage_map.coverage.files.entries.len),
293 .source_locations_len = @intCast(coverage_map.source_locations.len),
294 .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len),
295 .start_timestamp = coverage_map.start_timestamp,
296 .start_n_runs = coverage_map.start_n_runs,
297 };
298 var iovecs: [5][]const u8 = .{
299 @ptrCast(&header),
300 @ptrCast(coverage_map.coverage.directories.keys()),
301 @ptrCast(coverage_map.coverage.files.keys()),
302 @ptrCast(coverage_map.source_locations),
303 coverage_map.coverage.string_bytes.items,
304 };
305 try socket.writeMessageVec(&iovecs, .binary);
306 }
307
308 const header: abi.CoverageUpdateHeader = .{
309 .n_runs = n_runs,
310 .unique_runs = unique_runs,
311 };
312 var iovecs: [2][]const u8 = .{
313 @ptrCast(&header),
314 @ptrCast(seen_pcs),
315 };
316 try socket.writeMessageVec(&iovecs, .binary);
317
318 prev.unique_runs = unique_runs;
319 }
320
321 if (prev.entry_points != coverage_map.entry_points.items.len) {
322 const header: abi.EntryPointHeader = .init(@intCast(coverage_map.entry_points.items.len));
323 var iovecs: [2][]const u8 = .{
324 @ptrCast(&header),
325 @ptrCast(coverage_map.entry_points.items),
326 };
327 try socket.writeMessageVec(&iovecs, .binary);
328
329 prev.entry_points = coverage_map.entry_points.items.len;
330 }
331}
332
333fn coverageRun(fuzz: *Fuzz) void {
334 coverageRunCancelable(fuzz) catch |err| switch (err) {
335 error.Canceled => return,
336 };
337}
338
339fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void {
340 const io = fuzz.io;
341
342 try fuzz.queue_mutex.lock(io);
343 defer fuzz.queue_mutex.unlock(io);
344
345 while (true) {
346 try fuzz.queue_cond.wait(io, &fuzz.queue_mutex);
347 for (fuzz.msg_queue.items) |msg| switch (msg) {
348 .coverage => |coverage| prepareTables(fuzz, coverage.run, coverage.id) catch |err| switch (err) {
349 error.AlreadyReported => continue,
350 error.Canceled => return,
351 else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}),
352 },
353 .entry_point => |entry_point| addEntryPoint(fuzz, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) {
354 error.AlreadyReported => continue,
355 error.Canceled => return,
356 else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}),
357 },
358 };
359 fuzz.msg_queue.clearRetainingCapacity();
360 }
361}
362fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void {
363 if (true) @panic("TODO");
364 assert(fuzz.mode == .forever);
365 const ws = fuzz.mode.forever.ws;
366 const gpa = fuzz.gpa;
367 const io = fuzz.io;
368
369 try fuzz.coverage_mutex.lock(io);
370 defer fuzz.coverage_mutex.unlock(io);
371
372 const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id);
373 if (gop.found_existing) {
374 // We are fuzzing the same executable with multiple threads.
375 // Perhaps the same unit test; perhaps a different one. In any
376 // case, since the coverage file is the same, we only have to
377 // notice changes to that one file in order to learn coverage for
378 // this particular executable.
379 return;
380 }
381 errdefer _ = fuzz.coverage_files.pop();
382
383 gop.value_ptr.* = .{
384 .coverage = std.debug.Coverage.init,
385 .mapped_memory = undefined, // populated below
386 .source_locations = undefined, // populated below
387 .entry_points = .empty,
388 .start_timestamp = ws.now(),
389 .start_n_runs = undefined, // populated below
390 };
391 errdefer gop.value_ptr.coverage.deinit(gpa);
392
393 const rebuilt_exe_path = run_step_index.rebuilt_executable.?;
394 const target = run_step_index.producer.?.rootModuleTarget();
395 var debug_info = std.debug.Info.load(
396 gpa,
397 io,
398 rebuilt_exe_path,
399 &gop.value_ptr.coverage,
400 target.ofmt,
401 target.cpu.arch,
402 ) catch |err| {
403 log.err("step '{s}': failed to load debug information for '{f}': {t}", .{
404 run_step_index.step.name, rebuilt_exe_path, err,
405 });
406 return error.AlreadyReported;
407 };
408 defer debug_info.deinit(gpa);
409
410 const coverage_file_path: Build.Cache.Path = .{
411 .root_dir = run_step_index.step.owner.cache_root,
412 .sub_path = "v/" ++ std.fmt.hex(coverage_id),
413 };
414 var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| {
415 log.err("step '{s}': failed to load coverage file '{f}': {t}", .{
416 run_step_index.step.name, coverage_file_path, err,
417 });
418 return error.AlreadyReported;
419 };
420 defer coverage_file.close(io);
421
422 const file_size = coverage_file.length(io) catch |err| {
423 log.err("unable to check len of coverage file '{f}': {t}", .{ coverage_file_path, err });
424 return error.AlreadyReported;
425 };
426
427 const mapped_memory = std.posix.mmap(
428 null,
429 file_size,
430 .{ .READ = true },
431 .{ .TYPE = .SHARED },
432 coverage_file.handle,
433 0,
434 ) catch |err| {
435 log.err("failed to map coverage file '{f}': {t}", .{ coverage_file_path, err });
436 return error.AlreadyReported;
437 };
438 gop.value_ptr.mapped_memory = mapped_memory;
439
440 const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
441 const pcs = header.pcAddrs();
442 const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len);
443 errdefer gpa.free(source_locations);
444
445 // Unfortunately the PCs array that LLVM gives us from the 8-bit PC
446 // counters feature is not sorted.
447 var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .empty;
448 defer sorted_pcs.deinit(gpa);
449 try sorted_pcs.resize(gpa, pcs.len);
450 @memcpy(sorted_pcs.items(.pc), pcs);
451 for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i);
452 sorted_pcs.sortUnstable(struct {
453 addrs: []const u64,
454
455 pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
456 return ctx.addrs[a_index] < ctx.addrs[b_index];
457 }
458 }{ .addrs = sorted_pcs.items(.pc) });
459
460 debug_info.resolveAddresses(gpa, io, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {
461 log.err("failed to resolve addresses to source locations: {t}", .{err});
462 return error.AlreadyReported;
463 };
464
465 for (sorted_pcs.items(.index), sorted_pcs.items(.sl)) |i, sl| source_locations[i] = sl;
466 gop.value_ptr.source_locations = source_locations;
467 gop.value_ptr.start_n_runs = header.n_runs;
468
469 ws.notifyUpdate();
470}
471
472fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory, Canceled }!void {
473 const io = fuzz.io;
474
475 try fuzz.coverage_mutex.lock(io);
476 defer fuzz.coverage_mutex.unlock(io);
477
478 const coverage_map = fuzz.coverage_files.getPtr(coverage_id).?;
479 const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
480 const pcs = header.pcAddrs();
481
482 // Since this pcs list is unsorted, we must linear scan for the best index.
483 const index = i: {
484 var best: usize = 0;
485 for (pcs[1..], 1..) |elem_addr, i| {
486 if (elem_addr == addr) break :i i;
487 if (elem_addr > addr) continue;
488 if (elem_addr > pcs[best]) best = i;
489 }
490 break :i best;
491 };
492 if (index >= pcs.len) {
493 log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{
494 addr, pcs[0], pcs[pcs.len - 1],
495 });
496 return error.AlreadyReported;
497 }
498 if (false) {
499 const sl = coverage_map.source_locations[index];
500 const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename);
501 if (pcs.len == 1) {
502 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 (final)", .{
503 addr, file_name, sl.line, sl.column,
504 });
505 } else if (index == 0) {
506 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 before {x}", .{
507 addr, file_name, sl.line, sl.column, pcs[index + 1],
508 });
509 } else if (index == pcs.len - 1) {
510 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} (final) after {x}", .{
511 addr, file_name, sl.line, sl.column, index, pcs[index - 1],
512 });
513 } else {
514 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} between {x} and {x}", .{
515 addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1],
516 });
517 }
518 }
519 try coverage_map.entry_points.append(fuzz.gpa, @intCast(index));
520}
521
522pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void {
523 if (true) @panic("TODO");
524 assert(fuzz.mode == .limit);
525 const io = fuzz.io;
526
527 try fuzz.group.await(io);
528 fuzz.group = .init;
529
530 std.debug.print("======= FUZZING REPORT =======\n", .{});
531 for (fuzz.msg_queue.items) |msg| {
532 if (msg != .coverage) continue;
533
534 const cov = msg.coverage;
535 const coverage_file_path: std.Build.Cache.Path = .{
536 .root_dir = cov.run.step.owner.cache_root,
537 .sub_path = "v/" ++ std.fmt.hex(cov.id),
538 };
539 var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| {
540 fatal("step '{s}': failed to load coverage file '{f}': {t}", .{
541 cov.run.step.name, coverage_file_path, err,
542 });
543 };
544 defer coverage_file.close(io);
545
546 const fuzz_abi = std.Build.abi.fuzz;
547 var rbuf: [0x1000]u8 = undefined;
548 var r = coverage_file.reader(io, &rbuf);
549
550 var header: fuzz_abi.SeenPcsHeader = undefined;
551 r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| {
552 fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{
553 cov.run.step.name, coverage_file_path, err,
554 });
555 };
556
557 if (header.pcs_len == 0) {
558 fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{
559 cov.run.step.name, coverage_file_path,
560 });
561 }
562
563 var seen_count: usize = 0;
564 const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len);
565 for (0..chunk_count) |_| {
566 const seen = r.interface.takeInt(usize, .little) catch |err| {
567 fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{
568 cov.run.step.name, coverage_file_path, err,
569 });
570 };
571 seen_count += @popCount(seen);
572 }
573
574 const seen_f: f64 = @floatFromInt(seen_count);
575 const total_f: f64 = @floatFromInt(header.pcs_len);
576 const ratio = seen_f / total_f;
577 std.debug.print(
578 \\Step: {s}
579 \\Fuzz test: "{s}" ({x})
580 \\Runs: {} -> {}
581 \\Unique runs: {} -> {}
582 \\Coverage: {}/{} -> {}/{} ({:.02}%)
583 \\
584 , .{
585 cov.run.step.name,
586 cov.run.fuzz_tests.items[0],
587 cov.id,
588 cov.cumulative.runs,
589 header.n_runs,
590 cov.cumulative.unique,
591 header.unique_runs,
592 cov.cumulative.coverage,
593 header.pcs_len,
594 seen_count,
595 header.pcs_len,
596 ratio * 100,
597 });
598
599 std.debug.print("------------------------------\n", .{});
600 }
601 std.debug.print(
602 \\Values are accumulated across multiple runs when preserving the cache.
603 \\==============================
604 \\
605 , .{});
606}
lib/compiler/Maker/Graph.zig created+25
...@@ -0,0 +1,25 @@
1//! Shared maker state among all steps.
2const Graph = @This();
3
4const std = @import("std");
5const Io = std.Io;
6const Allocator = std.mem.Allocator;
7const Configuration = std.Build.Configuration;
8
9io: Io,
10/// Process lifetime.
11arena: Allocator,
12cache: std.Build.Cache,
13zig_exe: []const u8,
14environ_map: std.process.Environ.Map,
15global_cache_root: std.Build.Cache.Directory,
16zig_lib_directory: std.Build.Cache.Directory,
17
18debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null,
19incremental: ?bool = null,
20random_seed: u32 = 0,
21allow_so_scripts: ?bool = null,
22time_report: bool = false,
23/// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also
24/// respects the '--color' flag.
25stderr_mode: ?Io.Terminal.Mode = null,
lib/compiler/Maker/Step.zig created+840
...@@ -0,0 +1,840 @@
1//! The state that maker needs in order to process a step.
2const Step = @This();
3
4const builtin = @import("builtin");
5
6const std = @import("std");
7const Allocator = std.mem.Allocator;
8const Cache = std.Build.Cache;
9const Io = std.Io;
10const LazyPath = std.Build.Configuration.LazyPath;
11const Package = std.Build.Configuration.Package;
12const Path = std.Build.Cache.Path;
13const Configuration = std.Build.Configuration;
14const assert = std.debug.assert;
15
16const WebServer = @import("WebServer.zig");
17
18pub const Compile = void; // @import("Step/Compile.zig");
19pub const Run = void; // @import("Step/Run.zig");
20
21/// Avoid false sharing.
22_: void align(std.atomic.cache_line) = {},
23
24state: State = .precheck_unstarted,
25dependants: std.ArrayList(Configuration.Step.Index) = .empty,
26/// Collects the set of files that retrigger this step to run.
27///
28/// This is used by the build system's implementation of `--watch` but it can
29/// also be potentially useful for IDEs to know what effects editing a
30/// particular file has.
31///
32/// Populated within `make`. Implementation may choose to clear and repopulate,
33/// retain previous value, or update.
34inputs: Inputs = .init,
35pending_deps: u32 = undefined,
36
37result_error_msgs: std.ArrayList([]const u8) = .empty,
38result_error_bundle: std.zig.ErrorBundle = .empty,
39result_stderr: []const u8 = "",
40result_cached: bool = false,
41result_duration_ns: ?u64 = null,
42/// 0 means unavailable or not reported.
43result_peak_rss: usize = 0,
44/// If the step is failed and this field is populated, this is the command which failed.
45/// This field may be populated even if the step succeeded.
46result_failed_command: ?[]const u8 = null,
47test_results: TestResults = .{},
48
49pub const State = enum {
50 precheck_unstarted,
51 precheck_started,
52 /// This is also used to indicate "dirty" steps that have been modified
53 /// after a previous build completed, in which case, the step may or may
54 /// not have been completed before. Either way, one or more of its direct
55 /// file system inputs have been modified, meaning that the step needs to
56 /// be re-evaluated.
57 precheck_done,
58 dependency_failure,
59 success,
60 failure,
61 /// This state indicates that the step did not complete, however, it also did not fail,
62 /// and it is safe to continue executing its dependencies.
63 skipped,
64 /// This step was skipped because it specified a max_rss that exceeded the runner's maximum.
65 /// It is not safe to run its dependencies.
66 skipped_oom,
67};
68
69pub const Inputs = struct {
70 table: Table,
71
72 pub const init: Inputs = .{
73 .table = .{},
74 };
75
76 pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, Files, Cache.Path.TableAdapter, false);
77 /// The special file name "." means any changes inside the directory.
78 pub const Files = std.ArrayList([]const u8);
79
80 pub fn populated(inputs: *Inputs) bool {
81 return inputs.table.count() != 0;
82 }
83
84 pub fn clear(inputs: *Inputs, gpa: Allocator) void {
85 for (inputs.table.values()) |*files| files.deinit(gpa);
86 inputs.table.clearRetainingCapacity();
87 }
88};
89
90pub const TestResults = struct {
91 /// The total number of tests in the step. Every test has a "status" from the following:
92 /// * passed
93 /// * skipped
94 /// * failed cleanly
95 /// * crashed
96 /// * timed out
97 test_count: u32 = 0,
98
99 /// The number of tests which were skipped (`error.SkipZigTest`).
100 skip_count: u32 = 0,
101 /// The number of tests which failed cleanly.
102 fail_count: u32 = 0,
103 /// The number of tests which terminated unexpectedly, i.e. crashed.
104 crash_count: u32 = 0,
105 /// The number of tests which timed out.
106 timeout_count: u32 = 0,
107
108 /// The number of detected memory leaks. The associated test may still have passed; indeed, *all*
109 /// individual tests may have passed. However, the step as a whole fails if any test has leaks.
110 leak_count: u32 = 0,
111 /// The number of detected error logs. The associated test may still have passed; indeed, *all*
112 /// individual tests may have passed. However, the step as a whole fails if any test logs errors.
113 log_err_count: u32 = 0,
114
115 pub fn isSuccess(tr: TestResults) bool {
116 // all steps are success or skip
117 return tr.fail_count == 0 and
118 tr.crash_count == 0 and
119 tr.timeout_count == 0 and
120 // no (otherwise successful) step leaked memory or logged errors
121 tr.leak_count == 0 and
122 tr.log_err_count == 0;
123 }
124
125 /// Computes the number of tests which passed from the other values.
126 pub fn passCount(tr: TestResults) u32 {
127 return tr.test_count - tr.skip_count - tr.fail_count - tr.crash_count - tr.timeout_count;
128 }
129};
130
131pub const MakeOptions = struct {
132 progress_node: std.Progress.Node,
133 watch: bool,
134 web_server: ?*WebServer,
135 /// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds.
136 unit_test_timeout_ns: ?u64,
137 /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`.
138 gpa: Allocator,
139};
140
141pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void;
142
143/// If the Step's `make` function reports `error.MakeFailed`, it indicates they
144/// have already reported the error. Otherwise, we add a simple error report
145/// here.
146pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void {
147 if (true) @panic("TODO Step.make");
148 const arena = s.owner.allocator;
149 const graph = s.owner.graph;
150 const io = graph.io;
151
152 var start_ts: ?Io.Timestamp = t: {
153 if (!graph.time_report) break :t null;
154 if (s.id == .compile) break :t null;
155 if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null;
156 break :t Io.Clock.awake.now(io);
157 };
158 const make_result = s.makeFn(s, options);
159 if (start_ts) |*ts| {
160 const duration = ts.untilNow(io, .awake);
161 options.web_server.?.updateTimeReportGeneric(s, duration);
162 }
163
164 make_result catch |err| switch (err) {
165 error.MakeFailed, error.MakeSkipped => |e| return e,
166 else => {
167 s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM");
168 return error.MakeFailed;
169 },
170 };
171
172 if (!s.test_results.isSuccess()) {
173 return error.MakeFailed;
174 }
175
176 if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) {
177 const msg = std.fmt.allocPrint(arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{
178 s.result_peak_rss, s.max_rss,
179 }) catch @panic("OOM");
180 s.result_error_msgs.append(arena, msg) catch @panic("OOM");
181 }
182}
183
184/// Implementation detail of file watching. Prepares the step for being re-evaluated.
185/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated.
186pub fn invalidateResult(step: *Step, gpa: Allocator) bool {
187 if (true) @panic("TODO Step.invalidateResult");
188 if (step.state == .precheck_done) return false;
189 assert(step.pending_deps == 0);
190 step.state = .precheck_done;
191 step.reset(gpa);
192 for (step.dependants.items) |dependant| {
193 _ = dependant.invalidateResult(gpa);
194 dependant.pending_deps += 1;
195 }
196 return true;
197}
198
199/// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated.
200pub fn reset(step: *Step, gpa: Allocator) void {
201 assert(step.state == .precheck_done);
202
203 if (step.result_failed_command) |cmd| gpa.free(cmd);
204
205 step.result_error_msgs.clearRetainingCapacity();
206 step.result_stderr = "";
207 step.result_cached = false;
208 step.result_duration_ns = null;
209 step.result_peak_rss = 0;
210 step.result_failed_command = null;
211 step.test_results = .{};
212 step.clearWatchInputs();
213
214 step.result_error_bundle.deinit(gpa);
215 step.result_error_bundle = std.zig.ErrorBundle.empty;
216}
217
218/// Populates `s.result_failed_command`.
219pub fn captureChildProcess(
220 s: *Step,
221 gpa: Allocator,
222 progress_node: std.Progress.Node,
223 argv: []const []const u8,
224) !std.process.RunResult {
225 const graph = s.owner.graph;
226 const arena = graph.arena;
227 const io = graph.io;
228
229 // If an error occurs, it's happened in this command:
230 assert(s.result_failed_command == null);
231 s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv);
232
233 try handleChildProcUnsupported(s);
234 try handleVerbose(s, .inherit, argv);
235
236 const result = std.process.run(arena, io, .{
237 .argv = argv,
238 .environ_map = &graph.environ_map,
239 .progress_node = progress_node,
240 }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err });
241
242 if (result.stderr.len > 0) {
243 try s.result_error_msgs.append(arena, result.stderr);
244 }
245
246 return result;
247}
248
249pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } {
250 try step.addError(fmt, args);
251 return error.MakeFailed;
252}
253
254pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void {
255 const arena = step.owner.allocator;
256 const msg = try std.fmt.allocPrint(arena, fmt, args);
257 try step.result_error_msgs.append(arena, msg);
258}
259
260pub const ZigProcess = struct {
261 child: std.process.Child,
262 multi_reader_buffer: Io.File.MultiReader.Buffer(2),
263 multi_reader: Io.File.MultiReader,
264 progress_ipc_index: ?if (std.Progress.have_ipc) std.Progress.Ipc.Index else noreturn,
265
266 pub const StreamEnum = enum { stdout, stderr };
267
268 pub fn saveState(zp: *ZigProcess, prog_node: std.Progress.Node) void {
269 zp.progress_ipc_index = if (std.Progress.have_ipc) prog_node.takeIpcIndex() else null;
270 }
271
272 pub fn deinit(zp: *ZigProcess, io: Io) void {
273 zp.child.kill(io);
274 zp.multi_reader.deinit();
275 zp.* = undefined;
276 }
277};
278
279/// Assumes that argv contains `--listen=-` and that the process being spawned
280/// is the zig compiler - the same version that compiled the build runner.
281/// Populates `s.result_failed_command`.
282pub fn evalZigProcess(
283 s: *Step,
284 argv: []const []const u8,
285 prog_node: std.Progress.Node,
286 watch: bool,
287 web_server: ?*WebServer,
288 gpa: Allocator,
289) !?Cache.Path {
290 const b = s.owner;
291 const io = b.graph.io;
292
293 // If an error occurs, it's happened in this command:
294 assert(s.result_failed_command == null);
295 s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv);
296
297 if (s.getZigProcess()) |zp| update: {
298 assert(watch);
299 if (zp.progress_ipc_index) |ipc_index| prog_node.setIpcIndex(ipc_index);
300 zp.progress_ipc_index = null;
301 var exited = false;
302 defer if (exited) {
303 s.cast(Compile).?.zig_process = null;
304 zp.deinit(io);
305 gpa.destroy(zp);
306 } else zp.saveState(prog_node);
307 const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) {
308 error.BrokenPipe, error.EndOfStream => |reason| {
309 std.log.info("{s} restart required: {t}", .{ argv[0], reason });
310 // Process restart required.
311 const term = zp.child.wait(io) catch |e| {
312 return s.fail("unable to wait for {s}: {t}", .{ argv[0], e });
313 };
314 _ = term;
315 exited = true;
316 break :update;
317 },
318 else => |e| return e,
319 };
320
321 if (s.result_error_bundle.errorMessageCount() > 0) {
322 return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()});
323 }
324
325 if (s.result_error_msgs.items.len > 0 and result == null) {
326 // Crash detected.
327 const term = zp.child.wait(io) catch |e| {
328 return s.fail("unable to wait for {s}: {t}", .{ argv[0], e });
329 };
330 s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0;
331 exited = true;
332 try handleChildProcessTerm(s, term);
333 return error.MakeFailed;
334 }
335
336 return result;
337 }
338 assert(argv.len != 0);
339
340 try handleChildProcUnsupported(s);
341 try handleVerbose(s, .inherit, argv);
342
343 const zp = try gpa.create(ZigProcess);
344 defer if (!watch) gpa.destroy(zp);
345
346 zp.child = std.process.spawn(io, .{
347 .argv = argv,
348 .environ_map = &b.graph.environ_map,
349 .stdin = .pipe,
350 .stdout = .pipe,
351 .stderr = .pipe,
352 .request_resource_usage_statistics = true,
353 .progress_node = prog_node,
354 }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err });
355
356 zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{
357 zp.child.stdout.?, zp.child.stderr.?,
358 });
359 if (watch) s.cast(Compile).?.zig_process = zp;
360 defer if (!watch) zp.deinit(io);
361
362 const result = result: {
363 defer if (watch) zp.saveState(prog_node);
364 break :result try zigProcessUpdate(s, zp, watch, web_server, gpa);
365 };
366
367 if (!watch) {
368 // Send EOF to stdin.
369 zp.child.stdin.?.close(io);
370 zp.child.stdin = null;
371
372 const term = zp.child.wait(io) catch |err| {
373 return s.fail("unable to wait for {s}: {t}", .{ argv[0], err });
374 };
375 s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0;
376
377 // Special handling for Compile step that is expecting compile errors.
378 if (s.cast(Compile)) |compile| switch (term) {
379 .exited => {
380 // Note that the exit code may be 0 in this case due to the
381 // compiler server protocol.
382 if (compile.expect_errors != null) {
383 return error.NeedCompileErrorCheck;
384 }
385 },
386 else => {},
387 };
388
389 try handleChildProcessTerm(s, term);
390 }
391
392 if (s.result_error_bundle.errorMessageCount() > 0) {
393 return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()});
394 }
395
396 return result;
397}
398
399/// Wrapper around `Io.Dir.updateFile` that handles verbose and error output.
400pub fn installFile(s: *Step, src_lazy_path: LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus {
401 const b = s.owner;
402 const io = b.graph.io;
403 const src_path = src_lazy_path.getPath3(b, s);
404 try handleVerbose(s, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path });
405 return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err|
406 return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err });
407}
408
409/// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output.
410pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus {
411 const b = s.owner;
412 const io = b.graph.io;
413 try handleVerbose(s, .inherit, &.{ "install", "-d", dest_path });
414 return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err|
415 return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err });
416}
417
418fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebServer, gpa: Allocator) !?Path {
419 const b = s.owner;
420 const arena = b.allocator;
421 const io = b.graph.io;
422
423 const start_ts = Io.Clock.awake.now(io);
424
425 try sendMessage(io, zp.child.stdin.?, .update);
426 if (!watch) try sendMessage(io, zp.child.stdin.?, .exit);
427
428 var result: ?Path = null;
429 var eos_err: error{EndOfStream}!void = {};
430
431 const stdout = zp.multi_reader.fileReader(0);
432
433 while (true) {
434 const Header = std.zig.Server.Message.Header;
435 const header = stdout.interface.takeStruct(Header, .little) catch |err| switch (err) {
436 error.EndOfStream => break,
437 error.ReadFailed => return stdout.err.?,
438 };
439 const body = stdout.interface.take(header.bytes_len) catch |err| switch (err) {
440 error.EndOfStream => |e| {
441 // Better to report the crash with stderr below, but we set
442 // this in case the child exits successfully while violating
443 // this protocol.
444 eos_err = e;
445 break;
446 },
447 error.ReadFailed => return stdout.err.?,
448 };
449 switch (header.tag) {
450 .zig_version => {
451 if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
452 return s.fail(
453 "zig version mismatch build runner vs compiler: '{s}' vs '{s}'",
454 .{ builtin.zig_version_string, body },
455 );
456 }
457 },
458 .error_bundle => {
459 s.result_error_bundle = try std.zig.Server.allocErrorBundle(gpa, body);
460 // This message indicates the end of the update.
461 if (watch) break;
462 },
463 .emit_digest => {
464 const EmitDigest = std.zig.Server.Message.EmitDigest;
465 const emit_digest: *align(1) const EmitDigest = @ptrCast(body);
466 s.result_cached = emit_digest.flags.cache_hit;
467 const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len];
468 result = .{
469 .root_dir = b.cache_root,
470 .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)),
471 };
472 },
473 .file_system_inputs => {
474 s.clearWatchInputs();
475 var it = std.mem.splitScalar(u8, body, 0);
476 while (it.next()) |prefixed_path| {
477 const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1);
478 const sub_path = try arena.dupe(u8, prefixed_path[1..]);
479 const sub_path_dirname = std.fs.path.dirname(sub_path) orelse "";
480 switch (prefix_index) {
481 .cwd => {
482 const path: Cache.Path = .{
483 .root_dir = Cache.Directory.cwd(),
484 .sub_path = sub_path_dirname,
485 };
486 try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
487 },
488 .zig_lib => zl: {
489 if (s.cast(Step.Compile)) |compile| {
490 if (compile.zig_lib_dir) |zig_lib_dir| {
491 const lp = try zig_lib_dir.join(arena, sub_path);
492 try addWatchInput(s, lp);
493 break :zl;
494 }
495 }
496 const path: Cache.Path = .{
497 .root_dir = s.owner.graph.zig_lib_directory,
498 .sub_path = sub_path_dirname,
499 };
500 try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
501 },
502 .local_cache => {
503 const path: Cache.Path = .{
504 .root_dir = b.cache_root,
505 .sub_path = sub_path_dirname,
506 };
507 try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
508 },
509 .global_cache => {
510 const path: Cache.Path = .{
511 .root_dir = s.owner.graph.global_cache_root,
512 .sub_path = sub_path_dirname,
513 };
514 try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
515 },
516 }
517 }
518 },
519 .time_report => if (web_server) |ws| {
520 const TimeReport = std.zig.Server.Message.TimeReport;
521 const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]);
522 ws.updateTimeReportCompile(.{
523 .compile = s.cast(Step.Compile).?,
524 .use_llvm = tr.flags.use_llvm,
525 .stats = tr.stats,
526 .ns_total = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()),
527 .llvm_pass_timings_len = tr.llvm_pass_timings_len,
528 .files_len = tr.files_len,
529 .decls_len = tr.decls_len,
530 .trailing = body[@sizeOf(TimeReport)..],
531 });
532 },
533 else => {}, // ignore other messages
534 }
535 }
536
537 s.result_duration_ns = @intCast(start_ts.untilNow(io, .awake).toNanoseconds());
538
539 const stderr_contents = zp.multi_reader.reader(1).buffered();
540 if (stderr_contents.len > 0) {
541 try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents));
542 }
543
544 try eos_err;
545
546 return result;
547}
548
549pub fn getZigProcess(s: *Step) ?*ZigProcess {
550 if (true) @panic("TODO getZigProcess");
551 return switch (s.id) {
552 .compile => s.cast(Compile).?.zig_process,
553 else => null,
554 };
555}
556
557fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void {
558 const header: std.zig.Client.Message.Header = .{
559 .tag = tag,
560 .bytes_len = 0,
561 };
562 var w = file.writer(io, &.{});
563 w.interface.writeStruct(header, .little) catch |err| switch (err) {
564 error.WriteFailed => return w.err.?,
565 };
566}
567
568pub fn handleVerbose(
569 s: *Step,
570 arena: Allocator,
571 cwd: std.process.Child.Cwd,
572 opt_env: ?*const std.process.Environ.Map,
573 argv: []const []const u8,
574) error{OutOfMemory}!void {
575 if (!s.verbose) return;
576 const graph = s.graph;
577 // Intention of verbose is to print all sub-process command lines to
578 // stderr before spawning them.
579 const text = try allocPrintCmd(arena, cwd, if (opt_env) |env| .{
580 .child = env,
581 .parent = &graph.environ_map,
582 } else null, argv);
583 std.log.scoped(.verbose).info("{s}", .{text});
584}
585
586/// Asserts that the caller has already populated `s.result_failed_command`.
587pub inline fn handleChildProcUnsupported(s: *Step) error{ OutOfMemory, MakeFailed }!void {
588 if (!std.process.can_spawn) {
589 return s.fail("unable to spawn process: host cannot spawn child processes", .{});
590 }
591}
592
593/// Asserts that the caller has already populated `s.result_failed_command`.
594pub fn handleChildProcessTerm(s: *Step, term: std.process.Child.Term) error{ MakeFailed, OutOfMemory }!void {
595 assert(s.result_failed_command != null);
596 return switch (term) {
597 .exited => |code| if (code != 0) s.fail("process exited with error code {d}", .{code}),
598 .signal => |sig| s.fail("process terminated with signal {t}", .{sig}),
599 .stopped => |sig| s.fail("process stopped with signal {t}", .{sig}),
600 .unknown => s.fail("process terminated unexpectedly", .{}),
601 };
602}
603
604/// Prefer `cacheHitAndWatch` unless you already added watch inputs
605/// separately from using the cache system.
606pub fn cacheHit(s: *Step, man: *Cache.Manifest) !bool {
607 s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err);
608 return s.result_cached;
609}
610
611/// Clears previous watch inputs, if any, and then populates watch inputs from
612/// the full set of files picked up by the cache manifest.
613///
614/// Must be accompanied with `writeManifestAndWatch`.
615pub fn cacheHitAndWatch(s: *Step, man: *Cache.Manifest) !bool {
616 const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err);
617 s.result_cached = is_hit;
618 // The above call to hit() populates the manifest with files, so in case of
619 // a hit, we need to populate watch inputs.
620 if (is_hit) try setWatchInputsFromManifest(s, man);
621 return is_hit;
622}
623
624fn failWithCacheError(
625 s: *Step,
626 man: *const Cache.Manifest,
627 err: Cache.Manifest.HitError,
628) error{ OutOfMemory, Canceled, MakeFailed } {
629 switch (err) {
630 error.CacheCheckFailed => switch (man.diagnostic) {
631 .none => unreachable,
632 .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{
633 man.diagnostic, e,
634 }),
635 .file_open, .file_stat, .file_read, .file_hash => |op| {
636 const pp = man.files.keys()[op.file_index].prefixed_path;
637 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
638 return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{
639 prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err,
640 });
641 },
642 },
643 error.OutOfMemory, error.Canceled => |e| return e,
644 error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}),
645 }
646}
647
648/// Prefer `writeManifestAndWatch` unless you already added watch inputs
649/// separately from using the cache system.
650pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void {
651 if (s.test_results.isSuccess()) {
652 man.writeManifest() catch |err| {
653 try s.addError("unable to write cache manifest: {t}", .{err});
654 };
655 }
656}
657
658/// Clears previous watch inputs, if any, and then populates watch inputs from
659/// the full set of files picked up by the cache manifest.
660///
661/// Must be accompanied with `cacheHitAndWatch`.
662pub fn writeManifestAndWatch(s: *Step, man: *Cache.Manifest) !void {
663 try writeManifest(s, man);
664 try setWatchInputsFromManifest(s, man);
665}
666
667fn setWatchInputsFromManifest(s: *Step, man: *Cache.Manifest) !void {
668 const arena = s.owner.allocator;
669 const prefixes = man.cache.prefixes();
670 clearWatchInputs(s);
671 for (man.files.keys()) |file| {
672 // The file path data is freed when the cache manifest is cleaned up at the end of `make`.
673 const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path);
674 try addWatchInputFromPath(s, .{
675 .root_dir = prefixes[file.prefixed_path.prefix],
676 .sub_path = std.fs.path.dirname(sub_path) orelse "",
677 }, std.fs.path.basename(sub_path));
678 }
679}
680
681/// For steps that have a single input that never changes when re-running `make`.
682pub fn singleUnchangingWatchInput(step: *Step, lazy_path: LazyPath) Allocator.Error!void {
683 if (!step.inputs.populated()) try step.addWatchInput(lazy_path);
684}
685
686pub fn clearWatchInputs(step: *Step) void {
687 const gpa = step.owner.allocator;
688 step.inputs.clear(gpa);
689}
690
691/// Places a *file* dependency on the path.
692pub fn addWatchInput(step: *Step, lazy_file: LazyPath) Allocator.Error!void {
693 switch (lazy_file) {
694 .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path),
695 .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path),
696 .cwd_relative => |path_string| {
697 try addWatchInputFromPath(step, .{
698 .root_dir = .{
699 .path = null,
700 .handle = Io.Dir.cwd(),
701 },
702 .sub_path = std.fs.path.dirname(path_string) orelse "",
703 }, std.fs.path.basename(path_string));
704 },
705 // Nothing to watch because this dependency edge is modeled instead via `dependants`.
706 .generated => {},
707 }
708}
709
710/// Any changes inside the directory will trigger invalidation.
711///
712/// See also `addDirectoryWatchInputFromPath` which takes a `Cache.Path` instead.
713///
714/// Paths derived from this directory should also be manually added via
715/// `addDirectoryWatchInputFromPath` if and only if this function returns
716/// `true`.
717pub fn addDirectoryWatchInput(step: *Step, lazy_directory: LazyPath) Allocator.Error!bool {
718 switch (lazy_directory) {
719 .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path),
720 .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path),
721 .cwd_relative => |path_string| {
722 try addDirectoryWatchInputFromPath(step, .{
723 .root_dir = .{
724 .path = null,
725 .handle = Io.Dir.cwd(),
726 },
727 .sub_path = path_string,
728 });
729 },
730 // Nothing to watch because this dependency edge is modeled instead via `dependants`.
731 .generated => return false,
732 }
733 return true;
734}
735
736/// Any changes inside the directory will trigger invalidation.
737///
738/// See also `addDirectoryWatchInput` which takes a `LazyPath` instead.
739///
740/// This function should only be called when it has been verified that the
741/// dependency on `path` is not already accounted for by a `Step` dependency.
742/// In other words, before calling this function, first check that the
743/// `LazyPath` which this `path` is derived from is not `generated`.
744pub fn addDirectoryWatchInputFromPath(step: *Step, path: Cache.Path) !void {
745 return addWatchInputFromPath(step, path, ".");
746}
747
748fn addWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void {
749 return addWatchInputFromPath(step, .{
750 .root_dir = package.build_root,
751 .sub_path = std.fs.path.dirname(sub_path) orelse "",
752 }, std.fs.path.basename(sub_path));
753}
754
755fn addDirectoryWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void {
756 return addDirectoryWatchInputFromPath(step, .{
757 .root_dir = package.build_root,
758 .sub_path = sub_path,
759 });
760}
761
762fn addWatchInputFromPath(step: *Step, path: Cache.Path, basename: []const u8) !void {
763 const gpa = step.owner.allocator;
764 const gop = try step.inputs.table.getOrPut(gpa, path);
765 if (!gop.found_existing) gop.value_ptr.* = .empty;
766 try gop.value_ptr.append(gpa, basename);
767}
768
769pub fn allocPrintCmd(
770 gpa: Allocator,
771 cwd: std.process.Child.Cwd,
772 opt_env: ?struct {
773 child: *const std.process.Environ.Map,
774 parent: *const std.process.Environ.Map,
775 },
776 argv: []const []const u8,
777) Allocator.Error![]u8 {
778 const shell = struct {
779 fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void {
780 for (string) |c| {
781 if (switch (c) {
782 else => true,
783 '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false,
784 '=' => is_argv0,
785 }) break;
786 } else return writer.writeAll(string);
787
788 try writer.writeByte('"');
789 for (string) |c| {
790 if (switch (c) {
791 std.ascii.control_code.nul => break,
792 '!', '"', '$', '\\', '`' => true,
793 else => !std.ascii.isPrint(c),
794 }) try writer.writeByte('\\');
795 switch (c) {
796 std.ascii.control_code.nul => unreachable,
797 std.ascii.control_code.bel => try writer.writeByte('a'),
798 std.ascii.control_code.bs => try writer.writeByte('b'),
799 std.ascii.control_code.ht => try writer.writeByte('t'),
800 std.ascii.control_code.lf => try writer.writeByte('n'),
801 std.ascii.control_code.vt => try writer.writeByte('v'),
802 std.ascii.control_code.ff => try writer.writeByte('f'),
803 std.ascii.control_code.cr => try writer.writeByte('r'),
804 std.ascii.control_code.esc => try writer.writeByte('E'),
805 ' '...'~' => try writer.writeByte(c),
806 else => try writer.print("{o:0>3}", .{c}),
807 }
808 }
809 try writer.writeByte('"');
810 }
811 };
812
813 var aw: Io.Writer.Allocating = .init(gpa);
814 defer aw.deinit();
815 const writer = &aw.writer;
816 switch (cwd) {
817 .inherit => {},
818 .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory,
819 .dir => @panic("TODO"),
820 }
821 if (opt_env) |env| {
822 var it = env.child.iterator();
823 while (it.next()) |entry| {
824 const key = entry.key_ptr.*;
825 const value = entry.value_ptr.*;
826 if (env.parent.get(key)) |process_value| {
827 if (std.mem.eql(u8, value, process_value)) continue;
828 }
829 writer.print("{s}=", .{key}) catch return error.OutOfMemory;
830 shell.escape(writer, value, false) catch return error.OutOfMemory;
831 writer.writeByte(' ') catch return error.OutOfMemory;
832 }
833 }
834 shell.escape(writer, argv[0], true) catch return error.OutOfMemory;
835 for (argv[1..]) |arg| {
836 writer.writeByte(' ') catch return error.OutOfMemory;
837 shell.escape(writer, arg, false) catch return error.OutOfMemory;
838 }
839 return aw.toOwnedSlice();
840}
lib/compiler/Maker/Step/Compile.zig created+1200
...@@ -0,0 +1,1200 @@
1/// Populated during the make phase when there is a long-lived compiler process.
2/// Managed by the build runner, not user build script.
3zig_process: ?*Step.ZigProcess,
4
5fn make(step: *Step, options: Step.MakeOptions) !void {
6 const b = step.owner;
7 const compile: *Compile = @fieldParentPtr("step", step);
8
9 const zig_args = try getZigArgs(compile, false);
10
11 const maybe_output_dir = step.evalZigProcess(
12 zig_args,
13 options.progress_node,
14 (b.graph.incremental == true) and (options.watch or options.web_server != null),
15 options.web_server,
16 options.gpa,
17 ) catch |err| switch (err) {
18 error.NeedCompileErrorCheck => {
19 assert(compile.expect_errors != null);
20 try checkCompileErrors(compile);
21 return;
22 },
23 else => |e| return e,
24 };
25
26 // Update generated files
27 if (maybe_output_dir) |output_dir| {
28 if (compile.emit_directory) |lp| {
29 lp.path = b.fmt("{f}", .{output_dir});
30 }
31
32 // zig fmt: off
33 if (compile.generated_bin) |lp| lp.path = compile.outputPath(output_dir, .bin);
34 if (compile.generated_pdb) |lp| lp.path = compile.outputPath(output_dir, .pdb);
35 // hack for stage2_x86_64 + coff
36 if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib);
37 if (compile.generated_implib) |lp| lp.path = compile.outputPath(output_dir, .implib);
38 if (compile.generated_h) |lp| lp.path = compile.outputPath(output_dir, .h);
39 if (compile.generated_docs) |lp| lp.path = compile.outputPath(output_dir, .docs);
40 if (compile.generated_asm) |lp| lp.path = compile.outputPath(output_dir, .@"asm");
41 if (compile.generated_llvm_ir) |lp| lp.path = compile.outputPath(output_dir, .llvm_ir);
42 if (compile.generated_llvm_bc) |lp| lp.path = compile.outputPath(output_dir, .llvm_bc);
43 // zig fmt: on
44 }
45
46 if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic and
47 compile.version != null and compile.generated_bin != null and
48 std.Build.wantSharedLibSymLinks(compile.rootModuleTarget()))
49 {
50 try doAtomicSymLinks(
51 step,
52 compile.getEmittedBin().getPath2(b, step),
53 compile.major_only_filename.?,
54 compile.name_only_filename.?,
55 );
56 }
57}
58
59fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
60 const step = &compile.step;
61 const b = step.owner;
62 const arena = b.allocator;
63
64 var zig_args = std.array_list.Managed([]const u8).init(arena);
65 defer zig_args.deinit();
66
67 try zig_args.append(b.graph.zig_exe);
68
69 const cmd = switch (compile.kind) {
70 .lib => "build-lib",
71 .exe => "build-exe",
72 .obj => "build-obj",
73 .@"test" => "test",
74 .test_obj => "test-obj",
75 };
76 try zig_args.append(cmd);
77
78 if (b.reference_trace) |some| {
79 try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some}));
80 }
81 try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts);
82
83 try addFlag(&zig_args, "llvm", compile.use_llvm);
84 try addFlag(&zig_args, "lld", compile.use_lld);
85 try addFlag(&zig_args, "new-linker", compile.use_new_linker);
86
87 if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| {
88 try zig_args.append(try std.fmt.allocPrint(arena, "-ofmt={s}", .{@tagName(ofmt)}));
89 }
90
91 switch (compile.entry) {
92 .default => {},
93 .disabled => try zig_args.append("-fno-entry"),
94 .enabled => try zig_args.append("-fentry"),
95 .symbol_name => |entry_name| {
96 try zig_args.append(try std.fmt.allocPrint(arena, "-fentry={s}", .{entry_name}));
97 },
98 }
99
100 {
101 for (compile.force_undefined_symbols.keys()) |symbol_name| {
102 try zig_args.append("--force_undefined");
103 try zig_args.append(symbol_name.*);
104 }
105 }
106
107 if (compile.stack_size) |stack_size| {
108 try zig_args.append("--stack");
109 try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size}));
110 }
111
112 if (fuzz) {
113 try zig_args.append("-ffuzz");
114 }
115
116 {
117 // Stores system libraries that have already been seen for at least one
118 // module, along with any arguments that need to be passed to the
119 // compiler for each module individually.
120 var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty;
121 var frameworks: std.StringArrayHashMapUnmanaged(Module.LinkFrameworkOptions) = .empty;
122
123 var prev_has_cflags = false;
124 var prev_has_rcflags = false;
125 var prev_search_strategy: Module.SystemLib.SearchStrategy = .paths_first;
126 var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic;
127 // Track the number of positional arguments so that a nice error can be
128 // emitted if there is nothing to link.
129 var total_linker_objects: usize = @intFromBool(compile.root_module.root_source_file != null);
130
131 // Fully recursive iteration including dynamic libraries to detect
132 // libc and libc++ linkage.
133 for (compile.getCompileDependencies(true)) |some_compile| {
134 for (some_compile.root_module.getGraph().modules) |mod| {
135 if (mod.link_libc == true) compile.is_linking_libc = true;
136 if (mod.link_libcpp == true) compile.is_linking_libcpp = true;
137 }
138 }
139
140 var cli_named_modules = try CliNamedModules.init(arena, compile.root_module);
141
142 // For this loop, don't chase dynamic libraries because their link
143 // objects are already linked.
144 for (compile.getCompileDependencies(false)) |dep_compile| {
145 for (dep_compile.root_module.getGraph().modules) |mod| {
146 // While walking transitive dependencies, if a given link object is
147 // already included in a library, it should not redundantly be
148 // placed on the linker line of the dependee.
149 const my_responsibility = dep_compile == compile;
150 const already_linked = !my_responsibility and dep_compile.isDynamicLibrary();
151
152 // Inherit dependencies on darwin frameworks.
153 if (!already_linked) {
154 for (mod.frameworks.keys(), mod.frameworks.values()) |name, info| {
155 try frameworks.put(arena, name, info);
156 }
157 }
158
159 // Inherit dependencies on system libraries and static libraries.
160 for (mod.link_objects.items) |link_object| {
161 switch (link_object) {
162 .static_path => |static_path| {
163 if (my_responsibility) {
164 try zig_args.append(static_path.getPath2(mod.owner, step));
165 total_linker_objects += 1;
166 }
167 },
168 .system_lib => |system_lib| {
169 const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name);
170 if (system_lib_gop.found_existing) {
171 try zig_args.appendSlice(system_lib_gop.value_ptr.*);
172 continue;
173 } else {
174 system_lib_gop.value_ptr.* = &.{};
175 }
176
177 if (already_linked)
178 continue;
179
180 if ((system_lib.search_strategy != prev_search_strategy or
181 system_lib.preferred_link_mode != prev_preferred_link_mode) and
182 compile.linkage != .static)
183 {
184 switch (system_lib.search_strategy) {
185 .no_fallback => switch (system_lib.preferred_link_mode) {
186 .dynamic => try zig_args.append("-search_dylibs_only"),
187 .static => try zig_args.append("-search_static_only"),
188 },
189 .paths_first => switch (system_lib.preferred_link_mode) {
190 .dynamic => try zig_args.append("-search_paths_first"),
191 .static => try zig_args.append("-search_paths_first_static"),
192 },
193 .mode_first => switch (system_lib.preferred_link_mode) {
194 .dynamic => try zig_args.append("-search_dylibs_first"),
195 .static => try zig_args.append("-search_static_first"),
196 },
197 }
198 prev_search_strategy = system_lib.search_strategy;
199 prev_preferred_link_mode = system_lib.preferred_link_mode;
200 }
201
202 const prefix: []const u8 = prefix: {
203 if (system_lib.needed) break :prefix "-needed-l";
204 if (system_lib.weak) break :prefix "-weak-l";
205 break :prefix "-l";
206 };
207 switch (system_lib.use_pkg_config) {
208 .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })),
209 .yes, .force => {
210 if (compile.runPkgConfig(system_lib.name)) |result| {
211 try zig_args.appendSlice(result.cflags);
212 try zig_args.appendSlice(result.libs);
213 try seen_system_libs.put(arena, system_lib.name, result.cflags);
214 } else |err| switch (err) {
215 error.PkgConfigInvalidOutput,
216 error.PkgConfigCrashed,
217 error.PkgConfigFailed,
218 error.PkgConfigNotInstalled,
219 error.PackageNotFound,
220 => switch (system_lib.use_pkg_config) {
221 .yes => {
222 // pkg-config failed, so fall back to linking the library
223 // by name directly.
224 try zig_args.append(b.fmt("{s}{s}", .{
225 prefix,
226 system_lib.name,
227 }));
228 },
229 .force => {
230 panic("pkg-config failed for library {s}", .{system_lib.name});
231 },
232 .no => unreachable,
233 },
234
235 else => |e| return e,
236 }
237 },
238 }
239 },
240 .other_step => |other| {
241 switch (other.kind) {
242 .exe => return step.fail("cannot link with an executable build artifact", .{}),
243 .@"test" => return step.fail("cannot link with a test", .{}),
244 .obj, .test_obj => {
245 const included_in_lib_or_obj = !my_responsibility and
246 (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj);
247 if (!already_linked and !included_in_lib_or_obj) {
248 try zig_args.append(other.getEmittedBin().getPath2(b, step));
249 total_linker_objects += 1;
250 }
251 },
252 .lib => l: {
253 const other_produces_implib = other.producesImplib();
254 const other_is_static = other_produces_implib or other.isStaticLibrary();
255
256 if (compile.isStaticLibrary() and other_is_static) {
257 // Avoid putting a static library inside a static library.
258 break :l;
259 }
260
261 // For DLLs, we must link against the implib.
262 // For everything else, we directly link
263 // against the library file.
264 const full_path_lib = if (other_produces_implib)
265 try other.getGeneratedFilePath("generated_implib", &compile.step)
266 else
267 try other.getGeneratedFilePath("generated_bin", &compile.step);
268
269 try zig_args.append(full_path_lib);
270 total_linker_objects += 1;
271
272 if (other.linkage == .dynamic and
273 compile.rootModuleTarget().os.tag != .windows)
274 {
275 if (fs.path.dirname(full_path_lib)) |dirname| {
276 try zig_args.append("-rpath");
277 try zig_args.append(dirname);
278 }
279 }
280 },
281 }
282 },
283 .assembly_file => |asm_file| l: {
284 if (!my_responsibility) break :l;
285
286 if (prev_has_cflags) {
287 try zig_args.append("-cflags");
288 try zig_args.append("--");
289 prev_has_cflags = false;
290 }
291 try zig_args.append(asm_file.getPath2(mod.owner, step));
292 total_linker_objects += 1;
293 },
294
295 .c_source_file => |c_source_file| l: {
296 if (!my_responsibility) break :l;
297
298 if (prev_has_cflags or c_source_file.flags.len != 0) {
299 try zig_args.append("-cflags");
300 for (c_source_file.flags) |arg| {
301 try zig_args.append(arg);
302 }
303 try zig_args.append("--");
304 }
305 prev_has_cflags = (c_source_file.flags.len != 0);
306
307 if (c_source_file.language) |lang| {
308 try zig_args.append("-x");
309 try zig_args.append(lang.internalIdentifier());
310 }
311
312 try zig_args.append(c_source_file.file.getPath2(mod.owner, step));
313
314 if (c_source_file.language != null) {
315 try zig_args.append("-x");
316 try zig_args.append("none");
317 }
318 total_linker_objects += 1;
319 },
320
321 .c_source_files => |c_source_files| l: {
322 if (!my_responsibility) break :l;
323
324 if (prev_has_cflags or c_source_files.flags.len != 0) {
325 try zig_args.append("-cflags");
326 for (c_source_files.flags) |arg| {
327 try zig_args.append(arg);
328 }
329 try zig_args.append("--");
330 }
331 prev_has_cflags = (c_source_files.flags.len != 0);
332
333 if (c_source_files.language) |lang| {
334 try zig_args.append("-x");
335 try zig_args.append(lang.internalIdentifier());
336 }
337
338 const root_path = c_source_files.root.getPath2(mod.owner, step);
339 for (c_source_files.files) |file| {
340 try zig_args.append(b.pathJoin(&.{ root_path, file }));
341 }
342
343 if (c_source_files.language != null) {
344 try zig_args.append("-x");
345 try zig_args.append("none");
346 }
347
348 total_linker_objects += c_source_files.files.len;
349 },
350
351 .win32_resource_file => |rc_source_file| l: {
352 if (!my_responsibility) break :l;
353
354 if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) {
355 if (prev_has_rcflags) {
356 try zig_args.append("-rcflags");
357 try zig_args.append("--");
358 prev_has_rcflags = false;
359 }
360 } else {
361 try zig_args.append("-rcflags");
362 for (rc_source_file.flags) |arg| {
363 try zig_args.append(arg);
364 }
365 for (rc_source_file.include_paths) |include_path| {
366 try zig_args.append("/I");
367 try zig_args.append(include_path.getPath2(mod.owner, step));
368 }
369 try zig_args.append("--");
370 prev_has_rcflags = true;
371 }
372 try zig_args.append(rc_source_file.file.getPath2(mod.owner, step));
373 total_linker_objects += 1;
374 },
375 }
376 }
377
378 // We need to emit the --mod argument here so that the above link objects
379 // have the correct parent module, but only if the module is part of
380 // this compilation.
381 if (!my_responsibility) continue;
382 if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| {
383 const module_cli_name = cli_named_modules.names.keys()[module_cli_index];
384 try mod.appendZigProcessFlags(&zig_args, step);
385
386 // --dep arguments
387 try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2);
388 for (mod.import_table.keys(), mod.import_table.values()) |name, import| {
389 const import_index = cli_named_modules.modules.getIndex(import).?;
390 const import_cli_name = cli_named_modules.names.keys()[import_index];
391 zig_args.appendAssumeCapacity("--dep");
392 if (std.mem.eql(u8, import_cli_name, name)) {
393 zig_args.appendAssumeCapacity(import_cli_name);
394 } else {
395 zig_args.appendAssumeCapacity(b.fmt("{s}={s}", .{ name, import_cli_name }));
396 }
397 }
398
399 // When the CLI sees a -M argument, it determines whether it
400 // implies the existence of a Zig compilation unit based on
401 // whether there is a root source file. If there is no root
402 // source file, then this is not a zig compilation unit - it is
403 // perhaps a set of linker objects, or C source files instead.
404 // Linker objects are added to the CLI globally, while C source
405 // files must have a module parent.
406 if (mod.root_source_file) |lp| {
407 const src = lp.getPath2(mod.owner, step);
408 try zig_args.append(b.fmt("-M{s}={s}", .{ module_cli_name, src }));
409 } else if (moduleNeedsCliArg(mod)) {
410 try zig_args.append(b.fmt("-M{s}", .{module_cli_name}));
411 }
412 }
413 }
414 }
415
416 if (total_linker_objects == 0) {
417 return step.fail("the linker needs one or more objects to link", .{});
418 }
419
420 for (frameworks.keys(), frameworks.values()) |name, info| {
421 if (info.needed) {
422 try zig_args.append("-needed_framework");
423 } else if (info.weak) {
424 try zig_args.append("-weak_framework");
425 } else {
426 try zig_args.append("-framework");
427 }
428 try zig_args.append(name);
429 }
430
431 if (compile.is_linking_libcpp) {
432 try zig_args.append("-lc++");
433 }
434
435 if (compile.is_linking_libc) {
436 try zig_args.append("-lc");
437 }
438 }
439
440 if (compile.win32_manifest) |manifest_file| {
441 try zig_args.append(manifest_file.getPath2(b, step));
442 }
443
444 if (compile.win32_module_definition) |module_file| {
445 try zig_args.append(module_file.getPath2(b, step));
446 }
447
448 if (compile.image_base) |image_base| {
449 try zig_args.append("--image-base");
450 try zig_args.append(b.fmt("0x{x}", .{image_base}));
451 }
452
453 for (compile.filters) |filter| {
454 try zig_args.append("--test-filter");
455 try zig_args.append(filter);
456 }
457
458 if (compile.test_runner) |test_runner| {
459 try zig_args.append("--test-runner");
460 try zig_args.append(test_runner.path.getPath2(b, step));
461 }
462
463 for (b.debug_log_scopes) |log_scope| {
464 try zig_args.append("--debug-log");
465 try zig_args.append(log_scope);
466 }
467
468 if (b.debug_compile_errors) {
469 try zig_args.append("--debug-compile-errors");
470 }
471
472 if (b.debug_incremental) {
473 try zig_args.append("--debug-incremental");
474 }
475
476 if (b.verbose_air) try zig_args.append("--verbose-air");
477 if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path}));
478 if (b.verbose_llvm_bc) |path| try zig_args.append(b.fmt("--verbose-llvm-bc={s}", .{path}));
479 if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link");
480 if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc");
481 if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features");
482 if (b.graph.time_report) try zig_args.append("--time-report");
483
484 if (compile.generated_asm != null) try zig_args.append("-femit-asm");
485 if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin");
486 if (compile.generated_docs != null) try zig_args.append("-femit-docs");
487 if (compile.generated_implib != null) try zig_args.append("-femit-implib");
488 if (compile.generated_llvm_bc != null) try zig_args.append("-femit-llvm-bc");
489 if (compile.generated_llvm_ir != null) try zig_args.append("-femit-llvm-ir");
490 if (compile.generated_h != null) try zig_args.append("-femit-h");
491
492 try addFlag(&zig_args, "formatted-panics", compile.formatted_panics);
493
494 switch (compile.compress_debug_sections) {
495 .none => {},
496 .zlib => try zig_args.append("--compress-debug-sections=zlib"),
497 .zstd => try zig_args.append("--compress-debug-sections=zstd"),
498 }
499
500 if (compile.link_eh_frame_hdr) {
501 try zig_args.append("--eh-frame-hdr");
502 }
503 if (compile.link_emit_relocs) {
504 try zig_args.append("--emit-relocs");
505 }
506 if (compile.link_function_sections) {
507 try zig_args.append("-ffunction-sections");
508 }
509 if (compile.link_data_sections) {
510 try zig_args.append("-fdata-sections");
511 }
512 if (compile.link_gc_sections) |x| {
513 try zig_args.append(if (x) "--gc-sections" else "--no-gc-sections");
514 }
515 if (!compile.linker_dynamicbase) {
516 try zig_args.append("--no-dynamicbase");
517 }
518 if (compile.linker_allow_shlib_undefined) |x| {
519 try zig_args.append(if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined");
520 }
521 if (compile.link_z_notext) {
522 try zig_args.append("-z");
523 try zig_args.append("notext");
524 }
525 if (!compile.link_z_relro) {
526 try zig_args.append("-z");
527 try zig_args.append("norelro");
528 }
529 if (compile.link_z_lazy) {
530 try zig_args.append("-z");
531 try zig_args.append("lazy");
532 }
533 if (compile.link_z_common_page_size) |size| {
534 try zig_args.append("-z");
535 try zig_args.append(b.fmt("common-page-size={d}", .{size}));
536 }
537 if (compile.link_z_max_page_size) |size| {
538 try zig_args.append("-z");
539 try zig_args.append(b.fmt("max-page-size={d}", .{size}));
540 }
541 if (compile.link_z_defs) {
542 try zig_args.append("-z");
543 try zig_args.append("defs");
544 }
545
546 if (compile.libc_file) |libc_file| {
547 try zig_args.append("--libc");
548 try zig_args.append(libc_file.getPath2(b, step));
549 } else if (b.libc_file) |libc_file| {
550 try zig_args.append("--libc");
551 try zig_args.append(libc_file);
552 }
553
554 try zig_args.append("--cache-dir");
555 try zig_args.append(b.cache_root.path orelse ".");
556
557 try zig_args.append("--global-cache-dir");
558 try zig_args.append(b.graph.global_cache_root.path orelse ".");
559
560 if (b.graph.debug_compiler_runtime_libs) |mode|
561 try zig_args.append(b.fmt("--debug-rt={t}", .{mode}));
562
563 try zig_args.append("--name");
564 try zig_args.append(compile.name);
565
566 if (compile.linkage) |some| switch (some) {
567 .dynamic => try zig_args.append("-dynamic"),
568 .static => try zig_args.append("-static"),
569 };
570 if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) {
571 if (compile.version) |version| {
572 try zig_args.append("--version");
573 try zig_args.append(b.fmt("{f}", .{version}));
574 }
575
576 if (compile.rootModuleTarget().os.tag.isDarwin()) {
577 const install_name = compile.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{
578 compile.rootModuleTarget().libPrefix(),
579 compile.name,
580 compile.rootModuleTarget().dynamicLibSuffix(),
581 });
582 try zig_args.append("-install_name");
583 try zig_args.append(install_name);
584 }
585 }
586
587 if (compile.entitlements) |entitlements| {
588 try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements });
589 }
590 if (compile.pagezero_size) |pagezero_size| {
591 const size = try std.fmt.allocPrint(arena, "{x}", .{pagezero_size});
592 try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size });
593 }
594 if (compile.headerpad_size) |headerpad_size| {
595 const size = try std.fmt.allocPrint(arena, "{x}", .{headerpad_size});
596 try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size });
597 }
598 if (compile.headerpad_max_install_names) {
599 try zig_args.append("-headerpad_max_install_names");
600 }
601 if (compile.dead_strip_dylibs) {
602 try zig_args.append("-dead_strip_dylibs");
603 }
604 if (compile.force_load_objc) {
605 try zig_args.append("-ObjC");
606 }
607 if (compile.discard_local_symbols) {
608 try zig_args.append("--discard-all");
609 }
610
611 try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt);
612 try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt);
613 try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns);
614 if (compile.rdynamic) {
615 try zig_args.append("-rdynamic");
616 }
617 if (compile.import_memory) {
618 try zig_args.append("--import-memory");
619 }
620 if (compile.export_memory) {
621 try zig_args.append("--export-memory");
622 }
623 if (compile.import_symbols) {
624 try zig_args.append("--import-symbols");
625 }
626 if (compile.import_table) {
627 try zig_args.append("--import-table");
628 }
629 if (compile.export_table) {
630 try zig_args.append("--export-table");
631 }
632 if (compile.initial_memory) |initial_memory| {
633 try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory}));
634 }
635 if (compile.max_memory) |max_memory| {
636 try zig_args.append(b.fmt("--max-memory={d}", .{max_memory}));
637 }
638 if (compile.shared_memory) {
639 try zig_args.append("--shared-memory");
640 }
641 if (compile.global_base) |global_base| {
642 try zig_args.append(b.fmt("--global-base={d}", .{global_base}));
643 }
644
645 if (compile.wasi_exec_model) |model| {
646 try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)}));
647 }
648 if (compile.linker_script) |linker_script| {
649 try zig_args.append("--script");
650 try zig_args.append(linker_script.getPath2(b, step));
651 }
652
653 if (compile.version_script) |version_script| {
654 try zig_args.append("--version-script");
655 try zig_args.append(version_script.getPath2(b, step));
656 }
657 if (compile.linker_allow_undefined_version) |x| {
658 try zig_args.append(if (x) "--undefined-version" else "--no-undefined-version");
659 }
660
661 if (compile.linker_enable_new_dtags) |enabled| {
662 try zig_args.append(if (enabled) "--enable-new-dtags" else "--disable-new-dtags");
663 }
664
665 if (compile.kind == .@"test") {
666 if (compile.exec_cmd_args) |exec_cmd_args| {
667 for (exec_cmd_args) |cmd_arg| {
668 if (cmd_arg) |arg| {
669 try zig_args.append("--test-cmd");
670 try zig_args.append(arg);
671 } else {
672 try zig_args.append("--test-cmd-bin");
673 }
674 }
675 }
676 }
677
678 if (b.sysroot) |sysroot| {
679 try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot });
680 }
681
682 // -I and -L arguments that appear after the last --mod argument apply to all modules.
683 const cwd: Io.Dir = .cwd();
684 const io = b.graph.io;
685
686 for (b.search_prefixes.items) |search_prefix| {
687 var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| {
688 return step.fail("unable to open prefix directory '{s}': {s}", .{
689 search_prefix, @errorName(err),
690 });
691 };
692 defer prefix_dir.close(io);
693
694 // Avoid passing -L and -I flags for nonexistent directories.
695 // This prevents a warning, that should probably be upgraded to an error in Zig's
696 // CLI parsing code, when the linker sees an -L directory that does not exist.
697
698 if (prefix_dir.access(io, "lib", .{})) |_| {
699 try zig_args.appendSlice(&.{
700 "-L", b.pathJoin(&.{ search_prefix, "lib" }),
701 });
702 } else |err| switch (err) {
703 error.FileNotFound => {},
704 else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{
705 search_prefix, @errorName(e),
706 }),
707 }
708
709 if (prefix_dir.access(io, "include", .{})) |_| {
710 try zig_args.appendSlice(&.{
711 "-I", b.pathJoin(&.{ search_prefix, "include" }),
712 });
713 } else |err| switch (err) {
714 error.FileNotFound => {},
715 else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{
716 search_prefix, @errorName(e),
717 }),
718 }
719 }
720
721 if (compile.rc_includes != .any) {
722 try zig_args.append("-rcincludes");
723 try zig_args.append(@tagName(compile.rc_includes));
724 }
725
726 try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath);
727
728 if (compile.build_id orelse b.build_id) |build_id| {
729 try zig_args.append(switch (build_id) {
730 .hexstring => |hs| b.fmt("--build-id=0x{x}", .{hs.toSlice()}),
731 .none, .fast, .uuid, .sha1, .md5 => b.fmt("--build-id={s}", .{@tagName(build_id)}),
732 });
733 }
734
735 const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir|
736 dir.getPath2(b, step)
737 else if (b.graph.zig_lib_directory.path) |_|
738 b.fmt("{f}", .{b.graph.zig_lib_directory})
739 else
740 null;
741
742 if (opt_zig_lib_dir) |zig_lib_dir| {
743 try zig_args.append("--zig-lib-dir");
744 try zig_args.append(zig_lib_dir);
745 }
746
747 try addFlag(&zig_args, "PIE", compile.pie);
748
749 if (compile.lto) |lto| {
750 try zig_args.append(switch (lto) {
751 .full => "-flto=full",
752 .thin => "-flto=thin",
753 .none => "-fno-lto",
754 });
755 }
756
757 try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard);
758
759 if (compile.subsystem) |subsystem| {
760 try zig_args.append("--subsystem");
761 try zig_args.append(@tagName(subsystem));
762 }
763
764 if (compile.mingw_unicode_entry_point) {
765 try zig_args.append("-municode");
766 }
767
768 if (compile.error_limit) |err_limit| try zig_args.appendSlice(&.{
769 "--error-limit", b.fmt("{d}", .{err_limit}),
770 });
771
772 try addFlag(&zig_args, "incremental", b.graph.incremental);
773
774 try zig_args.append("--listen=-");
775
776 // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux
777 // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and
778 // pass that to zig, e.g. via 'zig build-lib @args.rsp'
779 // See @file syntax here: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html
780 var args_length: usize = 0;
781 for (zig_args.items) |arg| {
782 args_length += arg.len + 1; // +1 to account for null terminator
783 }
784 if (args_length >= 30 * 1024) {
785 try b.cache_root.handle.createDirPath(io, "args");
786
787 const args_to_escape = zig_args.items[2..];
788 var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len);
789 arg_blk: for (args_to_escape) |arg| {
790 for (arg, 0..) |c, arg_idx| {
791 if (c == '\\' or c == '"') {
792 // Slow path for arguments that need to be escaped. We'll need to allocate and copy
793 var escaped: std.ArrayList(u8) = .empty;
794 try escaped.ensureTotalCapacityPrecise(arena, arg.len + 1);
795 try escaped.appendSlice(arena, arg[0..arg_idx]);
796 for (arg[arg_idx..]) |to_escape| {
797 if (to_escape == '\\' or to_escape == '"') try escaped.append(arena, '\\');
798 try escaped.append(arena, to_escape);
799 }
800 escaped_args.appendAssumeCapacity(escaped.items);
801 continue :arg_blk;
802 }
803 }
804 escaped_args.appendAssumeCapacity(arg); // no escaping needed so just use original argument
805 }
806
807 // Write the args to zig-cache/args/<SHA256 hash of args> to avoid conflicts with
808 // other zig build commands running in parallel.
809 const partially_quoted = try std.mem.join(arena, "\" \"", escaped_args.items);
810 const args = try std.mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" });
811
812 var args_hash: [Sha256.digest_length]u8 = undefined;
813 Sha256.hash(args, &args_hash, .{});
814 var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined;
815 _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash});
816
817 const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash;
818 if (b.cache_root.handle.access(io, args_file, .{})) |_| {
819 // The args file is already present from a previous run.
820 } else |err| switch (err) {
821 error.FileNotFound => {
822 var af = b.cache_root.handle.createFileAtomic(io, args_file, .{
823 .replace = false,
824 .make_path = true,
825 }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{
826 b.cache_root, args_file, e,
827 });
828 defer af.deinit(io);
829
830 af.file.writeStreamingAll(io, args) catch |e| {
831 return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{
832 b.cache_root, args_file, e,
833 });
834 };
835 // Note we can't clean up this file, not even after build
836 // success, because that might interfere with another build
837 // process that needs the same file.
838 af.link(io) catch |e| switch (e) {
839 error.PathAlreadyExists => {
840 // The args file was created by another concurrent build process.
841 },
842 else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{
843 b.cache_root, args_file, other_err,
844 }),
845 };
846 },
847 else => |other_err| return other_err,
848 }
849
850 const resolved_args_file = try mem.concat(arena, u8, &.{
851 "@",
852 try b.cache_root.join(arena, &.{args_file}),
853 });
854
855 zig_args.shrinkRetainingCapacity(2);
856 try zig_args.append(resolved_args_file);
857 }
858
859 return try zig_args.toOwnedSlice();
860}
861
862pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path {
863 c.step.result_error_msgs.clearRetainingCapacity();
864 c.step.result_stderr = "";
865
866 c.step.result_error_bundle.deinit(gpa);
867 c.step.result_error_bundle = std.zig.ErrorBundle.empty;
868
869 if (c.step.result_failed_command) |cmd| {
870 gpa.free(cmd);
871 c.step.result_failed_command = null;
872 }
873
874 const zig_args = try getZigArgs(c, true);
875 const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa);
876 return maybe_output_bin_path.?;
877}
878
879pub fn doAtomicSymLinks(
880 step: *Step,
881 output_path: []const u8,
882 filename_major_only: []const u8,
883 filename_name_only: []const u8,
884) !void {
885 const b = step.owner;
886 const io = b.graph.io;
887 const out_dir = fs.path.dirname(output_path) orelse ".";
888 const out_basename = fs.path.basename(output_path);
889 // sym link for libfoo.so.1 to libfoo.so.1.2.3
890 const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only });
891 const cwd: Io.Dir = .cwd();
892 cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| {
893 return step.fail("unable to symlink {s} -> {s}: {s}", .{
894 major_only_path, out_basename, @errorName(err),
895 });
896 };
897 // sym link for libfoo.so to libfoo.so.1
898 const name_only_path = b.pathJoin(&.{ out_dir, filename_name_only });
899 cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| {
900 return step.fail("Unable to symlink {s} -> {s}: {s}", .{
901 name_only_path, filename_major_only, @errorName(err),
902 });
903 };
904}
905
906fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg {
907 const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config";
908 const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore);
909 var list = std.array_list.Managed(PkgConfigPkg).init(b.allocator);
910 errdefer list.deinit();
911 var line_it = mem.tokenizeAny(u8, stdout, "\r\n");
912 while (line_it.next()) |line| {
913 if (mem.trim(u8, line, " \t").len == 0) continue;
914 var tok_it = mem.tokenizeAny(u8, line, " \t");
915 try list.append(PkgConfigPkg{
916 .name = tok_it.next() orelse return error.PkgConfigInvalidOutput,
917 .desc = tok_it.rest(),
918 });
919 }
920 return list.toOwnedSlice();
921}
922
923fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg {
924 if (b.pkg_config_pkg_list) |res| {
925 return res;
926 }
927 var code: u8 = undefined;
928 if (execPkgConfigList(b, &code)) |list| {
929 b.pkg_config_pkg_list = list;
930 return list;
931 } else |err| {
932 const result = switch (err) {
933 error.ProcessTerminated => error.PkgConfigCrashed,
934 error.ExecNotSupported => error.PkgConfigFailed,
935 error.ExitCodeFailure => error.PkgConfigFailed,
936 error.FileNotFound => error.PkgConfigNotInstalled,
937 error.InvalidName => error.PkgConfigNotInstalled,
938 error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput,
939 else => return err,
940 };
941 b.pkg_config_pkg_list = result;
942 return result;
943 }
944}
945
946fn addFlag(args: *std.array_list.Managed([]const u8), comptime name: []const u8, opt: ?bool) !void {
947 const cond = opt orelse return;
948 try args.ensureUnusedCapacity(1);
949 if (cond) {
950 args.appendAssumeCapacity("-f" ++ name);
951 } else {
952 args.appendAssumeCapacity("-fno-" ++ name);
953 }
954}
955
956const PkgConfigResult = struct {
957 cflags: []const []const u8,
958 libs: []const []const u8,
959};
960
961/// Run pkg-config for the given library name and parse the output, returning the arguments
962/// that should be passed to zig to link the given library.
963fn runPkgConfig(compile: *Compile, lib_name: []const u8) !PkgConfigResult {
964 const wl_rpath_prefix = "-Wl,-rpath,";
965
966 const b = compile.step.owner;
967 const arena = b.allocator;
968 const pkg_name = match: {
969 // First we have to map the library name to pkg config name. Unfortunately,
970 // there are several examples where this is not straightforward:
971 // -lSDL2 -> pkg-config sdl2
972 // -lgdk-3 -> pkg-config gdk-3.0
973 // -latk-1.0 -> pkg-config atk
974 // -lpulse -> pkg-config libpulse
975 const pkgs = try getPkgConfigList(b);
976
977 // Exact match means instant winner.
978 for (pkgs) |pkg| {
979 if (mem.eql(u8, pkg.name, lib_name)) {
980 break :match pkg.name;
981 }
982 }
983
984 // Next we'll try ignoring case.
985 for (pkgs) |pkg| {
986 if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) {
987 break :match pkg.name;
988 }
989 }
990
991 // Prefixed "lib" or suffixed ".0".
992 for (pkgs) |pkg| {
993 if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| {
994 const prefix = pkg.name[0..pos];
995 const suffix = pkg.name[pos + lib_name.len ..];
996 if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue;
997 if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue;
998 break :match pkg.name;
999 }
1000 }
1001
1002 // Trimming "-1.0".
1003 if (mem.endsWith(u8, lib_name, "-1.0")) {
1004 const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len];
1005 for (pkgs) |pkg| {
1006 if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) {
1007 break :match pkg.name;
1008 }
1009 }
1010 }
1011
1012 return error.PackageNotFound;
1013 };
1014
1015 var code: u8 = undefined;
1016 const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config";
1017 const stdout = if (b.runAllowFail(&[_][]const u8{
1018 pkg_config_exe,
1019 pkg_name,
1020 "--cflags",
1021 "--libs",
1022 }, &code, .ignore)) |stdout| stdout else |err| switch (err) {
1023 error.ProcessTerminated => return error.PkgConfigCrashed,
1024 error.ExecNotSupported => return error.PkgConfigFailed,
1025 error.ExitCodeFailure => return error.PkgConfigFailed,
1026 error.FileNotFound => return error.PkgConfigNotInstalled,
1027 else => return err,
1028 };
1029
1030 var zig_cflags: std.ArrayList([]const u8) = .empty;
1031 defer zig_cflags.deinit(arena);
1032 var zig_libs: std.ArrayList([]const u8) = .empty;
1033 defer zig_libs.deinit(arena);
1034
1035 var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t");
1036 while (arg_it.next()) |arg| {
1037 if (mem.eql(u8, arg, "-I")) {
1038 const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput;
1039 try zig_cflags.appendSlice(arena, &.{ "-I", dir });
1040 } else if (mem.startsWith(u8, arg, "-I")) {
1041 try zig_cflags.append(arena, arg);
1042 } else if (mem.eql(u8, arg, "-L")) {
1043 const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput;
1044 try zig_libs.appendSlice(arena, &.{ "-L", dir });
1045 } else if (mem.startsWith(u8, arg, "-L")) {
1046 try zig_libs.append(arena, arg);
1047 } else if (mem.eql(u8, arg, "-l")) {
1048 const lib = arg_it.next() orelse return error.PkgConfigInvalidOutput;
1049 try zig_libs.appendSlice(arena, &.{ "-l", lib });
1050 } else if (mem.startsWith(u8, arg, "-l")) {
1051 try zig_libs.append(arena, arg);
1052 } else if (mem.eql(u8, arg, "-D")) {
1053 const macro = arg_it.next() orelse return error.PkgConfigInvalidOutput;
1054 try zig_cflags.appendSlice(arena, &.{ "-D", macro });
1055 } else if (mem.startsWith(u8, arg, "-D")) {
1056 try zig_cflags.append(arena, arg);
1057 } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) {
1058 try zig_cflags.appendSlice(arena, &.{ "-rpath", arg[wl_rpath_prefix.len..] });
1059 } else if (b.debug_pkg_config) {
1060 return compile.step.fail("unknown pkg-config flag '{s}'", .{arg});
1061 }
1062 }
1063
1064 try zig_cflags.shrinkToLen(arena);
1065 try zig_libs.shrinkToLen(arena);
1066
1067 return .{
1068 .cflags = zig_cflags.toOwnedSliceAssert(),
1069 .libs = zig_libs.toOwnedSliceAssert(),
1070 };
1071}
1072
1073fn checkCompileErrors(compile: *Compile) !void {
1074 // Clear this field so that it does not get printed by the build runner.
1075 const actual_eb = compile.step.result_error_bundle;
1076 compile.step.result_error_bundle = .empty;
1077
1078 const arena = compile.step.owner.allocator;
1079
1080 const actual_errors = ae: {
1081 var aw: std.Io.Writer.Allocating = .init(arena);
1082 defer aw.deinit();
1083 try actual_eb.renderToWriter(.{
1084 .include_reference_trace = false,
1085 .include_source_line = false,
1086 }, &aw.writer);
1087 break :ae try aw.toOwnedSlice();
1088 };
1089
1090 // Render the expected lines into a string that we can compare verbatim.
1091 var expected_generated: std.ArrayList(u8) = .empty;
1092 const expect_errors = compile.expect_errors.?;
1093
1094 var actual_line_it = mem.splitScalar(u8, actual_errors, '\n');
1095
1096 // TODO merge this with the testing.expectEqualStrings logic, and also CheckFile
1097 switch (expect_errors) {
1098 .starts_with => |expect_starts_with| {
1099 if (std.mem.startsWith(u8, actual_errors, expect_starts_with)) return;
1100 return compile.step.fail(
1101 \\
1102 \\========= should start with: ============
1103 \\{s}
1104 \\========= but not found: ================
1105 \\{s}
1106 \\=========================================
1107 , .{ expect_starts_with, actual_errors });
1108 },
1109 .contains => |expect_line| {
1110 while (actual_line_it.next()) |actual_line| {
1111 if (!matchCompileError(actual_line, expect_line)) continue;
1112 return;
1113 }
1114
1115 return compile.step.fail(
1116 \\
1117 \\========= should contain: ===============
1118 \\{s}
1119 \\========= but not found: ================
1120 \\{s}
1121 \\=========================================
1122 , .{ expect_line, actual_errors });
1123 },
1124 .stderr_contains => |expect_line| {
1125 const actual_stderr: []const u8 = if (compile.step.result_error_msgs.items.len > 0)
1126 compile.step.result_error_msgs.items[0]
1127 else
1128 &.{};
1129 compile.step.result_error_msgs.clearRetainingCapacity();
1130
1131 var stderr_line_it = mem.splitScalar(u8, actual_stderr, '\n');
1132
1133 while (stderr_line_it.next()) |actual_line| {
1134 if (!matchCompileError(actual_line, expect_line)) continue;
1135 return;
1136 }
1137
1138 return compile.step.fail(
1139 \\
1140 \\========= should contain: ===============
1141 \\{s}
1142 \\========= but not found: ================
1143 \\{s}
1144 \\=========================================
1145 , .{ expect_line, actual_stderr });
1146 },
1147 .exact => |expect_lines| {
1148 for (expect_lines) |expect_line| {
1149 const actual_line = actual_line_it.next() orelse {
1150 try expected_generated.appendSlice(arena, expect_line);
1151 try expected_generated.append(arena, '\n');
1152 continue;
1153 };
1154 if (matchCompileError(actual_line, expect_line)) {
1155 try expected_generated.appendSlice(arena, actual_line);
1156 try expected_generated.append(arena, '\n');
1157 continue;
1158 }
1159 try expected_generated.appendSlice(arena, expect_line);
1160 try expected_generated.append(arena, '\n');
1161 }
1162
1163 if (mem.eql(u8, expected_generated.items, actual_errors)) return;
1164
1165 return compile.step.fail(
1166 \\
1167 \\========= expected: =====================
1168 \\{s}
1169 \\========= but found: ====================
1170 \\{s}
1171 \\=========================================
1172 , .{ expected_generated.items, actual_errors });
1173 },
1174 }
1175}
1176
1177fn matchCompileError(actual: []const u8, expected: []const u8) bool {
1178 if (mem.endsWith(u8, actual, expected)) return true;
1179 if (mem.startsWith(u8, expected, ":?:?: ")) {
1180 if (mem.endsWith(u8, actual, expected[":?:?: ".len..])) return true;
1181 }
1182 // We scan for /?/ in expected line and if there is a match, we match everything
1183 // up to and after /?/.
1184 const expected_trim = mem.trim(u8, expected, " ");
1185 if (mem.find(u8, expected_trim, "/?/")) |index| {
1186 const actual_trim = mem.trim(u8, actual, " ");
1187 const lhs = expected_trim[0..index];
1188 const rhs = expected_trim[index + "/?/".len ..];
1189 if (mem.startsWith(u8, actual_trim, lhs) and mem.endsWith(u8, actual_trim, rhs)) return true;
1190 }
1191 return false;
1192}
1193
1194fn moduleNeedsCliArg(mod: *const Module) bool {
1195 return for (mod.link_objects.items) |o| switch (o) {
1196 .c_source_file, .c_source_files, .assembly_file, .win32_resource_file => break true,
1197 else => continue,
1198 } else false;
1199}
1200
lib/compiler/Maker/Step/InstallArtifact.zig created+96
...@@ -0,0 +1,96 @@
1
2fn make(step: *Step, options: Step.MakeOptions) !void {
3 _ = options;
4 const install_artifact: *InstallArtifact = @fieldParentPtr("step", step);
5 const b = step.owner;
6 const io = b.graph.io;
7
8 var all_cached = true;
9
10 if (install_artifact.dest_dir) |dest_dir| {
11 const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path);
12 const p = try step.installFile(install_artifact.emitted_bin.?, full_dest_path);
13 all_cached = all_cached and p == .fresh;
14
15 if (install_artifact.dylib_symlinks) |dls| {
16 try Step.Compile.doAtomicSymLinks(step, full_dest_path, dls.major_only_filename, dls.name_only_filename);
17 }
18
19 install_artifact.artifact.installed_path = full_dest_path;
20 }
21
22 if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| {
23 const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step));
24 const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path);
25 all_cached = all_cached and p == .fresh;
26 }
27
28 if (install_artifact.implib_dir) |implib_dir| {
29 const full_implib_path = b.getInstallPath(implib_dir, install_artifact.emitted_implib.?.basename(b, step));
30 const p = try step.installFile(install_artifact.emitted_implib.?, full_implib_path);
31 all_cached = all_cached and p == .fresh;
32 }
33
34 if (install_artifact.pdb_dir) |pdb_dir| {
35 const full_pdb_path = b.getInstallPath(pdb_dir, install_artifact.emitted_pdb.?.basename(b, step));
36 const p = try step.installFile(install_artifact.emitted_pdb.?, full_pdb_path);
37 all_cached = all_cached and p == .fresh;
38 }
39
40 if (install_artifact.h_dir) |h_dir| {
41 if (install_artifact.emitted_h) |emitted_h| {
42 const full_h_path = b.getInstallPath(h_dir, emitted_h.basename(b, step));
43 const p = try step.installFile(emitted_h, full_h_path);
44 all_cached = all_cached and p == .fresh;
45 }
46
47 for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) {
48 .file => |file| {
49 const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path);
50 const p = try step.installFile(file.source, full_h_path);
51 all_cached = all_cached and p == .fresh;
52 },
53 .directory => |dir| {
54 const src_dir_path = dir.source.getPath3(b, step);
55 const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path);
56
57 var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| {
58 return step.fail("unable to open source directory '{f}': {s}", .{
59 src_dir_path, @errorName(err),
60 });
61 };
62 defer src_dir.close(io);
63
64 var it = try src_dir.walk(b.allocator);
65 next_entry: while (try it.next(io)) |entry| {
66 for (dir.options.exclude_extensions) |ext| {
67 if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry;
68 }
69 if (dir.options.include_extensions) |incs| {
70 for (incs) |inc| {
71 if (std.mem.endsWith(u8, entry.path, inc)) break;
72 } else {
73 continue :next_entry;
74 }
75 }
76
77 const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path });
78 switch (entry.kind) {
79 .directory => {
80 try Step.handleVerbose(b, .inherit, &.{ "install", "-d", full_dest_path });
81 const p = try step.installDir(full_dest_path);
82 all_cached = all_cached and p == .existed;
83 },
84 .file => {
85 const p = try step.installFile(try dir.source.join(b.allocator, entry.path), full_dest_path);
86 all_cached = all_cached and p == .fresh;
87 },
88 else => continue,
89 }
90 }
91 },
92 };
93 }
94
95 step.result_cached = all_cached;
96}
lib/compiler/Maker/Step/Run.zig created+2130
...@@ -0,0 +1,2130 @@
1const Run = @This();
2
3const builtin = @import("builtin");
4
5const std = @import("std");
6const Io = std.Io;
7const Dir = std.Io.Dir;
8const mem = std.mem;
9const process = std.process;
10const EnvMap = std.process.Environ.Map;
11const assert = std.debug.assert;
12const Cache = std.Build.Cache;
13const Path = std.Build.Cache.Path;
14
15const Step = @import("../Step.zig");
16
17/// If this is a Zig unit test binary, this tracks the names of the unit
18/// tests that are also fuzz tests. Indexes cannot be used as they may
19/// change between reruns.
20fuzz_tests: std.ArrayList([]const u8),
21cached_test_metadata: ?CachedTestMetadata = null,
22
23/// Populated during the fuzz phase if this run step corresponds to a unit test
24/// executable that contains fuzz tests.
25rebuilt_executable: ?Path,
26
27fn make(step: *Step, options: Step.MakeOptions) !void {
28 const b = step.owner;
29 const io = b.graph.io;
30 const arena = b.allocator;
31 const run: *Run = @fieldParentPtr("step", step);
32 const has_side_effects = run.hasSideEffects();
33
34 var argv_list = std.array_list.Managed([]const u8).init(arena);
35 var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena);
36
37 var man = b.graph.cache.obtain();
38 defer man.deinit();
39
40 if (run.environ_map) |environ_map| {
41 for (environ_map.keys(), environ_map.values()) |key, value| {
42 man.hash.addBytes(key);
43 man.hash.addBytes(value);
44 }
45 }
46
47 man.hash.add(run.color);
48 man.hash.add(run.disable_zig_progress);
49
50 for (run.argv.items) |arg| {
51 switch (arg) {
52 .bytes => |bytes| {
53 try argv_list.append(bytes);
54 man.hash.addBytes(bytes);
55 },
56 .lazy_path => |file| {
57 const file_path = file.lazy_path.getPath3(b, step);
58 try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) }));
59 man.hash.addBytes(file.prefix);
60 _ = try man.addFilePath(file_path, null);
61 },
62 .decorated_directory => |dd| {
63 const file_path = dd.lazy_path.getPath3(b, step);
64 const resolved_arg = b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix });
65 try argv_list.append(resolved_arg);
66 man.hash.addBytes(resolved_arg);
67 },
68 .file_content => |file_plp| {
69 const file_path = file_plp.lazy_path.getPath3(b, step);
70
71 var result: std.Io.Writer.Allocating = .init(arena);
72 errdefer result.deinit();
73 result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory;
74
75 const file = file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}) catch |err| {
76 return step.fail(
77 "unable to open input file '{f}': {t}",
78 .{ file_path, err },
79 );
80 };
81 defer file.close(io);
82
83 var buf: [1024]u8 = undefined;
84 var file_reader = file.reader(io, &buf);
85 _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) {
86 error.ReadFailed => return step.fail(
87 "failed to read from '{f}': {t}",
88 .{ file_path, file_reader.err.? },
89 ),
90 error.WriteFailed => return error.OutOfMemory,
91 };
92
93 try argv_list.append(result.written());
94 man.hash.addBytes(file_plp.prefix);
95 _ = try man.addFilePath(file_path, null);
96 },
97 .artifact => |pa| {
98 const artifact = pa.artifact;
99
100 if (artifact.rootModuleTarget().os.tag == .windows) {
101 // On Windows we don't have rpaths so we have to add .dll search paths to PATH
102 run.addPathForDynLibs(artifact);
103 }
104 const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?;
105
106 try argv_list.append(b.fmt("{s}{s}", .{
107 pa.prefix,
108 run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }),
109 }));
110
111 _ = try man.addFile(file_path, null);
112 },
113 .output_file, .output_directory => |output| {
114 man.hash.addBytes(output.prefix);
115 man.hash.addBytes(output.basename);
116 // Add a placeholder into the argument list because we need the
117 // manifest hash to be updated with all arguments before the
118 // object directory is computed.
119 try output_placeholders.append(.{
120 .index = argv_list.items.len,
121 .tag = arg,
122 .output = output,
123 });
124 _ = try argv_list.addOne();
125 },
126 }
127 }
128
129 switch (run.stdin) {
130 .bytes => |bytes| {
131 man.hash.addBytes(bytes);
132 },
133 .lazy_path => |lazy_path| {
134 const file_path = lazy_path.getPath2(b, step);
135 _ = try man.addFile(file_path, null);
136 },
137 .none => {},
138 }
139
140 if (run.captured_stdout) |captured| {
141 man.hash.addBytes(captured.output.basename);
142 man.hash.add(captured.trim_whitespace);
143 }
144
145 if (run.captured_stderr) |captured| {
146 man.hash.addBytes(captured.output.basename);
147 man.hash.add(captured.trim_whitespace);
148 }
149
150 hashStdIo(&man.hash, run.stdio);
151
152 for (run.file_inputs.items) |lazy_path| {
153 _ = try man.addFile(lazy_path.getPath2(b, step), null);
154 }
155
156 if (run.cwd) |cwd| {
157 const cwd_path = cwd.getPath3(b, step);
158 _ = man.hash.addBytes(try cwd_path.toString(arena));
159 }
160
161 if (!has_side_effects and try step.cacheHitAndWatch(&man)) {
162 // cache hit, skip running command
163 const digest = man.final();
164
165 try populateGeneratedPaths(
166 arena,
167 output_placeholders.items,
168 run.captured_stdout,
169 run.captured_stderr,
170 b.cache_root,
171 &digest,
172 );
173
174 step.result_cached = true;
175 return;
176 }
177
178 const dep_output_file = run.dep_output_file orelse {
179 // We already know the final output paths, use them directly.
180 const digest = if (has_side_effects)
181 man.hash.final()
182 else
183 man.final();
184
185 try populateGeneratedPaths(
186 arena,
187 output_placeholders.items,
188 run.captured_stdout,
189 run.captured_stderr,
190 b.cache_root,
191 &digest,
192 );
193
194 const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest;
195 for (output_placeholders.items) |placeholder| {
196 const output_sub_path = b.pathJoin(&.{ output_dir_path, placeholder.output.basename });
197 const output_sub_dir_path = switch (placeholder.tag) {
198 .output_file => Dir.path.dirname(output_sub_path).?,
199 .output_directory => output_sub_path,
200 else => unreachable,
201 };
202 b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| {
203 return step.fail("unable to make path '{f}{s}': {s}", .{
204 b.cache_root, output_sub_dir_path, @errorName(err),
205 });
206 };
207 const arg_output_path = run.convertPathArg(.{
208 .root_dir = .cwd(),
209 .sub_path = placeholder.output.generated_file.getPath(),
210 });
211 argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0)
212 arg_output_path
213 else
214 b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path });
215 }
216
217 try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null);
218 if (!has_side_effects) try step.writeManifestAndWatch(&man);
219 return;
220 };
221
222 // We do not know the final output paths yet, use temp paths to run the command.
223 var rand_int: u64 = undefined;
224 io.random(@ptrCast(&rand_int));
225 const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int);
226
227 for (output_placeholders.items) |placeholder| {
228 const output_components = .{ tmp_dir_path, placeholder.output.basename };
229 const output_sub_path = b.pathJoin(&output_components);
230 const output_sub_dir_path = switch (placeholder.tag) {
231 .output_file => Dir.path.dirname(output_sub_path).?,
232 .output_directory => output_sub_path,
233 else => unreachable,
234 };
235 b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| {
236 return step.fail("unable to make path '{f}{s}': {s}", .{
237 b.cache_root, output_sub_dir_path, @errorName(err),
238 });
239 };
240 const raw_output_path: Cache.Path = .{
241 .root_dir = b.cache_root,
242 .sub_path = b.pathJoin(&output_components),
243 };
244 placeholder.output.generated_file.path = raw_output_path.toString(b.graph.arena) catch @panic("OOM");
245 argv_list.items[placeholder.index] = b.fmt("{s}{s}", .{
246 placeholder.output.prefix,
247 run.convertPathArg(raw_output_path),
248 });
249 }
250
251 try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null);
252
253 const dep_file_dir = Dir.cwd();
254 const dep_file_basename = dep_output_file.generated_file.getPath2(b, step);
255 if (has_side_effects)
256 try man.addDepFile(dep_file_dir, dep_file_basename)
257 else
258 try man.addDepFilePost(dep_file_dir, dep_file_basename);
259
260 const digest = if (has_side_effects)
261 man.hash.final()
262 else
263 man.final();
264
265 const any_output = output_placeholders.items.len > 0 or
266 run.captured_stdout != null or run.captured_stderr != null;
267
268 // Rename into place
269 if (any_output) {
270 const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest;
271
272 b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |err| switch (err) {
273 Dir.RenameError.DirNotEmpty => {
274 b.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| {
275 return step.fail("unable to remove dir '{f}'{s}: {t}", .{
276 b.cache_root, tmp_dir_path, del_err,
277 });
278 };
279 b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |retry_err| {
280 return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{
281 b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, retry_err,
282 });
283 };
284 },
285 else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{
286 b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, err,
287 }),
288 };
289 }
290
291 if (!has_side_effects) try step.writeManifestAndWatch(&man);
292
293 try populateGeneratedPaths(
294 arena,
295 output_placeholders.items,
296 run.captured_stdout,
297 run.captured_stderr,
298 b.cache_root,
299 &digest,
300 );
301}
302
303/// Reads stdout of a Zig test process until a termination condition is reached:
304/// * A write fails, indicating the child unexpectedly closed stdin
305/// * A test (or a response from the test runner) times out
306/// * The wait fails, indicating the child closed stdout and stderr
307fn waitZigTest(
308 run: *Run,
309 child: *process.Child,
310 options: Step.MakeOptions,
311 multi_reader: *Io.File.MultiReader,
312 opt_metadata: *?TestMetadata,
313 results: *Step.TestResults,
314) !union(enum) {
315 write_failed: anyerror,
316 no_poll: struct {
317 active_test_index: ?u32,
318 ns_elapsed: u64,
319 },
320 timeout: struct {
321 active_test_index: ?u32,
322 ns_elapsed: u64,
323 },
324} {
325 const gpa = run.step.owner.allocator;
326 const arena = run.step.owner.allocator;
327 const io = run.step.owner.graph.io;
328
329 var sub_prog_node: ?std.Progress.Node = null;
330 defer if (sub_prog_node) |n| n.end();
331
332 if (opt_metadata.*) |*md| {
333 // Previous unit test process died or was killed; we're continuing where it left off
334 requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err };
335 } else {
336 // Running unit tests normally
337 run.fuzz_tests.clearRetainingCapacity();
338 sendMessage(io, child.stdin.?, .query_test_metadata) catch |err| return .{ .write_failed = err };
339 }
340
341 var active_test_index: ?u32 = null;
342
343 var last_update: Io.Clock.Timestamp = .now(io, .awake);
344
345 // This timeout is used when we're waiting on the test runner itself rather than a user-specified
346 // test. For instance, if the test runner leaves this much time between us requesting a test to
347 // start and it acknowledging the test starting, we terminate the child and raise an error. This
348 // *should* never happen, but could in theory be caused by some very unlucky IB in a test.
349 const response_timeout: Io.Clock.Duration = t: {
350 if (fuzz_context != null) break :t null; // don't timeout fuzz tests
351 const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s);
352 break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) };
353 };
354 const test_timeout: ?Io.Clock.Duration = if (options.unit_test_timeout_ns) |ns| .{
355 .clock = .awake,
356 .raw = .fromNanoseconds(ns),
357 } else null;
358
359 const stdout = multi_reader.reader(0);
360 const stderr = multi_reader.reader(1);
361 const Header = std.zig.Server.Message.Header;
362
363 while (true) {
364 const timeout: Io.Timeout = t: {
365 const opt_duration = if (active_test_index == null) response_timeout else test_timeout;
366 const duration = opt_duration orelse break :t .none;
367 break :t .{ .deadline = last_update.addDuration(duration) };
368 };
369
370 // This block is exited when `stdout` contains enough bytes for a `Header`.
371 header_ready: {
372 if (stdout.buffered().len >= @sizeOf(Header)) {
373 // We already have one, no need to poll!
374 break :header_ready;
375 }
376
377 multi_reader.fill(64, timeout) catch |err| switch (err) {
378 error.Timeout => return .{ .timeout = .{
379 .active_test_index = active_test_index,
380 .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
381 } },
382 error.EndOfStream => return .{ .no_poll = .{
383 .active_test_index = active_test_index,
384 .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
385 } },
386 else => |e| return e,
387 };
388
389 continue;
390 }
391 // There is definitely a header available now -- read it.
392 const header = stdout.takeStruct(Header, .little) catch unreachable;
393
394 while (stdout.buffered().len < header.bytes_len) {
395 multi_reader.fill(64, timeout) catch |err| switch (err) {
396 error.Timeout => return .{ .timeout = .{
397 .active_test_index = active_test_index,
398 .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
399 } },
400 error.EndOfStream => return .{ .no_poll = .{
401 .active_test_index = active_test_index,
402 .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
403 } },
404 else => |e| return e,
405 };
406 }
407
408 const body = stdout.take(header.bytes_len) catch unreachable;
409 var body_r: std.Io.Reader = .fixed(body);
410 switch (header.tag) {
411 .zig_version => {
412 if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail(
413 "zig version mismatch build runner vs compiler: '{s}' vs '{s}'",
414 .{ builtin.zig_version_string, body },
415 );
416 },
417 .test_metadata => {
418 // `metadata` would only be populated if we'd already seen a `test_metadata`, but we
419 // only request it once (and importantly, we don't re-request it if we kill and
420 // restart the test runner).
421 assert(opt_metadata.* == null);
422
423 const tm_hdr = body_r.takeStruct(std.zig.Server.Message.TestMetadata, .little) catch unreachable;
424 results.test_count = tm_hdr.tests_len;
425
426 const names = try arena.alloc(u32, results.test_count);
427 for (names) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable;
428
429 const expected_panic_msgs = try arena.alloc(u32, results.test_count);
430 for (expected_panic_msgs) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable;
431
432 const string_bytes = body_r.take(tm_hdr.string_bytes_len) catch unreachable;
433
434 options.progress_node.setEstimatedTotalItems(names.len);
435 opt_metadata.* = .{
436 .string_bytes = try arena.dupe(u8, string_bytes),
437 .ns_per_test = try arena.alloc(u64, results.test_count),
438 .names = names,
439 .expected_panic_msgs = expected_panic_msgs,
440 .next_index = 0,
441 .prog_node = options.progress_node,
442 };
443 @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64));
444
445 active_test_index = null;
446 last_update = .now(io, .awake);
447
448 requestNextTest(io, child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch |err| return .{ .write_failed = err };
449 },
450 .test_started => {
451 active_test_index = opt_metadata.*.?.next_index - 1;
452 last_update = .now(io, .awake);
453 },
454 .test_results => {
455 const md = &opt_metadata.*.?;
456
457 const tr_hdr = body_r.takeStruct(std.zig.Server.Message.TestResults, .little) catch unreachable;
458 assert(tr_hdr.index == active_test_index);
459
460 switch (tr_hdr.flags.status) {
461 .pass => {},
462 .skip => results.skip_count +|= 1,
463 .fail => results.fail_count +|= 1,
464 }
465 const leak_count = tr_hdr.flags.leak_count;
466 const log_err_count = tr_hdr.flags.log_err_count;
467 results.leak_count +|= leak_count;
468 results.log_err_count +|= log_err_count;
469
470 if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index));
471
472 if (tr_hdr.flags.status == .fail) {
473 const name = md.testName(tr_hdr.index);
474 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
475 stderr.tossBuffered();
476 if (stderr_bytes.len == 0) {
477 try run.step.addError("'{s}' failed without output", .{name});
478 } else {
479 try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes });
480 }
481 } else if (leak_count > 0) {
482 const name = md.testName(tr_hdr.index);
483 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
484 stderr.tossBuffered();
485 try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes });
486 } else if (log_err_count > 0) {
487 const name = md.testName(tr_hdr.index);
488 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
489 stderr.tossBuffered();
490 try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes });
491 }
492
493 active_test_index = null;
494
495 const now: Io.Clock.Timestamp = .now(io, .awake);
496 md.ns_per_test[tr_hdr.index] = @intCast(last_update.durationTo(now).raw.nanoseconds);
497 last_update = now;
498
499 requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err };
500 },
501 else => {}, // ignore other messages
502 }
503 }
504}
505
506const FuzzTestRunner = struct {
507 run: *Run,
508 ctx: FuzzContext,
509 coverage_id: ?u64,
510
511 instances: []Instance,
512 /// The indexes of this are layed out such that it is effectively an array
513 /// of `[instances.len][3]Io.Operation.Storage` of stdin, stdout, stderr.
514 batch: Io.Batch,
515 /// LIFO. Stream of message bodies trailed by PendingBroadcastFooter.
516 pending_broadcasts: std.ArrayList(u8),
517 broadcast: std.ArrayList(u8),
518 broadcast_undelivered: u32,
519
520 const Instance = struct {
521 child: process.Child,
522 message: std.ArrayListAligned(u8, .@"4"),
523 broadcast_written: usize,
524 stderr: std.ArrayList(u8),
525 stdin_vec: [1][]u8,
526 stdout_vec: [1][]u8,
527 stderr_vec: [1][]u8,
528 progress_node: std.Progress.Node,
529
530 fn messageHeader(instance: *Instance) InHeader {
531 assert(instance.message.items.len >= @sizeOf(InHeader));
532 const header_ptr: *InHeader = @ptrCast(instance.message.items);
533 var header = header_ptr.*;
534 if (std.builtin.Endian.native != .little) {
535 std.mem.byteSwapAllFields(InHeader, &header);
536 }
537 return header;
538 }
539 };
540
541 const PendingBroadcastFooter = struct {
542 from_id: u32,
543 body_len: u32,
544 };
545
546 const InHeader = std.zig.Server.Message.Header;
547 const OutHeader = std.zig.Client.Message.Header;
548
549 const stdin_i = 0;
550 const stdout_i = 1;
551 const stderr_i = 2;
552
553 fn init(
554 run: *Run,
555 ctx: FuzzContext,
556 progress_node: std.Progress.Node,
557 spawn_options: process.SpawnOptions,
558 ) !FuzzTestRunner {
559 const step_owner = run.step.owner;
560 const gpa = step_owner.allocator;
561 const io = step_owner.graph.io;
562
563 const n_instances = switch (ctx.fuzz.mode) {
564 .forever => step_owner.graph.max_jobs orelse @min(
565 std.Thread.getCpuCount() catch 1,
566 (std.math.maxInt(u32) - 2) / 3,
567 ),
568 .limit => 1,
569 };
570 const instances = try gpa.alloc(Instance, n_instances);
571 errdefer gpa.free(instances);
572 const batch_storage = try gpa.alloc(Io.Operation.Storage, instances.len * 3);
573 errdefer gpa.free(batch_storage);
574
575 @memset(instances, .{
576 .child = undefined,
577 .message = .empty,
578 .broadcast_written = undefined,
579 .stderr = .empty,
580 .stdin_vec = undefined,
581 .stdout_vec = undefined,
582 .stderr_vec = undefined,
583 .progress_node = undefined,
584 });
585 for (0.., instances) |id, *instance| {
586 errdefer for (instances[0..id]) |*spawned| {
587 spawned.child.kill(io);
588 spawned.progress_node.end();
589 };
590 instance.child = try process.spawn(io, spawn_options);
591 instance.progress_node = progress_node.start("starting fuzzer", 0);
592 }
593
594 return .{
595 .run = run,
596 .ctx = ctx,
597 .coverage_id = null,
598
599 .instances = instances,
600 .batch = .init(batch_storage),
601 .pending_broadcasts = .empty,
602 .broadcast = .empty,
603 .broadcast_undelivered = 0,
604 };
605 }
606
607 fn deinit(f: *FuzzTestRunner) void {
608 const step_owner = f.run.step.owner;
609 const gpa = step_owner.allocator;
610 const io = step_owner.graph.io;
611
612 f.batch.cancel(io);
613 gpa.free(f.batch.storage);
614 var total_rss: usize = 0;
615 for (f.instances) |*instance| {
616 instance.child.kill(io);
617 instance.message.deinit(gpa);
618 instance.stderr.deinit(gpa);
619 instance.progress_node.end();
620 total_rss += instance.child.resource_usage_statistics.getMaxRss() orelse 0;
621 }
622 f.run.step.result_peak_rss = @max(f.run.step.result_peak_rss, total_rss);
623 gpa.free(f.instances);
624 }
625
626 fn startInstances(f: *FuzzTestRunner) !void {
627 const step_owner = f.run.step.owner;
628 const io = step_owner.graph.io;
629
630 for (0.., f.instances) |id, *instance| {
631 const id32: u32 = @intCast(id);
632 (switch (f.ctx.fuzz.mode) {
633 .forever => sendRunFuzzTestMessage(
634 io,
635 instance.child.stdin.?,
636 f.run.fuzz_tests.items,
637 .forever,
638 id32,
639 ),
640 .limit => |limit| sendRunFuzzTestMessage(
641 io,
642 instance.child.stdin.?,
643 f.run.fuzz_tests.items,
644 .iterations,
645 limit.amount,
646 ),
647 }) catch |write_err| {
648 // The runner unexpectedly closed stdin, which means it crashed during initialization.
649 // Clean up everything and wait for the child to exit.
650 instance.child.stdin.?.close(io);
651 instance.child.stdin = null;
652 const term = try instance.child.wait(io);
653 return f.run.step.fail(
654 "unable to write stdin ({t}); test process unexpectedly {f}",
655 .{ write_err, fmtTerm(term) },
656 );
657 };
658
659 try f.addStdoutRead(id32, @sizeOf(InHeader));
660 try f.addStderrRead(id32);
661 }
662 }
663
664 fn listen(f: *FuzzTestRunner) !void {
665 const step_owner = f.run.step.owner;
666 const io = step_owner.graph.io;
667
668 while (true) {
669 try f.batch.awaitConcurrent(io, .none);
670 while (f.batch.next()) |completion| {
671 const id = completion.index / 3;
672 const result = completion.result;
673 switch (completion.index % 3) {
674 0 => try f.completeStdinWrite(id, result.file_write_streaming catch |e| switch (e) {
675 // Avoid calling `instanceEos` until EndOfStream is seen with stderr so
676 // that all stderr is collected.
677 error.BrokenPipe => continue,
678 else => |write_e| return write_e,
679 }),
680 1 => try f.completeStdoutRead(id, result.file_read_streaming catch |e| switch (e) {
681 // Avoid calling `instanceEos` until EndOfStream is seen with stderr so
682 // that all stderr is collected.
683 error.EndOfStream => continue,
684 else => |read_e| return read_e,
685 }),
686 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) {
687 error.EndOfStream => return f.instanceEos(id),
688 else => |read_e| return read_e,
689 }),
690 else => unreachable,
691 }
692 }
693 }
694 }
695
696 fn completeStdoutRead(f: *FuzzTestRunner, id: u32, n: usize) !void {
697 const step_owner = f.run.step.owner;
698 const gpa = step_owner.allocator;
699 const io = step_owner.graph.io;
700 const instance = &f.instances[id];
701
702 instance.message.items.len += n;
703 const total_read = instance.message.items.len;
704 if (total_read < @sizeOf(InHeader)) {
705 try f.addStdoutRead(id, @sizeOf(InHeader));
706 return;
707 }
708
709 const header = instance.messageHeader();
710 const body = instance.message.items[@sizeOf(InHeader)..];
711 if (body.len != header.bytes_len) {
712 try f.addStdoutRead(id, @sizeOf(InHeader) + header.bytes_len);
713 return;
714 }
715
716 switch (header.tag) {
717 .zig_version => {
718 if (!std.mem.eql(u8, builtin.zig_version_string, body)) return f.run.step.fail(
719 "zig version mismatch build runner vs compiler: '{s}' vs '{s}'",
720 .{ builtin.zig_version_string, body },
721 );
722 },
723 .coverage_id => {
724 var body_r: Io.Reader = .fixed(body);
725 f.coverage_id = body_r.takeInt(u64, .little) catch unreachable;
726 const cumulative_runs = body_r.takeInt(u64, .little) catch unreachable;
727 const cumulative_unique = body_r.takeInt(u64, .little) catch unreachable;
728 const cumulative_coverage = body_r.takeInt(u64, .little) catch unreachable;
729
730 const fuzz = f.ctx.fuzz;
731 fuzz.queue_mutex.lockUncancelable(io);
732 defer fuzz.queue_mutex.unlock(io);
733 try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{
734 .id = f.coverage_id.?,
735 .cumulative = .{
736 .runs = cumulative_runs,
737 .unique = cumulative_unique,
738 .coverage = cumulative_coverage,
739 },
740 .run = f.run,
741 } });
742 fuzz.queue_cond.signal(io);
743 },
744 .fuzz_start_addr => {
745 var body_r: Io.Reader = .fixed(body);
746 const fuzz = f.ctx.fuzz;
747 const addr = body_r.takeInt(u64, .little) catch unreachable;
748
749 fuzz.queue_mutex.lockUncancelable(io);
750 defer fuzz.queue_mutex.unlock(io);
751 try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{
752 .addr = addr,
753 .coverage_id = f.coverage_id.?,
754 } });
755 fuzz.queue_cond.signal(io);
756 },
757 .fuzz_test_change => {
758 const test_i = std.mem.readInt(u32, body[0..4], .little);
759 instance.progress_node.setName(f.run.fuzz_tests.items[test_i]);
760 },
761 .broadcast_fuzz_input => {
762 if (f.instances.len == 1) {
763 // No other processes to broadcast to.
764 } else if (f.broadcast_undelivered == 0) {
765 try f.instanceBroadcast(id, body);
766 } else {
767 const footer: PendingBroadcastFooter = .{
768 .from_id = id,
769 .body_len = @intCast(body.len),
770 };
771 // There is another broadcast in progress so add this one to the queue.
772 const size = @sizeOf(PendingBroadcastFooter) + body.len;
773 try f.pending_broadcasts.ensureUnusedCapacity(gpa, size);
774 f.pending_broadcasts.appendSliceAssumeCapacity(body);
775 f.pending_broadcasts.appendSliceAssumeCapacity(@ptrCast(&footer));
776 }
777 },
778 else => {}, // ignore other messages
779 }
780
781 instance.message.clearRetainingCapacity();
782 try f.addStdoutRead(id, @sizeOf(InHeader));
783 }
784
785 fn completeStderrRead(f: *FuzzTestRunner, id: u32, n: usize) !void {
786 const instance = &f.instances[id];
787 instance.stderr.items.len += n;
788 try f.addStderrRead(id);
789 }
790
791 fn completeStdinWrite(f: *FuzzTestRunner, id: u32, n: usize) !void {
792 const instance = &f.instances[id];
793
794 instance.broadcast_written += n;
795 if (instance.broadcast_written == f.broadcast.items.len) {
796 f.broadcast_undelivered -= 1;
797 if (f.broadcast_undelivered == 0) {
798 try f.broadcastComplete();
799 }
800 } else {
801 f.addStdinWrite(id);
802 }
803 }
804
805 fn addStdoutRead(f: *FuzzTestRunner, id: u32, end: usize) !void {
806 const step_owner = f.run.step.owner;
807 const gpa = step_owner.allocator;
808 const instance = &f.instances[id];
809
810 try instance.message.ensureTotalCapacity(gpa, end);
811 const start = instance.message.items.len;
812 instance.stdout_vec = .{instance.message.allocatedSlice()[start..end]};
813 f.batch.addAt(id * 3 + stdout_i, .{ .file_read_streaming = .{
814 .file = instance.child.stdout.?,
815 .data = &instance.stdout_vec,
816 } });
817 }
818
819 fn addStderrRead(f: *FuzzTestRunner, id: u32) !void {
820 const step_owner = f.run.step.owner;
821 const gpa = step_owner.allocator;
822 const instance = &f.instances[id];
823
824 try instance.stderr.ensureUnusedCapacity(gpa, 1);
825 instance.stderr_vec = .{instance.stderr.unusedCapacitySlice()};
826 f.batch.addAt(id * 3 + stderr_i, .{ .file_read_streaming = .{
827 .file = instance.child.stderr.?,
828 .data = &instance.stderr_vec,
829 } });
830 }
831
832 fn addStdinWrite(f: *FuzzTestRunner, id: u32) void {
833 const instance = &f.instances[id];
834
835 assert(f.broadcast.items.len != instance.broadcast_written);
836 instance.stdin_vec = .{f.broadcast.items[instance.broadcast_written..]};
837 f.batch.addAt(id * 3 + stdin_i, .{ .file_write_streaming = .{
838 .file = instance.child.stdin.?,
839 .data = &instance.stdin_vec,
840 } });
841 }
842
843 fn instanceEos(f: *FuzzTestRunner, id: u32) !void {
844 const step_owner = f.run.step.owner;
845 const io = step_owner.graph.io;
846 const instance = &f.instances[id];
847
848 instance.child.stdin.?.close(io);
849 instance.child.stdin = null;
850 const term = try instance.child.wait(io);
851 if (!termMatches(.{ .exited = 0 }, term)) {
852 f.run.step.result_stderr = try f.mergedStderr();
853 try f.saveCrash(id, term);
854 return f.run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)});
855 }
856 }
857
858 fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void {
859 const step = &f.run.step;
860 const b = step.owner;
861 const io = b.graph.io;
862
863 if (f.coverage_id == null) return;
864
865 // Search for the input file corresponding to the instance
866 const InputHeader = Build.abi.fuzz.MmapInputHeader;
867 var in_r_buf: [@sizeOf(InputHeader)]u8 = undefined;
868 var in_r: Io.File.Reader = undefined;
869 var in_f: Io.File = undefined;
870 var in_name_buf: [12]u8 = undefined;
871 var in_name: []const u8 = undefined;
872 var i: u32 = 0;
873 const header: InputHeader = while (true) : ({
874 if (i == std.math.maxInt(u32)) return;
875 i += 1;
876 }) {
877 const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in";
878 in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable;
879 in_f = b.cache_root.handle.openFile(io, in_name, .{
880 .lock = .exclusive,
881 .lock_nonblocking = true,
882 }) catch |e| switch (e) {
883 error.FileNotFound => return,
884 error.WouldBlock => continue, // Can not be from
885 // the crashed instance since it is still locked.
886 else => return step.fail("failed to open file '{f}{s}': {t}", .{
887 b.cache_root, in_name, e,
888 }),
889 };
890
891 in_r = in_f.readerStreaming(io, &in_r_buf);
892 const header = in_r.interface.takeStruct(InputHeader, .little) catch |e| {
893 in_f.close(io);
894 switch (e) {
895 error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{
896 b.cache_root, in_name, in_r.err.?,
897 }),
898 error.EndOfStream => continue,
899 }
900 };
901
902 if (header.pc_digest == f.coverage_id.? and
903 header.instance_id == id and
904 header.test_i < f.run.fuzz_tests.items.len)
905 {
906 break header;
907 }
908
909 in_f.close(io);
910 };
911 defer in_f.close(io);
912
913 // Save it to a seperate file
914 const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash";
915 const out = b.cache_root.handle.createFile(io, crash_name, .{
916 .lock = .exclusive, // Multiple run steps could have found a crash at the same time
917 }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{
918 b.cache_root, crash_name, e,
919 });
920 defer out.close(io);
921
922 var out_w_buf: [512]u8 = undefined;
923 var out_w = out.writerStreaming(io, &out_w_buf);
924 _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) {
925 error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{
926 b.cache_root, in_name, in_r.err.?,
927 }),
928 error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{
929 b.cache_root, crash_name, out_w.err.?,
930 }),
931 };
932
933 return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{
934 f.run.fuzz_tests.items[header.test_i],
935 fmtTerm(term),
936 b.cache_root,
937 crash_name,
938 });
939 }
940
941 fn instanceBroadcast(f: *FuzzTestRunner, from_id: u32, bytes: []const u8) !void {
942 assert(f.instances.len > 1);
943 assert(f.broadcast_undelivered == 0); // no other broadcast is progress
944 assert(f.broadcast.items.len == 0);
945 assert(from_id < f.instances.len);
946
947 const step_owner = f.run.step.owner;
948 const gpa = step_owner.allocator;
949
950 var out_header: OutHeader = .{
951 .tag = .new_fuzz_input,
952 .bytes_len = @intCast(bytes.len),
953 };
954 if (std.builtin.Endian.native != .little) {
955 std.mem.byteSwapAllFields(OutHeader, &out_header);
956 }
957 try f.broadcast.ensureTotalCapacity(gpa, @sizeOf(OutHeader) + bytes.len);
958 f.broadcast.appendSliceAssumeCapacity(@ptrCast(&out_header));
959 f.broadcast.appendSliceAssumeCapacity(bytes);
960
961 f.broadcast_undelivered = @intCast(f.instances.len - 1);
962 for (0.., f.instances) |to_id, *instance| {
963 if (to_id == from_id) continue;
964 instance.broadcast_written = 0;
965 f.addStdinWrite(@intCast(to_id));
966 }
967 }
968
969 fn broadcastComplete(f: *FuzzTestRunner) !void {
970 assert(f.instances.len > 1);
971 assert(f.broadcast_undelivered == 0);
972 f.broadcast.clearRetainingCapacity();
973
974 const pending = &f.pending_broadcasts;
975 if (pending.items.len != 0) {
976 // Another broadcast is pending; copy it over to `broadcast`
977
978 const footer_len = @sizeOf(PendingBroadcastFooter);
979 const footer_bytes = pending.items[pending.items.len - footer_len ..];
980 const footer: *align(1) PendingBroadcastFooter = @ptrCast(footer_bytes);
981 pending.items.len -= footer_len;
982
983 const body = pending.items[pending.items.len - footer.body_len ..];
984 try f.instanceBroadcast(footer.from_id, body);
985 pending.items.len -= body.len;
986 }
987 }
988
989 fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 {
990 const step_owner = f.run.step.owner;
991 const arena = step_owner.allocator;
992
993 // Collect any available stderr
994 while (f.batch.next()) |completion| {
995 if (completion.index % 3 != 2) continue;
996 const len = completion.result.file_read_streaming catch continue;
997 f.instances[completion.index / 3].stderr.items.len += len;
998 }
999
1000 var stderr_len: usize = 0;
1001 for (f.instances) |*instance| stderr_len += instance.stderr.items.len;
1002 const stderr = try arena.alloc(u8, stderr_len);
1003
1004 stderr_len = 0;
1005 for (f.instances) |*instance| {
1006 @memcpy(stderr[stderr_len..][0..instance.stderr.items.len], instance.stderr.items);
1007 stderr_len += instance.stderr.items.len;
1008 }
1009 return stderr;
1010 }
1011};
1012
1013fn evalFuzzTest(
1014 run: *Run,
1015 spawn_options: process.SpawnOptions,
1016 options: Step.MakeOptions,
1017 fuzz_context: FuzzContext,
1018) !void {
1019 var f: FuzzTestRunner = try .init(run, fuzz_context, options.progress_node, spawn_options);
1020 defer f.deinit();
1021 try f.startInstances();
1022 try f.listen();
1023}
1024
1025const StdioPollEnum = enum { stdout, stderr };
1026
1027fn evalZigTest(
1028 run: *Run,
1029 spawn_options: process.SpawnOptions,
1030 options: Step.MakeOptions,
1031 fuzz_context: ?FuzzContext,
1032) !void {
1033 if (fuzz_context != null) {
1034 try evalFuzzTest(run, spawn_options, options, fuzz_context.?);
1035 return;
1036 }
1037
1038 const step_owner = run.step.owner;
1039 const gpa = step_owner.allocator;
1040 const arena = step_owner.allocator;
1041 const io = step_owner.graph.io;
1042
1043 // We will update this every time a child runs.
1044 run.step.result_peak_rss = 0;
1045
1046 var test_results: Step.TestResults = .{
1047 .test_count = 0,
1048 .skip_count = 0,
1049 .fail_count = 0,
1050 .crash_count = 0,
1051 .timeout_count = 0,
1052 .leak_count = 0,
1053 .log_err_count = 0,
1054 };
1055 var test_metadata: ?TestMetadata = null;
1056
1057 while (true) {
1058 var child = try process.spawn(io, spawn_options);
1059 var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined;
1060 var multi_reader: Io.File.MultiReader = undefined;
1061 multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? });
1062 var child_killed = false;
1063 defer if (!child_killed) {
1064 child.kill(io);
1065 multi_reader.deinit();
1066 run.step.result_peak_rss = @max(
1067 run.step.result_peak_rss,
1068 child.resource_usage_statistics.getMaxRss() orelse 0,
1069 );
1070 };
1071
1072 switch (try waitZigTest(
1073 run,
1074 &child,
1075 options,
1076 &multi_reader,
1077 &test_metadata,
1078 &test_results,
1079 )) {
1080 .write_failed => |err| {
1081 // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured
1082 // all available stderr to make our error output as useful as possible.
1083 const stderr_fr = multi_reader.fileReader(1);
1084 while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) {
1085 error.ReadFailed => return stderr_fr.err.?,
1086 error.EndOfStream => {},
1087 }
1088 run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered());
1089
1090 // Clean up everything and wait for the child to exit.
1091 child.stdin.?.close(io);
1092 child.stdin = null;
1093 multi_reader.deinit();
1094 child_killed = true;
1095 const term = try child.wait(io);
1096 run.step.result_peak_rss = @max(
1097 run.step.result_peak_rss,
1098 child.resource_usage_statistics.getMaxRss() orelse 0,
1099 );
1100
1101 // The individual unit test results are irrelevant: the test runner itself broke!
1102 // Fail immediately without populating `s.test_results`.
1103 return run.step.fail("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) });
1104 },
1105 .no_poll => |no_poll| {
1106 // This might be a success (we requested exit and the child dutifully closed stdout) or
1107 // a crash of some kind. Either way, the child will terminate by itself -- wait for it.
1108 const stderr_reader = multi_reader.reader(1);
1109 const stderr_owned = try arena.dupe(u8, stderr_reader.buffered());
1110
1111 // Clean up everything and wait for the child to exit.
1112 child.stdin.?.close(io);
1113 child.stdin = null;
1114 multi_reader.deinit();
1115 child_killed = true;
1116 const term = try child.wait(io);
1117 run.step.result_peak_rss = @max(
1118 run.step.result_peak_rss,
1119 child.resource_usage_statistics.getMaxRss() orelse 0,
1120 );
1121
1122 if (no_poll.active_test_index) |test_index| {
1123 // A test was running, so this is definitely a crash. Report it against that
1124 // test, and continue to the next test.
1125 test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed;
1126 test_results.crash_count += 1;
1127 try run.step.addError("'{s}' {f}{s}{s}", .{
1128 test_metadata.?.testName(test_index),
1129 fmtTerm(term),
1130 if (stderr_owned.len != 0) " with stderr:\n" else "",
1131 std.mem.trim(u8, stderr_owned, "\n"),
1132 });
1133 continue;
1134 }
1135
1136 // Report an error if the child terminated uncleanly or if we were still trying to run more tests.
1137 run.step.result_stderr = stderr_owned;
1138 const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32);
1139 if (!tests_done or !termMatches(.{ .exited = 0 }, term)) {
1140 // The individual unit test results are irrelevant: the test runner itself broke!
1141 // Fail immediately without populating `s.test_results`.
1142 return run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)});
1143 }
1144
1145 // We're done with all of the tests! Commit the test results and return.
1146 run.step.test_results = test_results;
1147 if (test_metadata) |tm| {
1148 run.cached_test_metadata = tm.toCachedTestMetadata();
1149 if (options.web_server) |ws| {
1150 if (run.step.owner.graph.time_report) {
1151 ws.updateTimeReportRunTest(
1152 run,
1153 &run.cached_test_metadata.?,
1154 tm.ns_per_test,
1155 );
1156 }
1157 }
1158 }
1159 return;
1160 },
1161 .timeout => |timeout| {
1162 const stderr_reader = multi_reader.reader(1);
1163 const stderr = stderr_reader.buffered();
1164 stderr_reader.tossBuffered();
1165 if (timeout.active_test_index) |test_index| {
1166 // A test was running. Report the timeout against that test, and continue on to
1167 // the next test.
1168 test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed;
1169 test_results.timeout_count += 1;
1170 try run.step.addError("'{s}' timed out after {f}{s}{s}", .{
1171 test_metadata.?.testName(test_index),
1172 Io.Duration{ .nanoseconds = timeout.ns_elapsed },
1173 if (stderr.len != 0) " with stderr:\n" else "",
1174 std.mem.trim(u8, stderr, "\n"),
1175 });
1176 continue;
1177 }
1178 // Just log an error and let the child be killed.
1179 run.step.result_stderr = try arena.dupe(u8, stderr);
1180 // The individual unit test results in `results` are irrelevant: the test runner
1181 // is broken! Fail immediately without populating `s.test_results`.
1182 return run.step.fail("test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }});
1183 },
1184 }
1185 comptime unreachable;
1186 }
1187}
1188
1189const TestMetadata = struct {
1190 names: []const u32,
1191 ns_per_test: []u64,
1192 expected_panic_msgs: []const u32,
1193 string_bytes: []const u8,
1194 next_index: u32,
1195 prog_node: std.Progress.Node,
1196
1197 fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata {
1198 return .{
1199 .names = tm.names,
1200 .string_bytes = tm.string_bytes,
1201 };
1202 }
1203
1204 fn testName(tm: TestMetadata, index: u32) []const u8 {
1205 return tm.toCachedTestMetadata().testName(index);
1206 }
1207};
1208
1209pub const CachedTestMetadata = struct {
1210 names: []const u32,
1211 string_bytes: []const u8,
1212
1213 pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 {
1214 return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0);
1215 }
1216};
1217
1218fn requestNextTest(io: Io, in: Io.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void {
1219 while (metadata.next_index < metadata.names.len) {
1220 const i = metadata.next_index;
1221 metadata.next_index += 1;
1222
1223 if (metadata.expected_panic_msgs[i] != 0) continue;
1224
1225 const name = metadata.testName(i);
1226 if (sub_prog_node.*) |n| n.end();
1227 sub_prog_node.* = metadata.prog_node.start(name, 0);
1228
1229 try sendRunTestMessage(io, in, .run_test, i);
1230 return;
1231 } else {
1232 metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done
1233 try sendMessage(io, in, .exit);
1234 }
1235}
1236
1237fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void {
1238 const header: std.zig.Client.Message.Header = .{
1239 .tag = tag,
1240 .bytes_len = 0,
1241 };
1242 var w = file.writerStreaming(io, &.{});
1243 w.interface.writeStruct(header, .little) catch |err| switch (err) {
1244 error.WriteFailed => return w.err.?,
1245 };
1246}
1247
1248fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, index: u32) !void {
1249 const header: std.zig.Client.Message.Header = .{
1250 .tag = tag,
1251 .bytes_len = 4,
1252 };
1253 var w = file.writerStreaming(io, &.{});
1254 w.interface.writeStruct(header, .little) catch |err| switch (err) {
1255 error.WriteFailed => return w.err.?,
1256 };
1257 w.interface.writeInt(u32, index, .little) catch |err| switch (err) {
1258 error.WriteFailed => return w.err.?,
1259 };
1260}
1261
1262fn sendRunFuzzTestMessage(
1263 io: Io,
1264 file: Io.File,
1265 test_names: []const []const u8,
1266 kind: std.Build.abi.fuzz.LimitKind,
1267 amount_or_instance: u64,
1268) !void {
1269 const header: std.zig.Client.Message.Header = .{
1270 .tag = .start_fuzzing,
1271 .bytes_len = 1 + 8 + 4 + count: {
1272 var c: u32 = @intCast(test_names.len * 4);
1273 for (test_names) |name| {
1274 c += @intCast(name.len);
1275 }
1276 break :count c;
1277 },
1278 };
1279 var w = file.writerStreaming(io, &.{});
1280 w.interface.writeStruct(header, .little) catch |err| switch (err) {
1281 error.WriteFailed => return w.err.?,
1282 };
1283 w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) {
1284 error.WriteFailed => return w.err.?,
1285 };
1286 w.interface.writeInt(u64, amount_or_instance, .little) catch |err| switch (err) {
1287 error.WriteFailed => return w.err.?,
1288 };
1289 w.interface.writeInt(u32, @intCast(test_names.len), .little) catch |err| switch (err) {
1290 error.WriteFailed => return w.err.?,
1291 };
1292 for (test_names) |test_name| {
1293 w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) {
1294 error.WriteFailed => return w.err.?,
1295 };
1296 w.interface.writeAll(test_name) catch |err| switch (err) {
1297 error.WriteFailed => return w.err.?,
1298 };
1299 }
1300}
1301
1302fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResult {
1303 const b = run.step.owner;
1304 const io = b.graph.io;
1305 const arena = b.allocator;
1306 const gpa = b.allocator;
1307
1308 var child = try process.spawn(io, spawn_options);
1309 defer child.kill(io);
1310
1311 switch (run.stdin) {
1312 .bytes => |bytes| {
1313 child.stdin.?.writeStreamingAll(io, bytes) catch |err| {
1314 return run.step.fail("unable to write stdin: {t}", .{err});
1315 };
1316 child.stdin.?.close(io);
1317 child.stdin = null;
1318 },
1319 .lazy_path => |lazy_path| {
1320 const path = lazy_path.getPath3(b, &run.step);
1321 const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| {
1322 return run.step.fail("unable to open stdin file: {t}", .{err});
1323 };
1324 defer file.close(io);
1325 // TODO https://github.com/ziglang/zig/issues/23955
1326 var read_buffer: [1024]u8 = undefined;
1327 var file_reader = file.reader(io, &read_buffer);
1328 var write_buffer: [1024]u8 = undefined;
1329 var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer);
1330 _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) {
1331 error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{
1332 path, file_reader.err.?,
1333 }),
1334 error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{
1335 stdin_writer.err.?,
1336 }),
1337 };
1338 stdin_writer.interface.flush() catch |err| switch (err) {
1339 error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{
1340 stdin_writer.err.?,
1341 }),
1342 };
1343 child.stdin.?.close(io);
1344 child.stdin = null;
1345 },
1346 .none => {},
1347 }
1348
1349 var stdout_bytes: ?[]const u8 = null;
1350 var stderr_bytes: ?[]const u8 = null;
1351
1352 if (child.stdout) |stdout| {
1353 if (child.stderr) |stderr| {
1354 var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined;
1355 var multi_reader: Io.File.MultiReader = undefined;
1356 multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr });
1357 defer multi_reader.deinit();
1358
1359 const stdout_reader = multi_reader.reader(0);
1360 const stderr_reader = multi_reader.reader(1);
1361
1362 while (multi_reader.fill(64, .none)) |_| {
1363 if (run.stdio_limit.toInt()) |limit| {
1364 if (stdout_reader.buffered().len > limit)
1365 return error.StdoutStreamTooLong;
1366 if (stderr_reader.buffered().len > limit)
1367 return error.StderrStreamTooLong;
1368 }
1369 } else |err| switch (err) {
1370 error.Timeout => unreachable,
1371 error.EndOfStream => {},
1372 else => |e| return e,
1373 }
1374
1375 try multi_reader.checkAnyError();
1376
1377 // TODO: this string can leak since alloc below can return error.
1378 stdout_bytes = try multi_reader.toOwnedSlice(0);
1379 // TODO: this string can leak since its allocated using gpa and `try child.wait(io)` below can fail.
1380 stderr_bytes = try multi_reader.toOwnedSlice(1);
1381 } else {
1382 var stdout_reader = stdout.readerStreaming(io, &.{});
1383 stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) {
1384 error.OutOfMemory => |e| return e,
1385 error.ReadFailed => return stdout_reader.err.?,
1386 error.StreamTooLong => return error.StdoutStreamTooLong,
1387 };
1388 }
1389 } else if (child.stderr) |stderr| {
1390 var stderr_reader = stderr.readerStreaming(io, &.{});
1391 stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) {
1392 error.OutOfMemory => |e| return e,
1393 error.ReadFailed => return stderr_reader.err.?,
1394 error.StreamTooLong => return error.StderrStreamTooLong,
1395 };
1396 }
1397
1398 if (stderr_bytes) |bytes| if (bytes.len > 0) {
1399 // Treat stderr as an error message.
1400 const stderr_is_diagnostic = run.captured_stderr == null and switch (run.stdio) {
1401 .check => |checks| !checksContainStderr(checks.items),
1402 else => true,
1403 };
1404 if (stderr_is_diagnostic) {
1405 run.step.result_stderr = bytes;
1406 }
1407 };
1408
1409 run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0;
1410
1411 return .{
1412 .term = try child.wait(io),
1413 .stdout = stdout_bytes,
1414 .stderr = stderr_bytes,
1415 };
1416}
1417
1418const IndexedOutput = struct {
1419 index: usize,
1420 tag: @typeInfo(Arg).@"union".tag_type.?,
1421 output: *Output,
1422};
1423
1424pub fn rerunInFuzzMode(
1425 run: *Run,
1426 fuzz: *std.Build.Fuzz,
1427 prog_node: std.Progress.Node,
1428) !void {
1429 const step = &run.step;
1430 const b = step.owner;
1431 const io = b.graph.io;
1432 const arena = b.allocator;
1433 var argv_list: std.ArrayList([]const u8) = .empty;
1434 for (run.argv.items) |arg| {
1435 switch (arg) {
1436 .bytes => |bytes| {
1437 try argv_list.append(arena, bytes);
1438 },
1439 .lazy_path => |file| {
1440 const file_path = file.lazy_path.getPath3(b, step);
1441 try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) }));
1442 },
1443 .decorated_directory => |dd| {
1444 const file_path = dd.lazy_path.getPath3(b, step);
1445 try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix }));
1446 },
1447 .file_content => |file_plp| {
1448 const file_path = file_plp.lazy_path.getPath3(b, step);
1449
1450 var result: std.Io.Writer.Allocating = .init(arena);
1451 errdefer result.deinit();
1452 result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory;
1453
1454 const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{});
1455 defer file.close(io);
1456
1457 var buf: [1024]u8 = undefined;
1458 var file_reader = file.reader(io, &buf);
1459 _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) {
1460 error.ReadFailed => return file_reader.err.?,
1461 error.WriteFailed => return error.OutOfMemory,
1462 };
1463
1464 try argv_list.append(arena, result.written());
1465 },
1466 .artifact => |pa| {
1467 const artifact = pa.artifact;
1468 const file_path: []const u8 = p: {
1469 if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?});
1470 break :p artifact.installed_path orelse artifact.generated_bin.?.path.?;
1471 };
1472 try argv_list.append(arena, b.fmt("{s}{s}", .{
1473 pa.prefix,
1474 run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }),
1475 }));
1476 },
1477 .output_file, .output_directory => unreachable,
1478 }
1479 }
1480
1481 if (run.step.result_failed_command) |cmd| {
1482 fuzz.gpa.free(cmd);
1483 run.step.result_failed_command = null;
1484 }
1485
1486 const has_side_effects = false;
1487 var rand_int: u64 = undefined;
1488 io.random(@ptrCast(&rand_int));
1489 const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int);
1490 try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{
1491 .progress_node = prog_node,
1492 .watch = undefined, // not used by `runCommand`
1493 .web_server = null, // only needed for time reports
1494 .unit_test_timeout_ns = null, // don't time out fuzz tests for now
1495 .gpa = fuzz.gpa,
1496 }, .{
1497 .fuzz = fuzz,
1498 });
1499}
1500
1501fn populateGeneratedPaths(
1502 arena: std.mem.Allocator,
1503 output_placeholders: []const IndexedOutput,
1504 captured_stdout: ?*CapturedStdIo,
1505 captured_stderr: ?*CapturedStdIo,
1506 cache_root: Cache.Directory,
1507 digest: *const Cache.HexDigest,
1508) !void {
1509 for (output_placeholders) |placeholder| {
1510 placeholder.output.generated_file.path = try cache_root.join(arena, &.{
1511 "o", digest, placeholder.output.basename,
1512 });
1513 }
1514
1515 if (captured_stdout) |captured| {
1516 captured.output.generated_file.path = try cache_root.join(arena, &.{
1517 "o", digest, captured.output.basename,
1518 });
1519 }
1520
1521 if (captured_stderr) |captured| {
1522 captured.output.generated_file.path = try cache_root.join(arena, &.{
1523 "o", digest, captured.output.basename,
1524 });
1525 }
1526}
1527
1528fn formatTerm(term: ?process.Child.Term, w: *std.Io.Writer) std.Io.Writer.Error!void {
1529 if (term) |t| switch (t) {
1530 .exited => |code| try w.print("exited with code {d}", .{code}),
1531 .signal => |sig| try w.print("terminated with signal {t}", .{sig}),
1532 .stopped => |sig| try w.print("stopped with signal {t}", .{sig}),
1533 .unknown => |code| try w.print("terminated for unknown reason with code {d}", .{code}),
1534 } else {
1535 try w.writeAll("exited with any code");
1536 }
1537}
1538fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTerm) {
1539 return .{ .data = term };
1540}
1541
1542const FuzzContext = struct {
1543 fuzz: *std.Build.Fuzz,
1544};
1545
1546fn runCommand(
1547 run: *Run,
1548 argv: []const []const u8,
1549 has_side_effects: bool,
1550 output_dir_path: []const u8,
1551 options: Step.MakeOptions,
1552 fuzz_context: ?FuzzContext,
1553) !void {
1554 const step = &run.step;
1555 const b = step.owner;
1556 const arena = b.allocator;
1557 const gpa = options.gpa;
1558 const io = b.graph.io;
1559
1560 const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit;
1561
1562 try step.handleChildProcUnsupported();
1563 try Step.handleVerbose2(step.owner, cwd, run.environ_map, argv);
1564
1565 const allow_skip = switch (run.stdio) {
1566 .check, .zig_test => run.skip_foreign_checks,
1567 else => false,
1568 };
1569
1570 var interp_argv = std.array_list.Managed([]const u8).init(b.allocator);
1571 defer interp_argv.deinit();
1572
1573 var environ_map: EnvMap = env: {
1574 const orig = run.environ_map orelse &b.graph.environ_map;
1575 break :env try orig.clone(gpa);
1576 };
1577 defer environ_map.deinit();
1578
1579 const opt_generic_result = spawnChildAndCollect(run, argv, &environ_map, has_side_effects, options, fuzz_context) catch |err| term: {
1580 // InvalidExe: cpu arch mismatch
1581 // FileNotFound: can happen with a wrong dynamic linker path
1582 if (err == error.InvalidExe or err == error.FileNotFound) interpret: {
1583 // TODO: learn the target from the binary directly rather than from
1584 // relying on it being a Compile step. This will make this logic
1585 // work even for the edge case that the binary was produced by a
1586 // third party.
1587 const exe = switch (run.argv.items[0]) {
1588 .artifact => |exe| exe.artifact,
1589 else => break :interpret,
1590 };
1591 switch (exe.kind) {
1592 .exe, .@"test" => {},
1593 else => break :interpret,
1594 }
1595
1596 const root_target = exe.rootModuleTarget();
1597 const need_cross_libc = exe.is_linking_libc and
1598 (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic));
1599 const other_target = exe.root_module.resolved_target.?.result;
1600 switch (std.zig.system.getExternalExecutor(io, &b.graph.host.result, &other_target, .{
1601 .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null,
1602 .link_libc = exe.is_linking_libc,
1603 })) {
1604 .native, .rosetta => {
1605 if (allow_skip) return error.MakeSkipped;
1606 break :interpret;
1607 },
1608 .wine => |bin_name| {
1609 if (b.enable_wine) {
1610 try interp_argv.append(bin_name);
1611 try interp_argv.appendSlice(argv);
1612
1613 // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but
1614 // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired.
1615 if (environ_map.get("WINEDEBUG") == null) {
1616 try environ_map.put("WINEDEBUG", "-all");
1617 }
1618 } else {
1619 return failForeign(run, "-fwine", argv[0], exe);
1620 }
1621 },
1622 .qemu => |bin_name| {
1623 if (b.enable_qemu) {
1624 try interp_argv.append(bin_name);
1625
1626 if (need_cross_libc) {
1627 if (b.libc_runtimes_dir) |dir| {
1628 try interp_argv.append("-L");
1629 try interp_argv.append(b.pathJoin(&.{
1630 dir,
1631 try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple(
1632 b.allocator,
1633 root_target.cpu.arch,
1634 root_target.os.tag,
1635 root_target.abi,
1636 ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple(
1637 b.allocator,
1638 root_target.cpu.arch,
1639 root_target.abi,
1640 ) else unreachable,
1641 }));
1642 } else return failForeign(run, "--libc-runtimes", argv[0], exe);
1643 }
1644
1645 try interp_argv.appendSlice(argv);
1646 } else return failForeign(run, "-fqemu", argv[0], exe);
1647 },
1648 .darling => |bin_name| {
1649 if (b.enable_darling) {
1650 try interp_argv.append(bin_name);
1651 try interp_argv.appendSlice(argv);
1652 } else {
1653 return failForeign(run, "-fdarling", argv[0], exe);
1654 }
1655 },
1656 .wasmtime => |bin_name| {
1657 if (b.enable_wasmtime) {
1658 try interp_argv.append(bin_name);
1659 try interp_argv.append("--dir=.");
1660 // Wasmtime doeesn't inherit environment variables from the parent process
1661 // by default. '-S inherit-env' was added in Wasmtime version 20.
1662 try interp_argv.append("-Sinherit-env");
1663 try interp_argv.append(argv[0]);
1664 try interp_argv.appendSlice(argv[1..]);
1665 } else {
1666 return failForeign(run, "-fwasmtime", argv[0], exe);
1667 }
1668 },
1669 .bad_dl => |foreign_dl| {
1670 if (allow_skip) return error.MakeSkipped;
1671
1672 const host_dl = b.graph.host.result.dynamic_linker.get() orelse "(none)";
1673
1674 return step.fail(
1675 \\the host system is unable to execute binaries from the target
1676 \\ because the host dynamic linker is '{s}',
1677 \\ while the target dynamic linker is '{s}'.
1678 \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step
1679 , .{ host_dl, foreign_dl });
1680 },
1681 .bad_os_or_cpu => {
1682 if (allow_skip) return error.MakeSkipped;
1683
1684 const host_name = try b.graph.host.result.zigTriple(b.allocator);
1685 const foreign_name = try root_target.zigTriple(b.allocator);
1686
1687 return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{
1688 host_name, foreign_name,
1689 });
1690 },
1691 }
1692
1693 if (root_target.os.tag == .windows) {
1694 // On Windows we don't have rpaths so we have to add .dll search paths to PATH
1695 run.addPathForDynLibs(exe);
1696 }
1697
1698 gpa.free(step.result_failed_command.?);
1699 step.result_failed_command = null;
1700 try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items);
1701
1702 break :term spawnChildAndCollect(run, interp_argv.items, &environ_map, has_side_effects, options, fuzz_context) catch |e| {
1703 if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped;
1704 if (e == error.MakeFailed) return error.MakeFailed; // error already reported
1705 return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e });
1706 };
1707 }
1708 if (err == error.MakeFailed) return error.MakeFailed; // error already reported
1709
1710 return step.fail("failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err });
1711 };
1712
1713 const generic_result = opt_generic_result orelse {
1714 assert(run.stdio == .zig_test);
1715 // Specific errors have already been reported, and test results are populated. All we need
1716 // to do is report step failure if any test failed.
1717 if (!step.test_results.isSuccess()) return error.MakeFailed;
1718 return;
1719 };
1720
1721 assert(fuzz_context == null);
1722 assert(run.stdio != .zig_test);
1723
1724 // Capture stdout and stderr to GeneratedFile objects.
1725 const Stream = struct {
1726 captured: ?*CapturedStdIo,
1727 bytes: ?[]const u8,
1728 };
1729 for ([_]Stream{
1730 .{
1731 .captured = run.captured_stdout,
1732 .bytes = generic_result.stdout,
1733 },
1734 .{
1735 .captured = run.captured_stderr,
1736 .bytes = generic_result.stderr,
1737 },
1738 }) |stream| {
1739 if (stream.captured) |captured| {
1740 const output_components = .{ output_dir_path, captured.output.basename };
1741 const output_path = try b.cache_root.join(arena, &output_components);
1742 captured.output.generated_file.path = output_path;
1743
1744 const sub_path = b.pathJoin(&output_components);
1745 const sub_path_dirname = Dir.path.dirname(sub_path).?;
1746 b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| {
1747 return step.fail("unable to make path '{f}{s}': {s}", .{
1748 b.cache_root, sub_path_dirname, @errorName(err),
1749 });
1750 };
1751 const data = switch (captured.trim_whitespace) {
1752 .none => stream.bytes.?,
1753 .all => mem.trim(u8, stream.bytes.?, &std.ascii.whitespace),
1754 .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace),
1755 .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace),
1756 };
1757 b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| {
1758 return step.fail("unable to write file '{f}{s}': {s}", .{
1759 b.cache_root, sub_path, @errorName(err),
1760 });
1761 };
1762 }
1763 }
1764
1765 switch (run.stdio) {
1766 .zig_test => unreachable,
1767 .check => |checks| for (checks.items) |check| switch (check) {
1768 .expect_stderr_exact => |expected_bytes| {
1769 if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) {
1770 return step.fail(
1771 \\========= expected this stderr: =========
1772 \\{s}
1773 \\========= but found: ====================
1774 \\{s}
1775 , .{
1776 expected_bytes,
1777 generic_result.stderr.?,
1778 });
1779 }
1780 },
1781 .expect_stderr_match => |match| {
1782 if (mem.find(u8, generic_result.stderr.?, match) == null) {
1783 return step.fail(
1784 \\========= expected to find in stderr: =========
1785 \\{s}
1786 \\========= but stderr does not contain it: =====
1787 \\{s}
1788 , .{
1789 match,
1790 generic_result.stderr.?,
1791 });
1792 }
1793 },
1794 .expect_stdout_exact => |expected_bytes| {
1795 if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) {
1796 return step.fail(
1797 \\========= expected this stdout: =========
1798 \\{s}
1799 \\========= but found: ====================
1800 \\{s}
1801 , .{
1802 expected_bytes,
1803 generic_result.stdout.?,
1804 });
1805 }
1806 },
1807 .expect_stdout_match => |match| {
1808 if (mem.find(u8, generic_result.stdout.?, match) == null) {
1809 return step.fail(
1810 \\========= expected to find in stdout: =========
1811 \\{s}
1812 \\========= but stdout does not contain it: =====
1813 \\{s}
1814 , .{
1815 match,
1816 generic_result.stdout.?,
1817 });
1818 }
1819 },
1820 .expect_term => |expected_term| {
1821 if (!termMatches(expected_term, generic_result.term)) {
1822 return step.fail("process {f} (expected {f})", .{
1823 fmtTerm(generic_result.term),
1824 fmtTerm(expected_term),
1825 });
1826 }
1827 },
1828 },
1829 else => {
1830 // On failure, report captured stderr like normal standard error output.
1831 const bad_exit = switch (generic_result.term) {
1832 .exited => |code| code != 0,
1833 .signal, .stopped, .unknown => true,
1834 };
1835 if (bad_exit) {
1836 if (generic_result.stderr) |bytes| {
1837 run.step.result_stderr = bytes;
1838 }
1839 }
1840
1841 try step.handleChildProcessTerm(generic_result.term);
1842 },
1843 }
1844}
1845
1846const EvalGenericResult = struct {
1847 term: process.Child.Term,
1848 stdout: ?[]const u8,
1849 stderr: ?[]const u8,
1850};
1851
1852fn spawnChildAndCollect(
1853 run: *Run,
1854 argv: []const []const u8,
1855 environ_map: *EnvMap,
1856 has_side_effects: bool,
1857 options: Step.MakeOptions,
1858 fuzz_context: ?FuzzContext,
1859) !?EvalGenericResult {
1860 const b = run.step.owner;
1861 const graph = b.graph;
1862 const io = graph.io;
1863
1864 if (fuzz_context != null) {
1865 assert(!has_side_effects);
1866 assert(run.stdio == .zig_test);
1867 }
1868
1869 const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit;
1870
1871 // If an error occurs, it's caused by this command:
1872 assert(run.step.result_failed_command == null);
1873 run.step.result_failed_command = try Step.allocPrintCmd(options.gpa, child_cwd, .{
1874 .child = environ_map,
1875 .parent = &graph.environ_map,
1876 }, argv);
1877
1878 var spawn_options: process.SpawnOptions = .{
1879 .argv = argv,
1880 .cwd = child_cwd,
1881 .environ_map = environ_map,
1882 .request_resource_usage_statistics = true,
1883 .stdin = if (run.stdin != .none) s: {
1884 assert(run.stdio != .inherit);
1885 break :s .pipe;
1886 } else switch (run.stdio) {
1887 .infer_from_args => if (has_side_effects) .inherit else .ignore,
1888 .inherit => .inherit,
1889 .check => .ignore,
1890 .zig_test => .pipe,
1891 },
1892 .stdout = if (run.captured_stdout != null) .pipe else switch (run.stdio) {
1893 .infer_from_args => if (has_side_effects) .inherit else .ignore,
1894 .inherit => .inherit,
1895 .check => |checks| if (checksContainStdout(checks.items)) .pipe else .ignore,
1896 .zig_test => .pipe,
1897 },
1898 .stderr = if (run.captured_stderr != null) .pipe else switch (run.stdio) {
1899 .infer_from_args => if (has_side_effects) .inherit else .pipe,
1900 .inherit => .inherit,
1901 .check => .pipe,
1902 .zig_test => .pipe,
1903 },
1904 };
1905
1906 if (run.stdio == .zig_test) {
1907 const started: Io.Clock.Timestamp = .now(io, .awake);
1908 const result = evalZigTest(run, spawn_options, options, fuzz_context) catch |err| switch (err) {
1909 error.Canceled => |e| return e,
1910 else => |e| e,
1911 };
1912 run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds);
1913 try result;
1914 return null;
1915 } else {
1916 const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit;
1917 if (!run.disable_zig_progress and !inherit) {
1918 spawn_options.progress_node = options.progress_node;
1919 }
1920 const terminal_mode: Io.Terminal.Mode = if (inherit) m: {
1921 const stderr = try io.lockStderr(&.{}, graph.stderr_mode);
1922 break :m stderr.terminal_mode;
1923 } else .no_color;
1924 defer if (inherit) io.unlockStderr();
1925 try setColorEnvironmentVariables(run, environ_map, terminal_mode);
1926
1927 const started: Io.Clock.Timestamp = .now(io, .awake);
1928 const result = evalGeneric(run, spawn_options) catch |err| switch (err) {
1929 error.Canceled => |e| return e,
1930 else => |e| e,
1931 };
1932 run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds);
1933 return try result;
1934 }
1935}
1936
1937fn hashStdIo(hh: *Cache.HashHelper, stdio: StdIo) void {
1938 switch (stdio) {
1939 .infer_from_args, .inherit, .zig_test => {},
1940 .check => |checks| for (checks.items) |check| {
1941 hh.add(@as(std.meta.Tag(StdIo.Check), check));
1942 switch (check) {
1943 .expect_stderr_exact,
1944 .expect_stderr_match,
1945 .expect_stdout_exact,
1946 .expect_stdout_match,
1947 => |s| hh.addBytes(s),
1948
1949 .expect_term => |term| {
1950 hh.add(@as(std.meta.Tag(process.Child.Term), term));
1951 switch (term) {
1952 inline .exited, .signal, .stopped => |x| hh.add(x),
1953 .unknown => |x| hh.add(x),
1954 }
1955 },
1956 }
1957 },
1958 }
1959}
1960fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool {
1961 return if (expected) |e| switch (e) {
1962 .exited => |expected_code| switch (actual) {
1963 .exited => |actual_code| expected_code == actual_code,
1964 else => false,
1965 },
1966 .signal => |expected_sig| switch (actual) {
1967 .signal => |actual_sig| expected_sig == actual_sig,
1968 else => false,
1969 },
1970 .stopped => |expected_sig| switch (actual) {
1971 .stopped => |actual_sig| expected_sig == actual_sig,
1972 else => false,
1973 },
1974 .unknown => |expected_code| switch (actual) {
1975 .unknown => |actual_code| expected_code == actual_code,
1976 else => false,
1977 },
1978 } else switch (actual) {
1979 .exited => true,
1980 else => false,
1981 };
1982}
1983
1984fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: Io.Terminal.Mode) !void {
1985 color: switch (run.color) {
1986 .manual => {},
1987 .enable => {
1988 try environ_map.put("CLICOLOR_FORCE", "1");
1989 _ = environ_map.swapRemove("NO_COLOR");
1990 },
1991 .disable => {
1992 try environ_map.put("NO_COLOR", "1");
1993 _ = environ_map.swapRemove("CLICOLOR_FORCE");
1994 },
1995 .inherit => switch (terminal_mode) {
1996 .no_color, .windows_api => continue :color .disable,
1997 .escape_codes => continue :color .enable,
1998 },
1999 .auto => {
2000 const capture_stderr = run.captured_stderr != null or switch (run.stdio) {
2001 .check => |checks| checksContainStderr(checks.items),
2002 .infer_from_args, .inherit, .zig_test => false,
2003 };
2004 if (capture_stderr) {
2005 continue :color .disable;
2006 } else {
2007 continue :color .inherit;
2008 }
2009 },
2010 }
2011}
2012
2013fn checksContainStdout(checks: []const StdIo.Check) bool {
2014 for (checks) |check| switch (check) {
2015 .expect_stderr_exact,
2016 .expect_stderr_match,
2017 .expect_term,
2018 => continue,
2019
2020 .expect_stdout_exact,
2021 .expect_stdout_match,
2022 => return true,
2023 };
2024 return false;
2025}
2026
2027fn checksContainStderr(checks: []const StdIo.Check) bool {
2028 for (checks) |check| switch (check) {
2029 .expect_stdout_exact,
2030 .expect_stdout_match,
2031 .expect_term,
2032 => continue,
2033
2034 .expect_stderr_exact,
2035 .expect_stderr_match,
2036 => return true,
2037 };
2038 return false;
2039}
2040
2041/// Returns whether the Run step has side effects *other than* updating the output arguments.
2042fn hasSideEffects(run: Run) bool {
2043 if (run.has_side_effects) return true;
2044 return switch (run.stdio) {
2045 .infer_from_args => !run.hasAnyOutputArgs(),
2046 .inherit => true,
2047 .check => false,
2048 .zig_test => false,
2049 };
2050}
2051
2052fn hasAnyOutputArgs(run: Run) bool {
2053 if (run.captured_stdout != null) return true;
2054 if (run.captured_stderr != null) return true;
2055 for (run.argv.items) |arg| switch (arg) {
2056 .output_file, .output_directory => return true,
2057 else => continue,
2058 };
2059 return false;
2060}
2061
2062/// If `path` is cwd-relative, make it relative to the cwd of the child instead.
2063///
2064/// Whenever a path is included in the argv of a child, it should be put through this function first
2065/// to make sure the child doesn't see paths relative to a cwd other than its own.
2066fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 {
2067 const b = run.step.owner;
2068 const graph = b.graph;
2069 const arena = graph.arena;
2070
2071 const path_str = path.toString(arena) catch @panic("OOM");
2072 if (Dir.path.isAbsolute(path_str)) {
2073 // Absolute paths don't need changing.
2074 return path_str;
2075 }
2076 const child_cwd_rel: []const u8 = rel: {
2077 const child_lazy_cwd = run.cwd orelse break :rel path_str;
2078 const child_cwd = child_lazy_cwd.getPath3(b, &run.step).toString(arena) catch @panic("OOM");
2079 // Convert it from relative to *our* cwd, to relative to the *child's* cwd.
2080 break :rel Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str) catch @panic("OOM");
2081 };
2082 // Not every path can be made relative, e.g. if the path and the child cwd are on different
2083 // disk designators on Windows. In that case, `relative` will return an absolute path which we can
2084 // just return.
2085 if (Dir.path.isAbsolute(child_cwd_rel)) return child_cwd_rel;
2086
2087 // We're not done yet. In some cases this path must be prefixed with './':
2088 // * On POSIX, the executable name cannot be a single component like 'foo'
2089 // * Some executables might treat a leading '-' like a flag, which we must avoid
2090 // There's no harm in it, so just *always* apply this prefix.
2091 return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM");
2092}
2093
2094fn addPathForDynLibs(run: *Run, artifact: *Step.Compile) void {
2095 const b = run.step.owner;
2096 const compiles = artifact.getCompileDependencies(true);
2097 for (compiles) |compile| {
2098 if (compile.root_module.resolved_target.?.result.os.tag == .windows and
2099 compile.isDynamicLibrary())
2100 {
2101 addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?);
2102 }
2103 }
2104}
2105
2106fn failForeign(
2107 run: *Run,
2108 suggested_flag: []const u8,
2109 argv0: []const u8,
2110 exe: *Step.Compile,
2111) error{ MakeFailed, MakeSkipped, OutOfMemory } {
2112 switch (run.stdio) {
2113 .check, .zig_test => {
2114 if (run.skip_foreign_checks)
2115 return error.MakeSkipped;
2116
2117 const b = run.step.owner;
2118 const host_name = try b.graph.host.result.zigTriple(b.allocator);
2119 const foreign_name = try exe.rootModuleTarget().zigTriple(b.allocator);
2120
2121 return run.step.fail(
2122 \\unable to spawn foreign binary '{s}' ({s}) on host system ({s})
2123 \\ consider using {s} or enabling skip_foreign_checks in the Run step
2124 , .{ argv0, foreign_name, host_name, suggested_flag });
2125 },
2126 else => {
2127 return run.step.fail("unable to spawn foreign binary '{s}'", .{argv0});
2128 },
2129 }
2130}
lib/compiler/Maker/Step/WriteFile.zig created+206
...@@ -0,0 +1,206 @@
1
2fn make(step: *Step, options: Step.MakeOptions) !void {
3 _ = options;
4 const b = step.owner;
5 const graph = b.graph;
6 const io = graph.io;
7 const arena = b.allocator;
8 const gpa = graph.cache.gpa;
9 const write_file: *WriteFile = @fieldParentPtr("step", step);
10
11 const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len);
12 var open_dirs_count: usize = 0;
13 defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]);
14
15 switch (write_file.mode) {
16 .whole_cached => {
17 step.clearWatchInputs();
18
19 // The cache is used here not really as a way to speed things up - because writing
20 // the data to a file would probably be very fast - but as a way to find a canonical
21 // location to put build artifacts.
22
23 // If, for example, a hard-coded path was used as the location to put WriteFile
24 // files, then two WriteFiles executing in parallel might clobber each other.
25
26 var man = b.graph.cache.obtain();
27 defer man.deinit();
28
29 for (write_file.files.items) |file| {
30 man.hash.addBytes(file.sub_path);
31
32 switch (file.contents) {
33 .bytes => |bytes| {
34 man.hash.addBytes(bytes);
35 },
36 .copy => |lazy_path| {
37 const path = lazy_path.getPath3(b, step);
38 _ = try man.addFilePath(path, null);
39 try step.addWatchInput(lazy_path);
40 },
41 }
42 }
43
44 for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| {
45 man.hash.addBytes(dir.sub_path);
46 for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext);
47 if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc);
48
49 const need_derived_inputs = try step.addDirectoryWatchInput(dir.source);
50 const src_dir_path = dir.source.getPath3(b, step);
51
52 var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| {
53 return step.fail("unable to open source directory '{f}': {s}", .{
54 src_dir_path, @errorName(err),
55 });
56 };
57 open_dir_cache_elem.* = src_dir;
58 open_dirs_count += 1;
59
60 var it = try src_dir.walk(gpa);
61 defer it.deinit();
62 while (try it.next(io)) |entry| {
63 if (!dir.options.pathIncluded(entry.path)) continue;
64
65 switch (entry.kind) {
66 .directory => {
67 if (need_derived_inputs) {
68 const entry_path = try src_dir_path.join(arena, entry.path);
69 try step.addDirectoryWatchInputFromPath(entry_path);
70 }
71 },
72 .file => {
73 const entry_path = try src_dir_path.join(arena, entry.path);
74 _ = try man.addFilePath(entry_path, null);
75 },
76 else => continue,
77 }
78 }
79 }
80
81 if (try step.cacheHit(&man)) {
82 const digest = man.final();
83 write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest });
84 assert(step.result_cached);
85 return;
86 }
87
88 const digest = man.final();
89 const cache_path = "o" ++ Dir.path.sep_str ++ digest;
90
91 write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path});
92
93 try operate(write_file, open_dir_cache, .{
94 .root_dir = b.cache_root,
95 .sub_path = cache_path,
96 });
97
98 try step.writeManifest(&man);
99 },
100 .tmp => {
101 step.result_cached = false;
102
103 var rand_int: u64 = undefined;
104 io.random(@ptrCast(&rand_int));
105 const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int);
106
107 write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path});
108
109 try operate(write_file, open_dir_cache, .{
110 .root_dir = b.cache_root,
111 .sub_path = tmp_dir_sub_path,
112 });
113 },
114 .mutate => |lp| {
115 step.result_cached = false;
116 const root_path = try lp.getPath4(b, step);
117 write_file.generated_directory.path = try root_path.toString(arena);
118 try operate(write_file, open_dir_cache, root_path);
119 },
120 }
121}
122
123fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void {
124 const step = &write_file.step;
125 const b = step.owner;
126 const io = b.graph.io;
127 const gpa = b.graph.cache.gpa;
128 const arena = b.allocator;
129
130 var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err|
131 return step.fail("unable to make path {f}: {t}", .{ root_path, err });
132 defer cache_dir.close(io);
133
134 for (write_file.files.items) |file| {
135 if (Dir.path.dirname(file.sub_path)) |dirname| {
136 cache_dir.createDirPath(io, dirname) catch |err| {
137 return step.fail("unable to make path '{f}{c}{s}': {t}", .{
138 root_path, Dir.path.sep, dirname, err,
139 });
140 };
141 }
142 switch (file.contents) {
143 .bytes => |bytes| {
144 cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| {
145 return step.fail("unable to write file '{f}{c}{s}': {t}", .{
146 root_path, Dir.path.sep, file.sub_path, err,
147 });
148 };
149 },
150 .copy => |file_source| {
151 const source_path = file_source.getPath2(b, step);
152 const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| {
153 return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{
154 source_path, root_path, Dir.path.sep, file.sub_path, err,
155 });
156 };
157 // At this point we already will mark the step as a cache miss.
158 // But this is kind of a partial cache hit since individual
159 // file copies may be avoided. Oh well, this information is
160 // discarded.
161 _ = prev_status;
162 },
163 }
164 }
165
166 for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| {
167 const src_dir_path = dir.source.getPath3(b, step);
168 const dest_dirname = dir.sub_path;
169
170 if (dest_dirname.len != 0) {
171 cache_dir.createDirPath(io, dest_dirname) catch |err| {
172 return step.fail("unable to make path '{f}{c}{s}': {t}", .{
173 root_path, Dir.path.sep, dest_dirname, err,
174 });
175 };
176 }
177
178 var it = try already_open_dir.walk(gpa);
179 defer it.deinit();
180 while (try it.next(io)) |entry| {
181 if (!dir.options.pathIncluded(entry.path)) continue;
182
183 const src_entry_path = try src_dir_path.join(arena, entry.path);
184 const dest_path = b.pathJoin(&.{ dest_dirname, entry.path });
185 switch (entry.kind) {
186 .directory => try cache_dir.createDirPath(io, dest_path),
187 .file => {
188 const prev_status = Io.Dir.updateFile(
189 src_entry_path.root_dir.handle,
190 io,
191 src_entry_path.sub_path,
192 cache_dir,
193 dest_path,
194 .{},
195 ) catch |err| {
196 return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{
197 src_entry_path, root_path, Dir.path.sep, dest_path, err,
198 });
199 };
200 _ = prev_status;
201 },
202 else => continue,
203 }
204 }
205 }
206}
lib/compiler/Maker/Watch.zig created+976
...@@ -0,0 +1,976 @@
1const Watch = @This();
2const builtin = @import("builtin");
3
4const std = @import("std");
5const Io = std.Io;
6const Allocator = std.mem.Allocator;
7const assert = std.debug.assert;
8const fatal = std.process.fatal;
9const Configuration = std.Build.Configuration;
10
11const FsEvents = @import("Watch/FsEvents.zig");
12const Step = @import("Step.zig");
13
14os: Os,
15/// The number to show as the number of directories being watched.
16dir_count: usize,
17// These fields are common to most implementations so are kept here for simplicity.
18// They are `undefined` on implementations which do not utilize then.
19dir_table: DirTable,
20generation: Generation,
21configuration: *const Configuration,
22make_steps: []Step,
23
24pub const have_impl = Os != void;
25
26/// Key is the directory to watch which contains one or more files we are
27/// interested in noticing changes to.
28///
29/// Value is generation.
30const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAdapter, false);
31
32/// Special key of "." means any changes in this directory trigger the steps.
33const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet);
34const StepSet = std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, Generation);
35
36const Generation = u8;
37
38const Hash = std.hash.Wyhash;
39const Cache = std.Build.Cache;
40
41const Os = switch (builtin.os.tag) {
42 .linux => struct {
43 const posix = std.posix;
44
45 /// Keyed differently but indexes correspond 1:1 with `dir_table`.
46 handle_table: HandleTable,
47 /// fanotify file descriptors are keyed by mount id since marks
48 /// are limited to a single filesystem.
49 poll_fds: std.AutoArrayHashMapUnmanaged(MountId, posix.pollfd),
50
51 const MountId = i32;
52 const HandleTable = std.ArrayHashMapUnmanaged(FileHandle, struct { mount_id: MountId, reaction_set: ReactionSet }, FileHandle.Adapter, false);
53
54 const fan_mask: std.os.linux.fanotify.MarkMask = .{
55 .CLOSE_WRITE = true,
56 .CREATE = true,
57 .DELETE = true,
58 .DELETE_SELF = true,
59 .EVENT_ON_CHILD = true,
60 .MOVED_FROM = true,
61 .MOVED_TO = true,
62 .MOVE_SELF = true,
63 .ONDIR = true,
64 };
65
66 const FileHandle = struct {
67 handle: *align(1) std.os.linux.file_handle,
68
69 fn clone(lfh: FileHandle, gpa: Allocator) Allocator.Error!FileHandle {
70 const bytes = lfh.slice();
71 const new_ptr = try gpa.alignedAlloc(
72 u8,
73 .of(std.os.linux.file_handle),
74 @sizeOf(std.os.linux.file_handle) + bytes.len,
75 );
76 const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr);
77 new_header.* = lfh.handle.*;
78 const new: FileHandle = .{ .handle = new_header };
79 @memcpy(new.slice(), lfh.slice());
80 return new;
81 }
82
83 fn destroy(lfh: FileHandle, gpa: Allocator) void {
84 const ptr: [*]u8 = @ptrCast(lfh.handle);
85 const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes];
86 return gpa.free(allocated_slice);
87 }
88
89 fn slice(lfh: FileHandle) []u8 {
90 const ptr: [*]u8 = &lfh.handle.f_handle;
91 return ptr[0..lfh.handle.handle_bytes];
92 }
93
94 const Adapter = struct {
95 pub fn hash(self: Adapter, a: FileHandle) u32 {
96 _ = self;
97 const unsigned_type: u32 = @bitCast(a.handle.handle_type);
98 return @truncate(Hash.hash(unsigned_type, a.slice()));
99 }
100 pub fn eql(self: Adapter, a: FileHandle, b: FileHandle, b_index: usize) bool {
101 _ = self;
102 _ = b_index;
103 return a.handle.handle_type == b.handle.handle_type and std.mem.eql(u8, a.slice(), b.slice());
104 }
105 };
106 };
107
108 fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch {
109 _ = cwd_path;
110 return .{
111 .dir_table = .{},
112 .dir_count = 0,
113 .os = switch (builtin.os.tag) {
114 .linux => .{
115 .handle_table = .{},
116 .poll_fds = .{},
117 },
118 else => {},
119 },
120 .generation = 0,
121 .make_steps = make_steps,
122 .configuration = configuration,
123 };
124 }
125
126 fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path, mount_id: *MountId) !FileHandle {
127 var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined;
128 var buf: [std.fs.max_path_bytes]u8 = undefined;
129 const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{
130 path.sub_path,
131 }) catch return error.NameTooLong;
132 const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer);
133 stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle);
134 try posix.name_to_handle_at(path.root_dir.handle.handle, adjusted_path, stack_ptr, mount_id, std.os.linux.AT.HANDLE_FID);
135 const stack_lfh: FileHandle = .{ .handle = stack_ptr };
136 return stack_lfh.clone(gpa);
137 }
138
139 fn markDirtySteps(w: *Watch, gpa: Allocator, fan_fd: posix.fd_t) !bool {
140 const fanotify = std.os.linux.fanotify;
141 const M = fanotify.event_metadata;
142 var events_buf: [256 + 4096]u8 = undefined;
143 var any_dirty = false;
144 while (true) {
145 var len = posix.read(fan_fd, &events_buf) catch |err| switch (err) {
146 error.WouldBlock => return any_dirty,
147 else => |e| return e,
148 };
149 var meta: [*]align(1) M = @ptrCast(&events_buf);
150 while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({
151 len -= meta[0].event_len;
152 meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len);
153 }) {
154 assert(meta[0].vers == M.VERSION);
155 if (meta[0].mask.Q_OVERFLOW) {
156 any_dirty = true;
157 std.log.warn("file system watch queue overflowed; falling back to fstat", .{});
158 markAllFilesDirty(w, gpa);
159 return true;
160 }
161 const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1);
162 switch (fid.hdr.info_type) {
163 .DFID_NAME => {
164 const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
165 const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes);
166 const file_name = std.mem.span(file_name_z);
167 const lfh: FileHandle = .{ .handle = file_handle };
168 if (w.os.handle_table.getPtr(lfh)) |value| {
169 if (value.reaction_set.getPtr(".")) |glob_set|
170 any_dirty = markStepSetDirty(gpa, w.make_steps, glob_set, any_dirty);
171 if (value.reaction_set.getPtr(file_name)) |step_set|
172 any_dirty = markStepSetDirty(gpa, w.make_steps, step_set, any_dirty);
173 }
174 },
175 else => |t| std.log.warn("unexpected fanotify event '{t}'", .{t}),
176 }
177 }
178 }
179 }
180
181 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
182 // Add missing marks and note persisted ones.
183 for (steps) |step_index| {
184 const step = &w.make_steps[@intFromEnum(step_index)];
185 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
186 const reaction_set = rs: {
187 const gop = try w.dir_table.getOrPut(gpa, path);
188 if (!gop.found_existing) {
189 var mount_id: MountId = undefined;
190 const dir_handle = getDirHandle(gpa, path, &mount_id) catch |err| switch (err) {
191 error.FileNotFound => {
192 std.debug.assert(w.dir_table.swapRemove(path));
193 continue;
194 },
195 else => return err,
196 };
197 const fan_fd = blk: {
198 const fd_gop = try w.os.poll_fds.getOrPut(gpa, mount_id);
199 if (!fd_gop.found_existing) {
200 const fan_fd = std.posix.fanotify_init(.{
201 .CLASS = .NOTIF,
202 .CLOEXEC = true,
203 .NONBLOCK = true,
204 .REPORT_NAME = true,
205 .REPORT_DIR_FID = true,
206 .REPORT_FID = true,
207 .REPORT_TARGET_FID = true,
208 }, 0) catch |err| switch (err) {
209 error.UnsupportedFlags => fatal("fanotify_init failed due to old kernel; requires 5.17+", .{}),
210 else => |e| return e,
211 };
212 fd_gop.value_ptr.* = .{
213 .fd = fan_fd,
214 .events = std.posix.POLL.IN,
215 .revents = undefined,
216 };
217 }
218 break :blk fd_gop.value_ptr.*.fd;
219 };
220 // `dir_handle` may already be present in the table in
221 // the case that we have multiple Cache.Path instances
222 // that compare inequal but ultimately point to the same
223 // directory on the file system.
224 // In such case, we must revert adding this directory, but keep
225 // the additions to the step set.
226 const dh_gop = try w.os.handle_table.getOrPut(gpa, dir_handle);
227 if (dh_gop.found_existing) {
228 _ = w.dir_table.pop();
229 } else {
230 assert(dh_gop.index == gop.index);
231 dh_gop.value_ptr.* = .{ .mount_id = mount_id, .reaction_set = .{} };
232 posix.fanotify_mark(fan_fd, .{
233 .ADD = true,
234 .ONLYDIR = true,
235 }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| {
236 fatal("unable to watch {f}: {s}", .{ path, @errorName(err) });
237 };
238 }
239 break :rs &dh_gop.value_ptr.reaction_set;
240 }
241 break :rs &w.os.handle_table.values()[gop.index].reaction_set;
242 };
243 for (files.items) |basename| {
244 const gop = try reaction_set.getOrPut(gpa, basename);
245 if (!gop.found_existing) gop.value_ptr.* = .{};
246 try gop.value_ptr.put(gpa, step_index, w.generation);
247 }
248 }
249 }
250
251 {
252 // Remove marks for files that are no longer inputs.
253 var i: usize = 0;
254 while (i < w.os.handle_table.entries.len) {
255 {
256 const reaction_set = &w.os.handle_table.values()[i].reaction_set;
257 var step_set_i: usize = 0;
258 while (step_set_i < reaction_set.entries.len) {
259 const step_set = &reaction_set.values()[step_set_i];
260 var dirent_i: usize = 0;
261 while (dirent_i < step_set.entries.len) {
262 const generations = step_set.values();
263 if (generations[dirent_i] == w.generation) {
264 dirent_i += 1;
265 continue;
266 }
267 step_set.swapRemoveAt(dirent_i);
268 }
269 if (step_set.entries.len > 0) {
270 step_set_i += 1;
271 continue;
272 }
273 reaction_set.swapRemoveAt(step_set_i);
274 }
275 if (reaction_set.entries.len > 0) {
276 i += 1;
277 continue;
278 }
279 }
280
281 const path = w.dir_table.keys()[i];
282
283 const mount_id = w.os.handle_table.values()[i].mount_id;
284 const fan_fd = w.os.poll_fds.getEntry(mount_id).?.value_ptr.fd;
285 posix.fanotify_mark(fan_fd, .{
286 .REMOVE = true,
287 .ONLYDIR = true,
288 }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| switch (err) {
289 error.FileNotFound => {}, // Expected, harmless.
290 else => |e| std.log.warn("unable to unwatch '{f}': {s}", .{ path, @errorName(e) }),
291 };
292
293 w.dir_table.swapRemoveAt(i);
294 w.os.handle_table.swapRemoveAt(i);
295 }
296 w.generation +%= 1;
297 }
298 w.dir_count = w.dir_table.count();
299 }
300
301 fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult {
302 _ = io;
303 const events_len = try std.posix.poll(w.os.poll_fds.values(), timeout.to_i32_ms());
304 if (events_len == 0)
305 return .timeout;
306 for (w.os.poll_fds.values()) |poll_fd| {
307 if (poll_fd.revents & std.posix.POLL.IN == std.posix.POLL.IN and try markDirtySteps(w, gpa, poll_fd.fd))
308 return .dirty;
309 }
310 return .clean;
311 }
312 },
313 .windows => struct {
314 const windows = std.os.windows;
315
316 /// Keyed differently but indexes correspond 1:1 with `dir_table`.
317 handle_table: std.ArrayHashMapUnmanaged(*Directory, void, Directory.TableAdapter, false),
318 ready_dirs: std.DoublyLinkedList,
319
320 const FileId = struct {
321 volumeSerialNumber: windows.ULONG,
322 indexNumber: windows.LARGE_INTEGER,
323 };
324
325 const Directory = struct {
326 reaction_set: ReactionSet,
327 id: FileId,
328 file: Io.File,
329 state: enum { idle, listening, ready },
330 iosb: windows.IO_STATUS_BLOCK,
331 // 64 KB is the packet size limit when monitoring over a network.
332 // https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-readdirectorychangesw#remarks
333 buffer: [64 * 1024]u8 align(@alignOf(windows.FILE.NOTIFY.INFORMATION)),
334 ready_node: std.DoublyLinkedList.Node,
335
336 /// Start listening for events, buffer field will be overwritten eventually.
337 fn startListening(dir: *Directory, w: *Watch) !void {
338 assert(dir.file.flags.nonblocking);
339 assert(dir.state == .idle);
340 switch (windows.ntdll.NtNotifyChangeDirectoryFileEx(
341 dir.file.handle,
342 null,
343 &notifyApc,
344 w,
345 &dir.iosb,
346 &dir.buffer,
347 dir.buffer.len,
348 .{
349 .FILE_NAME = true,
350 .DIR_NAME = true,
351 .SIZE = true,
352 .LAST_WRITE = true,
353 .CREATION = true,
354 },
355 .FALSE,
356 .Notify,
357 )) {
358 .SUCCESS, .PENDING => dir.state = .listening,
359 .ILLEGAL_FUNCTION => return error.ReadDirectoryChangesUnsupported,
360 else => |status| return windows.unexpectedStatus(status),
361 }
362 }
363
364 fn notifyApc(apc_context: ?*anyopaque, iosb: *windows.IO_STATUS_BLOCK, _: windows.ULONG) align(std.Io.Threaded.apc_align) callconv(.winapi) void {
365 const w: *Watch = @ptrCast(@alignCast(apc_context));
366 const dir: *Directory = @fieldParentPtr("iosb", iosb);
367 assert(iosb.u.Status != .PENDING);
368 assert(dir.state == .listening);
369 w.os.ready_dirs.append(&dir.ready_node);
370 dir.state = .ready;
371 }
372
373 fn init(gpa: Allocator, path: Cache.Path) !*Directory {
374 // The following code is a drawn out NtCreateFile call. (mostly adapted from Io.Dir.makeOpenDirAccessMaskW)
375 // It's necessary in order to get the specific flags that are required when calling ReadDirectoryChangesW.
376 var dir_handle: windows.HANDLE = undefined;
377 const root_fd = path.root_dir.handle.handle;
378 const sub_path = path.subPathOrDot();
379 const sub_path_w = try Io.Threaded.sliceToPrefixedFileW(root_fd, sub_path, .{}); // TODO eliminate this call
380 var iosb: windows.IO_STATUS_BLOCK = undefined;
381 switch (windows.ntdll.NtCreateFile(
382 &dir_handle,
383 .{
384 .SPECIFIC = .{ .FILE_DIRECTORY = .{
385 .LIST = true,
386 } },
387 .STANDARD = .{ .SYNCHRONIZE = true },
388 .GENERIC = .{ .READ = true },
389 },
390 &.{
391 .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(sub_path_w.span())) null else root_fd,
392 .ObjectName = @constCast(&sub_path_w.string()),
393 },
394 &iosb,
395 null,
396 .{},
397 .VALID_FLAGS,
398 .OPEN,
399 .{
400 .DIRECTORY_FILE = true,
401 .IO = .ASYNCHRONOUS,
402 .OPEN_FOR_BACKUP_INTENT = true,
403 },
404 null,
405 0,
406 )) {
407 .SUCCESS => {},
408 .OBJECT_NAME_INVALID => return error.BadPathName,
409 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
410 .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
411 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
412 .NOT_A_DIRECTORY => return error.NotDir,
413 // This can happen if the directory has 'List folder contents' permission set to 'Deny'
414 .ACCESS_DENIED => return error.AccessDenied,
415 .INVALID_PARAMETER => unreachable,
416 else => |rc| return windows.unexpectedStatus(rc),
417 }
418 assert(dir_handle != windows.INVALID_HANDLE_VALUE);
419 errdefer windows.CloseHandle(dir_handle);
420
421 const dir_id = try getFileId(dir_handle);
422
423 const dir = try gpa.create(Directory);
424 dir.* = .{
425 .reaction_set = .empty,
426 .id = dir_id,
427 .file = .{ .handle = dir_handle, .flags = .{ .nonblocking = true } },
428 .state = .idle,
429 .iosb = undefined,
430 .buffer = undefined,
431 .ready_node = undefined,
432 };
433 return dir;
434 }
435
436 fn deinit(dir: *Directory, gpa: Allocator, w: *Watch) void {
437 state: switch (dir.state) {
438 .idle => {},
439 .listening => {
440 var cancel_iosb: windows.IO_STATUS_BLOCK = undefined;
441 _ = windows.ntdll.NtCancelIoFileEx(dir.file.handle, &dir.iosb, &cancel_iosb);
442 while (switch (dir.state) {
443 .idle => unreachable,
444 .listening => true,
445 .ready => false,
446 }) Io.Threaded.waitForApcOrAlert();
447 continue :state .ready;
448 },
449 .ready => w.os.ready_dirs.remove(&dir.ready_node),
450 }
451 windows.CloseHandle(dir.file.handle);
452 gpa.destroy(dir);
453 }
454
455 /// Useful to make `*Directory` a key in `std.ArrayHashMap`.
456 const TableAdapter = struct {
457 pub fn hash(_: TableAdapter, lhs_dir: *Directory) u32 {
458 return @truncate(Hash.hash(lhs_dir.id.volumeSerialNumber, @ptrCast(&lhs_dir.id.indexNumber)));
459 }
460 pub fn eql(_: TableAdapter, lhs_dir: *Directory, rhs_dir: *Directory, rhs_index: usize) bool {
461 _ = rhs_index;
462 return lhs_dir.id.volumeSerialNumber == rhs_dir.id.volumeSerialNumber and
463 lhs_dir.id.indexNumber == rhs_dir.id.indexNumber;
464 }
465 };
466 };
467
468 fn init(cwd_path: []const u8) !Watch {
469 _ = cwd_path;
470 return .{
471 .dir_table = .{},
472 .dir_count = 0,
473 .os = switch (builtin.os.tag) {
474 .windows => .{
475 .handle_table = .empty,
476 .ready_dirs = .{},
477 },
478 else => {},
479 },
480 .generation = 0,
481 };
482 }
483
484 fn getFileId(handle: windows.HANDLE) !FileId {
485 var file_id: FileId = undefined;
486 var io_status: windows.IO_STATUS_BLOCK = undefined;
487 var volume_info: windows.FILE.FS_VOLUME_INFORMATION = undefined;
488 switch (windows.ntdll.NtQueryVolumeInformationFile(
489 handle,
490 &io_status,
491 &volume_info,
492 @sizeOf(windows.FILE.FS_VOLUME_INFORMATION),
493 .Volume,
494 )) {
495 .SUCCESS => {},
496 // Buffer overflow here indicates that there is more information available than was able to be stored in the buffer
497 // size provided. This is treated as success because the type of variable-length information that this would be relevant for
498 // (name, volume name, etc) we don't care about.
499 .BUFFER_OVERFLOW => {},
500 else => |rc| return windows.unexpectedStatus(rc),
501 }
502 file_id.volumeSerialNumber = volume_info.VolumeSerialNumber;
503 var internal_info: windows.FILE.INTERNAL_INFORMATION = undefined;
504 switch (windows.ntdll.NtQueryInformationFile(
505 handle,
506 &io_status,
507 &internal_info,
508 @sizeOf(windows.FILE.INTERNAL_INFORMATION),
509 .Internal,
510 )) {
511 .SUCCESS => {},
512 else => |rc| return windows.unexpectedStatus(rc),
513 }
514 file_id.indexNumber = internal_info.IndexNumber;
515 return file_id;
516 }
517
518 fn markDirtySteps(w: *Watch, gpa: Allocator, dir: *Directory) !bool {
519 var any_dirty = false;
520 const bytes_returned = dir.iosb.Information;
521 if (bytes_returned == 0) {
522 std.log.warn("file system watch queue overflowed; falling back to fstat", .{});
523 markAllFilesDirty(w, gpa);
524 try dir.startListening(w);
525 return true;
526 }
527 var file_name_buf: [std.fs.max_path_bytes]u8 = undefined;
528 var offset: usize = 0;
529 while (true) {
530 const notify: *windows.FILE.NOTIFY.INFORMATION = @ptrCast(@alignCast(&dir.buffer[offset]));
531 const file_name = file_name_buf[0..std.unicode.wtf16LeToWtf8(&file_name_buf, notify.fileName())];
532 if (dir.reaction_set.getPtr(".")) |glob_set|
533 any_dirty = markStepSetDirty(gpa, glob_set, any_dirty);
534 if (dir.reaction_set.getPtr(file_name)) |step_set|
535 any_dirty = markStepSetDirty(gpa, step_set, any_dirty);
536 if (notify.NextEntryOffset == 0)
537 break;
538
539 offset += notify.NextEntryOffset;
540 }
541
542 // We call this now since at this point we have finished reading dir.buffer.
543 try dir.startListening(w);
544 return any_dirty;
545 }
546
547 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
548 // Add missing marks and note persisted ones.
549 for (steps) |step| {
550 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
551 const dir = dir: {
552 const gop = try w.dir_table.getOrPut(gpa, path);
553 if (!gop.found_existing) {
554 const dir: *Directory = try .init(gpa, path);
555 errdefer dir.deinit(gpa, w);
556 // `dir.id` may already be present in the table in
557 // the case that we have multiple Cache.Path instances
558 // that compare inequal but ultimately point to the same
559 // directory on the file system.
560 // In such case, we must revert adding this directory, but keep
561 // the additions to the step set.
562 const dh_gop = try w.os.handle_table.getOrPut(gpa, dir);
563 if (dh_gop.found_existing) {
564 dir.deinit(gpa, w);
565 _ = w.dir_table.pop();
566 break :dir w.os.handle_table.keys()[dh_gop.index];
567 } else {
568 assert(dh_gop.index == gop.index);
569 try dir.startListening(w);
570 break :dir dir;
571 }
572 }
573 break :dir w.os.handle_table.keys()[gop.index];
574 };
575 for (files.items) |basename| {
576 const gop = try dir.reaction_set.getOrPut(gpa, basename);
577 if (!gop.found_existing) gop.value_ptr.* = .{};
578 try gop.value_ptr.put(gpa, step, w.generation);
579 }
580 }
581 }
582
583 {
584 // Remove marks for files that are no longer inputs.
585 var i: usize = 0;
586 while (i < w.os.handle_table.entries.len) {
587 const dir = w.os.handle_table.keys()[i];
588 {
589 var step_set_i: usize = 0;
590 while (step_set_i < dir.reaction_set.entries.len) {
591 const step_set = &dir.reaction_set.values()[step_set_i];
592 var dirent_i: usize = 0;
593 while (dirent_i < step_set.entries.len) {
594 const generations = step_set.values();
595 if (generations[dirent_i] == w.generation) {
596 dirent_i += 1;
597 continue;
598 }
599 step_set.swapRemoveAt(dirent_i);
600 }
601 if (step_set.entries.len > 0) {
602 step_set_i += 1;
603 continue;
604 }
605 dir.reaction_set.swapRemoveAt(step_set_i);
606 }
607 if (dir.reaction_set.entries.len > 0) {
608 i += 1;
609 continue;
610 }
611 }
612
613 w.dir_table.swapRemoveAt(i);
614 w.os.handle_table.swapRemoveAt(i);
615 dir.deinit(gpa, w);
616 }
617 w.generation +%= 1;
618 }
619 w.dir_count = w.dir_table.count();
620 }
621
622 fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult {
623 for (0..2) |attempt| {
624 while (w.os.ready_dirs.popFirst()) |ready_node| {
625 const dir: *Directory = @fieldParentPtr("ready_node", ready_node);
626 assert(dir.state == .ready);
627 dir.state = .idle;
628 switch (dir.iosb.u.Status) {
629 .SUCCESS => return if (try markDirtySteps(w, gpa, dir)) .dirty else .clean,
630 .PENDING => unreachable,
631 .CANCELLED => {},
632 else => |status| return windows.unexpectedStatus(status),
633 }
634 try dir.startListening(w);
635 }
636 try io.checkCancel();
637 if (attempt == 1) return .timeout;
638 const delay_interval: windows.LARGE_INTEGER = switch (timeout) {
639 .none => std.math.minInt(windows.LARGE_INTEGER),
640 .ms => |ms| -@as(windows.LARGE_INTEGER, ms) * (std.time.ns_per_ms / 100),
641 };
642 _ = windows.ntdll.NtDelayExecution(.TRUE, &delay_interval);
643 } else unreachable;
644 }
645 },
646 .dragonfly, .freebsd, .netbsd, .openbsd, .ios, .tvos, .visionos, .watchos => struct {
647 const posix = std.posix;
648
649 kq_fd: i32,
650 /// Indexes correspond 1:1 with `dir_table`.
651 handles: std.MultiArrayList(struct {
652 rs: ReactionSet,
653 /// If the corresponding dir_table Path has sub_path == "", then it
654 /// suffices as the open directory handle, and this value will be
655 /// -1. Otherwise, it needs to be opened in update(), and will be
656 /// stored here.
657 dir_fd: i32,
658 }),
659
660 const dir_open_flags: posix.O = f: {
661 var f: posix.O = .{
662 .ACCMODE = .RDONLY,
663 .NOFOLLOW = false,
664 .DIRECTORY = true,
665 .CLOEXEC = true,
666 };
667 if (@hasField(posix.O, "EVTONLY")) f.EVTONLY = true;
668 if (@hasField(posix.O, "PATH")) f.PATH = true;
669 break :f f;
670 };
671
672 const EV = std.c.EV;
673 const NOTE = std.c.NOTE;
674
675 fn init(cwd_path: []const u8) !Watch {
676 _ = cwd_path;
677 return .{
678 .dir_table = .{},
679 .dir_count = 0,
680 .os = .{
681 .kq_fd = try Io.Kqueue.createFileDescriptor(),
682 .handles = .empty,
683 },
684 .generation = 0,
685 };
686 }
687
688 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
689 const handles = &w.os.handles;
690 for (steps) |step| {
691 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
692 const reaction_set = rs: {
693 const gop = try w.dir_table.getOrPut(gpa, path);
694 if (!gop.found_existing) {
695 const skip_open_dir = path.sub_path.len == 0;
696 const dir_fd = if (skip_open_dir)
697 path.root_dir.handle.handle
698 else
699 posix.openat(path.root_dir.handle.handle, path.sub_path, dir_open_flags, 0) catch |err| {
700 fatal("failed to open directory {f}: {t}", .{ path, err });
701 };
702 // Empirically the dir has to stay open or else no events are triggered.
703 errdefer if (!skip_open_dir) std.Io.Threaded.closeFd(dir_fd);
704 const changes = [1]posix.Kevent{.{
705 .ident = @bitCast(@as(isize, dir_fd)),
706 .filter = std.c.EVFILT.VNODE,
707 .flags = EV.ADD | EV.ENABLE | EV.CLEAR,
708 .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE,
709 .data = 0,
710 .udata = gop.index,
711 }};
712 _ = try Io.Kqueue.kevent(w.os.kq_fd, &changes, &.{}, null);
713 assert(handles.len == gop.index);
714 try handles.append(gpa, .{
715 .rs = .{},
716 .dir_fd = if (skip_open_dir) -1 else dir_fd,
717 });
718 }
719
720 break :rs &handles.items(.rs)[gop.index];
721 };
722 for (files.items) |basename| {
723 const gop = try reaction_set.getOrPut(gpa, basename);
724 if (!gop.found_existing) gop.value_ptr.* = .{};
725 try gop.value_ptr.put(gpa, step, w.generation);
726 }
727 }
728 }
729
730 {
731 // Remove marks for files that are no longer inputs.
732 var i: usize = 0;
733 while (i < handles.len) {
734 {
735 const reaction_set = &handles.items(.rs)[i];
736 var step_set_i: usize = 0;
737 while (step_set_i < reaction_set.entries.len) {
738 const step_set = &reaction_set.values()[step_set_i];
739 var dirent_i: usize = 0;
740 while (dirent_i < step_set.entries.len) {
741 const generations = step_set.values();
742 if (generations[dirent_i] == w.generation) {
743 dirent_i += 1;
744 continue;
745 }
746 step_set.swapRemoveAt(dirent_i);
747 }
748 if (step_set.entries.len > 0) {
749 step_set_i += 1;
750 continue;
751 }
752 reaction_set.swapRemoveAt(step_set_i);
753 }
754 if (reaction_set.entries.len > 0) {
755 i += 1;
756 continue;
757 }
758 }
759
760 // If the sub_path == "" then this patch has already the
761 // dir fd that we need to use as the ident to remove the
762 // event. If it was opened above with openat() then we need
763 // to access that data via the dir_fd field.
764 const path = w.dir_table.keys()[i];
765 const dir_fd = if (path.sub_path.len == 0)
766 path.root_dir.handle.handle
767 else
768 handles.items(.dir_fd)[i];
769 assert(dir_fd != -1);
770
771 // The changelist also needs to update the udata field of the last
772 // event, since we are doing a swap remove, and we store the dir_table
773 // index in the udata field.
774 const last_dir_fd = fd: {
775 const last_path = w.dir_table.keys()[handles.len - 1];
776 const last_dir_fd = if (last_path.sub_path.len == 0)
777 last_path.root_dir.handle.handle
778 else
779 handles.items(.dir_fd)[handles.len - 1];
780 assert(last_dir_fd != -1);
781 break :fd last_dir_fd;
782 };
783 const changes = [_]posix.Kevent{
784 .{
785 .ident = @bitCast(@as(isize, dir_fd)),
786 .filter = std.c.EVFILT.VNODE,
787 .flags = EV.DELETE,
788 .fflags = 0,
789 .data = 0,
790 .udata = i,
791 },
792 .{
793 .ident = @bitCast(@as(isize, last_dir_fd)),
794 .filter = std.c.EVFILT.VNODE,
795 .flags = EV.ADD,
796 .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE,
797 .data = 0,
798 .udata = i,
799 },
800 };
801 const filtered_changes = if (i == handles.len - 1) changes[0..1] else &changes;
802 _ = try Io.Kqueue.kevent(w.os.kq_fd, filtered_changes, &.{}, null);
803 if (path.sub_path.len != 0) std.Io.Threaded.closeFd(dir_fd);
804
805 w.dir_table.swapRemoveAt(i);
806 handles.swapRemove(i);
807 }
808 w.generation +%= 1;
809 }
810 w.dir_count = w.dir_table.count();
811 }
812
813 fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult {
814 _ = io;
815 var timespec_buffer: posix.timespec = undefined;
816 var event_buffer: [100]posix.Kevent = undefined;
817 var n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, timeout.toTimespec(&timespec_buffer));
818 if (n == 0) return .timeout;
819 const reaction_sets = w.os.handles.items(.rs);
820 var any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], false);
821 timespec_buffer = .{ .sec = 0, .nsec = 0 };
822 while (n == event_buffer.len) {
823 n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, &timespec_buffer);
824 if (n == 0) break;
825 any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], any_dirty);
826 }
827 return if (any_dirty) .dirty else .clean;
828 }
829
830 fn markDirtySteps(
831 gpa: Allocator,
832 reaction_sets: []ReactionSet,
833 events: []const std.c.Kevent,
834 start_any_dirty: bool,
835 ) bool {
836 var any_dirty = start_any_dirty;
837 for (events) |event| {
838 const index: usize = @intCast(event.udata);
839 const reaction_set = &reaction_sets[index];
840 // If we knew the basename of the changed file, here we would
841 // mark only the step set dirty, and possibly the glob set:
842 //if (reaction_set.getPtr(".")) |glob_set|
843 // any_dirty = markStepSetDirty(gpa, glob_set, any_dirty);
844 //if (reaction_set.getPtr(file_name)) |step_set|
845 // any_dirty = markStepSetDirty(gpa, step_set, any_dirty);
846 // However we don't know the file name so just mark all the
847 // sets dirty for this directory.
848 for (reaction_set.values()) |*step_set| {
849 any_dirty = markStepSetDirty(gpa, step_set, any_dirty);
850 }
851 }
852 return any_dirty;
853 }
854 },
855 .macos => struct {
856 fse: FsEvents,
857
858 fn init(cwd_path: []const u8) !Watch {
859 return .{
860 .os = .{ .fse = try .init(cwd_path) },
861 .dir_count = 0,
862 .dir_table = undefined,
863 .generation = undefined,
864 };
865 }
866 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
867 try w.os.fse.setPaths(gpa, steps);
868 w.dir_count = w.os.fse.watch_roots.len;
869 }
870 fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult {
871 _ = io;
872 return w.os.fse.wait(gpa, switch (timeout) {
873 .none => null,
874 .ms => |ms| @as(u64, ms) * std.time.ns_per_ms,
875 });
876 }
877 },
878 else => void,
879};
880
881pub fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch {
882 return Os.init(cwd_path, configuration, make_steps);
883}
884
885pub const Match = struct {
886 /// Relative to the watched directory, the file path that triggers this
887 /// match.
888 basename: []const u8,
889 /// The step to re-run when file corresponding to `basename` is changed.
890 step_index: Configuration.Step.Index,
891
892 pub const Context = struct {
893 pub fn hash(self: Context, a: Match) u32 {
894 _ = self;
895 var hasher = Hash.init(@intFromEnum(a.step_index));
896 hasher.update(a.basename);
897 return @truncate(hasher.final());
898 }
899 pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool {
900 _ = self;
901 _ = b_index;
902 return a.step_index == b.step_index and std.mem.eql(u8, a.basename, b.basename);
903 }
904 };
905};
906
907fn markAllFilesDirty(w: *Watch, gpa: Allocator) void {
908 for (switch (builtin.os.tag) {
909 .windows => w.os.handle_table.keys(),
910 else => w.os.handle_table.values(),
911 }) |item| {
912 const reaction_set = switch (builtin.os.tag) {
913 .linux, .windows => item.reaction_set,
914 else => item,
915 };
916 for (reaction_set.values()) |step_set| {
917 for (step_set.keys()) |step_index| {
918 const step = &w.make_steps[@intFromEnum(step_index)];
919 _ = step.invalidateResult(gpa);
920 }
921 }
922 }
923}
924
925fn markStepSetDirty(gpa: Allocator, make_steps: []Step, step_set: *StepSet, any_dirty: bool) bool {
926 var this_any_dirty = false;
927 for (step_set.keys()) |step_index| {
928 const step = &make_steps[@intFromEnum(step_index)];
929 if (step.invalidateResult(gpa)) this_any_dirty = true;
930 }
931 return any_dirty or this_any_dirty;
932}
933
934pub fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
935 return Os.update(w, gpa, steps);
936}
937
938pub const Timeout = union(enum) {
939 none,
940 ms: u16,
941
942 pub fn to_i32_ms(t: Timeout) i32 {
943 return switch (t) {
944 .none => -1,
945 .ms => |ms| ms,
946 };
947 }
948
949 pub fn toTimespec(t: Timeout, buf: *std.posix.timespec) ?*std.posix.timespec {
950 return switch (t) {
951 .none => null,
952 .ms => |ms_u16| {
953 const ms: isize = ms_u16;
954 buf.* = .{
955 .sec = @divTrunc(ms, std.time.ms_per_s),
956 .nsec = @rem(ms, std.time.ms_per_s) * std.time.ns_per_ms,
957 };
958 return buf;
959 },
960 };
961 }
962};
963
964pub const WaitResult = enum {
965 timeout,
966 /// File system watching triggered on files that were marked as inputs to at least one Step.
967 /// Relevant steps have been marked dirty.
968 dirty,
969 /// File system watching triggered but none of the events were relevant to
970 /// what we are listening to. There is nothing to do.
971 clean,
972};
973
974pub fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult {
975 return Os.wait(w, gpa, io, timeout);
976}
lib/compiler/Maker/Watch/FsEvents.zig created+479
...@@ -0,0 +1,479 @@
1//! An implementation of file-system watching based on the `FSEventStream` API in macOS.
2//! While macOS supports kqueue, it does not allow detecting changes to files without
3//! placing watches on each individual file, meaning FD limits are reached incredibly
4//! quickly. The File System Events API works differently: it implements *recursive*
5//! directory watches, managed by a system service. Rather than being in libc, the API is
6//! exposed by the CoreServices framework. To avoid a compile dependency on the framework
7//! bundle, we dynamically load CoreServices with `std.DynLib`.
8//!
9//! While the logic in this file *is* specialized to `std.Build.Watch`, efforts have been
10//! made to keep that specialization to a minimum. Other use cases could be served with
11//! relatively minimal modifications to the `watch_paths` field and its usages (in
12//! particular the `setPaths` function). We avoid using the global GCD dispatch queue in
13//! favour of creating our own and synchronizing with an explicit semaphore, meaning this
14//! logic is thread-safe and does not affect process-global state.
15//!
16//! In theory, this API is quite good at avoiding filesystem race conditions. In practice,
17//! the logic that would avoid them is currently disabled, because the build system kind
18//! of relies on them at the time of writing to avoid redundant work -- see the comment at
19//! the top of `wait` for details.
20
21const enable_debug_logs = false;
22
23core_services: std.DynLib,
24resolved_symbols: ResolvedSymbols,
25
26paths_arena: std.heap.ArenaAllocator.State,
27/// The roots of the recursive watches. FSEvents has relatively small limits on the number
28/// of watched paths, so this slice must not be too long. The paths themselves are allocated
29/// into `paths_arena`, but this slice is allocated into the GPA.
30watch_roots: [][:0]const u8,
31/// All of the paths being watched. Value is the set of steps which depend on the file/directory.
32/// Keys and values are in `paths_arena`, but this map is allocated into the GPA.
33watch_paths: std.StringArrayHashMapUnmanaged([]const *std.Build.Step),
34
35/// The semaphore we use to block the thread calling `wait` until the callback determines a relevant
36/// event has occurred. This is retained across `wait` calls for simplicity and efficiency.
37waiting_semaphore: dispatch.semaphore_t,
38/// This dispatch queue is created by us and executes serially. It exists exclusively to trigger the
39/// callbacks of the FSEventStream we create. This is not in use outside of `wait`, but is retained
40/// across `wait` calls for simplicity and efficiency.
41dispatch_queue: dispatch.queue_t,
42/// In theory, this field avoids race conditions. In practice, it is essentially unused at the time
43/// of writing. See the comment at the start of `wait` for details.
44since_event: FSEventStreamEventId,
45
46cwd_path: []const u8,
47
48/// All of the symbols we pull from the `dlopen`ed CoreServices framework. If any of these symbols
49/// is not present, `init` will close the framework and return an error.
50const ResolvedSymbols = struct {
51 FSEventStreamCreate: *const fn (
52 allocator: CFAllocatorRef,
53 callback: FSEventStreamCallback,
54 ctx: ?*const FSEventStreamContext,
55 paths_to_watch: CFArrayRef,
56 since_when: FSEventStreamEventId,
57 latency: CFTimeInterval,
58 flags: FSEventStreamCreateFlags,
59 ) callconv(.c) FSEventStreamRef,
60 FSEventStreamSetDispatchQueue: *const fn (stream: FSEventStreamRef, queue: dispatch.queue_t) callconv(.c) void,
61 FSEventStreamStart: *const fn (stream: FSEventStreamRef) callconv(.c) bool,
62 FSEventStreamStop: *const fn (stream: FSEventStreamRef) callconv(.c) void,
63 FSEventStreamInvalidate: *const fn (stream: FSEventStreamRef) callconv(.c) void,
64 FSEventStreamRelease: *const fn (stream: FSEventStreamRef) callconv(.c) void,
65 FSEventStreamGetLatestEventId: *const fn (stream: ConstFSEventStreamRef) callconv(.c) FSEventStreamEventId,
66 FSEventsGetCurrentEventId: *const fn () callconv(.c) FSEventStreamEventId,
67 CFRelease: *const fn (cf: *const anyopaque) callconv(.c) void,
68 CFArrayCreate: *const fn (
69 allocator: CFAllocatorRef,
70 values: [*]const usize,
71 num_values: CFIndex,
72 call_backs: ?*const CFArrayCallBacks,
73 ) callconv(.c) CFArrayRef,
74 CFStringCreateWithCString: *const fn (
75 alloc: CFAllocatorRef,
76 c_str: [*:0]const u8,
77 encoding: CFStringEncoding,
78 ) callconv(.c) CFStringRef,
79 CFAllocatorCreate: *const fn (allocator: CFAllocatorRef, context: *const CFAllocatorContext) callconv(.c) CFAllocatorRef,
80 kCFAllocatorUseContext: *const CFAllocatorRef,
81};
82
83pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreServicesSymbol, SystemResources }!FsEvents {
84 var core_services = std.DynLib.open("/System/Library/Frameworks/CoreServices.framework/CoreServices") catch
85 return error.OpenFrameworkFailed;
86 errdefer core_services.close();
87
88 var resolved_symbols: ResolvedSymbols = undefined;
89 inline for (@typeInfo(ResolvedSymbols).@"struct".fields) |f| {
90 @field(resolved_symbols, f.name) = core_services.lookup(f.type, f.name) orelse return error.MissingCoreServicesSymbol;
91 }
92
93 return .{
94 .core_services = core_services,
95 .resolved_symbols = resolved_symbols,
96 .paths_arena = .{},
97 .watch_roots = &.{},
98 .watch_paths = .empty,
99 .waiting_semaphore = dispatch.semaphore_create(0) orelse return error.SystemResources,
100 .dispatch_queue = dispatch.queue_create("zig-watch", .SERIAL()) orelse return error.SystemResources,
101 // Not `.since_now`, because this means we can init `FsEvents` *before* we do work in order
102 // to notice any changes which happened during said work.
103 .since_event = resolved_symbols.FSEventsGetCurrentEventId(),
104 .cwd_path = cwd_path,
105 };
106}
107
108pub fn deinit(fse: *FsEvents, gpa: Allocator, io: Io) void {
109 fse.waiting_semaphore.as_object().release();
110 fse.dispatch_queue.as_object().release();
111 fse.core_services.close(io);
112
113 gpa.free(fse.watch_roots);
114 fse.watch_paths.deinit(gpa);
115 {
116 var paths_arena = fse.paths_arena.promote(gpa);
117 paths_arena.deinit();
118 }
119}
120
121pub fn setPaths(fse: *FsEvents, gpa: Allocator, steps: []const *std.Build.Step) !void {
122 var paths_arena_instance = fse.paths_arena.promote(gpa);
123 defer fse.paths_arena = paths_arena_instance.state;
124 const paths_arena = paths_arena_instance.allocator();
125
126 var need_dirs: std.StringArrayHashMapUnmanaged(void) = .empty;
127 defer need_dirs.deinit(gpa);
128
129 fse.watch_paths.clearRetainingCapacity();
130
131 // We take `step` by pointer for a slight memory optimization in a moment.
132 for (steps) |*step| {
133 for (step.*.inputs.table.keys(), step.*.inputs.table.values()) |path, *files| {
134 const resolved_dir = try std.fs.path.resolvePosix(paths_arena, &.{
135 fse.cwd_path, path.root_dir.path orelse ".", path.sub_path,
136 });
137 try need_dirs.put(gpa, resolved_dir, {});
138 for (files.items) |file_name| {
139 const watch_path = if (std.mem.eql(u8, file_name, "."))
140 resolved_dir
141 else
142 try std.fs.path.join(paths_arena, &.{ resolved_dir, file_name });
143 const gop = try fse.watch_paths.getOrPut(gpa, watch_path);
144 if (gop.found_existing) {
145 const old_steps = gop.value_ptr.*;
146 const new_steps = try paths_arena.alloc(*std.Build.Step, old_steps.len + 1);
147 @memcpy(new_steps[0..old_steps.len], old_steps);
148 new_steps[old_steps.len] = step.*;
149 gop.value_ptr.* = new_steps;
150 } else {
151 // This is why we captured `step` by pointer! We can avoid allocating a slice of one
152 // step in the arena in the common case where a file is referenced by only one step.
153 gop.value_ptr.* = step[0..1];
154 }
155 }
156 }
157 }
158
159 {
160 // There's no point looking at directories inside other ones (e.g. "/foo" and "/foo/bar").
161 // To eliminate these, we'll re-add directories in order of path length with a redundancy check.
162 const old_dirs = try gpa.dupe([]const u8, need_dirs.keys());
163 defer gpa.free(old_dirs);
164 std.mem.sort([]const u8, old_dirs, {}, struct {
165 fn lessThan(ctx: void, a: []const u8, b: []const u8) bool {
166 ctx;
167 return std.mem.lessThan(u8, a, b);
168 }
169 }.lessThan);
170 need_dirs.clearRetainingCapacity();
171 for (old_dirs) |dir_path| {
172 var it: std.fs.path.ComponentIterator(.posix, u8) = .init(dir_path);
173 while (it.next()) |component| {
174 if (need_dirs.contains(component.path)) {
175 // this path is '/foo/bar/qux', but '/foo' or '/foo/bar' was already added
176 break;
177 }
178 } else {
179 need_dirs.putAssumeCapacityNoClobber(dir_path, {});
180 }
181 }
182 }
183
184 // `need_dirs` is now a set of directories to watch with no redundancy. In practice, this is very
185 // likely to have reduced it to a quite small set (e.g. it'll typically coalesce a full `src/`
186 // directory into one entry). However, the FSEventStream API has a fairly low undocumented limit
187 // on total watches (supposedly 4096), so we should handle the case where we exceed it. To be
188 // safe, because this API can be a little unpredictable, we'll cap ourselves a little *below*
189 // that known limit.
190 if (need_dirs.count() > 2048) {
191 // Fallback: watch the whole filesystem. This is excessive, but... it *works* :P
192 if (enable_debug_logs) watch_log.debug("too many dirs; recursively watching root", .{});
193 fse.watch_roots = try gpa.realloc(fse.watch_roots, 1);
194 fse.watch_roots[0] = "/";
195 } else {
196 fse.watch_roots = try gpa.realloc(fse.watch_roots, need_dirs.count());
197 for (fse.watch_roots, need_dirs.keys()) |*out, in| {
198 out.* = try paths_arena.dupeSentinel(u8, in, 0);
199 }
200 }
201 if (enable_debug_logs) {
202 watch_log.debug("watching {d} paths using {d} recursive watches:", .{ fse.watch_paths.count(), fse.watch_roots.len });
203 for (fse.watch_roots) |dir_path| {
204 watch_log.debug("- '{s}'", .{dir_path});
205 }
206 }
207}
208
209pub fn wait(fse: *FsEvents, gpa: Allocator, timeout_ns: ?u64) error{ OutOfMemory, StartFailed }!std.Build.Watch.WaitResult {
210 if (fse.watch_roots.len == 0) @panic("nothing to watch");
211
212 const rs = fse.resolved_symbols;
213
214 // At the time of writing, using `since_event` in the obvious way causes redundant rebuilds
215 // to occur, because one step modifies a file which is an input to another step. The solution
216 // to this problem will probably be either:
217 //
218 // a) Don't include the output of one step as a watch input of another; only mark external
219 // files as watch inputs. Or...
220 //
221 // b) Note the current event ID when a step begins, and disregard events preceding that ID
222 // when considering whether to dirty that step in `eventCallback`.
223 //
224 // For now, to avoid the redundant rebuilds, we bypass this `since_event` mechanism. This does
225 // introduce race conditions, but the other `std.Build.Watch` implementations suffer from those
226 // too at the time of writing, so this is kind of expected.
227 fse.since_event = .since_now;
228
229 const cf_allocator = rs.CFAllocatorCreate(rs.kCFAllocatorUseContext.*, &.{
230 .version = 0,
231 .info = @constCast(&gpa),
232 .retain = null,
233 .release = null,
234 .copy_description = null,
235 .allocate = &cf_alloc_callbacks.allocate,
236 .reallocate = &cf_alloc_callbacks.reallocate,
237 .deallocate = &cf_alloc_callbacks.deallocate,
238 .preferred_size = null,
239 }) orelse return error.OutOfMemory;
240 defer rs.CFRelease(cf_allocator);
241
242 const cf_paths = try gpa.alloc(?CFStringRef, fse.watch_roots.len);
243 @memset(cf_paths, null);
244 defer {
245 for (cf_paths) |o| if (o) |p| rs.CFRelease(p);
246 gpa.free(cf_paths);
247 }
248 for (fse.watch_roots, cf_paths) |raw_path, *cf_path| {
249 cf_path.* = rs.CFStringCreateWithCString(cf_allocator, raw_path, .utf8);
250 }
251 const cf_paths_array = rs.CFArrayCreate(cf_allocator, @ptrCast(cf_paths), @intCast(cf_paths.len), null);
252 defer rs.CFRelease(cf_paths_array);
253
254 const callback_ctx: EventCallbackCtx = .{
255 .fse = fse,
256 .gpa = gpa,
257 };
258 const event_stream = rs.FSEventStreamCreate(
259 null,
260 &eventCallback,
261 &.{
262 .version = 0,
263 .info = @constCast(&callback_ctx),
264 .retain = null,
265 .release = null,
266 .copy_description = null,
267 },
268 cf_paths_array,
269 fse.since_event,
270 0.05, // 0.05s latency; higher values increase efficiency by coalescing more events
271 .{ .watch_root = true, .file_events = true },
272 );
273 defer rs.FSEventStreamRelease(event_stream);
274 rs.FSEventStreamSetDispatchQueue(event_stream, fse.dispatch_queue);
275 defer rs.FSEventStreamInvalidate(event_stream);
276 if (!rs.FSEventStreamStart(event_stream)) return error.StartFailed;
277 defer rs.FSEventStreamStop(event_stream);
278 const result = fse.waiting_semaphore.wait(timeout: {
279 const ns = timeout_ns orelse break :timeout .FOREVER;
280 break :timeout .time(.NOW, @intCast(ns));
281 });
282 return switch (result) {
283 0 => .dirty,
284 else => .timeout,
285 };
286}
287
288const cf_alloc_callbacks = struct {
289 const log = std.log.scoped(.cf_alloc);
290 fn allocate(size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque {
291 if (enable_debug_logs) log.debug("allocate {d}", .{size});
292 _ = hint;
293 const gpa: *const Allocator = @ptrCast(@alignCast(info));
294 const mem = gpa.alignedAlloc(u8, .of(usize), @intCast(size + @sizeOf(usize))) catch return null;
295 const metadata: *usize = @ptrCast(mem);
296 metadata.* = @intCast(size);
297 return mem[@sizeOf(usize)..].ptr;
298 }
299 fn reallocate(ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque {
300 if (enable_debug_logs) log.debug("reallocate @{*} {d}", .{ ptr, new_size });
301 _ = hint;
302 if (ptr == null or new_size == 0) return null; // not a bug: documentation explicitly states that realloc on NULL should return NULL
303 const gpa: *const Allocator = @ptrCast(@alignCast(info));
304 const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize));
305 const old_size = @as(*const usize, @ptrCast(old_base)).*;
306 const old_mem = old_base[0 .. old_size + @sizeOf(usize)];
307 const new_mem = gpa.realloc(old_mem, @intCast(new_size + @sizeOf(usize))) catch return null;
308 const metadata: *usize = @ptrCast(new_mem);
309 metadata.* = @intCast(new_size);
310 return new_mem[@sizeOf(usize)..].ptr;
311 }
312 fn deallocate(ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void {
313 if (enable_debug_logs) log.debug("deallocate @{*}", .{ptr});
314 const gpa: *const Allocator = @ptrCast(@alignCast(info));
315 const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize));
316 const old_size = @as(*const usize, @ptrCast(old_base)).*;
317 const old_mem = old_base[0 .. old_size + @sizeOf(usize)];
318 gpa.free(old_mem);
319 }
320};
321
322const EventCallbackCtx = struct {
323 fse: *FsEvents,
324 gpa: Allocator,
325};
326
327fn eventCallback(
328 stream: ConstFSEventStreamRef,
329 client_callback_info: ?*anyopaque,
330 num_events: usize,
331 events_paths_ptr: *anyopaque,
332 events_flags_ptr: [*]const FSEventStreamEventFlags,
333 events_ids_ptr: [*]const FSEventStreamEventId,
334) callconv(.c) void {
335 const ctx: *const EventCallbackCtx = @ptrCast(@alignCast(client_callback_info));
336 const fse = ctx.fse;
337 const gpa = ctx.gpa;
338 const rs = fse.resolved_symbols;
339 const events_paths_ptr_casted: [*]const [*:0]const u8 = @ptrCast(@alignCast(events_paths_ptr));
340 const events_paths = events_paths_ptr_casted[0..num_events];
341 const events_ids = events_ids_ptr[0..num_events];
342 const events_flags = events_flags_ptr[0..num_events];
343 var any_dirty = false;
344 for (events_paths, events_ids, events_flags) |event_path_nts, event_id, event_flags| {
345 _ = event_id;
346 if (event_flags.history_done) continue; // sentinel
347 const event_path = std.mem.span(event_path_nts);
348 switch (event_flags.must_scan_sub_dirs) {
349 false => {
350 if (fse.watch_paths.get(event_path)) |steps| {
351 assert(steps.len > 0);
352 for (steps) |s| {
353 if (s.invalidateResult(gpa)) any_dirty = true;
354 }
355 }
356 if (std.fs.path.dirname(event_path)) |event_dirname| {
357 // Modifying '/foo/bar' triggers the watch on '/foo'.
358 if (fse.watch_paths.get(event_dirname)) |steps| {
359 assert(steps.len > 0);
360 for (steps) |s| {
361 if (s.invalidateResult(gpa)) any_dirty = true;
362 }
363 }
364 }
365 },
366 true => {
367 // This is unlikely, but can occasionally happen when bottlenecked: events have been
368 // coalesced into one. We want to see if any of these events are actually relevant
369 // to us. The only way we can reasonably do that in this rare edge case is iterate
370 // the watch paths and see if any is under this directory. That's acceptable because
371 // we would otherwise kick off a rebuild which would be clearing those paths anyway.
372 const changed_path = std.fs.path.dirname(event_path) orelse event_path;
373 for (fse.watch_paths.keys(), fse.watch_paths.values()) |watching_path, steps| {
374 if (dirStartsWith(watching_path, changed_path)) {
375 for (steps) |s| {
376 if (s.invalidateResult(gpa)) any_dirty = true;
377 }
378 }
379 }
380 },
381 }
382 }
383 if (any_dirty) {
384 fse.since_event = rs.FSEventStreamGetLatestEventId(stream);
385 _ = fse.waiting_semaphore.signal();
386 }
387}
388fn dirStartsWith(path: []const u8, prefix: []const u8) bool {
389 if (std.mem.eql(u8, path, prefix)) return true;
390 if (!std.mem.startsWith(u8, path, prefix)) return false;
391 if (path[prefix.len] != '/') return false; // `path` is `/foo/barx`, `prefix` is `/foo/bar`
392 return true; // `path` is `/foo/bar/...`, `prefix` is `/foo/bar`
393}
394
395const CFAllocatorRef = ?*const opaque {};
396const CFArrayRef = *const opaque {};
397const CFStringRef = *const opaque {};
398const CFTimeInterval = f64;
399const CFIndex = i32;
400const CFOptionFlags = enum(u32) { _ };
401const CFAllocatorRetainCallBack = *const fn (info: ?*const anyopaque) callconv(.c) *const anyopaque;
402const CFAllocatorReleaseCallBack = *const fn (info: ?*const anyopaque) callconv(.c) void;
403const CFAllocatorCopyDescriptionCallBack = *const fn (info: ?*const anyopaque) callconv(.c) CFStringRef;
404const CFAllocatorAllocateCallBack = *const fn (alloc_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque;
405const CFAllocatorReallocateCallBack = *const fn (ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque;
406const CFAllocatorDeallocateCallBack = *const fn (ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void;
407const CFAllocatorPreferredSizeCallBack = *const fn (size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) CFIndex;
408const CFAllocatorContext = extern struct {
409 version: CFIndex,
410 info: ?*anyopaque,
411 retain: ?CFAllocatorRetainCallBack,
412 release: ?CFAllocatorReleaseCallBack,
413 copy_description: ?CFAllocatorCopyDescriptionCallBack,
414 allocate: CFAllocatorAllocateCallBack,
415 reallocate: ?CFAllocatorReallocateCallBack,
416 deallocate: ?CFAllocatorDeallocateCallBack,
417 preferred_size: ?CFAllocatorPreferredSizeCallBack,
418};
419const CFArrayCallBacks = opaque {};
420const CFStringEncoding = enum(u32) {
421 invalid_id = std.math.maxInt(u32),
422 mac_roman = 0,
423 windows_latin_1 = 0x500,
424 iso_latin_1 = 0x201,
425 next_step_latin = 0xB01,
426 ascii = 0x600,
427 unicode = 0x100,
428 utf8 = 0x8000100,
429 non_lossy_ascii = 0xBFF,
430};
431
432const FSEventStreamRef = *opaque {};
433const ConstFSEventStreamRef = *const @typeInfo(FSEventStreamRef).pointer.child;
434const FSEventStreamCallback = *const fn (
435 stream: ConstFSEventStreamRef,
436 client_callback_info: ?*anyopaque,
437 num_events: usize,
438 event_paths: *anyopaque,
439 event_flags: [*]const FSEventStreamEventFlags,
440 event_ids: [*]const FSEventStreamEventId,
441) callconv(.c) void;
442const FSEventStreamContext = extern struct {
443 version: CFIndex,
444 info: ?*anyopaque,
445 retain: ?CFAllocatorRetainCallBack,
446 release: ?CFAllocatorReleaseCallBack,
447 copy_description: ?CFAllocatorCopyDescriptionCallBack,
448};
449const FSEventStreamEventId = enum(u64) {
450 since_now = std.math.maxInt(u64),
451 _,
452};
453const FSEventStreamCreateFlags = packed struct(u32) {
454 use_cf_types: bool = false,
455 no_defer: bool = false,
456 watch_root: bool = false,
457 ignore_self: bool = false,
458 file_events: bool = false,
459 _: u27 = 0,
460};
461const FSEventStreamEventFlags = packed struct(u32) {
462 must_scan_sub_dirs: bool,
463 user_dropped: bool,
464 kernel_dropped: bool,
465 event_ids_wrapped: bool,
466 history_done: bool,
467 root_changed: bool,
468 mount: bool,
469 unmount: bool,
470 _: u24 = 0,
471};
472
473const dispatch = std.c.dispatch;
474const std = @import("std");
475const Io = std.Io;
476const assert = std.debug.assert;
477const Allocator = std.mem.Allocator;
478const watch_log = std.log.scoped(.watch);
479const FsEvents = @This();
lib/compiler/Maker/WebServer.zig created+940
...@@ -0,0 +1,940 @@
1const WebServer = @This();
2
3const builtin = @import("builtin");
4
5const std = @import("std");
6const Allocator = std.mem.Allocator;
7const Cache = std.Build.Cache;
8const Configuration = std.Build.Configuration;
9const Io = std.Io;
10const abi = std.Build.abi;
11const assert = std.debug.assert;
12const http = std.http;
13const log = std.log.scoped(.web_server);
14const mem = std.mem;
15const net = std.Io.net;
16
17const Fuzz = @import("Fuzz.zig");
18const Graph = @import("Graph.zig");
19const Step = @import("Step.zig");
20
21gpa: Allocator,
22graph: *const Graph,
23all_steps: []const Configuration.Step.Index,
24listen_address: net.IpAddress,
25root_prog_node: std.Progress.Node,
26watch: bool,
27
28tcp_server: ?net.Server,
29serve_task: ?Io.Future(Io.Cancelable!void),
30
31/// Uses `Io.Clock.awake`.
32base_timestamp: Io.Timestamp,
33/// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`.
34step_names_trailing: []u8,
35
36/// The bit-packed "step status" data. Values are `abi.StepUpdate.Status`. LSBs are earlier steps.
37/// Accessed atomically.
38step_status_bits: []u8,
39
40fuzz: ?Fuzz,
41time_report_mutex: Io.Mutex,
42time_report_msgs: [][]u8,
43time_report_update_times: []i64,
44
45build_status: std.atomic.Value(abi.BuildStatus),
46/// When an event occurs which means WebSocket clients should be sent updates, call `notifyUpdate`
47/// to increment this value. Each client thread waits for this increment with `Io.futexWaitTimeout`, so
48/// `notifyUpdate` will wake those threads. Updates are sent on a short interval regardless, so it
49/// is recommended to only use `notifyUpdate` for changes which the user should see immediately. For
50/// instance, we do not call `notifyUpdate` when the number of "unique runs" in the fuzzer changes,
51/// because this value changes quickly so this would result in constantly spamming all clients with
52/// an unreasonable number of packets.
53update_id: std.atomic.Value(u32),
54
55runner_request_mutex: Io.Mutex,
56runner_request_ready_cond: Io.Condition,
57runner_request_empty_cond: Io.Condition,
58runner_request: ?RunnerRequest,
59
60/// If a client is not explicitly notified of changes with `notifyUpdate`, it will be sent updates
61/// on a fixed interval of this many milliseconds.
62const default_update_interval_ms = 500;
63
64pub const base_clock: Io.Clock = .awake;
65
66/// Thread-safe. Triggers updates to be sent to connected WebSocket clients; see `update_id`.
67pub fn notifyUpdate(ws: *WebServer) void {
68 _ = ws.update_id.rmw(.Add, 1, .release);
69 ws.graph.io.futexWake(u32, &ws.update_id.raw, 16);
70}
71
72pub const Options = struct {
73 gpa: Allocator,
74 graph: *const Graph,
75 all_steps: []const Configuration.Step.Index,
76 root_prog_node: std.Progress.Node,
77 watch: bool,
78 listen_address: net.IpAddress,
79 base_timestamp: Io.Clock.Timestamp,
80 configuration: *const Configuration,
81};
82pub fn init(opts: Options) WebServer {
83 // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent`
84 // instead of threads, so that the web server can function in single-threaded builds.
85 comptime assert(!builtin.single_threaded);
86 assert(opts.base_timestamp.clock == base_clock);
87
88 const all_steps = opts.all_steps;
89 const c = opts.configuration;
90
91 const step_names_trailing = opts.gpa.alloc(u8, len: {
92 var name_bytes: usize = 0;
93 for (all_steps) |step_index| name_bytes += step_index.ptr(c).name.slice(c).len;
94 break :len name_bytes + all_steps.len * 4;
95 }) catch @panic("out of memory");
96 {
97 const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]);
98 var idx: usize = all_steps.len * 4;
99 for (all_steps, step_name_lens) |step_index, *name_len| {
100 const step_name = step_index.ptr(c).name.slice(c);
101 name_len.* = @intCast(step_name.len);
102 @memcpy(step_names_trailing[idx..][0..step_name.len], step_name);
103 idx += step_name.len;
104 }
105 assert(idx == step_names_trailing.len);
106 }
107
108 const step_status_bits = opts.gpa.alloc(
109 u8,
110 std.math.divCeil(usize, all_steps.len, 4) catch unreachable,
111 ) catch @panic("out of memory");
112 @memset(step_status_bits, 0);
113
114 const time_reports_len: usize = if (opts.graph.time_report) all_steps.len else 0;
115 const time_report_msgs = opts.gpa.alloc([]u8, time_reports_len) catch @panic("out of memory");
116 const time_report_update_times = opts.gpa.alloc(i64, time_reports_len) catch @panic("out of memory");
117 @memset(time_report_msgs, &.{});
118 @memset(time_report_update_times, std.math.minInt(i64));
119
120 return .{
121 .gpa = opts.gpa,
122 .graph = opts.graph,
123 .all_steps = all_steps,
124 .listen_address = opts.listen_address,
125 .root_prog_node = opts.root_prog_node,
126 .watch = opts.watch,
127
128 .tcp_server = null,
129 .serve_task = null,
130
131 .base_timestamp = opts.base_timestamp.raw,
132 .step_names_trailing = step_names_trailing,
133
134 .step_status_bits = step_status_bits,
135
136 .fuzz = null,
137 .time_report_mutex = .init,
138 .time_report_msgs = time_report_msgs,
139 .time_report_update_times = time_report_update_times,
140
141 .build_status = .init(.idle),
142 .update_id = .init(0),
143
144 .runner_request_mutex = .init,
145 .runner_request_ready_cond = .init,
146 .runner_request_empty_cond = .init,
147 .runner_request = null,
148 };
149}
150pub fn deinit(ws: *WebServer) void {
151 const gpa = ws.gpa;
152 const io = ws.graph.io;
153
154 gpa.free(ws.step_names_trailing);
155 gpa.free(ws.step_status_bits);
156
157 if (ws.fuzz) |*f| f.deinit();
158 for (ws.time_report_msgs) |msg| gpa.free(msg);
159 gpa.free(ws.time_report_msgs);
160 gpa.free(ws.time_report_update_times);
161
162 if (ws.serve_task) |t| {
163 if (ws.tcp_server) |*s| s.stream.close(io);
164 t.await();
165 }
166 if (ws.tcp_server) |*s| s.deinit();
167
168 gpa.free(ws.step_names_trailing);
169}
170pub fn start(ws: *WebServer) error{AlreadyReported}!void {
171 assert(ws.tcp_server == null);
172 assert(ws.serve_task == null);
173 const io = ws.graph.io;
174
175 ws.tcp_server = ws.listen_address.listen(io, .{ .reuse_address = true }) catch |err| {
176 log.err("failed to listen to port {d}: {t}", .{ ws.listen_address.getPort(), err });
177 return error.AlreadyReported;
178 };
179 ws.serve_task = io.concurrent(serve, .{ws}) catch |err| {
180 log.err("unable to spawn web server thread: {t}", .{err});
181 ws.tcp_server.?.deinit(io);
182 ws.tcp_server = null;
183 return error.AlreadyReported;
184 };
185
186 log.info("web interface listening at http://{f}/", .{ws.tcp_server.?.socket.address});
187 if (ws.listen_address.getPort() == 0) {
188 log.info("hint: pass '--webui={f}' to use the same port next time", .{ws.tcp_server.?.socket.address});
189 }
190}
191fn serve(ws: *WebServer) Io.Cancelable!void {
192 const io = ws.graph.io;
193 var group: Io.Group = .init;
194 defer group.cancel(io);
195 while (true) {
196 var stream = ws.tcp_server.?.accept(io) catch |err| switch (err) {
197 error.Canceled => |e| return e,
198 else => |e| {
199 log.err("failed to accept connection: {t}", .{e});
200 return;
201 },
202 };
203 group.concurrent(io, accept, .{ ws, stream }) catch |err| {
204 log.err("unable to spawn connection thread: {t}", .{err});
205 stream.close(io);
206 continue;
207 };
208 }
209}
210
211pub fn startBuild(ws: *WebServer) void {
212 if (ws.fuzz) |*fuzz| {
213 fuzz.deinit();
214 ws.fuzz = null;
215 }
216 for (ws.step_status_bits) |*bits| @atomicStore(u8, bits, 0, .monotonic);
217 ws.build_status.store(.running, .monotonic);
218 ws.notifyUpdate();
219}
220
221pub fn updateStepStatus(
222 ws: *WebServer,
223 step_index: Configuration.Step.Index,
224 new_status: abi.StepUpdate.Status,
225) void {
226 // TODO don't do linear search, especially in a hot loop like this
227 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
228 if (s == step_index) break @intCast(i);
229 } else unreachable;
230 const ptr = &ws.step_status_bits[step_idx / 4];
231 const bit_offset: u3 = @intCast((step_idx % 4) * 2);
232 const old_bits: u2 = @truncate(@atomicLoad(u8, ptr, .monotonic) >> bit_offset);
233 const mask = @as(u8, @intFromEnum(new_status) ^ old_bits) << bit_offset;
234 _ = @atomicRmw(u8, ptr, .Xor, mask, .monotonic);
235 ws.notifyUpdate();
236}
237
238pub fn finishBuild(ws: *WebServer, opts: struct {
239 fuzz: bool,
240}) void {
241 if (opts.fuzz) {
242 switch (builtin.os.tag) {
243 // Current implementation depends on two things that need to be ported to Windows:
244 // * Memory-mapping to share data between the fuzzer and build runner.
245 // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
246 // many addresses to source locations).
247 .windows => std.process.fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}),
248 else => {},
249 }
250 if (@bitSizeOf(usize) != 64) {
251 // Current implementation depends on posix.mmap()'s second
252 // parameter, `length: usize`, being compatible with file system's
253 // u64 return value. This is not the case on 32-bit platforms.
254 // Affects or affected by issues #5185, #22523, and #22464.
255 std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
256 }
257
258 assert(ws.fuzz == null);
259
260 ws.build_status.store(.fuzz_init, .monotonic);
261 ws.notifyUpdate();
262
263 ws.fuzz = Fuzz.init(
264 ws.gpa,
265 ws.graph.io,
266 ws.all_steps,
267 ws.root_prog_node,
268 .{ .forever = .{ .ws = ws } },
269 ) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)});
270 ws.fuzz.?.start();
271 }
272
273 ws.build_status.store(if (ws.watch) .watching else .idle, .monotonic);
274 ws.notifyUpdate();
275}
276
277pub fn now(s: *const WebServer) i64 {
278 const io = s.graph.io;
279 const ts = base_clock.now(io);
280 return @intCast(s.base_timestamp.durationTo(ts).toNanoseconds());
281}
282
283fn accept(ws: *WebServer, stream: net.Stream) void {
284 const io = ws.graph.io;
285 defer {
286 // `net.Stream.close` wants to helpfully overwrite `stream` with
287 // `undefined`, but it cannot do so since it is an immutable parameter.
288 var copy = stream;
289 copy.close(io);
290 }
291 var send_buffer: [4096]u8 = undefined;
292 var recv_buffer: [4096]u8 = undefined;
293 var connection_reader = stream.reader(io, &recv_buffer);
294 var connection_writer = stream.writer(io, &send_buffer);
295 var server: http.Server = .init(&connection_reader.interface, &connection_writer.interface);
296
297 while (true) {
298 var request = server.receiveHead() catch |err| switch (err) {
299 error.HttpConnectionClosing => return,
300 else => return log.err("failed to receive http request: {t}", .{err}),
301 };
302 switch (request.upgradeRequested()) {
303 .websocket => |opt_key| {
304 const key = opt_key orelse return log.err("missing websocket key", .{});
305 var web_socket = request.respondWebSocket(.{ .key = key }) catch {
306 return log.err("failed to respond web socket: {t}", .{connection_writer.err.?});
307 };
308 ws.serveWebSocket(&web_socket) catch |err| {
309 log.err("failed to serve websocket: {t}", .{err});
310 return;
311 };
312 comptime unreachable;
313 },
314 .other => |name| return log.err("unknown upgrade request: {s}", .{name}),
315 .none => {
316 ws.serveRequest(&request) catch |err| switch (err) {
317 error.AlreadyReported => return,
318 else => {
319 log.err("failed to serve '{s}': {t}", .{ request.head.target, err });
320 return;
321 },
322 };
323 },
324 }
325 }
326}
327
328fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn {
329 const io = ws.graph.io;
330
331 var prev_build_status = ws.build_status.load(.monotonic);
332
333 const prev_step_status_bits = try ws.gpa.alloc(u8, ws.step_status_bits.len);
334 defer ws.gpa.free(prev_step_status_bits);
335 for (prev_step_status_bits, ws.step_status_bits) |*copy, *shared| {
336 copy.* = @atomicLoad(u8, shared, .monotonic);
337 }
338
339 var recv_thread = try io.concurrent(recvWebSocketMessages, .{ ws, sock });
340 defer recv_thread.cancel(io);
341
342 {
343 const hello_header: abi.Hello = .{
344 .status = prev_build_status,
345 .flags = .{
346 .time_report = ws.graph.time_report,
347 },
348 .timestamp = ws.now(),
349 .steps_len = @intCast(ws.all_steps.len),
350 };
351 var bufs: [3][]const u8 = .{ @ptrCast(&hello_header), ws.step_names_trailing, prev_step_status_bits };
352 try sock.writeMessageVec(&bufs, .binary);
353 }
354
355 var prev_fuzz: Fuzz.Previous = .init;
356 var prev_time: i64 = std.math.minInt(i64);
357 while (true) {
358 const start_time = ws.now();
359 const start_update_id = ws.update_id.load(.acquire);
360
361 if (ws.fuzz) |*fuzz| {
362 try fuzz.sendUpdate(sock, &prev_fuzz);
363 }
364
365 {
366 try ws.time_report_mutex.lock(io);
367 defer ws.time_report_mutex.unlock(io);
368 for (ws.time_report_msgs, ws.time_report_update_times) |msg, update_time| {
369 if (update_time <= prev_time) continue;
370 // We want to send `msg`, but shouldn't block `ws.time_report_mutex` while we do, so
371 // that we don't hold up the build system on the client accepting this packet.
372 const owned_msg = try ws.gpa.dupe(u8, msg);
373 defer ws.gpa.free(owned_msg);
374 // Temporarily unlock, then re-lock after the message is sent.
375 ws.time_report_mutex.unlock(io);
376 defer ws.time_report_mutex.lockUncancelable(io);
377 try sock.writeMessage(owned_msg, .binary);
378 }
379 }
380
381 {
382 const build_status = ws.build_status.load(.monotonic);
383 if (build_status != prev_build_status) {
384 prev_build_status = build_status;
385 const msg: abi.StatusUpdate = .{ .new = build_status };
386 try sock.writeMessage(@ptrCast(&msg), .binary);
387 }
388 }
389
390 for (prev_step_status_bits, ws.step_status_bits, 0..) |*prev_byte, *shared, byte_idx| {
391 const cur_byte = @atomicLoad(u8, shared, .monotonic);
392 if (prev_byte.* == cur_byte) continue;
393 const cur: [4]abi.StepUpdate.Status = .{
394 @enumFromInt(@as(u2, @truncate(cur_byte >> 0))),
395 @enumFromInt(@as(u2, @truncate(cur_byte >> 2))),
396 @enumFromInt(@as(u2, @truncate(cur_byte >> 4))),
397 @enumFromInt(@as(u2, @truncate(cur_byte >> 6))),
398 };
399 const prev: [4]abi.StepUpdate.Status = .{
400 @enumFromInt(@as(u2, @truncate(prev_byte.* >> 0))),
401 @enumFromInt(@as(u2, @truncate(prev_byte.* >> 2))),
402 @enumFromInt(@as(u2, @truncate(prev_byte.* >> 4))),
403 @enumFromInt(@as(u2, @truncate(prev_byte.* >> 6))),
404 };
405 for (cur, prev, byte_idx * 4..) |cur_status, prev_status, step_idx| {
406 const msg: abi.StepUpdate = .{ .step_idx = @intCast(step_idx), .bits = .{ .status = cur_status } };
407 if (cur_status != prev_status) try sock.writeMessage(@ptrCast(&msg), .binary);
408 }
409 prev_byte.* = cur_byte;
410 }
411
412 prev_time = start_time;
413
414 const old_cp = io.swapCancelProtection(.blocked);
415 defer _ = io.swapCancelProtection(old_cp);
416 io.futexWaitTimeout(
417 u32,
418 &ws.update_id.raw,
419 start_update_id,
420 .{ .duration = .{
421 .clock = .awake,
422 .raw = .fromMilliseconds(default_update_interval_ms),
423 } },
424 ) catch |err| switch (err) {
425 error.Canceled => unreachable,
426 };
427 }
428}
429fn recvWebSocketMessages(ws: *WebServer, sock: *http.Server.WebSocket) void {
430 const io = ws.graph.io;
431
432 while (true) {
433 const msg = sock.readSmallMessage() catch return;
434 if (msg.opcode != .binary) continue;
435 if (msg.data.len == 0) continue;
436 const tag: abi.ToServerTag = @enumFromInt(msg.data[0]);
437 switch (tag) {
438 _ => continue,
439 .rebuild => while (true) {
440 ws.runner_request_mutex.lock(io) catch |err| switch (err) {
441 error.Canceled => return,
442 };
443 defer ws.runner_request_mutex.unlock(io);
444 if (ws.runner_request == null) {
445 ws.runner_request = .rebuild;
446 ws.runner_request_ready_cond.signal(io);
447 break;
448 }
449 ws.runner_request_empty_cond.wait(io, &ws.runner_request_mutex) catch return;
450 },
451 }
452 }
453}
454
455fn serveRequest(ws: *WebServer, req: *http.Server.Request) !void {
456 // Strip an optional leading '/debug' component from the request.
457 const target: []const u8, const debug: bool = target: {
458 if (mem.eql(u8, req.head.target, "/debug")) break :target .{ "/", true };
459 if (mem.eql(u8, req.head.target, "/debug/")) break :target .{ "/", true };
460 if (mem.startsWith(u8, req.head.target, "/debug/")) break :target .{ req.head.target["/debug".len..], true };
461 break :target .{ req.head.target, false };
462 };
463
464 if (mem.eql(u8, target, "/")) return serveLibFile(ws, req, "build-web/index.html", "text/html");
465 if (mem.eql(u8, target, "/main.js")) return serveLibFile(ws, req, "build-web/main.js", "application/javascript");
466 if (mem.eql(u8, target, "/style.css")) return serveLibFile(ws, req, "build-web/style.css", "text/css");
467 if (mem.eql(u8, target, "/time_report.css")) return serveLibFile(ws, req, "build-web/time_report.css", "text/css");
468 if (mem.eql(u8, target, "/main.wasm")) return serveClientWasm(ws, req, if (debug) .Debug else .ReleaseFast);
469
470 if (ws.fuzz) |*fuzz| {
471 if (mem.eql(u8, target, "/sources.tar")) return fuzz.serveSourcesTar(req);
472 }
473
474 try req.respond("not found", .{
475 .status = .not_found,
476 .extra_headers = &.{
477 .{ .name = "Content-Type", .value = "text/plain" },
478 },
479 });
480}
481
482fn serveLibFile(
483 ws: *WebServer,
484 request: *http.Server.Request,
485 sub_path: []const u8,
486 content_type: []const u8,
487) !void {
488 return serveFile(ws, request, .{
489 .root_dir = ws.graph.zig_lib_directory,
490 .sub_path = sub_path,
491 }, content_type);
492}
493fn serveClientWasm(
494 ws: *WebServer,
495 req: *http.Server.Request,
496 optimize_mode: std.builtin.OptimizeMode,
497) !void {
498 var arena_state: std.heap.ArenaAllocator = .init(ws.gpa);
499 defer arena_state.deinit();
500 const arena = arena_state.allocator();
501
502 // We always rebuild the wasm on-the-fly, so that if it is edited the user can just refresh the page.
503 const bin_path = try buildClientWasm(ws, arena, optimize_mode);
504 return serveFile(ws, req, bin_path, "application/wasm");
505}
506
507pub fn serveFile(
508 ws: *WebServer,
509 request: *http.Server.Request,
510 path: Cache.Path,
511 content_type: []const u8,
512) !void {
513 const gpa = ws.gpa;
514 const io = ws.graph.io;
515 // The desired API is actually sendfile, which will require enhancing http.Server.
516 // We load the file with every request so that the user can make changes to the file
517 // and refresh the HTML page without restarting this server.
518 const file_contents = path.root_dir.handle.readFileAlloc(io, path.sub_path, gpa, .limited(10 * 1024 * 1024)) catch |err| {
519 log.err("failed to read '{f}': {t}", .{ path, err });
520 return error.AlreadyReported;
521 };
522 defer gpa.free(file_contents);
523 try request.respond(file_contents, .{
524 .extra_headers = &.{
525 .{ .name = "Content-Type", .value = content_type },
526 cache_control_header,
527 },
528 });
529}
530pub fn serveTarFile(ws: *WebServer, request: *http.Server.Request, paths: []const Cache.Path) !void {
531 const graph = ws.graph;
532 const io = graph.io;
533
534 var send_buffer: [0x4000]u8 = undefined;
535 var response = try request.respondStreaming(&send_buffer, .{
536 .respond_options = .{
537 .extra_headers = &.{
538 .{ .name = "Content-Type", .value = "application/x-tar" },
539 cache_control_header,
540 },
541 },
542 });
543
544 var archiver: std.tar.Writer = .{ .underlying_writer = &response.writer };
545
546 for (paths) |path| {
547 var file = path.root_dir.handle.openFile(io, path.sub_path, .{}) catch |err| {
548 log.err("failed to open '{f}': {s}", .{ path, @errorName(err) });
549 continue;
550 };
551 defer file.close(io);
552 const stat = try file.stat(io);
553 var read_buffer: [1024]u8 = undefined;
554 var file_reader: Io.File.Reader = .initSize(file, io, &read_buffer, stat.size);
555
556 // TODO: this logic is completely bogus -- obviously so, because `path.root_dir.path` can
557 // be cwd-relative. This is also related to why linkification doesn't work in the fuzzer UI:
558 // it turns out the WASM treats the first path component as the module name, typically
559 // resulting in modules named "" and "src". The compiler needs to tell the build system
560 // about the module graph so that the build system can correctly encode this information in
561 // the tar file.
562 //
563 // Additionally, this needs to ensure that all path separators for both prefix and
564 // sub_path are using the POSIX-style `/` on platforms that don't use it as their native
565 // path separator.
566 archiver.prefix = path.root_dir.path orelse graph.cache.cwd;
567 try archiver.writeFile(path.sub_path, &file_reader, @intCast(stat.mtime.toSeconds()));
568 }
569
570 // intentionally not calling `archiver.finishPedantically`
571 try response.end();
572}
573
574fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.OptimizeMode) !Cache.Path {
575 const root_name = "build-web";
576 const arch_os_abi = "wasm32-freestanding";
577 const cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext";
578
579 const gpa = ws.gpa;
580 const graph = ws.graph;
581 const io = graph.io;
582
583 const main_src_path: Cache.Path = .{
584 .root_dir = graph.zig_lib_directory,
585 .sub_path = "build-web/main.zig",
586 };
587 const walk_src_path: Cache.Path = .{
588 .root_dir = graph.zig_lib_directory,
589 .sub_path = "docs/wasm/Walk.zig",
590 };
591 const html_render_src_path: Cache.Path = .{
592 .root_dir = graph.zig_lib_directory,
593 .sub_path = "docs/wasm/html_render.zig",
594 };
595
596 var argv: std.ArrayList([]const u8) = .empty;
597
598 try argv.appendSlice(arena, &.{
599 graph.zig_exe, "build-exe", //
600 "-fno-entry", //
601 "-O", @tagName(optimize), //
602 "-target", arch_os_abi, //
603 "-mcpu", cpu_features, //
604 "--cache-dir", graph.global_cache_root.path orelse ".", //
605 "--global-cache-dir", graph.global_cache_root.path orelse ".", //
606 "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", //
607 "--name", root_name, //
608 "-rdynamic", //
609 "-fsingle-threaded", //
610 "--dep", "Walk", //
611 "--dep", "html_render", //
612 try std.fmt.allocPrint(arena, "-Mroot={f}", .{main_src_path}), //
613 try std.fmt.allocPrint(arena, "-MWalk={f}", .{walk_src_path}), //
614 "--dep", "Walk", //
615 try std.fmt.allocPrint(arena, "-Mhtml_render={f}", .{html_render_src_path}), //
616 "--listen=-",
617 });
618
619 var child = try std.process.spawn(io, .{
620 .argv = argv.items,
621 .environ_map = &graph.environ_map,
622 .stdin = .pipe,
623 .stdout = .pipe,
624 .stderr = .pipe,
625 });
626 defer child.kill(io);
627
628 var stderr_task = try io.concurrent(readStreamAlloc, .{ gpa, io, child.stderr.?, .unlimited });
629 defer if (stderr_task.cancel(io)) |slice| gpa.free(slice) else |_| {};
630
631 var stdout_buffer: [512]u8 = undefined;
632 var stdout_reader: Io.File.Reader = .initStreaming(child.stdout.?, io, &stdout_buffer);
633 const stdout = &stdout_reader.interface;
634
635 {
636 var w = child.stdin.?.writer(io, &.{});
637 w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .update, .bytes_len = 0 }, .little) catch |err| switch (err) {
638 error.WriteFailed => return w.err.?,
639 };
640 w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .exit, .bytes_len = 0 }, .little) catch |err| switch (err) {
641 error.WriteFailed => return w.err.?,
642 };
643 }
644
645 const Header = std.zig.Server.Message.Header;
646
647 var result: ?Cache.Path = null;
648 var result_error_bundle = std.zig.ErrorBundle.empty;
649 var body_buffer: std.ArrayList(u8) = .empty;
650 defer body_buffer.deinit(gpa);
651
652 while (true) {
653 const header = stdout.takeStruct(Header, .little) catch |err| switch (err) {
654 error.ReadFailed => |e| return e,
655 error.EndOfStream => break,
656 };
657 body_buffer.clearRetainingCapacity();
658 try stdout.appendExact(gpa, &body_buffer, header.bytes_len);
659 const body = body_buffer.items;
660
661 switch (header.tag) {
662 .zig_version => {
663 if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
664 return error.ZigProtocolVersionMismatch;
665 }
666 },
667 .error_bundle => {
668 result_error_bundle = try std.zig.Server.allocErrorBundle(arena, body);
669 },
670 .emit_digest => {
671 const EmitDigest = std.zig.Server.Message.EmitDigest;
672 const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body);
673 if (!ebp_hdr.flags.cache_hit) {
674 log.info("source changes detected; rebuilt wasm component", .{});
675 }
676 const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len];
677 result = .{
678 .root_dir = graph.global_cache_root,
679 .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)),
680 };
681 },
682 else => {}, // ignore other messages
683 }
684 }
685
686 const stderr_contents = try stderr_task.await(io);
687 if (stderr_contents.len > 0) {
688 std.debug.print("{s}", .{stderr_contents});
689 }
690
691 // Send EOF to stdin.
692 child.stdin.?.close(io);
693 child.stdin = null;
694
695 switch (try child.wait(io)) {
696 .exited => |code| {
697 if (code != 0) {
698 log.err(
699 "the following command exited with error code {d}:\n{s}",
700 .{ code, try Step.allocPrintCmd(arena, .inherit, null, argv.items) },
701 );
702 return error.WasmCompilationFailed;
703 }
704 },
705 .signal => |sig| {
706 log.err(
707 "the following command terminated with signal {t}:\n{s}",
708 .{ sig, try Step.allocPrintCmd(arena, .inherit, null, argv.items) },
709 );
710 return error.WasmCompilationFailed;
711 },
712 .stopped => |sig| {
713 log.err(
714 "the following command stopped unexpectedly with signal {t}:\n{s}",
715 .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) },
716 );
717 return error.WasmCompilationFailed;
718 },
719 .unknown => {
720 log.err(
721 "the following command terminated unexpectedly:\n{s}",
722 .{try Step.allocPrintCmd(arena, .inherit, null, argv.items)},
723 );
724 return error.WasmCompilationFailed;
725 },
726 }
727
728 if (result_error_bundle.errorMessageCount() > 0) {
729 try result_error_bundle.renderToStderr(io, .{}, .auto);
730 log.err("the following command failed with {d} compilation errors:\n{s}", .{
731 result_error_bundle.errorMessageCount(),
732 try Step.allocPrintCmd(arena, .inherit, null, argv.items),
733 });
734 return error.WasmCompilationFailed;
735 }
736
737 const base_path = result orelse {
738 log.err("child process failed to report result\n{s}", .{
739 try Step.allocPrintCmd(arena, .inherit, null, argv.items),
740 });
741 return error.WasmCompilationFailed;
742 };
743 const bin_name = try std.zig.binNameAlloc(arena, .{
744 .root_name = root_name,
745 .target = &(std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{
746 .arch_os_abi = arch_os_abi,
747 .cpu_features = cpu_features,
748 }) catch unreachable) catch unreachable),
749 .output_mode = .Exe,
750 });
751 return base_path.join(arena, bin_name);
752}
753
754fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 {
755 var file_reader: Io.File.Reader = .initStreaming(file, io, &.{});
756 return file_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) {
757 error.ReadFailed => return file_reader.err.?,
758 else => |e| return e,
759 };
760}
761
762pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
763 compile_step: Configuration.Step.Index,
764
765 use_llvm: bool,
766 stats: abi.time_report.CompileResult.Stats,
767 ns_total: u64,
768
769 llvm_pass_timings_len: u32,
770 files_len: u32,
771 decls_len: u32,
772
773 /// The trailing data of `abi.time_report.CompileResult`, except the step name.
774 trailing: []const u8,
775}) void {
776 const gpa = ws.gpa;
777 const io = ws.graph.io;
778
779 // TODO don't do linear search
780 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
781 if (s == opts.compile_step) break @intCast(i);
782 } else unreachable;
783
784 const old_buf = old: {
785 ws.time_report_mutex.lock(io) catch return;
786 defer ws.time_report_mutex.unlock(io);
787 const old = ws.time_report_msgs[step_idx];
788 ws.time_report_msgs[step_idx] = &.{};
789 break :old old;
790 };
791 const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.CompileResult) + opts.trailing.len) catch @panic("out of memory");
792
793 const out_header: *align(1) abi.time_report.CompileResult = @ptrCast(buf[0..@sizeOf(abi.time_report.CompileResult)]);
794 out_header.* = .{
795 .step_idx = step_idx,
796 .flags = .{
797 .use_llvm = opts.use_llvm,
798 },
799 .stats = opts.stats,
800 .ns_total = opts.ns_total,
801 .llvm_pass_timings_len = opts.llvm_pass_timings_len,
802 .files_len = opts.files_len,
803 .decls_len = opts.decls_len,
804 };
805 @memcpy(buf[@sizeOf(abi.time_report.CompileResult)..], opts.trailing);
806
807 {
808 ws.time_report_mutex.lock(io) catch return;
809 defer ws.time_report_mutex.unlock(io);
810 assert(ws.time_report_msgs[step_idx].len == 0);
811 ws.time_report_msgs[step_idx] = buf;
812 ws.time_report_update_times[step_idx] = ws.now();
813 }
814 ws.notifyUpdate();
815}
816
817pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.Index, duration: Io.Duration) void {
818 const gpa = ws.gpa;
819 const io = ws.graph.io;
820
821 // TODO don't do linear search
822 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
823 if (s == step_index) break @intCast(i);
824 } else unreachable;
825
826 const old_buf = old: {
827 ws.time_report_mutex.lock(io) catch return;
828 defer ws.time_report_mutex.unlock(io);
829 const old = ws.time_report_msgs[step_idx];
830 ws.time_report_msgs[step_idx] = &.{};
831 break :old old;
832 };
833 const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.GenericResult)) catch @panic("out of memory");
834 const out: *align(1) abi.time_report.GenericResult = @ptrCast(buf);
835 out.* = .{
836 .step_idx = step_idx,
837 .ns_total = @intCast(duration.toNanoseconds()),
838 };
839 {
840 ws.time_report_mutex.lock(io) catch return;
841 defer ws.time_report_mutex.unlock(io);
842 assert(ws.time_report_msgs[step_idx].len == 0);
843 ws.time_report_msgs[step_idx] = buf;
844 ws.time_report_update_times[step_idx] = ws.now();
845 }
846 ws.notifyUpdate();
847}
848
849pub fn updateTimeReportRunTest(
850 ws: *WebServer,
851 run_step_index: Configuration.Step.Index,
852 tests: *const Step.Run.CachedTestMetadata,
853 ns_per_test: []const u64,
854) void {
855 const gpa = ws.gpa;
856 const io = ws.graph.io;
857
858 // TODO don't do linear search
859 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
860 if (s == run_step_index) break @intCast(i);
861 } else unreachable;
862
863 assert(tests.names.len == ns_per_test.len);
864 const tests_len: u32 = @intCast(tests.names.len);
865
866 const new_len: u64 = len: {
867 var names_len: u64 = 0;
868 for (0..tests_len) |i| {
869 names_len += tests.testName(@intCast(i)).len + 1;
870 }
871 break :len @sizeOf(abi.time_report.RunTestResult) + names_len + 8 * tests_len;
872 };
873 const old_buf = old: {
874 ws.time_report_mutex.lock(io) catch return;
875 defer ws.time_report_mutex.unlock(io);
876 const old = ws.time_report_msgs[step_idx];
877 ws.time_report_msgs[step_idx] = &.{};
878 break :old old;
879 };
880 const buf = gpa.realloc(old_buf, new_len) catch @panic("out of memory");
881
882 const out_header: *align(1) abi.time_report.RunTestResult = @ptrCast(buf[0..@sizeOf(abi.time_report.RunTestResult)]);
883 out_header.* = .{
884 .step_idx = step_idx,
885 .tests_len = tests_len,
886 };
887 var offset: usize = @sizeOf(abi.time_report.RunTestResult);
888 const ns_per_test_out: []align(1) u64 = @ptrCast(buf[offset..][0 .. tests_len * 8]);
889 @memcpy(ns_per_test_out, ns_per_test);
890 offset += tests_len * 8;
891 for (0..tests_len) |i| {
892 const name = tests.testName(@intCast(i));
893 @memcpy(buf[offset..][0..name.len], name);
894 buf[offset..][name.len] = 0;
895 offset += name.len + 1;
896 }
897 assert(offset == buf.len);
898
899 {
900 ws.time_report_mutex.lock(io) catch return;
901 defer ws.time_report_mutex.unlock(io);
902 assert(ws.time_report_msgs[step_idx].len == 0);
903 ws.time_report_msgs[step_idx] = buf;
904 ws.time_report_update_times[step_idx] = ws.now();
905 }
906 ws.notifyUpdate();
907}
908
909const RunnerRequest = union(enum) {
910 rebuild,
911};
912pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest {
913 const io = ws.graph.io;
914 ws.runner_request_mutex.lock(io) catch return;
915 defer ws.runner_request_mutex.unlock(io);
916 if (ws.runner_request) |req| {
917 ws.runner_request = null;
918 ws.runner_request_empty_cond.signal();
919 return req;
920 }
921 return null;
922}
923pub fn wait(ws: *WebServer) Io.Cancelable!RunnerRequest {
924 const io = ws.graph.io;
925 try ws.runner_request_mutex.lock(io);
926 defer ws.runner_request_mutex.unlock(io);
927 while (true) {
928 if (ws.runner_request) |req| {
929 ws.runner_request = null;
930 ws.runner_request_empty_cond.signal(io);
931 return req;
932 }
933 try ws.runner_request_ready_cond.wait(io, &ws.runner_request_mutex);
934 }
935}
936
937const cache_control_header: http.Header = .{
938 .name = "Cache-Control",
939 .value = "max-age=0, must-revalidate",
940};
lib/compiler/maker.zig deleted-1848
...@@ -1,1848 +0,0 @@
1const Maker = @This();
2const builtin = @import("builtin");
3
4const std = @import("std");
5const Allocator = std.mem.Allocator;
6const Cache = std.Build.Cache;
7const Configuration = std.Build.Configuration;
8const File = std.Io.File;
9const Io = std.Io;
10const Path = std.Build.Cache.Path;
11const Writer = std.Io.Writer;
12const assert = std.debug.assert;
13const fatal = std.process.fatal;
14const fmt = std.fmt;
15const log = std.log;
16const mem = std.mem;
17const process = std.process;
18
19const Fuzz = @import("maker/Fuzz.zig");
20const Graph = @import("maker/Graph.zig");
21const Step = @import("maker/Step.zig");
22const Watch = @import("maker/Watch.zig");
23const WebServer = @import("maker/WebServer.zig");
24
25pub const std_options: std.Options = .{
26 .side_channels_mitigations = .none,
27 .http_disable_tls = true,
28};
29
30gpa: Allocator,
31graph: *Graph,
32install_paths: InstallPaths,
33scanned_config: *const ScannedConfig,
34steps: []Step,
35
36available_rss: usize,
37max_rss_is_default: bool,
38max_rss_mutex: Io.Mutex,
39skip_oom_steps: bool,
40unit_test_timeout_ns: ?u64,
41watch: bool,
42web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn,
43/// Allocated into `gpa`.
44memory_blocked_steps: std.ArrayList(Configuration.Step.Index),
45/// Allocated into `gpa`.
46step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void),
47
48error_style: ErrorStyle,
49multiline_errors: MultilineErrors,
50summary: Summary,
51
52pub fn main(init: process.Init.Minimal) !void {
53 // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not
54 // always the case. So, we do need a true gpa for some things.
55 var safe_gpa_state: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{});
56 defer _ = safe_gpa_state.deinit();
57 const gpa = safe_gpa_state.allocator();
58
59 var threaded: std.Io.Threaded = .init(gpa, .{
60 .environ = init.environ,
61 .argv0 = .init(init.args),
62 });
63 defer threaded.deinit();
64 const io = threaded.io();
65
66 // ...but we'll back our arena by `std.heap.page_allocator` for efficiency.
67 var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
68 defer arena_instance.deinit();
69 const arena = arena_instance.allocator();
70
71 const args = try init.args.toSlice(arena);
72
73 // skip my own exe name
74 var arg_idx: usize = 1;
75
76 const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig");
77 const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir");
78 const build_root = expectArgOrFatal(args, &arg_idx, "--build-root");
79 const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache");
80 const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache");
81 const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration");
82
83 const cwd: Io.Dir = .cwd();
84
85 const zig_lib_directory: Cache.Directory = .{
86 .path = zig_lib_dir,
87 .handle = try cwd.openDir(io, zig_lib_dir, .{}),
88 };
89
90 const build_root_directory: Cache.Directory = .{
91 .path = build_root,
92 .handle = try cwd.openDir(io, build_root, .{}),
93 };
94
95 const local_cache_directory: Cache.Directory = .{
96 .path = local_cache_root,
97 .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}),
98 };
99
100 const global_cache_directory: Cache.Directory = .{
101 .path = global_cache_root,
102 .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}),
103 };
104
105 var graph: Graph = .{
106 .io = io,
107 .arena = arena,
108 .cache = .{
109 .io = io,
110 .gpa = gpa,
111 .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}),
112 .cwd = try process.currentPathAlloc(io, arena),
113 },
114 .zig_exe = zig_exe,
115 .environ_map = try init.environ.createMap(arena),
116 .global_cache_root = global_cache_directory,
117 .zig_lib_directory = zig_lib_directory,
118 };
119
120 graph.cache.addPrefix(.{ .path = null, .handle = cwd });
121 graph.cache.addPrefix(build_root_directory);
122 graph.cache.addPrefix(local_cache_directory);
123 graph.cache.addPrefix(global_cache_directory);
124 graph.cache.hash.addBytes(builtin.zig_version_string);
125
126 var step_names: std.ArrayList([]const u8) = .empty;
127 var debug_log_scopes: std.ArrayList([]const u8) = .empty;
128 var help_menu = false;
129 var steps_menu = false;
130 var print_configuration = false;
131 var override_install_prefix: ?[]const u8 = null;
132 var override_lib_dir: ?[]const u8 = null;
133 var override_bin_dir: ?[]const u8 = null;
134 var override_include_dir: ?[]const u8 = null;
135 var error_style: ErrorStyle = .verbose;
136 var multiline_errors: MultilineErrors = .indent;
137 var summary: ?Summary = null;
138 var max_rss: u64 = 0;
139 var skip_oom_steps = false;
140 var test_timeout_ns: ?u64 = null;
141 var color: Color = .auto;
142 var watch = false;
143 var fuzz: ?Fuzz.Mode = null;
144 var debounce_interval_ms: u16 = 50;
145 var webui_listen: ?Io.net.IpAddress = null;
146 var verbose = false;
147 var sysroot: ?[]const u8 = null;
148 var search_prefixes: std.ArrayList([]const u8) = .empty;
149 var libc_file: ?[]const u8 = null;
150 var debug_pkg_config: bool = false;
151 // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md,
152 // this will be the directory $glibc-build-dir/install/glibcs
153 // Given the example of the aarch64 target, this is the directory
154 // that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`.
155 // Also works for dynamic musl.
156 var libc_runtimes_dir: ?[]const u8 = null;
157 var enable_wine = false;
158 var enable_qemu = false;
159 var enable_wasmtime = false;
160 var enable_darling = false;
161 var enable_rosetta = false;
162 var reference_trace: ?u32 = null;
163 var run_args: ?[]const []const u8 = null;
164
165 if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| {
166 if (std.meta.stringToEnum(ErrorStyle, str)) |style| {
167 error_style = style;
168 }
169 }
170
171 if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| {
172 if (std.meta.stringToEnum(MultilineErrors, str)) |style| {
173 multiline_errors = style;
174 }
175 }
176
177 while (nextArg(args, &arg_idx)) |arg| {
178 if (mem.startsWith(u8, arg, "-")) {
179 if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
180 help_menu = true;
181 } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) {
182 steps_menu = true;
183 } else if (mem.eql(u8, arg, "--print-configuration")) {
184 print_configuration = true;
185 } else if (mem.eql(u8, arg, "--verbose")) {
186 verbose = true;
187 } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) {
188 override_install_prefix = nextArgOrFatal(args, &arg_idx);
189 } else if (mem.eql(u8, arg, "--prefix-lib-dir")) {
190 override_lib_dir = nextArgOrFatal(args, &arg_idx);
191 } else if (mem.eql(u8, arg, "--prefix-exe-dir")) {
192 override_bin_dir = nextArgOrFatal(args, &arg_idx);
193 } else if (mem.eql(u8, arg, "--prefix-include-dir")) {
194 override_include_dir = nextArgOrFatal(args, &arg_idx);
195 } else if (mem.eql(u8, arg, "--sysroot")) {
196 sysroot = nextArgOrFatal(args, &arg_idx);
197 } else if (mem.eql(u8, arg, "--maxrss")) {
198 const max_rss_text = nextArgOrFatal(args, &arg_idx);
199 max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err|
200 fatal("invalid byte size: '{s}': {t}", .{ max_rss_text, err });
201 } else if (mem.eql(u8, arg, "--skip-oom-steps")) {
202 skip_oom_steps = true;
203 } else if (mem.eql(u8, arg, "--test-timeout")) {
204 const units: []const struct { []const u8, u64 } = &.{
205 .{ "ns", 1 },
206 .{ "nanosecond", 1 },
207 .{ "us", std.time.ns_per_us },
208 .{ "microsecond", std.time.ns_per_us },
209 .{ "ms", std.time.ns_per_ms },
210 .{ "millisecond", std.time.ns_per_ms },
211 .{ "s", std.time.ns_per_s },
212 .{ "second", std.time.ns_per_s },
213 .{ "m", std.time.ns_per_min },
214 .{ "minute", std.time.ns_per_min },
215 .{ "h", std.time.ns_per_hour },
216 .{ "hour", std.time.ns_per_hour },
217 };
218 const timeout_str = nextArgOrFatal(args, &arg_idx);
219 const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal(
220 "invalid timeout '{s}': expected unit (ns, us, ms, s, m, h)",
221 .{timeout_str},
222 );
223 const num_str = timeout_str[0 .. num_end_idx + 1];
224 const unit_str = timeout_str[num_end_idx + 1 ..];
225 const unit_factor: f64 = for (units) |unit_and_factor| {
226 if (std.mem.eql(u8, unit_str, unit_and_factor[0])) {
227 break @floatFromInt(unit_and_factor[1]);
228 }
229 } else fatal(
230 "invalid timeout '{s}': invalid unit '{s}' (expected ns, us, ms, s, m, h)",
231 .{ timeout_str, unit_str },
232 );
233 const num_parsed = std.fmt.parseFloat(f64, num_str) catch |err| fatal(
234 "invalid timeout '{s}': invalid number '{s}' ({t})",
235 .{ timeout_str, num_str, err },
236 );
237 test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed);
238 } else if (mem.eql(u8, arg, "--search-prefix")) {
239 try search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx));
240 } else if (mem.eql(u8, arg, "--libc")) {
241 libc_file = nextArgOrFatal(args, &arg_idx);
242 } else if (mem.eql(u8, arg, "--color")) {
243 const next_arg = nextArg(args, &arg_idx) orelse
244 fatalWithHint("expected [auto|on|off] after '{s}'", .{arg});
245 color = std.meta.stringToEnum(Color, next_arg) orelse {
246 fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{
247 arg, next_arg,
248 });
249 };
250 } else if (mem.eql(u8, arg, "--error-style")) {
251 const next_arg = nextArg(args, &arg_idx) orelse
252 fatalWithHint("expected style after '{s}'", .{arg});
253 error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse {
254 fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg });
255 };
256 } else if (mem.eql(u8, arg, "--multiline-errors")) {
257 const next_arg = nextArg(args, &arg_idx) orelse
258 fatalWithHint("expected style after '{s}'", .{arg});
259 multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse {
260 fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg });
261 };
262 } else if (mem.eql(u8, arg, "--summary")) {
263 const next_arg = nextArg(args, &arg_idx) orelse
264 fatalWithHint("expected [all|new|failures|line|none] after '{s}'", .{arg});
265 summary = std.meta.stringToEnum(Summary, next_arg) orelse {
266 fatalWithHint("expected [all|new|failures|line|none] after '{s}', found '{s}'", .{
267 arg, next_arg,
268 });
269 };
270 } else if (mem.eql(u8, arg, "--seed")) {
271 const next_arg = nextArg(args, &arg_idx) orelse
272 fatalWithHint("expected u32 after '{s}'", .{arg});
273 graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| {
274 fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {t}", .{ next_arg, err });
275 };
276 } else if (mem.eql(u8, arg, "--debounce")) {
277 const next_arg = nextArg(args, &arg_idx) orelse
278 fatalWithHint("expected u16 after '{s}'", .{arg});
279 debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| {
280 fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {t}\n", .{
281 next_arg, err,
282 });
283 };
284 } else if (mem.eql(u8, arg, "--webui")) {
285 if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) };
286 } else if (mem.startsWith(u8, arg, "--webui=")) {
287 const addr_str = arg["--webui=".len..];
288 if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{});
289 webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| {
290 fatal("invalid web UI address '{s}': {t}", .{ addr_str, err });
291 };
292 } else if (mem.eql(u8, arg, "--debug-log")) {
293 const next_arg = nextArgOrFatal(args, &arg_idx);
294 try debug_log_scopes.append(arena, next_arg);
295 } else if (mem.eql(u8, arg, "--debug-pkg-config")) {
296 debug_pkg_config = true;
297 } else if (mem.eql(u8, arg, "--debug-rt")) {
298 graph.debug_compiler_runtime_libs = .Debug;
299 } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| {
300 graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse
301 fatal("unrecognized optimization mode: {s}", .{rest});
302 } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) {
303 // --glibc-runtimes was the old name of the flag; kept for compatibility for now.
304 libc_runtimes_dir = nextArgOrFatal(args, &arg_idx);
305 } else if (mem.eql(u8, arg, "--watch")) {
306 watch = true;
307 } else if (mem.eql(u8, arg, "--time-report")) {
308 graph.time_report = true;
309 if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) };
310 } else if (mem.eql(u8, arg, "--fuzz")) {
311 fuzz = .{ .forever = undefined };
312 if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) };
313 } else if (mem.startsWith(u8, arg, "--fuzz=")) {
314 const value = arg["--fuzz=".len..];
315 if (value.len == 0) fatal("missing argument to --fuzz", .{});
316
317 const unit: u8 = value[value.len - 1];
318 const digits = switch (unit) {
319 '0'...'9' => value,
320 'K', 'M', 'G' => value[0 .. value.len - 1],
321 else => fatal(
322 "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
323 .{},
324 ),
325 };
326
327 const amount = std.fmt.parseInt(u64, digits, 10) catch {
328 fatal(
329 "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
330 .{},
331 );
332 };
333
334 const normalized_amount = std.math.mul(u64, amount, switch (unit) {
335 else => unreachable,
336 '0'...'9' => 1,
337 'K' => 1000,
338 'M' => 1_000_000,
339 'G' => 1_000_000_000,
340 }) catch fatal("fuzzing limit amount overflows u64", .{});
341
342 fuzz = .{
343 .limit = .{
344 .amount = normalized_amount,
345 },
346 };
347 } else if (mem.eql(u8, arg, "-fincremental")) {
348 graph.incremental = true;
349 } else if (mem.eql(u8, arg, "-fno-incremental")) {
350 graph.incremental = false;
351 } else if (mem.eql(u8, arg, "-fwine")) {
352 enable_wine = true;
353 } else if (mem.eql(u8, arg, "-fno-wine")) {
354 enable_wine = false;
355 } else if (mem.eql(u8, arg, "-fqemu")) {
356 enable_qemu = true;
357 } else if (mem.eql(u8, arg, "-fno-qemu")) {
358 enable_qemu = false;
359 } else if (mem.eql(u8, arg, "-fwasmtime")) {
360 enable_wasmtime = true;
361 } else if (mem.eql(u8, arg, "-fno-wasmtime")) {
362 enable_wasmtime = false;
363 } else if (mem.eql(u8, arg, "-frosetta")) {
364 enable_rosetta = true;
365 } else if (mem.eql(u8, arg, "-fno-rosetta")) {
366 enable_rosetta = false;
367 } else if (mem.eql(u8, arg, "-fdarling")) {
368 enable_darling = true;
369 } else if (mem.eql(u8, arg, "-fno-darling")) {
370 enable_darling = false;
371 } else if (mem.eql(u8, arg, "-fallow-so-scripts")) {
372 graph.allow_so_scripts = true;
373 } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) {
374 graph.allow_so_scripts = false;
375 } else if (mem.eql(u8, arg, "-freference-trace")) {
376 reference_trace = 256;
377 } else if (mem.startsWith(u8, arg, "-freference-trace=")) {
378 const num = arg["-freference-trace=".len..];
379 reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| {
380 std.debug.print("unable to parse reference_trace count '{s}': {t}", .{ num, err });
381 process.exit(1);
382 };
383 } else if (mem.eql(u8, arg, "-fno-reference-trace")) {
384 reference_trace = null;
385 } else if (mem.cutPrefix(u8, arg, "-j")) |text| {
386 const n = std.fmt.parseUnsigned(u32, text, 10) catch |err|
387 fatal("unable to parse jobs count '{s}': {t}", .{ text, err });
388 if (n < 1) fatal("number of jobs must be at least 1", .{});
389 threaded.setAsyncLimit(.limited(n));
390 graph.max_jobs = n;
391 } else if (mem.eql(u8, arg, "--")) {
392 run_args = argsRest(args, arg_idx);
393 break;
394 } else {
395 fatalWithHint("unrecognized argument: '{s}'", .{arg});
396 }
397 } else {
398 try step_names.append(arena, arg);
399 }
400 }
401
402 const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map);
403 const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map);
404
405 graph.stderr_mode = switch (color) {
406 .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE),
407 .on => .escape_codes,
408 .off => .no_color,
409 };
410
411 const scanned_config: ScannedConfig = sc: {
412 const configuration = c: {
413 var file = cwd.openFile(io, configure_path, .{}) catch |err|
414 fatal("failed to open configuration file {s}: {t}", .{ configure_path, err });
415 defer file.close(io);
416 break :c Configuration.loadFile(arena, io, file) catch |err|
417 fatal("failed to load configuration file {s}: {t}", .{ configure_path, err });
418 };
419 var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty;
420 for (configuration.steps, 0..) |*conf_step, step_index_usize| {
421 const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize);
422 const flags: Configuration.Step.Flags = @bitCast(configuration.extra[conf_step.extra_index]);
423 if (flags.tag == .top_level) {
424 const name = step_index.ptr(&configuration).name.slice(&configuration);
425 try top_level_steps.put(arena, name, step_index);
426 }
427 }
428 break :sc .{
429 .configuration = configuration,
430 .top_level_steps = top_level_steps,
431 };
432 };
433
434 if (help_menu) {
435 var w = initStdoutWriter(io);
436 scanned_config.printUsage(&graph, w) catch |err| switch (err) {
437 error.WriteFailed => return stdout_writer_allocation.err.?,
438 else => |e| return e,
439 };
440 w.flush() catch return stdout_writer_allocation.err.?;
441 return;
442 } else if (steps_menu) {
443 var w = initStdoutWriter(io);
444 scanned_config.printSteps(&graph, w) catch |err| switch (err) {
445 error.WriteFailed => return stdout_writer_allocation.err.?,
446 else => |e| return e,
447 };
448 w.flush() catch return stdout_writer_allocation.err.?;
449 return;
450 } else if (print_configuration) {
451 var w = initStdoutWriter(io);
452 scanned_config.print(w) catch return stdout_writer_allocation.err.?;
453 w.flush() catch return stdout_writer_allocation.err.?;
454 return;
455 }
456
457 if (webui_listen != null) {
458 if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{});
459 if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{});
460 }
461
462 const main_progress_node = std.Progress.start(io, .{
463 .disable_printing = (color == .off),
464 });
465 defer main_progress_node.end();
466
467 const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{
468 .root_dir = .cwd(),
469 .sub_path = try Io.Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }),
470 } else if (override_install_prefix) |cwd_relative| .{
471 .root_dir = .cwd(),
472 .sub_path = cwd_relative,
473 } else .{
474 .root_dir = build_root_directory,
475 .sub_path = "zig-out",
476 };
477
478 const install_lib_path: Path = if (override_lib_dir) |cwd_relative| .{
479 .root_dir = .cwd(),
480 .sub_path = cwd_relative,
481 } else try install_prefix_path.join(arena, "lib");
482
483 const install_bin_path: Path = if (override_bin_dir) |cwd_relative| .{
484 .root_dir = .cwd(),
485 .sub_path = cwd_relative,
486 } else try install_prefix_path.join(arena, "bin");
487
488 const install_include_path: Path = if (override_include_dir) |cwd_relative| .{
489 .root_dir = .cwd(),
490 .sub_path = cwd_relative,
491 } else try install_prefix_path.join(arena, "include");
492
493 var maker: Maker = .{
494 .gpa = gpa,
495 .graph = &graph,
496 .scanned_config = &scanned_config,
497 .install_paths = .{
498 .prefix = install_prefix_path,
499 .lib = install_lib_path,
500 .bin = install_bin_path,
501 .include = install_include_path,
502 },
503 .steps = try arena.alloc(Step, scanned_config.configuration.steps.len),
504
505 .available_rss = max_rss,
506 .max_rss_is_default = false,
507 .max_rss_mutex = .init,
508 .skip_oom_steps = skip_oom_steps,
509 .unit_test_timeout_ns = test_timeout_ns,
510
511 .watch = watch,
512 .web_server = undefined, // set after `prepare`
513 .memory_blocked_steps = .empty,
514 .step_stack = .empty,
515
516 .error_style = error_style,
517 .multiline_errors = multiline_errors,
518 .summary = summary orelse if (watch or webui_listen != null) .line else .failures,
519 };
520 defer {
521 maker.memory_blocked_steps.deinit(gpa);
522 maker.step_stack.deinit(gpa);
523 }
524
525 if (maker.available_rss == 0) {
526 maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64);
527 maker.max_rss_is_default = true;
528 }
529
530 maker.prepare(step_names.items) catch |err| switch (err) {
531 error.DependencyLoopDetected, error.InsufficientMemory => {
532 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
533 process.exit(1);
534 },
535 else => |e| return e,
536 };
537
538 var w: Watch = w: {
539 if (!watch) break :w undefined;
540 if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag});
541 break :w try .init(graph.cache.cwd, &scanned_config.configuration, maker.steps);
542 };
543
544 const now = Io.Clock.Timestamp.now(io, .awake);
545
546 maker.web_server = if (webui_listen) |listen_address| ws: {
547 if (builtin.single_threaded) unreachable; // `fatal` above
548 break :ws .init(.{
549 .gpa = gpa,
550 .graph = &graph,
551 .all_steps = maker.step_stack.keys(),
552 .root_prog_node = main_progress_node,
553 .watch = watch,
554 .listen_address = listen_address,
555 .base_timestamp = now,
556 .configuration = &scanned_config.configuration,
557 });
558 } else null;
559
560 if (maker.web_server) |*ws| {
561 ws.start() catch |err| fatal("failed to start web server: {t}", .{err});
562 }
563
564 rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) {
565 const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
566 defer io.unlockStderr();
567 try stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H");
568 }) {
569 if (maker.web_server) |*ws| ws.startBuild();
570
571 try maker.makeStepNames(step_names.items, main_progress_node, fuzz);
572
573 if (maker.web_server) |*web_server| {
574 if (fuzz) |mode| if (mode != .forever) fatal(
575 "error: limited fuzzing is not implemented yet for --webui",
576 .{},
577 );
578
579 web_server.finishBuild(.{ .fuzz = fuzz != null });
580 }
581
582 if (maker.web_server) |*ws| {
583 const c = &scanned_config.configuration;
584 assert(!watch); // fatal error after CLI parsing
585 while (true) switch (try ws.wait()) {
586 .rebuild => {
587 for (maker.step_stack.keys()) |step_index| {
588 const step = maker.stepByIndex(step_index);
589 step.state = .precheck_done;
590 const deps = step_index.ptr(c).deps.slice(c);
591 step.pending_deps = @intCast(deps.len);
592 step.reset(gpa);
593 }
594 continue :rebuild;
595 },
596 };
597 }
598
599 // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`.
600 if (!Watch.have_impl) unreachable;
601
602 try w.update(gpa, maker.step_stack.keys());
603
604 // Wait until a file system notification arrives. Read all such events
605 // until the buffer is empty. Then wait for a debounce interval, resetting
606 // if any more events come in. After the debounce interval has passed,
607 // trigger a rebuild on all steps with modified inputs, as well as their
608 // recursive dependants.
609 var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined;
610 const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{
611 w.dir_count, countSubProcesses(maker.steps, maker.step_stack.keys()),
612 }) catch &caption_buf;
613 var debouncing_node = main_progress_node.start(caption, 0);
614 var in_debounce = false;
615 while (true) switch (try w.wait(gpa, io, if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) {
616 .timeout => {
617 assert(in_debounce);
618 debouncing_node.end();
619 markFailedStepsDirty(gpa, maker.steps, maker.step_stack.keys());
620 continue :rebuild;
621 },
622 .dirty => if (!in_debounce) {
623 in_debounce = true;
624 debouncing_node.end();
625 debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0);
626 },
627 .clean => {},
628 };
629 }
630}
631
632fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const Configuration.Step.Index) void {
633 for (all_steps) |step_index| {
634 const step = &make_steps[@intFromEnum(step_index)];
635 switch (step.state) {
636 .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa),
637 else => continue,
638 }
639 }
640 // Now that all dirty steps have been found, the remaining steps that
641 // succeeded from last run shall be marked "cached".
642 for (all_steps) |step_index| {
643 const step = &make_steps[@intFromEnum(step_index)];
644 switch (step.state) {
645 .success => step.result_cached = true,
646 else => continue,
647 }
648 }
649}
650
651fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.Index) usize {
652 var count: usize = 0;
653 for (all_steps) |step_index| {
654 const s = &make_steps[@intFromEnum(step_index)];
655 count += @intFromBool(s.getZigProcess() != null);
656 }
657 return count;
658}
659
660const InstallPaths = struct {
661 prefix: Path,
662 lib: Path,
663 bin: Path,
664 include: Path,
665};
666
667fn stepByIndex(maker: *const Maker, i: Configuration.Step.Index) *Step {
668 return &maker.steps[@intFromEnum(i)];
669}
670
671fn prepare(maker: *Maker, step_names: []const []const u8) !void {
672 const gpa = maker.gpa;
673 const graph = maker.graph;
674 const arena = graph.arena;
675 const seed: u32 = graph.random_seed;
676 const step_stack = &maker.step_stack;
677 const c = &maker.scanned_config.configuration;
678
679 @memset(maker.steps, .{});
680
681 if (step_names.len == 0) {
682 try step_stack.put(gpa, c.default_step, {});
683 } else {
684 try step_stack.ensureUnusedCapacity(gpa, step_names.len);
685 for (0..step_names.len) |i| {
686 const step_name = step_names[step_names.len - i - 1];
687 const s = maker.scanned_config.top_level_steps.get(step_name) orelse {
688 log.info("to list available steps: zig build -l", .{});
689 fatal("no such step: {s}", .{step_name});
690 };
691 step_stack.putAssumeCapacity(s, {});
692 }
693 }
694
695 const starting_steps = try arena.dupe(Configuration.Step.Index, step_stack.keys());
696
697 var rng = std.Random.DefaultPrng.init(seed);
698 const rand = rng.random();
699 rand.shuffle(Configuration.Step.Index, starting_steps);
700
701 for (starting_steps) |s| {
702 try constructGraphAndCheckForDependencyLoop(gpa, c, maker.steps, s, &maker.step_stack, rand);
703 }
704
705 {
706 // Check that we have enough memory to complete the build.
707 var any_problems = false;
708 var max_needed: usize = 0;
709 for (step_stack.keys()) |step_index| {
710 const make_step = maker.stepByIndex(step_index);
711 const conf_step = step_index.ptr(c);
712 const max_rss = conf_step.max_rss.toBytes();
713 if (max_rss == 0) continue;
714 max_needed = @max(max_needed, max_rss);
715 if (max_rss > maker.available_rss) {
716 if (maker.skip_oom_steps) {
717 make_step.state = .skipped_oom;
718 for (make_step.dependants.items) |dependant| {
719 maker.stepByIndex(dependant).pending_deps -= 1;
720 }
721 } else {
722 log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{
723 conf_step.owner.depPrefixSlice(c),
724 conf_step.name.slice(c),
725 max_rss,
726 maker.available_rss,
727 });
728 any_problems = true;
729 }
730 }
731 }
732 if (any_problems) {
733 if (maker.max_rss_is_default) {
734 std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{
735 max_needed,
736 });
737 }
738 return error.InsufficientMemory;
739 }
740 }
741}
742
743fn makeStepNames(
744 maker: *Maker,
745 step_names: []const []const u8,
746 parent_prog_node: std.Progress.Node,
747 fuzz: ?Fuzz.Mode,
748) !void {
749 const graph = maker.graph;
750 const gpa = maker.gpa;
751 const io = graph.io;
752 const step_stack = &maker.step_stack;
753 const top_level_steps = &maker.scanned_config.top_level_steps;
754 const c = &maker.scanned_config.configuration;
755
756 {
757 // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer,
758 // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking
759 // a step is initial when it actually became ready due to an earlier initial step.
760 var initial_set: std.ArrayList(Configuration.Step.Index) = .empty;
761 defer initial_set.deinit(gpa);
762 try initial_set.ensureUnusedCapacity(gpa, step_stack.count());
763 for (step_stack.keys()) |step_index| {
764 const s = maker.stepByIndex(step_index);
765 if (s.state == .precheck_done and s.pending_deps == 0) {
766 initial_set.appendAssumeCapacity(step_index);
767 }
768 }
769
770 const step_prog = parent_prog_node.start("steps", step_stack.count());
771 defer step_prog.end();
772
773 var group: Io.Group = .init;
774 defer group.cancel(io);
775 // Start working on all of the initial steps...
776 for (initial_set.items) |step_index| try stepReady(maker, &group, step_index, step_prog);
777 // ...and `makeStep` will trigger every other step when their last dependency finishes.
778 try group.await(io);
779 }
780
781 assert(maker.memory_blocked_steps.items.len == 0);
782
783 var test_pass_count: usize = 0;
784 var test_skip_count: usize = 0;
785 var test_fail_count: usize = 0;
786 var test_crash_count: usize = 0;
787 var test_timeout_count: usize = 0;
788
789 var test_count: usize = 0;
790
791 var success_count: usize = 0;
792 var skipped_count: usize = 0;
793 var failure_count: usize = 0;
794 var pending_count: usize = 0;
795 var total_compile_errors: usize = 0;
796
797 var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() });
798 defer cleanup_task.await(io);
799
800 for (step_stack.keys()) |step_index| {
801 const make_step = maker.stepByIndex(step_index);
802 test_pass_count += make_step.test_results.passCount();
803 test_skip_count += make_step.test_results.skip_count;
804 test_fail_count += make_step.test_results.fail_count;
805 test_crash_count += make_step.test_results.crash_count;
806 test_timeout_count += make_step.test_results.timeout_count;
807
808 test_count += make_step.test_results.test_count;
809
810 switch (make_step.state) {
811 .precheck_unstarted => unreachable,
812 .precheck_started => unreachable,
813 .precheck_done => unreachable,
814 .dependency_failure => pending_count += 1,
815 .success => success_count += 1,
816 .skipped, .skipped_oom => skipped_count += 1,
817 .failure => {
818 failure_count += 1;
819 const compile_errors_len = make_step.result_error_bundle.errorMessageCount();
820 if (compile_errors_len > 0) {
821 total_compile_errors += compile_errors_len;
822 }
823 },
824 }
825 }
826
827 if (fuzz) |mode| blk: {
828 switch (builtin.os.tag) {
829 // Current implementation depends on two things that need to be ported to Windows:
830 // * Memory-mapping to share data between the fuzzer and build runner.
831 // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
832 // many addresses to source locations).
833 .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}),
834 else => {},
835 }
836 if (@bitSizeOf(usize) != 64) {
837 // Current implementation depends on posix.mmap()'s second parameter, `length: usize`,
838 // being compatible with file system's u64 return value. This is not the case
839 // on 32-bit platforms.
840 // Affects or affected by issues #5185, #22523, and #22464.
841 fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
842 }
843
844 switch (mode) {
845 .forever => break :blk,
846 .limit => {},
847 }
848
849 assert(mode == .limit);
850 var f = Fuzz.init(
851 gpa,
852 io,
853 step_stack.keys(),
854 parent_prog_node,
855 mode,
856 ) catch |err| fatal("failed to start fuzzer: {t}", .{err});
857 defer f.deinit();
858
859 f.start();
860 try f.waitAndPrintReport();
861 }
862
863 // Every test has a state
864 assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count);
865
866 if (failure_count == 0) {
867 std.Progress.setStatus(.success);
868 } else {
869 std.Progress.setStatus(.failure);
870 }
871
872 summary: {
873 switch (maker.summary) {
874 .all, .new, .line => {},
875 .failures => if (failure_count == 0) break :summary,
876 .none => break :summary,
877 }
878
879 const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
880 defer io.unlockStderr();
881 const t = stderr.terminal();
882 const w = &stderr.file_writer.interface;
883
884 const total_count = success_count + failure_count + pending_count + skipped_count;
885 t.setColor(.cyan) catch {};
886 t.setColor(.bold) catch {};
887 w.writeAll("Build Summary: ") catch {};
888 t.setColor(.reset) catch {};
889 w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {};
890 {
891 t.setColor(.dim) catch {};
892 var first = true;
893 if (skipped_count > 0) {
894 w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {};
895 first = false;
896 }
897 if (failure_count > 0) {
898 w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {};
899 first = false;
900 }
901 if (!first) w.writeByte(')') catch {};
902 t.setColor(.reset) catch {};
903 }
904
905 if (test_count > 0) {
906 w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {};
907 t.setColor(.dim) catch {};
908 var first = true;
909 if (test_skip_count > 0) {
910 w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {};
911 first = false;
912 }
913 if (test_fail_count > 0) {
914 w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {};
915 first = false;
916 }
917 if (test_crash_count > 0) {
918 w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {};
919 first = false;
920 }
921 if (test_timeout_count > 0) {
922 w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {};
923 first = false;
924 }
925 if (!first) w.writeByte(')') catch {};
926 t.setColor(.reset) catch {};
927 }
928
929 w.writeAll("\n") catch {};
930
931 if (maker.summary == .line) break :summary;
932
933 // Print a fancy tree with build results.
934 var step_stack_copy = try step_stack.clone(gpa);
935 defer step_stack_copy.deinit(gpa);
936
937 var print_node: PrintNode = .{ .parent = null };
938 if (step_names.len == 0) {
939 print_node.last = true;
940 printTreeStep(maker, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) {
941 error.Canceled => |e| return e,
942 else => {},
943 };
944 } else {
945 const last_index = if (maker.summary == .all) top_level_steps.count() else blk: {
946 var i: usize = step_names.len;
947 while (i > 0) {
948 i -= 1;
949 const step_index = top_level_steps.get(step_names[i]).?;
950 const step = maker.stepByIndex(step_index);
951 const found = switch (maker.summary) {
952 .all, .line, .none => unreachable,
953 .failures => step.state != .success,
954 .new => !step.result_cached,
955 };
956 if (found) break :blk i;
957 }
958 break :blk top_level_steps.count();
959 };
960 for (step_names, 0..) |step_name, i| {
961 const step_index = top_level_steps.get(step_name).?;
962 print_node.last = i + 1 == last_index;
963 printTreeStep(maker, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) {
964 error.Canceled => |e| return e,
965 else => {},
966 };
967 }
968 }
969 w.writeByte('\n') catch {};
970 }
971
972 if (maker.watch or maker.web_server != null) return;
973
974 // Perhaps in the future there could be an Advanced Options flag such as
975 // --debug-build-runner-leaks which would make this code return instead of
976 // calling exit.
977
978 const code: u8 = code: {
979 if (failure_count == 0) break :code 0; // success
980 if (maker.error_style.verboseContext()) break :code 1; // failure; print build command
981 break :code 2; // failure; do not print build command
982 };
983 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
984 process.exit(code);
985}
986
987fn stepReady(
988 maker: *Maker,
989 group: *Io.Group,
990 step_index: Configuration.Step.Index,
991 root_prog_node: std.Progress.Node,
992) Io.Cancelable!void {
993 const graph = maker.graph;
994 const io = graph.io;
995 const c = &maker.scanned_config.configuration;
996 const max_rss = step_index.ptr(c).max_rss.toBytes();
997 if (max_rss != 0) {
998 try maker.max_rss_mutex.lock(io);
999 defer maker.max_rss_mutex.unlock(io);
1000 if (maker.available_rss < max_rss) {
1001 // Running this step right now could possibly exceed the allotted RSS.
1002 maker.memory_blocked_steps.append(maker.gpa, step_index) catch
1003 @panic("TODO eliminate memory allocation here");
1004 return;
1005 }
1006 maker.available_rss -= max_rss;
1007 }
1008 group.async(io, makeStep, .{ maker, group, step_index, root_prog_node });
1009}
1010
1011/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready
1012/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must
1013/// have already subtracted this value from `maker.available_rss`. This function will release the RSS
1014/// claim (i.e. add `s.max_rss` back into `maker.available_rss`) and queue any viable memory-blocked
1015/// steps after "make" completes for `s`.
1016fn makeStep(
1017 maker: *Maker,
1018 group: *Io.Group,
1019 step_index: Configuration.Step.Index,
1020 root_prog_node: std.Progress.Node,
1021) Io.Cancelable!void {
1022 const graph = maker.graph;
1023 const io = graph.io;
1024 const gpa = maker.gpa;
1025 const c = &maker.scanned_config.configuration;
1026 const conf_step = step_index.ptr(c);
1027 const step_name = conf_step.name.slice(c);
1028 const deps = conf_step.deps.slice(c);
1029 const make_step = maker.stepByIndex(step_index);
1030
1031 {
1032 const step_prog_node = root_prog_node.start(step_name, 0);
1033 defer step_prog_node.end();
1034
1035 if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .wip);
1036
1037 const new_state: Step.State = for (deps) |dep_index| {
1038 const dep_make_step = maker.stepByIndex(dep_index);
1039 switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) {
1040 .precheck_unstarted => unreachable,
1041 .precheck_started => unreachable,
1042 .precheck_done => unreachable,
1043
1044 .failure,
1045 .dependency_failure,
1046 .skipped_oom,
1047 => break .dependency_failure,
1048
1049 .success, .skipped => {},
1050 }
1051 } else if (make_step.make(.{
1052 .progress_node = step_prog_node,
1053 .watch = maker.watch,
1054 .web_server = if (maker.web_server) |*ws| ws else null,
1055 .unit_test_timeout_ns = maker.unit_test_timeout_ns,
1056 .gpa = gpa,
1057 })) state: {
1058 break :state .success;
1059 } else |err| switch (err) {
1060 error.MakeFailed => .failure,
1061 error.MakeSkipped => .skipped,
1062 };
1063
1064 @atomicStore(Step.State, &make_step.state, new_state, .monotonic);
1065
1066 switch (new_state) {
1067 .precheck_unstarted => unreachable,
1068 .precheck_started => unreachable,
1069 .precheck_done => unreachable,
1070
1071 .failure,
1072 .dependency_failure,
1073 .skipped_oom,
1074 => {
1075 if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .failure);
1076 std.Progress.setStatus(.failure_working);
1077 },
1078
1079 .success,
1080 .skipped,
1081 => {
1082 if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .success);
1083 },
1084 }
1085 }
1086
1087 // No matter the result, we want to display error/warning messages.
1088 if (make_step.result_error_bundle.errorMessageCount() > 0 or
1089 make_step.result_error_msgs.items.len > 0 or
1090 make_step.result_stderr.len > 0)
1091 {
1092 const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
1093 defer io.unlockStderr();
1094 printErrorMessages(gpa, c, maker.steps, step_index, .{}, stderr.terminal(), maker.error_style, maker.multiline_errors) catch |err| switch (err) {
1095 error.Canceled => |e| return e,
1096 error.WriteFailed => switch (stderr.file_writer.err.?) {
1097 error.Canceled => |e| return e,
1098 else => {},
1099 },
1100 else => {},
1101 };
1102 }
1103
1104 const max_rss = conf_step.max_rss.toBytes();
1105 if (max_rss != 0) {
1106 var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty;
1107 defer dispatch_set.deinit(gpa);
1108
1109 // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set`
1110 // as a staging buffer to avoid recursing into `makeStep` while `maker.max_rss_mutex` is held.
1111 {
1112 try maker.max_rss_mutex.lock(io);
1113 defer maker.max_rss_mutex.unlock(io);
1114 maker.available_rss += max_rss;
1115 dispatch_set.ensureUnusedCapacity(gpa, maker.memory_blocked_steps.items.len) catch
1116 @panic("TODO eliminate memory allocation here");
1117 while (maker.memory_blocked_steps.getLast()) |candidate_index| {
1118 const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes();
1119 if (maker.available_rss < candidate_max_rss) break;
1120 assert(maker.memory_blocked_steps.pop() == candidate_index);
1121 dispatch_set.appendAssumeCapacity(candidate_index);
1122 }
1123 }
1124 for (dispatch_set.items) |candidate| {
1125 group.async(io, makeStep, .{ maker, group, candidate, root_prog_node });
1126 }
1127 }
1128
1129 for (make_step.dependants.items) |dependant_index| {
1130 const dependant = maker.stepByIndex(dependant_index);
1131 // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0.
1132 if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) {
1133 try stepReady(maker, group, dependant_index, root_prog_node);
1134 }
1135 }
1136}
1137
1138fn printTreeStep(
1139 maker: *const Maker,
1140 step_index: Configuration.Step.Index,
1141 stderr: Io.Terminal,
1142 parent_node: *PrintNode,
1143 step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void),
1144) !void {
1145 const writer = stderr.writer;
1146 const first = step_stack.swapRemove(step_index);
1147 const summary = maker.summary;
1148 const c = &maker.scanned_config.configuration;
1149 const conf_step = step_index.ptr(c);
1150 const make_step = maker.stepByIndex(step_index);
1151 const skip = switch (summary) {
1152 .none, .line => unreachable,
1153 .all => false,
1154 .new => make_step.result_cached,
1155 .failures => make_step.state == .success,
1156 };
1157 if (skip) return;
1158 try printPrefix(parent_node, stderr);
1159
1160 if (parent_node.parent != null) {
1161 if (parent_node.last) {
1162 try printChildNodePrefix(stderr);
1163 } else {
1164 try writer.writeAll(switch (stderr.mode) {
1165 .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─
1166 else => "+- ",
1167 });
1168 }
1169 }
1170
1171 if (!first) try stderr.setColor(.dim);
1172
1173 // dep_prefix omitted here because it is redundant with the tree.
1174 try writer.writeAll(conf_step.name.slice(c));
1175
1176 const deps = conf_step.deps.slice(c);
1177
1178 if (first) {
1179 try printStepStatus(maker, step_index, stderr);
1180
1181 const last_index = if (summary == .all) deps.len -| 1 else blk: {
1182 var i: usize = deps.len;
1183 while (i > 0) {
1184 i -= 1;
1185
1186 const dep_index = deps[i];
1187 const dep = maker.stepByIndex(dep_index);
1188 const found = switch (summary) {
1189 .all, .line, .none => unreachable,
1190 .failures => dep.state != .success,
1191 .new => !dep.result_cached,
1192 };
1193 if (found) break :blk i;
1194 }
1195 break :blk deps.len -| 1;
1196 };
1197 for (deps, 0..) |dep, i| {
1198 var print_node: PrintNode = .{
1199 .parent = parent_node,
1200 .last = i == last_index,
1201 };
1202 try printTreeStep(maker, dep, stderr, &print_node, step_stack);
1203 }
1204 } else {
1205 if (deps.len == 0) {
1206 try writer.writeAll(" (reused)\n");
1207 } else {
1208 try writer.print(" (+{d} more reused dependencies)\n", .{deps.len});
1209 }
1210 try stderr.setColor(.reset);
1211 }
1212}
1213
1214fn printStepStatus(maker: *const Maker, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void {
1215 const s = maker.stepByIndex(step_index);
1216 const writer = stderr.writer;
1217 switch (s.state) {
1218 .precheck_unstarted => unreachable,
1219 .precheck_started => unreachable,
1220 .precheck_done => unreachable,
1221
1222 .dependency_failure => {
1223 try stderr.setColor(.dim);
1224 try writer.writeAll(" transitive failure\n");
1225 try stderr.setColor(.reset);
1226 },
1227
1228 .success => {
1229 try stderr.setColor(.green);
1230 if (s.result_cached) {
1231 try writer.writeAll(" cached");
1232 } else if (s.test_results.test_count > 0) {
1233 const pass_count = s.test_results.passCount();
1234 assert(s.test_results.test_count == pass_count + s.test_results.skip_count);
1235 try writer.print(" {d} pass", .{pass_count});
1236 if (s.test_results.skip_count > 0) {
1237 try stderr.setColor(.reset);
1238 try writer.writeAll(", ");
1239 try stderr.setColor(.yellow);
1240 try writer.print("{d} skip", .{s.test_results.skip_count});
1241 }
1242 try stderr.setColor(.reset);
1243 try writer.print(" ({d} total)", .{s.test_results.test_count});
1244 } else {
1245 try writer.writeAll(" success");
1246 }
1247 try stderr.setColor(.reset);
1248 if (s.result_duration_ns) |ns| {
1249 try stderr.setColor(.dim);
1250 if (ns >= std.time.ns_per_min) {
1251 try writer.print(" {d}m", .{ns / std.time.ns_per_min});
1252 } else if (ns >= std.time.ns_per_s) {
1253 try writer.print(" {d}s", .{ns / std.time.ns_per_s});
1254 } else if (ns >= std.time.ns_per_ms) {
1255 try writer.print(" {d}ms", .{ns / std.time.ns_per_ms});
1256 } else if (ns >= std.time.ns_per_us) {
1257 try writer.print(" {d}us", .{ns / std.time.ns_per_us});
1258 } else {
1259 try writer.print(" {d}ns", .{ns});
1260 }
1261 try stderr.setColor(.reset);
1262 }
1263 if (s.result_peak_rss != 0) {
1264 const rss = s.result_peak_rss;
1265 try stderr.setColor(.dim);
1266 if (rss >= 1000_000_000) {
1267 try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000});
1268 } else if (rss >= 1000_000) {
1269 try writer.print(" MaxRSS:{d}M", .{rss / 1000_000});
1270 } else if (rss >= 1000) {
1271 try writer.print(" MaxRSS:{d}K", .{rss / 1000});
1272 } else {
1273 try writer.print(" MaxRSS:{d}B", .{rss});
1274 }
1275 try stderr.setColor(.reset);
1276 }
1277 try writer.writeAll("\n");
1278 },
1279 .skipped => {
1280 try stderr.setColor(.yellow);
1281 try writer.writeAll(" skipped\n");
1282 try stderr.setColor(.reset);
1283 },
1284 .skipped_oom => {
1285 const c = &maker.scanned_config.configuration;
1286 const max_rss = step_index.ptr(c).max_rss.toBytes();
1287 try stderr.setColor(.yellow);
1288 try writer.writeAll(" skipped (not enough memory)");
1289 try stderr.setColor(.dim);
1290 try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{
1291 max_rss, maker.available_rss,
1292 });
1293 try stderr.setColor(.reset);
1294 },
1295 .failure => {
1296 try printStepFailure(maker.steps, step_index, stderr, false);
1297 try stderr.setColor(.reset);
1298 },
1299 }
1300}
1301
1302fn printStepFailure(
1303 make_steps: []Step,
1304 step_index: Configuration.Step.Index,
1305 stderr: Io.Terminal,
1306 dim: bool,
1307) !void {
1308 const w = stderr.writer;
1309 const s = &make_steps[@intFromEnum(step_index)];
1310 if (s.result_error_bundle.errorMessageCount() > 0) {
1311 try stderr.setColor(.red);
1312 try w.print(" {d} errors\n", .{
1313 s.result_error_bundle.errorMessageCount(),
1314 });
1315 } else if (!s.test_results.isSuccess()) {
1316 // These first values include all of the test "statuses". Every test is either passsed,
1317 // skipped, failed, crashed, or timed out.
1318 try stderr.setColor(.green);
1319 try w.print(" {d} pass", .{s.test_results.passCount()});
1320 try stderr.setColor(.reset);
1321 if (dim) try stderr.setColor(.dim);
1322 if (s.test_results.skip_count > 0) {
1323 try w.writeAll(", ");
1324 try stderr.setColor(.yellow);
1325 try w.print("{d} skip", .{s.test_results.skip_count});
1326 try stderr.setColor(.reset);
1327 if (dim) try stderr.setColor(.dim);
1328 }
1329 if (s.test_results.fail_count > 0) {
1330 try w.writeAll(", ");
1331 try stderr.setColor(.red);
1332 try w.print("{d} fail", .{s.test_results.fail_count});
1333 try stderr.setColor(.reset);
1334 if (dim) try stderr.setColor(.dim);
1335 }
1336 if (s.test_results.crash_count > 0) {
1337 try w.writeAll(", ");
1338 try stderr.setColor(.red);
1339 try w.print("{d} crash", .{s.test_results.crash_count});
1340 try stderr.setColor(.reset);
1341 if (dim) try stderr.setColor(.dim);
1342 }
1343 if (s.test_results.timeout_count > 0) {
1344 try w.writeAll(", ");
1345 try stderr.setColor(.red);
1346 try w.print("{d} timeout", .{s.test_results.timeout_count});
1347 try stderr.setColor(.reset);
1348 if (dim) try stderr.setColor(.dim);
1349 }
1350 try w.print(" ({d} total)", .{s.test_results.test_count});
1351
1352 // Memory leaks are intentionally written after the total, because is isn't a test *status*,
1353 // but just a flag that any tests -- even passed ones -- can have. We also use a different
1354 // separator, so it looks like:
1355 // 2 pass, 1 skip, 2 fail (5 total); 2 leaks
1356 if (s.test_results.leak_count > 0) {
1357 try w.writeAll("; ");
1358 try stderr.setColor(.red);
1359 try w.print("{d} leaks", .{s.test_results.leak_count});
1360 try stderr.setColor(.reset);
1361 if (dim) try stderr.setColor(.dim);
1362 }
1363
1364 // It's usually not helpful to know how many error logs there were because they tend to
1365 // just come with other errors (e.g. crashes and leaks print stack traces, and clean
1366 // failures print error traces). So only mention them if they're the only thing causing
1367 // the failure.
1368 const show_err_logs: bool = show: {
1369 var alt_results = s.test_results;
1370 alt_results.log_err_count = 0;
1371 break :show alt_results.isSuccess();
1372 };
1373 if (show_err_logs) {
1374 try w.writeAll("; ");
1375 try stderr.setColor(.red);
1376 try w.print("{d} error logs", .{s.test_results.log_err_count});
1377 try stderr.setColor(.reset);
1378 if (dim) try stderr.setColor(.dim);
1379 }
1380
1381 try w.writeAll("\n");
1382 } else if (s.result_error_msgs.items.len > 0) {
1383 try stderr.setColor(.red);
1384 try w.writeAll(" failure\n");
1385 } else {
1386 assert(s.result_stderr.len > 0);
1387 try stderr.setColor(.red);
1388 try w.writeAll(" w\n");
1389 }
1390}
1391
1392const PrintNode = struct {
1393 parent: ?*PrintNode,
1394 last: bool = false,
1395};
1396
1397fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void {
1398 const parent = node.parent orelse return;
1399 const writer = stderr.writer;
1400 if (parent.parent == null) return;
1401 try printPrefix(parent, stderr);
1402 if (parent.last) {
1403 try writer.writeAll(" ");
1404 } else {
1405 try writer.writeAll(switch (stderr.mode) {
1406 .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │
1407 else => "| ",
1408 });
1409 }
1410}
1411
1412fn printChildNodePrefix(stderr: Io.Terminal) !void {
1413 try stderr.writer.writeAll(switch (stderr.mode) {
1414 .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─
1415 else => "+- ",
1416 });
1417}
1418
1419/// Traverse the dependency graph depth-first and make it undirected by having
1420/// steps know their dependants (they only know dependencies at start).
1421/// Along the way, check that there is no dependency loop, and record the steps
1422/// in traversal order in `step_stack`.
1423/// Each step has its dependencies traversed in random order, this accomplishes
1424/// two things:
1425/// - `step_stack` will be in randomized-depth-first order, so the build runner
1426/// spawns initial steps in a random order
1427/// - each step's `dependants` list is also filled in a random order, so that
1428/// when it finishes executing in `makeStep`, it spawns next steps to run in
1429/// random order
1430fn constructGraphAndCheckForDependencyLoop(
1431 gpa: Allocator,
1432 c: *const Configuration,
1433 steps: []Step,
1434 step_index: Configuration.Step.Index,
1435 step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void),
1436 rand: std.Random,
1437) error{ DependencyLoopDetected, OutOfMemory }!void {
1438 const make_step: *Step = &steps[@intFromEnum(step_index)];
1439 switch (make_step.state) {
1440 .precheck_started => {
1441 log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)});
1442 return error.DependencyLoopDetected;
1443 },
1444 .precheck_unstarted => {
1445 make_step.state = .precheck_started;
1446
1447 const step = step_index.ptr(c);
1448 const dependencies = step.deps.slice(c);
1449 try step_stack.ensureUnusedCapacity(gpa, dependencies.len);
1450
1451 // We dupe to avoid shuffling the steps in the summary, it depends
1452 // on dependencies' order.
1453 const deps = try gpa.dupe(Configuration.Step.Index, dependencies);
1454 defer gpa.free(deps);
1455
1456 rand.shuffle(Configuration.Step.Index, deps);
1457
1458 for (deps) |dep| {
1459 const dep_step: *Step = &steps[@intFromEnum(dep)];
1460 try step_stack.put(gpa, dep, {});
1461 try dep_step.dependants.append(gpa, step_index);
1462 constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) {
1463 error.DependencyLoopDetected => {
1464 log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)});
1465 return err;
1466 },
1467 else => return err,
1468 };
1469 }
1470
1471 make_step.state = .precheck_done;
1472 make_step.pending_deps = @intCast(dependencies.len);
1473 },
1474 .precheck_done => {},
1475
1476 // These don't happen until we actually run the step graph.
1477 .dependency_failure => unreachable,
1478 .success => unreachable,
1479 .failure => unreachable,
1480 .skipped => unreachable,
1481 .skipped_oom => unreachable,
1482 }
1483}
1484
1485pub fn printErrorMessages(
1486 gpa: Allocator,
1487 c: *const Configuration,
1488 make_steps: []Step,
1489 failing_step_index: Configuration.Step.Index,
1490 options: std.zig.ErrorBundle.RenderOptions,
1491 stderr: Io.Terminal,
1492 error_style: ErrorStyle,
1493 multiline_errors: MultilineErrors,
1494) !void {
1495 const writer = stderr.writer;
1496 if (error_style.verboseContext()) {
1497 // Provide context for where these error messages are coming from by
1498 // printing the corresponding Step subtree.
1499 var step_stack: std.ArrayList(Configuration.Step.Index) = .empty;
1500 defer step_stack.deinit(gpa);
1501 try step_stack.append(gpa, failing_step_index);
1502 while (true) {
1503 const last_step = &make_steps[@intFromEnum(step_stack.items[step_stack.items.len - 1])];
1504 if (last_step.dependants.items.len == 0) break;
1505 try step_stack.append(gpa, last_step.dependants.items[0]);
1506 }
1507
1508 // Now, `step_stack` has the subtree that we want to print, in reverse order.
1509 try stderr.setColor(.dim);
1510 var indent: usize = 0;
1511 while (step_stack.pop()) |step_index| : (indent += 1) {
1512 if (indent > 0) {
1513 try writer.splatByteAll(' ', (indent - 1) * 3);
1514 try printChildNodePrefix(stderr);
1515 }
1516
1517 try writer.writeAll(step_index.ptr(c).name.slice(c));
1518
1519 if (step_index == failing_step_index) {
1520 try printStepFailure(make_steps, step_index, stderr, true);
1521 } else {
1522 try writer.writeAll("\n");
1523 }
1524 }
1525 try stderr.setColor(.reset);
1526 } else {
1527 // Just print the failing step itself.
1528 try stderr.setColor(.dim);
1529 try writer.writeAll(failing_step_index.ptr(c).name.slice(c));
1530 try printStepFailure(make_steps, failing_step_index, stderr, true);
1531 try stderr.setColor(.reset);
1532 }
1533
1534 const failing_step = &make_steps[@intFromEnum(failing_step_index)];
1535
1536 if (failing_step.result_stderr.len > 0) {
1537 try writer.writeAll(failing_step.result_stderr);
1538 if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) {
1539 try writer.writeAll("\n");
1540 }
1541 }
1542
1543 try failing_step.result_error_bundle.renderToTerminal(options, stderr);
1544
1545 for (failing_step.result_error_msgs.items) |msg| {
1546 try stderr.setColor(.red);
1547 try writer.writeAll("error:");
1548 try stderr.setColor(.reset);
1549 if (std.mem.indexOfScalar(u8, msg, '\n') == null) {
1550 try writer.print(" {s}\n", .{msg});
1551 } else switch (multiline_errors) {
1552 .indent => {
1553 var it = std.mem.splitScalar(u8, msg, '\n');
1554 try writer.print(" {s}\n", .{it.first()});
1555 while (it.next()) |line| {
1556 try writer.print(" {s}\n", .{line});
1557 }
1558 },
1559 .newline => try writer.print("\n{s}\n", .{msg}),
1560 .none => try writer.print(" {s}\n", .{msg}),
1561 }
1562 }
1563
1564 if (error_style.verboseContext()) {
1565 if (failing_step.result_failed_command) |cmd_str| {
1566 try stderr.setColor(.red);
1567 try writer.writeAll("failed command: ");
1568 try stderr.setColor(.reset);
1569 try writer.writeAll(cmd_str);
1570 try writer.writeByte('\n');
1571 }
1572 }
1573
1574 try writer.writeByte('\n');
1575}
1576
1577fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 {
1578 if (idx.* >= args.len) return null;
1579 defer idx.* += 1;
1580 return args[idx.*];
1581}
1582
1583fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 {
1584 return nextArg(args, idx) orelse {
1585 fatalWithHint("expected argument after {q}", .{args[idx.* - 1]});
1586 };
1587}
1588
1589fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 {
1590 const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first});
1591 if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg });
1592 const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first});
1593 return arg;
1594}
1595
1596fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 {
1597 if (idx >= args.len) return null;
1598 return args[idx..];
1599}
1600
1601const Color = std.zig.Color;
1602const ErrorStyle = enum {
1603 verbose,
1604 minimal,
1605 verbose_clear,
1606 minimal_clear,
1607 fn verboseContext(s: ErrorStyle) bool {
1608 return switch (s) {
1609 .verbose, .verbose_clear => true,
1610 .minimal, .minimal_clear => false,
1611 };
1612 }
1613 fn clearOnUpdate(s: ErrorStyle) bool {
1614 return switch (s) {
1615 .verbose, .minimal => false,
1616 .verbose_clear, .minimal_clear => true,
1617 };
1618 }
1619};
1620const MultilineErrors = enum { indent, newline, none };
1621const Summary = enum { all, new, failures, line, none };
1622
1623fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn {
1624 log.info("to access the help menu: zig build -h", .{});
1625 fatal(f, args);
1626}
1627
1628fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void {
1629 for (steps) |step_index| {
1630 if (true) @panic("TODO");
1631 const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue;
1632 if (wf.mode != .tmp) continue;
1633 const path = wf.generated_directory.path orelse continue;
1634 Io.Dir.cwd().deleteTree(io, path) catch |err| {
1635 log.warn("failed to delete {s}: {t}", .{ path, err });
1636 };
1637 }
1638}
1639
1640var stdio_buffer_allocation: [256]u8 = undefined;
1641var stdout_writer_allocation: Io.File.Writer = undefined;
1642
1643fn initStdoutWriter(io: Io) *Writer {
1644 stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation);
1645 return &stdout_writer_allocation.interface;
1646}
1647
1648const ScannedConfig = struct {
1649 configuration: Configuration,
1650 top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index),
1651
1652 fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void {
1653 const c = &sc.configuration;
1654 var serializer: std.zon.Serializer = .{ .writer = w };
1655 var s = try serializer.beginStruct(.{});
1656
1657 try s.field("default_step", @intFromEnum(c.default_step), .{});
1658 {
1659 var ss = try s.beginStructField("top_level_steps", .{});
1660 for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step| {
1661 try ss.field(name, @intFromEnum(step), .{});
1662 }
1663 try ss.end();
1664 }
1665
1666 try s.end();
1667 }
1668
1669 fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void {
1670 const arena = graph.arena;
1671 const c = &sc.configuration;
1672 for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step_index| {
1673 const step = step_index.ptr(c);
1674 const decorated_name = if (step_index == c.default_step)
1675 try fmt.allocPrint(arena, "{s} (default)", .{name})
1676 else
1677 name;
1678 const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index);
1679 const description = top_level.description.slice(c);
1680 try w.print(" {s:<28} {s}\n", .{ decorated_name, description });
1681 }
1682 }
1683
1684 fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void {
1685 const arena = graph.arena;
1686
1687 try w.print(
1688 \\Usage: {s} build [steps] [options]
1689 \\
1690 \\Steps:
1691 \\
1692 , .{graph.zig_exe});
1693 try printSteps(sc, graph, w);
1694 try w.writeAll(
1695 \\
1696 \\Project-Specific Options:
1697 \\
1698 );
1699
1700 const available_options = sc.configuration.available_options;
1701 if (available_options.len == 0) {
1702 try w.print(" (none)\n", .{});
1703 } else {
1704 for (available_options) |option| {
1705 const name = option.name.slice(&sc.configuration);
1706 const description = option.description.slice(&sc.configuration);
1707 const help = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ name, option.type });
1708 try w.print("{s:<30} {s}\n", .{ help, description });
1709 if (option.enum_options.slice(&sc.configuration)) |enum_options| {
1710 const padding: [33]u8 = @splat(' ');
1711 try w.writeAll(padding ++ "Supported Values:\n");
1712 for (enum_options) |enum_option_index| {
1713 const enum_option = enum_option_index.slice(&sc.configuration);
1714 try w.print(padding ++ " {s}\n", .{enum_option});
1715 }
1716 }
1717 }
1718 }
1719
1720 try w.writeAll(
1721 \\
1722 \\System Integration Options:
1723 \\ --search-prefix [path] Add a path to look for binaries, libraries, headers
1724 \\ --sysroot [path] Set the system root directory (usually /)
1725 \\ --libc [file] Provide a file which specifies libc paths
1726 \\
1727 \\ --system [pkgdir] Disable package fetching; enable all integrations
1728 \\ -fsys=[name] Enable a system integration
1729 \\ -fno-sys=[name] Disable a system integration
1730 \\
1731 \\ -fdarling, -fno-darling Integration with system-installed Darling to
1732 \\ execute macOS programs on Linux hosts
1733 \\ (default: no)
1734 \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute
1735 \\ foreign-architecture programs on Linux hosts
1736 \\ (default: no)
1737 \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc
1738 \\ (e.g. glibc or musl) built for multiple foreign
1739 \\ architectures, allowing execution of non-native
1740 \\ programs that link with libc.
1741 \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on
1742 \\ ARM64 macOS hosts. (default: no)
1743 \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to
1744 \\ execute WASI binaries. (default: no)
1745 \\ -fwine, -fno-wine Integration with system-installed Wine to execute
1746 \\ Windows programs on Linux hosts. (default: no)
1747 \\
1748 \\ Available System Integrations: Enabled:
1749 \\
1750 );
1751 if (sc.configuration.system_integrations.len == 0) {
1752 try w.writeAll(" (none) -\n");
1753 } else {
1754 for (sc.configuration.system_integrations) |system_integration| {
1755 const name = system_integration.name.slice(&sc.configuration);
1756 const status = switch (system_integration.status) {
1757 .disabled => "no",
1758 .enabled => "yes",
1759 };
1760 try w.print(" {s:<43} {s}\n", .{ name, status });
1761 }
1762 }
1763
1764 try w.writeAll(
1765 \\
1766 \\General Options:
1767 \\ -h, --help Print this help to stdout and exit
1768 \\ -l, --list-steps Print available steps to stdout and exit
1769 \\
1770 \\ -p, --prefix [path] Where to install files (default: zig-out)
1771 \\ --prefix-lib-dir [path] Where to install libraries
1772 \\ --prefix-exe-dir [path] Where to install executables
1773 \\ --prefix-include-dir [path] Where to install C header files
1774 \\ --release[=mode] Request release mode, optionally specifying a
1775 \\ preferred optimization mode: fast, safe, small
1776 \\
1777 \\ --verbose Print commands before executing them
1778 \\ --color [auto|off|on] Enable or disable colored error messages
1779 \\ --error-style [style] Control how build errors are printed
1780 \\ verbose (Default) Report errors with full context
1781 \\ minimal Report errors after summary, excluding context like command lines
1782 \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update
1783 \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update
1784 \\ --multiline-errors [style] Control how multi-line error messages are printed
1785 \\ indent (Default) Indent non-initial lines to align with initial line
1786 \\ newline Include a leading newline so that the error message is on its own lines
1787 \\ none Print as usual so the first line is misaligned
1788 \\ --summary [mode] Control the printing of the build summary
1789 \\ all Print the build summary in its entirety
1790 \\ new Omit cached steps
1791 \\ failures (Default if short-lived) Only print failed steps
1792 \\ line (Default if long-lived) Only print the single-line summary
1793 \\ none Do not print the build summary
1794 \\ -j<N> Limit concurrent jobs (default is to use all CPU cores)
1795 \\ --maxrss <bytes> Limit memory usage (default is to use available memory)
1796 \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss
1797 \\ --test-timeout <timeout> Limit execution time of unit tests, terminating if exceeded.
1798 \\ The timeout must include a unit: ns, us, ms, s, m, h
1799 \\ --watch Continuously rebuild when source files are modified
1800 \\ --debounce <ms> Delay before rebuilding after changed file detected
1801 \\ --webui[=ip] Enable the web interface on the given IP address
1802 \\ --fuzz[=limit] Continuously search for unit test failures with an optional
1803 \\ limit to the max number of iterations. The argument supports
1804 \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies
1805 \\ '--webui' when no limit is specified.
1806 \\ --time-report Force full rebuild and provide detailed information on
1807 \\ compilation time of Zig source code (implies '--webui')
1808 \\ -fincremental Enable incremental compilation
1809 \\ -fno-incremental Disable incremental compilation
1810 \\
1811 \\Package Management Options:
1812 \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit
1813 \\ needed (Default) Lazy dependencies are fetched as needed
1814 \\ all Lazy dependencies are always fetched
1815 \\ --fork=[path] Override one or more projects from dependency tree
1816 \\
1817 \\Advanced Options:
1818 \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error
1819 \\ -fno-reference-trace Disable reference trace
1820 \\ -fallow-so-scripts Allows .so files to be GNU ld scripts
1821 \\ -fno-allow-so-scripts (default) .so files must be ELF files
1822 \\ --build-file [file] Override path to build.zig
1823 \\ --cache-dir [path] Override path to local Zig cache directory
1824 \\ --global-cache-dir [path] Override path to global Zig cache directory
1825 \\ --zig-lib-dir [arg] Override path to Zig lib directory
1826 \\ --build-runner [file] Override path to build runner
1827 \\ --seed [integer] For shuffling dependency traversal order (default: random)
1828 \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries
1829 \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM)
1830 \\ sha1, tree 20-byte cryptographic hash (ELF, WASM)
1831 \\ md5 16-byte cryptographic hash (ELF)
1832 \\ uuid 16-byte random UUID (ELF, WASM)
1833 \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM)
1834 \\ none (default) No build ID
1835 \\ --debug-log [scope] Enable debugging the compiler
1836 \\ --debug-pkg-config Fail if unknown pkg-config flags encountered
1837 \\ --debug-rt Debug compiler runtime libraries
1838 \\ --verbose-link Enable compiler debug output for linking
1839 \\ --verbose-air Enable compiler debug output for Zig AIR
1840 \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR
1841 \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC
1842 \\ --verbose-cimport Enable compiler debug output for C imports
1843 \\ --verbose-cc Enable compiler debug output for C compilation
1844 \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features
1845 \\
1846 );
1847 }
1848};
lib/compiler/maker/Fuzz.zig deleted-606
...@@ -1,606 +0,0 @@
1const Fuzz = @This();
2
3const std = @import("std");
4const Allocator = std.mem.Allocator;
5const Build = std.Build;
6const Cache = std.Build.Cache;
7const Coverage = std.debug.Coverage;
8const Configuration = std.Build.Configuration;
9const Io = std.Io;
10const abi = std.Build.abi.fuzz;
11const assert = std.debug.assert;
12const fatal = std.process.fatal;
13const log = std.log;
14
15const maker = @import("../maker.zig");
16const WebServer = @import("WebServer.zig");
17
18gpa: Allocator,
19io: Io,
20mode: Mode,
21
22/// Allocated into `gpa`.
23run_steps: []const Configuration.Step.Index,
24
25group: Io.Group,
26root_prog_node: std.Progress.Node,
27prog_node: std.Progress.Node,
28
29/// Protects `coverage_files`.
30coverage_mutex: Io.Mutex,
31coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap),
32
33queue_mutex: Io.Mutex,
34queue_cond: Io.Condition,
35msg_queue: std.ArrayList(Msg),
36
37pub const Mode = union(enum) {
38 forever: struct { ws: *WebServer },
39 limit: Limited,
40
41 pub const Limited = struct {
42 amount: u64,
43 };
44};
45
46const Msg = union(enum) {
47 coverage: struct {
48 id: u64,
49 cumulative: struct {
50 runs: u64,
51 unique: u64,
52 coverage: u64,
53 },
54 run: Configuration.Step.Index,
55 },
56 entry_point: struct {
57 coverage_id: u64,
58 addr: u64,
59 },
60};
61
62const CoverageMap = struct {
63 mapped_memory: []align(std.heap.page_size_min) const u8,
64 coverage: Coverage,
65 source_locations: []Coverage.SourceLocation,
66 /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested.
67 entry_points: std.ArrayList(u32),
68 start_timestamp: i64,
69 start_n_runs: u64,
70
71 fn deinit(cm: *CoverageMap, gpa: Allocator) void {
72 std.posix.munmap(cm.mapped_memory);
73 cm.coverage.deinit(gpa);
74 cm.* = undefined;
75 }
76};
77
78pub fn init(
79 gpa: Allocator,
80 io: Io,
81 all_steps: []const Configuration.Step.Index,
82 root_prog_node: std.Progress.Node,
83 mode: Mode,
84) error{ OutOfMemory, Canceled }!Fuzz {
85 const run_steps: []const Configuration.Step.Index = steps: {
86 var steps: std.ArrayList(Configuration.Step.Index) = .empty;
87 defer steps.deinit(gpa);
88 const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);
89 defer rebuild_node.end();
90 var rebuild_group: Io.Group = .init;
91 defer rebuild_group.cancel(io);
92
93 for (all_steps) |step| {
94 if (true) @panic("TODO");
95 const run = step.cast(std.Build.Step.Run) orelse continue;
96 if (run.producer == null) continue;
97 if (run.fuzz_tests.items.len == 0) continue;
98 try steps.append(gpa, run);
99 rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, rebuild_node });
100 }
101
102 if (steps.items.len == 0) fatal("no fuzz tests found", .{});
103 rebuild_node.setEstimatedTotalItems(steps.items.len);
104 const run_steps = try gpa.dupe(Configuration.Step.Index, steps.items);
105 try rebuild_group.await(io);
106 break :steps run_steps;
107 };
108 errdefer gpa.free(run_steps);
109
110 for (run_steps) |run_step_index| {
111 if (true) @panic("TODO");
112 assert(run_step_index.fuzz_tests.items.len > 0);
113 if (run_step_index.rebuilt_executable == null)
114 fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{});
115 }
116
117 return .{
118 .gpa = gpa,
119 .io = io,
120 .mode = mode,
121 .run_steps = run_steps,
122 .group = .init,
123 .root_prog_node = root_prog_node,
124 .prog_node = .none,
125 .coverage_files = .empty,
126 .coverage_mutex = .init,
127 .queue_mutex = .init,
128 .queue_cond = .init,
129 .msg_queue = .empty,
130 };
131}
132
133pub fn start(fuzz: *Fuzz) void {
134 const io = fuzz.io;
135 fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", 0);
136
137 if (fuzz.mode == .forever) {
138 // For polling messages and sending updates to subscribers.
139 fuzz.group.concurrent(io, coverageRun, .{fuzz}) catch |err|
140 fatal("unable to spawn coverage task: {t}", .{err});
141 }
142
143 if (true) @panic("TODO");
144
145 for (fuzz.run_steps) |run| {
146 assert(run.rebuilt_executable != null);
147 fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run });
148 }
149}
150
151pub fn deinit(fuzz: *Fuzz) void {
152 const io = fuzz.io;
153 fuzz.group.cancel(io);
154 fuzz.prog_node.end();
155 fuzz.gpa.free(fuzz.run_steps);
156}
157
158fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void {
159 rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| {
160 const compile = run.producer.?;
161 log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err });
162 };
163}
164
165fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) !void {
166 const graph = run.step.owner.graph;
167 const io = graph.io;
168 const compile = run.producer.?;
169 const prog_node = parent_prog_node.start(compile.step.name, 0);
170 defer prog_node.end();
171
172 const result = compile.rebuildInFuzzMode(gpa, prog_node);
173
174 const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0;
175 const show_error_msgs = compile.step.result_error_msgs.items.len > 0;
176 const show_stderr = compile.step.result_stderr.len > 0;
177
178 if (show_error_msgs or show_compile_errors or show_stderr) {
179 var buf: [256]u8 = undefined;
180 const stderr = try io.lockStderr(&buf, graph.stderr_mode);
181 defer io.unlockStderr();
182 maker.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {};
183 }
184
185 const rebuilt_bin_path = result catch |err| switch (err) {
186 error.MakeFailed => return,
187 else => |other| return other,
188 };
189 run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);
190}
191
192fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void {
193 const owner = run.step.owner;
194 const gpa = owner.allocator;
195 const graph = owner.graph;
196 const io = graph.io;
197
198 run.rerunInFuzzMode(fuzz, fuzz.prog_node) catch |err| switch (err) {
199 error.MakeFailed => {
200 var buf: [256]u8 = undefined;
201 const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) {
202 error.Canceled => return,
203 };
204 defer io.unlockStderr();
205 maker.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {};
206 return;
207 },
208 else => {
209 log.err("step '{s}': failed to rerun in fuzz mode: {t}", .{ run.step.name, err });
210 return;
211 },
212 };
213}
214
215pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
216 if (true) @panic("TODO");
217 assert(fuzz.mode == .forever);
218
219 var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa);
220 defer arena_state.deinit();
221 const arena = arena_state.allocator();
222
223 const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
224 var dedup_table: DedupTable = .empty;
225 defer dedup_table.deinit(fuzz.gpa);
226
227 for (fuzz.run_steps) |run_step| {
228 const compile_inputs = run_step.producer.?.step.inputs.table;
229 for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| {
230 try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len);
231 for (file_list.items) |sub_path| {
232 if (!std.mem.endsWith(u8, sub_path, ".zig")) continue;
233 const joined_path = try dir_path.join(arena, sub_path);
234 dedup_table.putAssumeCapacity(joined_path, {});
235 }
236 }
237 }
238
239 const deduped_paths = dedup_table.keys();
240 const SortContext = struct {
241 pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool {
242 _ = this;
243 return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) {
244 .lt => true,
245 .gt => false,
246 .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path),
247 };
248 }
249 };
250 std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
251 return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths);
252}
253
254pub const Previous = struct {
255 unique_runs: usize,
256 entry_points: usize,
257 sent_source_index: bool,
258 pub const init: Previous = .{
259 .unique_runs = 0,
260 .entry_points = 0,
261 .sent_source_index = false,
262 };
263};
264pub fn sendUpdate(
265 fuzz: *Fuzz,
266 socket: *std.http.Server.WebSocket,
267 prev: *Previous,
268) !void {
269 const io = fuzz.io;
270
271 try fuzz.coverage_mutex.lock(io);
272 defer fuzz.coverage_mutex.unlock(io);
273
274 const coverage_maps = fuzz.coverage_files.values();
275 if (coverage_maps.len == 0) return;
276 // TODO: handle multiple fuzz steps in the WebSocket packets
277 const coverage_map = &coverage_maps[0];
278 const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
279 // TODO: this isn't sound! We need to do volatile reads of these bits rather than handing the
280 // buffer off to the kernel, because we might race with the fuzzer process[es]. This brings the
281 // whole mmap strategy into question. Incidentally, I wonder if post-writergate we could pass
282 // this data straight to the socket with sendfile...
283 const seen_pcs = cov_header.seenBits();
284 const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic);
285 const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic);
286 {
287 if (!prev.sent_source_index) {
288 prev.sent_source_index = true;
289 // We need to send initial context.
290 const header: abi.SourceIndexHeader = .{
291 .directories_len = @intCast(coverage_map.coverage.directories.entries.len),
292 .files_len = @intCast(coverage_map.coverage.files.entries.len),
293 .source_locations_len = @intCast(coverage_map.source_locations.len),
294 .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len),
295 .start_timestamp = coverage_map.start_timestamp,
296 .start_n_runs = coverage_map.start_n_runs,
297 };
298 var iovecs: [5][]const u8 = .{
299 @ptrCast(&header),
300 @ptrCast(coverage_map.coverage.directories.keys()),
301 @ptrCast(coverage_map.coverage.files.keys()),
302 @ptrCast(coverage_map.source_locations),
303 coverage_map.coverage.string_bytes.items,
304 };
305 try socket.writeMessageVec(&iovecs, .binary);
306 }
307
308 const header: abi.CoverageUpdateHeader = .{
309 .n_runs = n_runs,
310 .unique_runs = unique_runs,
311 };
312 var iovecs: [2][]const u8 = .{
313 @ptrCast(&header),
314 @ptrCast(seen_pcs),
315 };
316 try socket.writeMessageVec(&iovecs, .binary);
317
318 prev.unique_runs = unique_runs;
319 }
320
321 if (prev.entry_points != coverage_map.entry_points.items.len) {
322 const header: abi.EntryPointHeader = .init(@intCast(coverage_map.entry_points.items.len));
323 var iovecs: [2][]const u8 = .{
324 @ptrCast(&header),
325 @ptrCast(coverage_map.entry_points.items),
326 };
327 try socket.writeMessageVec(&iovecs, .binary);
328
329 prev.entry_points = coverage_map.entry_points.items.len;
330 }
331}
332
333fn coverageRun(fuzz: *Fuzz) void {
334 coverageRunCancelable(fuzz) catch |err| switch (err) {
335 error.Canceled => return,
336 };
337}
338
339fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void {
340 const io = fuzz.io;
341
342 try fuzz.queue_mutex.lock(io);
343 defer fuzz.queue_mutex.unlock(io);
344
345 while (true) {
346 try fuzz.queue_cond.wait(io, &fuzz.queue_mutex);
347 for (fuzz.msg_queue.items) |msg| switch (msg) {
348 .coverage => |coverage| prepareTables(fuzz, coverage.run, coverage.id) catch |err| switch (err) {
349 error.AlreadyReported => continue,
350 error.Canceled => return,
351 else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}),
352 },
353 .entry_point => |entry_point| addEntryPoint(fuzz, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) {
354 error.AlreadyReported => continue,
355 error.Canceled => return,
356 else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}),
357 },
358 };
359 fuzz.msg_queue.clearRetainingCapacity();
360 }
361}
362fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void {
363 if (true) @panic("TODO");
364 assert(fuzz.mode == .forever);
365 const ws = fuzz.mode.forever.ws;
366 const gpa = fuzz.gpa;
367 const io = fuzz.io;
368
369 try fuzz.coverage_mutex.lock(io);
370 defer fuzz.coverage_mutex.unlock(io);
371
372 const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id);
373 if (gop.found_existing) {
374 // We are fuzzing the same executable with multiple threads.
375 // Perhaps the same unit test; perhaps a different one. In any
376 // case, since the coverage file is the same, we only have to
377 // notice changes to that one file in order to learn coverage for
378 // this particular executable.
379 return;
380 }
381 errdefer _ = fuzz.coverage_files.pop();
382
383 gop.value_ptr.* = .{
384 .coverage = std.debug.Coverage.init,
385 .mapped_memory = undefined, // populated below
386 .source_locations = undefined, // populated below
387 .entry_points = .empty,
388 .start_timestamp = ws.now(),
389 .start_n_runs = undefined, // populated below
390 };
391 errdefer gop.value_ptr.coverage.deinit(gpa);
392
393 const rebuilt_exe_path = run_step_index.rebuilt_executable.?;
394 const target = run_step_index.producer.?.rootModuleTarget();
395 var debug_info = std.debug.Info.load(
396 gpa,
397 io,
398 rebuilt_exe_path,
399 &gop.value_ptr.coverage,
400 target.ofmt,
401 target.cpu.arch,
402 ) catch |err| {
403 log.err("step '{s}': failed to load debug information for '{f}': {t}", .{
404 run_step_index.step.name, rebuilt_exe_path, err,
405 });
406 return error.AlreadyReported;
407 };
408 defer debug_info.deinit(gpa);
409
410 const coverage_file_path: Build.Cache.Path = .{
411 .root_dir = run_step_index.step.owner.cache_root,
412 .sub_path = "v/" ++ std.fmt.hex(coverage_id),
413 };
414 var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| {
415 log.err("step '{s}': failed to load coverage file '{f}': {t}", .{
416 run_step_index.step.name, coverage_file_path, err,
417 });
418 return error.AlreadyReported;
419 };
420 defer coverage_file.close(io);
421
422 const file_size = coverage_file.length(io) catch |err| {
423 log.err("unable to check len of coverage file '{f}': {t}", .{ coverage_file_path, err });
424 return error.AlreadyReported;
425 };
426
427 const mapped_memory = std.posix.mmap(
428 null,
429 file_size,
430 .{ .READ = true },
431 .{ .TYPE = .SHARED },
432 coverage_file.handle,
433 0,
434 ) catch |err| {
435 log.err("failed to map coverage file '{f}': {t}", .{ coverage_file_path, err });
436 return error.AlreadyReported;
437 };
438 gop.value_ptr.mapped_memory = mapped_memory;
439
440 const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
441 const pcs = header.pcAddrs();
442 const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len);
443 errdefer gpa.free(source_locations);
444
445 // Unfortunately the PCs array that LLVM gives us from the 8-bit PC
446 // counters feature is not sorted.
447 var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .empty;
448 defer sorted_pcs.deinit(gpa);
449 try sorted_pcs.resize(gpa, pcs.len);
450 @memcpy(sorted_pcs.items(.pc), pcs);
451 for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i);
452 sorted_pcs.sortUnstable(struct {
453 addrs: []const u64,
454
455 pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
456 return ctx.addrs[a_index] < ctx.addrs[b_index];
457 }
458 }{ .addrs = sorted_pcs.items(.pc) });
459
460 debug_info.resolveAddresses(gpa, io, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {
461 log.err("failed to resolve addresses to source locations: {t}", .{err});
462 return error.AlreadyReported;
463 };
464
465 for (sorted_pcs.items(.index), sorted_pcs.items(.sl)) |i, sl| source_locations[i] = sl;
466 gop.value_ptr.source_locations = source_locations;
467 gop.value_ptr.start_n_runs = header.n_runs;
468
469 ws.notifyUpdate();
470}
471
472fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory, Canceled }!void {
473 const io = fuzz.io;
474
475 try fuzz.coverage_mutex.lock(io);
476 defer fuzz.coverage_mutex.unlock(io);
477
478 const coverage_map = fuzz.coverage_files.getPtr(coverage_id).?;
479 const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
480 const pcs = header.pcAddrs();
481
482 // Since this pcs list is unsorted, we must linear scan for the best index.
483 const index = i: {
484 var best: usize = 0;
485 for (pcs[1..], 1..) |elem_addr, i| {
486 if (elem_addr == addr) break :i i;
487 if (elem_addr > addr) continue;
488 if (elem_addr > pcs[best]) best = i;
489 }
490 break :i best;
491 };
492 if (index >= pcs.len) {
493 log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{
494 addr, pcs[0], pcs[pcs.len - 1],
495 });
496 return error.AlreadyReported;
497 }
498 if (false) {
499 const sl = coverage_map.source_locations[index];
500 const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename);
501 if (pcs.len == 1) {
502 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 (final)", .{
503 addr, file_name, sl.line, sl.column,
504 });
505 } else if (index == 0) {
506 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 before {x}", .{
507 addr, file_name, sl.line, sl.column, pcs[index + 1],
508 });
509 } else if (index == pcs.len - 1) {
510 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} (final) after {x}", .{
511 addr, file_name, sl.line, sl.column, index, pcs[index - 1],
512 });
513 } else {
514 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} between {x} and {x}", .{
515 addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1],
516 });
517 }
518 }
519 try coverage_map.entry_points.append(fuzz.gpa, @intCast(index));
520}
521
522pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void {
523 if (true) @panic("TODO");
524 assert(fuzz.mode == .limit);
525 const io = fuzz.io;
526
527 try fuzz.group.await(io);
528 fuzz.group = .init;
529
530 std.debug.print("======= FUZZING REPORT =======\n", .{});
531 for (fuzz.msg_queue.items) |msg| {
532 if (msg != .coverage) continue;
533
534 const cov = msg.coverage;
535 const coverage_file_path: std.Build.Cache.Path = .{
536 .root_dir = cov.run.step.owner.cache_root,
537 .sub_path = "v/" ++ std.fmt.hex(cov.id),
538 };
539 var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| {
540 fatal("step '{s}': failed to load coverage file '{f}': {t}", .{
541 cov.run.step.name, coverage_file_path, err,
542 });
543 };
544 defer coverage_file.close(io);
545
546 const fuzz_abi = std.Build.abi.fuzz;
547 var rbuf: [0x1000]u8 = undefined;
548 var r = coverage_file.reader(io, &rbuf);
549
550 var header: fuzz_abi.SeenPcsHeader = undefined;
551 r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| {
552 fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{
553 cov.run.step.name, coverage_file_path, err,
554 });
555 };
556
557 if (header.pcs_len == 0) {
558 fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{
559 cov.run.step.name, coverage_file_path,
560 });
561 }
562
563 var seen_count: usize = 0;
564 const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len);
565 for (0..chunk_count) |_| {
566 const seen = r.interface.takeInt(usize, .little) catch |err| {
567 fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{
568 cov.run.step.name, coverage_file_path, err,
569 });
570 };
571 seen_count += @popCount(seen);
572 }
573
574 const seen_f: f64 = @floatFromInt(seen_count);
575 const total_f: f64 = @floatFromInt(header.pcs_len);
576 const ratio = seen_f / total_f;
577 std.debug.print(
578 \\Step: {s}
579 \\Fuzz test: "{s}" ({x})
580 \\Runs: {} -> {}
581 \\Unique runs: {} -> {}
582 \\Coverage: {}/{} -> {}/{} ({:.02}%)
583 \\
584 , .{
585 cov.run.step.name,
586 cov.run.fuzz_tests.items[0],
587 cov.id,
588 cov.cumulative.runs,
589 header.n_runs,
590 cov.cumulative.unique,
591 header.unique_runs,
592 cov.cumulative.coverage,
593 header.pcs_len,
594 seen_count,
595 header.pcs_len,
596 ratio * 100,
597 });
598
599 std.debug.print("------------------------------\n", .{});
600 }
601 std.debug.print(
602 \\Values are accumulated across multiple runs when preserving the cache.
603 \\==============================
604 \\
605 , .{});
606}
lib/compiler/maker/Graph.zig deleted-25
...@@ -1,25 +0,0 @@
1//! Shared maker state among all steps.
2const Graph = @This();
3
4const std = @import("std");
5const Io = std.Io;
6const Allocator = std.mem.Allocator;
7const Configuration = std.Build.Configuration;
8
9io: Io,
10/// Process lifetime.
11arena: Allocator,
12cache: std.Build.Cache,
13zig_exe: []const u8,
14environ_map: std.process.Environ.Map,
15global_cache_root: std.Build.Cache.Directory,
16zig_lib_directory: std.Build.Cache.Directory,
17
18debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null,
19incremental: ?bool = null,
20random_seed: u32 = 0,
21allow_so_scripts: ?bool = null,
22time_report: bool = false,
23/// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also
24/// respects the '--color' flag.
25stderr_mode: ?Io.Terminal.Mode = null,
lib/compiler/maker/Step.zig deleted-840
...@@ -1,840 +0,0 @@
1//! The state that maker needs in order to process a step.
2const Step = @This();
3
4const builtin = @import("builtin");
5
6const std = @import("std");
7const Allocator = std.mem.Allocator;
8const Cache = std.Build.Cache;
9const Io = std.Io;
10const LazyPath = std.Build.Configuration.LazyPath;
11const Package = std.Build.Configuration.Package;
12const Path = std.Build.Cache.Path;
13const Configuration = std.Build.Configuration;
14const assert = std.debug.assert;
15
16const WebServer = @import("WebServer.zig");
17
18pub const Compile = void; // @import("Step/Compile.zig");
19pub const Run = void; // @import("Step/Run.zig");
20
21/// Avoid false sharing.
22_: void align(std.atomic.cache_line) = {},
23
24state: State = .precheck_unstarted,
25dependants: std.ArrayList(Configuration.Step.Index) = .empty,
26/// Collects the set of files that retrigger this step to run.
27///
28/// This is used by the build system's implementation of `--watch` but it can
29/// also be potentially useful for IDEs to know what effects editing a
30/// particular file has.
31///
32/// Populated within `make`. Implementation may choose to clear and repopulate,
33/// retain previous value, or update.
34inputs: Inputs = .init,
35pending_deps: u32 = undefined,
36
37result_error_msgs: std.ArrayList([]const u8) = .empty,
38result_error_bundle: std.zig.ErrorBundle = .empty,
39result_stderr: []const u8 = "",
40result_cached: bool = false,
41result_duration_ns: ?u64 = null,
42/// 0 means unavailable or not reported.
43result_peak_rss: usize = 0,
44/// If the step is failed and this field is populated, this is the command which failed.
45/// This field may be populated even if the step succeeded.
46result_failed_command: ?[]const u8 = null,
47test_results: TestResults = .{},
48
49pub const State = enum {
50 precheck_unstarted,
51 precheck_started,
52 /// This is also used to indicate "dirty" steps that have been modified
53 /// after a previous build completed, in which case, the step may or may
54 /// not have been completed before. Either way, one or more of its direct
55 /// file system inputs have been modified, meaning that the step needs to
56 /// be re-evaluated.
57 precheck_done,
58 dependency_failure,
59 success,
60 failure,
61 /// This state indicates that the step did not complete, however, it also did not fail,
62 /// and it is safe to continue executing its dependencies.
63 skipped,
64 /// This step was skipped because it specified a max_rss that exceeded the runner's maximum.
65 /// It is not safe to run its dependencies.
66 skipped_oom,
67};
68
69pub const Inputs = struct {
70 table: Table,
71
72 pub const init: Inputs = .{
73 .table = .{},
74 };
75
76 pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, Files, Cache.Path.TableAdapter, false);
77 /// The special file name "." means any changes inside the directory.
78 pub const Files = std.ArrayList([]const u8);
79
80 pub fn populated(inputs: *Inputs) bool {
81 return inputs.table.count() != 0;
82 }
83
84 pub fn clear(inputs: *Inputs, gpa: Allocator) void {
85 for (inputs.table.values()) |*files| files.deinit(gpa);
86 inputs.table.clearRetainingCapacity();
87 }
88};
89
90pub const TestResults = struct {
91 /// The total number of tests in the step. Every test has a "status" from the following:
92 /// * passed
93 /// * skipped
94 /// * failed cleanly
95 /// * crashed
96 /// * timed out
97 test_count: u32 = 0,
98
99 /// The number of tests which were skipped (`error.SkipZigTest`).
100 skip_count: u32 = 0,
101 /// The number of tests which failed cleanly.
102 fail_count: u32 = 0,
103 /// The number of tests which terminated unexpectedly, i.e. crashed.
104 crash_count: u32 = 0,
105 /// The number of tests which timed out.
106 timeout_count: u32 = 0,
107
108 /// The number of detected memory leaks. The associated test may still have passed; indeed, *all*
109 /// individual tests may have passed. However, the step as a whole fails if any test has leaks.
110 leak_count: u32 = 0,
111 /// The number of detected error logs. The associated test may still have passed; indeed, *all*
112 /// individual tests may have passed. However, the step as a whole fails if any test logs errors.
113 log_err_count: u32 = 0,
114
115 pub fn isSuccess(tr: TestResults) bool {
116 // all steps are success or skip
117 return tr.fail_count == 0 and
118 tr.crash_count == 0 and
119 tr.timeout_count == 0 and
120 // no (otherwise successful) step leaked memory or logged errors
121 tr.leak_count == 0 and
122 tr.log_err_count == 0;
123 }
124
125 /// Computes the number of tests which passed from the other values.
126 pub fn passCount(tr: TestResults) u32 {
127 return tr.test_count - tr.skip_count - tr.fail_count - tr.crash_count - tr.timeout_count;
128 }
129};
130
131pub const MakeOptions = struct {
132 progress_node: std.Progress.Node,
133 watch: bool,
134 web_server: ?*WebServer,
135 /// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds.
136 unit_test_timeout_ns: ?u64,
137 /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`.
138 gpa: Allocator,
139};
140
141pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void;
142
143/// If the Step's `make` function reports `error.MakeFailed`, it indicates they
144/// have already reported the error. Otherwise, we add a simple error report
145/// here.
146pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void {
147 if (true) @panic("TODO Step.make");
148 const arena = s.owner.allocator;
149 const graph = s.owner.graph;
150 const io = graph.io;
151
152 var start_ts: ?Io.Timestamp = t: {
153 if (!graph.time_report) break :t null;
154 if (s.id == .compile) break :t null;
155 if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null;
156 break :t Io.Clock.awake.now(io);
157 };
158 const make_result = s.makeFn(s, options);
159 if (start_ts) |*ts| {
160 const duration = ts.untilNow(io, .awake);
161 options.web_server.?.updateTimeReportGeneric(s, duration);
162 }
163
164 make_result catch |err| switch (err) {
165 error.MakeFailed, error.MakeSkipped => |e| return e,
166 else => {
167 s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM");
168 return error.MakeFailed;
169 },
170 };
171
172 if (!s.test_results.isSuccess()) {
173 return error.MakeFailed;
174 }
175
176 if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) {
177 const msg = std.fmt.allocPrint(arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{
178 s.result_peak_rss, s.max_rss,
179 }) catch @panic("OOM");
180 s.result_error_msgs.append(arena, msg) catch @panic("OOM");
181 }
182}
183
184/// Implementation detail of file watching. Prepares the step for being re-evaluated.
185/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated.
186pub fn invalidateResult(step: *Step, gpa: Allocator) bool {
187 if (true) @panic("TODO Step.invalidateResult");
188 if (step.state == .precheck_done) return false;
189 assert(step.pending_deps == 0);
190 step.state = .precheck_done;
191 step.reset(gpa);
192 for (step.dependants.items) |dependant| {
193 _ = dependant.invalidateResult(gpa);
194 dependant.pending_deps += 1;
195 }
196 return true;
197}
198
199/// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated.
200pub fn reset(step: *Step, gpa: Allocator) void {
201 assert(step.state == .precheck_done);
202
203 if (step.result_failed_command) |cmd| gpa.free(cmd);
204
205 step.result_error_msgs.clearRetainingCapacity();
206 step.result_stderr = "";
207 step.result_cached = false;
208 step.result_duration_ns = null;
209 step.result_peak_rss = 0;
210 step.result_failed_command = null;
211 step.test_results = .{};
212 step.clearWatchInputs();
213
214 step.result_error_bundle.deinit(gpa);
215 step.result_error_bundle = std.zig.ErrorBundle.empty;
216}
217
218/// Populates `s.result_failed_command`.
219pub fn captureChildProcess(
220 s: *Step,
221 gpa: Allocator,
222 progress_node: std.Progress.Node,
223 argv: []const []const u8,
224) !std.process.RunResult {
225 const graph = s.owner.graph;
226 const arena = graph.arena;
227 const io = graph.io;
228
229 // If an error occurs, it's happened in this command:
230 assert(s.result_failed_command == null);
231 s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv);
232
233 try handleChildProcUnsupported(s);
234 try handleVerbose(s, .inherit, argv);
235
236 const result = std.process.run(arena, io, .{
237 .argv = argv,
238 .environ_map = &graph.environ_map,
239 .progress_node = progress_node,
240 }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err });
241
242 if (result.stderr.len > 0) {
243 try s.result_error_msgs.append(arena, result.stderr);
244 }
245
246 return result;
247}
248
249pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } {
250 try step.addError(fmt, args);
251 return error.MakeFailed;
252}
253
254pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void {
255 const arena = step.owner.allocator;
256 const msg = try std.fmt.allocPrint(arena, fmt, args);
257 try step.result_error_msgs.append(arena, msg);
258}
259
260pub const ZigProcess = struct {
261 child: std.process.Child,
262 multi_reader_buffer: Io.File.MultiReader.Buffer(2),
263 multi_reader: Io.File.MultiReader,
264 progress_ipc_index: ?if (std.Progress.have_ipc) std.Progress.Ipc.Index else noreturn,
265
266 pub const StreamEnum = enum { stdout, stderr };
267
268 pub fn saveState(zp: *ZigProcess, prog_node: std.Progress.Node) void {
269 zp.progress_ipc_index = if (std.Progress.have_ipc) prog_node.takeIpcIndex() else null;
270 }
271
272 pub fn deinit(zp: *ZigProcess, io: Io) void {
273 zp.child.kill(io);
274 zp.multi_reader.deinit();
275 zp.* = undefined;
276 }
277};
278
279/// Assumes that argv contains `--listen=-` and that the process being spawned
280/// is the zig compiler - the same version that compiled the build runner.
281/// Populates `s.result_failed_command`.
282pub fn evalZigProcess(
283 s: *Step,
284 argv: []const []const u8,
285 prog_node: std.Progress.Node,
286 watch: bool,
287 web_server: ?*WebServer,
288 gpa: Allocator,
289) !?Cache.Path {
290 const b = s.owner;
291 const io = b.graph.io;
292
293 // If an error occurs, it's happened in this command:
294 assert(s.result_failed_command == null);
295 s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv);
296
297 if (s.getZigProcess()) |zp| update: {
298 assert(watch);
299 if (zp.progress_ipc_index) |ipc_index| prog_node.setIpcIndex(ipc_index);
300 zp.progress_ipc_index = null;
301 var exited = false;
302 defer if (exited) {
303 s.cast(Compile).?.zig_process = null;
304 zp.deinit(io);
305 gpa.destroy(zp);
306 } else zp.saveState(prog_node);
307 const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) {
308 error.BrokenPipe, error.EndOfStream => |reason| {
309 std.log.info("{s} restart required: {t}", .{ argv[0], reason });
310 // Process restart required.
311 const term = zp.child.wait(io) catch |e| {
312 return s.fail("unable to wait for {s}: {t}", .{ argv[0], e });
313 };
314 _ = term;
315 exited = true;
316 break :update;
317 },
318 else => |e| return e,
319 };
320
321 if (s.result_error_bundle.errorMessageCount() > 0) {
322 return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()});
323 }
324
325 if (s.result_error_msgs.items.len > 0 and result == null) {
326 // Crash detected.
327 const term = zp.child.wait(io) catch |e| {
328 return s.fail("unable to wait for {s}: {t}", .{ argv[0], e });
329 };
330 s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0;
331 exited = true;
332 try handleChildProcessTerm(s, term);
333 return error.MakeFailed;
334 }
335
336 return result;
337 }
338 assert(argv.len != 0);
339
340 try handleChildProcUnsupported(s);
341 try handleVerbose(s, .inherit, argv);
342
343 const zp = try gpa.create(ZigProcess);
344 defer if (!watch) gpa.destroy(zp);
345
346 zp.child = std.process.spawn(io, .{
347 .argv = argv,
348 .environ_map = &b.graph.environ_map,
349 .stdin = .pipe,
350 .stdout = .pipe,
351 .stderr = .pipe,
352 .request_resource_usage_statistics = true,
353 .progress_node = prog_node,
354 }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err });
355
356 zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{
357 zp.child.stdout.?, zp.child.stderr.?,
358 });
359 if (watch) s.cast(Compile).?.zig_process = zp;
360 defer if (!watch) zp.deinit(io);
361
362 const result = result: {
363 defer if (watch) zp.saveState(prog_node);
364 break :result try zigProcessUpdate(s, zp, watch, web_server, gpa);
365 };
366
367 if (!watch) {
368 // Send EOF to stdin.
369 zp.child.stdin.?.close(io);
370 zp.child.stdin = null;
371
372 const term = zp.child.wait(io) catch |err| {
373 return s.fail("unable to wait for {s}: {t}", .{ argv[0], err });
374 };
375 s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0;
376
377 // Special handling for Compile step that is expecting compile errors.
378 if (s.cast(Compile)) |compile| switch (term) {
379 .exited => {
380 // Note that the exit code may be 0 in this case due to the
381 // compiler server protocol.
382 if (compile.expect_errors != null) {
383 return error.NeedCompileErrorCheck;
384 }
385 },
386 else => {},
387 };
388
389 try handleChildProcessTerm(s, term);
390 }
391
392 if (s.result_error_bundle.errorMessageCount() > 0) {
393 return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()});
394 }
395
396 return result;
397}
398
399/// Wrapper around `Io.Dir.updateFile` that handles verbose and error output.
400pub fn installFile(s: *Step, src_lazy_path: LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus {
401 const b = s.owner;
402 const io = b.graph.io;
403 const src_path = src_lazy_path.getPath3(b, s);
404 try handleVerbose(s, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path });
405 return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err|
406 return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err });
407}
408
409/// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output.
410pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus {
411 const b = s.owner;
412 const io = b.graph.io;
413 try handleVerbose(s, .inherit, &.{ "install", "-d", dest_path });
414 return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err|
415 return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err });
416}
417
418fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebServer, gpa: Allocator) !?Path {
419 const b = s.owner;
420 const arena = b.allocator;
421 const io = b.graph.io;
422
423 const start_ts = Io.Clock.awake.now(io);
424
425 try sendMessage(io, zp.child.stdin.?, .update);
426 if (!watch) try sendMessage(io, zp.child.stdin.?, .exit);
427
428 var result: ?Path = null;
429 var eos_err: error{EndOfStream}!void = {};
430
431 const stdout = zp.multi_reader.fileReader(0);
432
433 while (true) {
434 const Header = std.zig.Server.Message.Header;
435 const header = stdout.interface.takeStruct(Header, .little) catch |err| switch (err) {
436 error.EndOfStream => break,
437 error.ReadFailed => return stdout.err.?,
438 };
439 const body = stdout.interface.take(header.bytes_len) catch |err| switch (err) {
440 error.EndOfStream => |e| {
441 // Better to report the crash with stderr below, but we set
442 // this in case the child exits successfully while violating
443 // this protocol.
444 eos_err = e;
445 break;
446 },
447 error.ReadFailed => return stdout.err.?,
448 };
449 switch (header.tag) {
450 .zig_version => {
451 if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
452 return s.fail(
453 "zig version mismatch build runner vs compiler: '{s}' vs '{s}'",
454 .{ builtin.zig_version_string, body },
455 );
456 }
457 },
458 .error_bundle => {
459 s.result_error_bundle = try std.zig.Server.allocErrorBundle(gpa, body);
460 // This message indicates the end of the update.
461 if (watch) break;
462 },
463 .emit_digest => {
464 const EmitDigest = std.zig.Server.Message.EmitDigest;
465 const emit_digest: *align(1) const EmitDigest = @ptrCast(body);
466 s.result_cached = emit_digest.flags.cache_hit;
467 const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len];
468 result = .{
469 .root_dir = b.cache_root,
470 .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)),
471 };
472 },
473 .file_system_inputs => {
474 s.clearWatchInputs();
475 var it = std.mem.splitScalar(u8, body, 0);
476 while (it.next()) |prefixed_path| {
477 const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1);
478 const sub_path = try arena.dupe(u8, prefixed_path[1..]);
479 const sub_path_dirname = std.fs.path.dirname(sub_path) orelse "";
480 switch (prefix_index) {
481 .cwd => {
482 const path: Cache.Path = .{
483 .root_dir = Cache.Directory.cwd(),
484 .sub_path = sub_path_dirname,
485 };
486 try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
487 },
488 .zig_lib => zl: {
489 if (s.cast(Step.Compile)) |compile| {
490 if (compile.zig_lib_dir) |zig_lib_dir| {
491 const lp = try zig_lib_dir.join(arena, sub_path);
492 try addWatchInput(s, lp);
493 break :zl;
494 }
495 }
496 const path: Cache.Path = .{
497 .root_dir = s.owner.graph.zig_lib_directory,
498 .sub_path = sub_path_dirname,
499 };
500 try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
501 },
502 .local_cache => {
503 const path: Cache.Path = .{
504 .root_dir = b.cache_root,
505 .sub_path = sub_path_dirname,
506 };
507 try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
508 },
509 .global_cache => {
510 const path: Cache.Path = .{
511 .root_dir = s.owner.graph.global_cache_root,
512 .sub_path = sub_path_dirname,
513 };
514 try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
515 },
516 }
517 }
518 },
519 .time_report => if (web_server) |ws| {
520 const TimeReport = std.zig.Server.Message.TimeReport;
521 const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]);
522 ws.updateTimeReportCompile(.{
523 .compile = s.cast(Step.Compile).?,
524 .use_llvm = tr.flags.use_llvm,
525 .stats = tr.stats,
526 .ns_total = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()),
527 .llvm_pass_timings_len = tr.llvm_pass_timings_len,
528 .files_len = tr.files_len,
529 .decls_len = tr.decls_len,
530 .trailing = body[@sizeOf(TimeReport)..],
531 });
532 },
533 else => {}, // ignore other messages
534 }
535 }
536
537 s.result_duration_ns = @intCast(start_ts.untilNow(io, .awake).toNanoseconds());
538
539 const stderr_contents = zp.multi_reader.reader(1).buffered();
540 if (stderr_contents.len > 0) {
541 try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents));
542 }
543
544 try eos_err;
545
546 return result;
547}
548
549pub fn getZigProcess(s: *Step) ?*ZigProcess {
550 if (true) @panic("TODO getZigProcess");
551 return switch (s.id) {
552 .compile => s.cast(Compile).?.zig_process,
553 else => null,
554 };
555}
556
557fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void {
558 const header: std.zig.Client.Message.Header = .{
559 .tag = tag,
560 .bytes_len = 0,
561 };
562 var w = file.writer(io, &.{});
563 w.interface.writeStruct(header, .little) catch |err| switch (err) {
564 error.WriteFailed => return w.err.?,
565 };
566}
567
568pub fn handleVerbose(
569 s: *Step,
570 arena: Allocator,
571 cwd: std.process.Child.Cwd,
572 opt_env: ?*const std.process.Environ.Map,
573 argv: []const []const u8,
574) error{OutOfMemory}!void {
575 if (!s.verbose) return;
576 const graph = s.graph;
577 // Intention of verbose is to print all sub-process command lines to
578 // stderr before spawning them.
579 const text = try allocPrintCmd(arena, cwd, if (opt_env) |env| .{
580 .child = env,
581 .parent = &graph.environ_map,
582 } else null, argv);
583 std.log.scoped(.verbose).info("{s}", .{text});
584}
585
586/// Asserts that the caller has already populated `s.result_failed_command`.
587pub inline fn handleChildProcUnsupported(s: *Step) error{ OutOfMemory, MakeFailed }!void {
588 if (!std.process.can_spawn) {
589 return s.fail("unable to spawn process: host cannot spawn child processes", .{});
590 }
591}
592
593/// Asserts that the caller has already populated `s.result_failed_command`.
594pub fn handleChildProcessTerm(s: *Step, term: std.process.Child.Term) error{ MakeFailed, OutOfMemory }!void {
595 assert(s.result_failed_command != null);
596 return switch (term) {
597 .exited => |code| if (code != 0) s.fail("process exited with error code {d}", .{code}),
598 .signal => |sig| s.fail("process terminated with signal {t}", .{sig}),
599 .stopped => |sig| s.fail("process stopped with signal {t}", .{sig}),
600 .unknown => s.fail("process terminated unexpectedly", .{}),
601 };
602}
603
604/// Prefer `cacheHitAndWatch` unless you already added watch inputs
605/// separately from using the cache system.
606pub fn cacheHit(s: *Step, man: *Cache.Manifest) !bool {
607 s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err);
608 return s.result_cached;
609}
610
611/// Clears previous watch inputs, if any, and then populates watch inputs from
612/// the full set of files picked up by the cache manifest.
613///
614/// Must be accompanied with `writeManifestAndWatch`.
615pub fn cacheHitAndWatch(s: *Step, man: *Cache.Manifest) !bool {
616 const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err);
617 s.result_cached = is_hit;
618 // The above call to hit() populates the manifest with files, so in case of
619 // a hit, we need to populate watch inputs.
620 if (is_hit) try setWatchInputsFromManifest(s, man);
621 return is_hit;
622}
623
624fn failWithCacheError(
625 s: *Step,
626 man: *const Cache.Manifest,
627 err: Cache.Manifest.HitError,
628) error{ OutOfMemory, Canceled, MakeFailed } {
629 switch (err) {
630 error.CacheCheckFailed => switch (man.diagnostic) {
631 .none => unreachable,
632 .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{
633 man.diagnostic, e,
634 }),
635 .file_open, .file_stat, .file_read, .file_hash => |op| {
636 const pp = man.files.keys()[op.file_index].prefixed_path;
637 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
638 return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{
639 prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err,
640 });
641 },
642 },
643 error.OutOfMemory, error.Canceled => |e| return e,
644 error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}),
645 }
646}
647
648/// Prefer `writeManifestAndWatch` unless you already added watch inputs
649/// separately from using the cache system.
650pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void {
651 if (s.test_results.isSuccess()) {
652 man.writeManifest() catch |err| {
653 try s.addError("unable to write cache manifest: {t}", .{err});
654 };
655 }
656}
657
658/// Clears previous watch inputs, if any, and then populates watch inputs from
659/// the full set of files picked up by the cache manifest.
660///
661/// Must be accompanied with `cacheHitAndWatch`.
662pub fn writeManifestAndWatch(s: *Step, man: *Cache.Manifest) !void {
663 try writeManifest(s, man);
664 try setWatchInputsFromManifest(s, man);
665}
666
667fn setWatchInputsFromManifest(s: *Step, man: *Cache.Manifest) !void {
668 const arena = s.owner.allocator;
669 const prefixes = man.cache.prefixes();
670 clearWatchInputs(s);
671 for (man.files.keys()) |file| {
672 // The file path data is freed when the cache manifest is cleaned up at the end of `make`.
673 const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path);
674 try addWatchInputFromPath(s, .{
675 .root_dir = prefixes[file.prefixed_path.prefix],
676 .sub_path = std.fs.path.dirname(sub_path) orelse "",
677 }, std.fs.path.basename(sub_path));
678 }
679}
680
681/// For steps that have a single input that never changes when re-running `make`.
682pub fn singleUnchangingWatchInput(step: *Step, lazy_path: LazyPath) Allocator.Error!void {
683 if (!step.inputs.populated()) try step.addWatchInput(lazy_path);
684}
685
686pub fn clearWatchInputs(step: *Step) void {
687 const gpa = step.owner.allocator;
688 step.inputs.clear(gpa);
689}
690
691/// Places a *file* dependency on the path.
692pub fn addWatchInput(step: *Step, lazy_file: LazyPath) Allocator.Error!void {
693 switch (lazy_file) {
694 .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path),
695 .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path),
696 .cwd_relative => |path_string| {
697 try addWatchInputFromPath(step, .{
698 .root_dir = .{
699 .path = null,
700 .handle = Io.Dir.cwd(),
701 },
702 .sub_path = std.fs.path.dirname(path_string) orelse "",
703 }, std.fs.path.basename(path_string));
704 },
705 // Nothing to watch because this dependency edge is modeled instead via `dependants`.
706 .generated => {},
707 }
708}
709
710/// Any changes inside the directory will trigger invalidation.
711///
712/// See also `addDirectoryWatchInputFromPath` which takes a `Cache.Path` instead.
713///
714/// Paths derived from this directory should also be manually added via
715/// `addDirectoryWatchInputFromPath` if and only if this function returns
716/// `true`.
717pub fn addDirectoryWatchInput(step: *Step, lazy_directory: LazyPath) Allocator.Error!bool {
718 switch (lazy_directory) {
719 .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path),
720 .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path),
721 .cwd_relative => |path_string| {
722 try addDirectoryWatchInputFromPath(step, .{
723 .root_dir = .{
724 .path = null,
725 .handle = Io.Dir.cwd(),
726 },
727 .sub_path = path_string,
728 });
729 },
730 // Nothing to watch because this dependency edge is modeled instead via `dependants`.
731 .generated => return false,
732 }
733 return true;
734}
735
736/// Any changes inside the directory will trigger invalidation.
737///
738/// See also `addDirectoryWatchInput` which takes a `LazyPath` instead.
739///
740/// This function should only be called when it has been verified that the
741/// dependency on `path` is not already accounted for by a `Step` dependency.
742/// In other words, before calling this function, first check that the
743/// `LazyPath` which this `path` is derived from is not `generated`.
744pub fn addDirectoryWatchInputFromPath(step: *Step, path: Cache.Path) !void {
745 return addWatchInputFromPath(step, path, ".");
746}
747
748fn addWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void {
749 return addWatchInputFromPath(step, .{
750 .root_dir = package.build_root,
751 .sub_path = std.fs.path.dirname(sub_path) orelse "",
752 }, std.fs.path.basename(sub_path));
753}
754
755fn addDirectoryWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void {
756 return addDirectoryWatchInputFromPath(step, .{
757 .root_dir = package.build_root,
758 .sub_path = sub_path,
759 });
760}
761
762fn addWatchInputFromPath(step: *Step, path: Cache.Path, basename: []const u8) !void {
763 const gpa = step.owner.allocator;
764 const gop = try step.inputs.table.getOrPut(gpa, path);
765 if (!gop.found_existing) gop.value_ptr.* = .empty;
766 try gop.value_ptr.append(gpa, basename);
767}
768
769pub fn allocPrintCmd(
770 gpa: Allocator,
771 cwd: std.process.Child.Cwd,
772 opt_env: ?struct {
773 child: *const std.process.Environ.Map,
774 parent: *const std.process.Environ.Map,
775 },
776 argv: []const []const u8,
777) Allocator.Error![]u8 {
778 const shell = struct {
779 fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void {
780 for (string) |c| {
781 if (switch (c) {
782 else => true,
783 '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false,
784 '=' => is_argv0,
785 }) break;
786 } else return writer.writeAll(string);
787
788 try writer.writeByte('"');
789 for (string) |c| {
790 if (switch (c) {
791 std.ascii.control_code.nul => break,
792 '!', '"', '$', '\\', '`' => true,
793 else => !std.ascii.isPrint(c),
794 }) try writer.writeByte('\\');
795 switch (c) {
796 std.ascii.control_code.nul => unreachable,
797 std.ascii.control_code.bel => try writer.writeByte('a'),
798 std.ascii.control_code.bs => try writer.writeByte('b'),
799 std.ascii.control_code.ht => try writer.writeByte('t'),
800 std.ascii.control_code.lf => try writer.writeByte('n'),
801 std.ascii.control_code.vt => try writer.writeByte('v'),
802 std.ascii.control_code.ff => try writer.writeByte('f'),
803 std.ascii.control_code.cr => try writer.writeByte('r'),
804 std.ascii.control_code.esc => try writer.writeByte('E'),
805 ' '...'~' => try writer.writeByte(c),
806 else => try writer.print("{o:0>3}", .{c}),
807 }
808 }
809 try writer.writeByte('"');
810 }
811 };
812
813 var aw: Io.Writer.Allocating = .init(gpa);
814 defer aw.deinit();
815 const writer = &aw.writer;
816 switch (cwd) {
817 .inherit => {},
818 .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory,
819 .dir => @panic("TODO"),
820 }
821 if (opt_env) |env| {
822 var it = env.child.iterator();
823 while (it.next()) |entry| {
824 const key = entry.key_ptr.*;
825 const value = entry.value_ptr.*;
826 if (env.parent.get(key)) |process_value| {
827 if (std.mem.eql(u8, value, process_value)) continue;
828 }
829 writer.print("{s}=", .{key}) catch return error.OutOfMemory;
830 shell.escape(writer, value, false) catch return error.OutOfMemory;
831 writer.writeByte(' ') catch return error.OutOfMemory;
832 }
833 }
834 shell.escape(writer, argv[0], true) catch return error.OutOfMemory;
835 for (argv[1..]) |arg| {
836 writer.writeByte(' ') catch return error.OutOfMemory;
837 shell.escape(writer, arg, false) catch return error.OutOfMemory;
838 }
839 return aw.toOwnedSlice();
840}
lib/compiler/maker/Step/Compile.zig deleted-1200
...@@ -1,1200 +0,0 @@
1/// Populated during the make phase when there is a long-lived compiler process.
2/// Managed by the build runner, not user build script.
3zig_process: ?*Step.ZigProcess,
4
5fn make(step: *Step, options: Step.MakeOptions) !void {
6 const b = step.owner;
7 const compile: *Compile = @fieldParentPtr("step", step);
8
9 const zig_args = try getZigArgs(compile, false);
10
11 const maybe_output_dir = step.evalZigProcess(
12 zig_args,
13 options.progress_node,
14 (b.graph.incremental == true) and (options.watch or options.web_server != null),
15 options.web_server,
16 options.gpa,
17 ) catch |err| switch (err) {
18 error.NeedCompileErrorCheck => {
19 assert(compile.expect_errors != null);
20 try checkCompileErrors(compile);
21 return;
22 },
23 else => |e| return e,
24 };
25
26 // Update generated files
27 if (maybe_output_dir) |output_dir| {
28 if (compile.emit_directory) |lp| {
29 lp.path = b.fmt("{f}", .{output_dir});
30 }
31
32 // zig fmt: off
33 if (compile.generated_bin) |lp| lp.path = compile.outputPath(output_dir, .bin);
34 if (compile.generated_pdb) |lp| lp.path = compile.outputPath(output_dir, .pdb);
35 // hack for stage2_x86_64 + coff
36 if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib);
37 if (compile.generated_implib) |lp| lp.path = compile.outputPath(output_dir, .implib);
38 if (compile.generated_h) |lp| lp.path = compile.outputPath(output_dir, .h);
39 if (compile.generated_docs) |lp| lp.path = compile.outputPath(output_dir, .docs);
40 if (compile.generated_asm) |lp| lp.path = compile.outputPath(output_dir, .@"asm");
41 if (compile.generated_llvm_ir) |lp| lp.path = compile.outputPath(output_dir, .llvm_ir);
42 if (compile.generated_llvm_bc) |lp| lp.path = compile.outputPath(output_dir, .llvm_bc);
43 // zig fmt: on
44 }
45
46 if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic and
47 compile.version != null and compile.generated_bin != null and
48 std.Build.wantSharedLibSymLinks(compile.rootModuleTarget()))
49 {
50 try doAtomicSymLinks(
51 step,
52 compile.getEmittedBin().getPath2(b, step),
53 compile.major_only_filename.?,
54 compile.name_only_filename.?,
55 );
56 }
57}
58
59fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
60 const step = &compile.step;
61 const b = step.owner;
62 const arena = b.allocator;
63
64 var zig_args = std.array_list.Managed([]const u8).init(arena);
65 defer zig_args.deinit();
66
67 try zig_args.append(b.graph.zig_exe);
68
69 const cmd = switch (compile.kind) {
70 .lib => "build-lib",
71 .exe => "build-exe",
72 .obj => "build-obj",
73 .@"test" => "test",
74 .test_obj => "test-obj",
75 };
76 try zig_args.append(cmd);
77
78 if (b.reference_trace) |some| {
79 try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some}));
80 }
81 try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts);
82
83 try addFlag(&zig_args, "llvm", compile.use_llvm);
84 try addFlag(&zig_args, "lld", compile.use_lld);
85 try addFlag(&zig_args, "new-linker", compile.use_new_linker);
86
87 if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| {
88 try zig_args.append(try std.fmt.allocPrint(arena, "-ofmt={s}", .{@tagName(ofmt)}));
89 }
90
91 switch (compile.entry) {
92 .default => {},
93 .disabled => try zig_args.append("-fno-entry"),
94 .enabled => try zig_args.append("-fentry"),
95 .symbol_name => |entry_name| {
96 try zig_args.append(try std.fmt.allocPrint(arena, "-fentry={s}", .{entry_name}));
97 },
98 }
99
100 {
101 for (compile.force_undefined_symbols.keys()) |symbol_name| {
102 try zig_args.append("--force_undefined");
103 try zig_args.append(symbol_name.*);
104 }
105 }
106
107 if (compile.stack_size) |stack_size| {
108 try zig_args.append("--stack");
109 try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size}));
110 }
111
112 if (fuzz) {
113 try zig_args.append("-ffuzz");
114 }
115
116 {
117 // Stores system libraries that have already been seen for at least one
118 // module, along with any arguments that need to be passed to the
119 // compiler for each module individually.
120 var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty;
121 var frameworks: std.StringArrayHashMapUnmanaged(Module.LinkFrameworkOptions) = .empty;
122
123 var prev_has_cflags = false;
124 var prev_has_rcflags = false;
125 var prev_search_strategy: Module.SystemLib.SearchStrategy = .paths_first;
126 var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic;
127 // Track the number of positional arguments so that a nice error can be
128 // emitted if there is nothing to link.
129 var total_linker_objects: usize = @intFromBool(compile.root_module.root_source_file != null);
130
131 // Fully recursive iteration including dynamic libraries to detect
132 // libc and libc++ linkage.
133 for (compile.getCompileDependencies(true)) |some_compile| {
134 for (some_compile.root_module.getGraph().modules) |mod| {
135 if (mod.link_libc == true) compile.is_linking_libc = true;
136 if (mod.link_libcpp == true) compile.is_linking_libcpp = true;
137 }
138 }
139
140 var cli_named_modules = try CliNamedModules.init(arena, compile.root_module);
141
142 // For this loop, don't chase dynamic libraries because their link
143 // objects are already linked.
144 for (compile.getCompileDependencies(false)) |dep_compile| {
145 for (dep_compile.root_module.getGraph().modules) |mod| {
146 // While walking transitive dependencies, if a given link object is
147 // already included in a library, it should not redundantly be
148 // placed on the linker line of the dependee.
149 const my_responsibility = dep_compile == compile;
150 const already_linked = !my_responsibility and dep_compile.isDynamicLibrary();
151
152 // Inherit dependencies on darwin frameworks.
153 if (!already_linked) {
154 for (mod.frameworks.keys(), mod.frameworks.values()) |name, info| {
155 try frameworks.put(arena, name, info);
156 }
157 }
158
159 // Inherit dependencies on system libraries and static libraries.
160 for (mod.link_objects.items) |link_object| {
161 switch (link_object) {
162 .static_path => |static_path| {
163 if (my_responsibility) {
164 try zig_args.append(static_path.getPath2(mod.owner, step));
165 total_linker_objects += 1;
166 }
167 },
168 .system_lib => |system_lib| {
169 const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name);
170 if (system_lib_gop.found_existing) {
171 try zig_args.appendSlice(system_lib_gop.value_ptr.*);
172 continue;
173 } else {
174 system_lib_gop.value_ptr.* = &.{};
175 }
176
177 if (already_linked)
178 continue;
179
180 if ((system_lib.search_strategy != prev_search_strategy or
181 system_lib.preferred_link_mode != prev_preferred_link_mode) and
182 compile.linkage != .static)
183 {
184 switch (system_lib.search_strategy) {
185 .no_fallback => switch (system_lib.preferred_link_mode) {
186 .dynamic => try zig_args.append("-search_dylibs_only"),
187 .static => try zig_args.append("-search_static_only"),
188 },
189 .paths_first => switch (system_lib.preferred_link_mode) {
190 .dynamic => try zig_args.append("-search_paths_first"),
191 .static => try zig_args.append("-search_paths_first_static"),
192 },
193 .mode_first => switch (system_lib.preferred_link_mode) {
194 .dynamic => try zig_args.append("-search_dylibs_first"),
195 .static => try zig_args.append("-search_static_first"),
196 },
197 }
198 prev_search_strategy = system_lib.search_strategy;
199 prev_preferred_link_mode = system_lib.preferred_link_mode;
200 }
201
202 const prefix: []const u8 = prefix: {
203 if (system_lib.needed) break :prefix "-needed-l";
204 if (system_lib.weak) break :prefix "-weak-l";
205 break :prefix "-l";
206 };
207 switch (system_lib.use_pkg_config) {
208 .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })),
209 .yes, .force => {
210 if (compile.runPkgConfig(system_lib.name)) |result| {
211 try zig_args.appendSlice(result.cflags);
212 try zig_args.appendSlice(result.libs);
213 try seen_system_libs.put(arena, system_lib.name, result.cflags);
214 } else |err| switch (err) {
215 error.PkgConfigInvalidOutput,
216 error.PkgConfigCrashed,
217 error.PkgConfigFailed,
218 error.PkgConfigNotInstalled,
219 error.PackageNotFound,
220 => switch (system_lib.use_pkg_config) {
221 .yes => {
222 // pkg-config failed, so fall back to linking the library
223 // by name directly.
224 try zig_args.append(b.fmt("{s}{s}", .{
225 prefix,
226 system_lib.name,
227 }));
228 },
229 .force => {
230 panic("pkg-config failed for library {s}", .{system_lib.name});
231 },
232 .no => unreachable,
233 },
234
235 else => |e| return e,
236 }
237 },
238 }
239 },
240 .other_step => |other| {
241 switch (other.kind) {
242 .exe => return step.fail("cannot link with an executable build artifact", .{}),
243 .@"test" => return step.fail("cannot link with a test", .{}),
244 .obj, .test_obj => {
245 const included_in_lib_or_obj = !my_responsibility and
246 (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj);
247 if (!already_linked and !included_in_lib_or_obj) {
248 try zig_args.append(other.getEmittedBin().getPath2(b, step));
249 total_linker_objects += 1;
250 }
251 },
252 .lib => l: {
253 const other_produces_implib = other.producesImplib();
254 const other_is_static = other_produces_implib or other.isStaticLibrary();
255
256 if (compile.isStaticLibrary() and other_is_static) {
257 // Avoid putting a static library inside a static library.
258 break :l;
259 }
260
261 // For DLLs, we must link against the implib.
262 // For everything else, we directly link
263 // against the library file.
264 const full_path_lib = if (other_produces_implib)
265 try other.getGeneratedFilePath("generated_implib", &compile.step)
266 else
267 try other.getGeneratedFilePath("generated_bin", &compile.step);
268
269 try zig_args.append(full_path_lib);
270 total_linker_objects += 1;
271
272 if (other.linkage == .dynamic and
273 compile.rootModuleTarget().os.tag != .windows)
274 {
275 if (fs.path.dirname(full_path_lib)) |dirname| {
276 try zig_args.append("-rpath");
277 try zig_args.append(dirname);
278 }
279 }
280 },
281 }
282 },
283 .assembly_file => |asm_file| l: {
284 if (!my_responsibility) break :l;
285
286 if (prev_has_cflags) {
287 try zig_args.append("-cflags");
288 try zig_args.append("--");
289 prev_has_cflags = false;
290 }
291 try zig_args.append(asm_file.getPath2(mod.owner, step));
292 total_linker_objects += 1;
293 },
294
295 .c_source_file => |c_source_file| l: {
296 if (!my_responsibility) break :l;
297
298 if (prev_has_cflags or c_source_file.flags.len != 0) {
299 try zig_args.append("-cflags");
300 for (c_source_file.flags) |arg| {
301 try zig_args.append(arg);
302 }
303 try zig_args.append("--");
304 }
305 prev_has_cflags = (c_source_file.flags.len != 0);
306
307 if (c_source_file.language) |lang| {
308 try zig_args.append("-x");
309 try zig_args.append(lang.internalIdentifier());
310 }
311
312 try zig_args.append(c_source_file.file.getPath2(mod.owner, step));
313
314 if (c_source_file.language != null) {
315 try zig_args.append("-x");
316 try zig_args.append("none");
317 }
318 total_linker_objects += 1;
319 },
320
321 .c_source_files => |c_source_files| l: {
322 if (!my_responsibility) break :l;
323
324 if (prev_has_cflags or c_source_files.flags.len != 0) {
325 try zig_args.append("-cflags");
326 for (c_source_files.flags) |arg| {
327 try zig_args.append(arg);
328 }
329 try zig_args.append("--");
330 }
331 prev_has_cflags = (c_source_files.flags.len != 0);
332
333 if (c_source_files.language) |lang| {
334 try zig_args.append("-x");
335 try zig_args.append(lang.internalIdentifier());
336 }
337
338 const root_path = c_source_files.root.getPath2(mod.owner, step);
339 for (c_source_files.files) |file| {
340 try zig_args.append(b.pathJoin(&.{ root_path, file }));
341 }
342
343 if (c_source_files.language != null) {
344 try zig_args.append("-x");
345 try zig_args.append("none");
346 }
347
348 total_linker_objects += c_source_files.files.len;
349 },
350
351 .win32_resource_file => |rc_source_file| l: {
352 if (!my_responsibility) break :l;
353
354 if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) {
355 if (prev_has_rcflags) {
356 try zig_args.append("-rcflags");
357 try zig_args.append("--");
358 prev_has_rcflags = false;
359 }
360 } else {
361 try zig_args.append("-rcflags");
362 for (rc_source_file.flags) |arg| {
363 try zig_args.append(arg);
364 }
365 for (rc_source_file.include_paths) |include_path| {
366 try zig_args.append("/I");
367 try zig_args.append(include_path.getPath2(mod.owner, step));
368 }
369 try zig_args.append("--");
370 prev_has_rcflags = true;
371 }
372 try zig_args.append(rc_source_file.file.getPath2(mod.owner, step));
373 total_linker_objects += 1;
374 },
375 }
376 }
377
378 // We need to emit the --mod argument here so that the above link objects
379 // have the correct parent module, but only if the module is part of
380 // this compilation.
381 if (!my_responsibility) continue;
382 if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| {
383 const module_cli_name = cli_named_modules.names.keys()[module_cli_index];
384 try mod.appendZigProcessFlags(&zig_args, step);
385
386 // --dep arguments
387 try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2);
388 for (mod.import_table.keys(), mod.import_table.values()) |name, import| {
389 const import_index = cli_named_modules.modules.getIndex(import).?;
390 const import_cli_name = cli_named_modules.names.keys()[import_index];
391 zig_args.appendAssumeCapacity("--dep");
392 if (std.mem.eql(u8, import_cli_name, name)) {
393 zig_args.appendAssumeCapacity(import_cli_name);
394 } else {
395 zig_args.appendAssumeCapacity(b.fmt("{s}={s}", .{ name, import_cli_name }));
396 }
397 }
398
399 // When the CLI sees a -M argument, it determines whether it
400 // implies the existence of a Zig compilation unit based on
401 // whether there is a root source file. If there is no root
402 // source file, then this is not a zig compilation unit - it is
403 // perhaps a set of linker objects, or C source files instead.
404 // Linker objects are added to the CLI globally, while C source
405 // files must have a module parent.
406 if (mod.root_source_file) |lp| {
407 const src = lp.getPath2(mod.owner, step);
408 try zig_args.append(b.fmt("-M{s}={s}", .{ module_cli_name, src }));
409 } else if (moduleNeedsCliArg(mod)) {
410 try zig_args.append(b.fmt("-M{s}", .{module_cli_name}));
411 }
412 }
413 }
414 }
415
416 if (total_linker_objects == 0) {
417 return step.fail("the linker needs one or more objects to link", .{});
418 }
419
420 for (frameworks.keys(), frameworks.values()) |name, info| {
421 if (info.needed) {
422 try zig_args.append("-needed_framework");
423 } else if (info.weak) {
424 try zig_args.append("-weak_framework");
425 } else {
426 try zig_args.append("-framework");
427 }
428 try zig_args.append(name);
429 }
430
431 if (compile.is_linking_libcpp) {
432 try zig_args.append("-lc++");
433 }
434
435 if (compile.is_linking_libc) {
436 try zig_args.append("-lc");
437 }
438 }
439
440 if (compile.win32_manifest) |manifest_file| {
441 try zig_args.append(manifest_file.getPath2(b, step));
442 }
443
444 if (compile.win32_module_definition) |module_file| {
445 try zig_args.append(module_file.getPath2(b, step));
446 }
447
448 if (compile.image_base) |image_base| {
449 try zig_args.append("--image-base");
450 try zig_args.append(b.fmt("0x{x}", .{image_base}));
451 }
452
453 for (compile.filters) |filter| {
454 try zig_args.append("--test-filter");
455 try zig_args.append(filter);
456 }
457
458 if (compile.test_runner) |test_runner| {
459 try zig_args.append("--test-runner");
460 try zig_args.append(test_runner.path.getPath2(b, step));
461 }
462
463 for (b.debug_log_scopes) |log_scope| {
464 try zig_args.append("--debug-log");
465 try zig_args.append(log_scope);
466 }
467
468 if (b.debug_compile_errors) {
469 try zig_args.append("--debug-compile-errors");
470 }
471
472 if (b.debug_incremental) {
473 try zig_args.append("--debug-incremental");
474 }
475
476 if (b.verbose_air) try zig_args.append("--verbose-air");
477 if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path}));
478 if (b.verbose_llvm_bc) |path| try zig_args.append(b.fmt("--verbose-llvm-bc={s}", .{path}));
479 if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link");
480 if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc");
481 if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features");
482 if (b.graph.time_report) try zig_args.append("--time-report");
483
484 if (compile.generated_asm != null) try zig_args.append("-femit-asm");
485 if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin");
486 if (compile.generated_docs != null) try zig_args.append("-femit-docs");
487 if (compile.generated_implib != null) try zig_args.append("-femit-implib");
488 if (compile.generated_llvm_bc != null) try zig_args.append("-femit-llvm-bc");
489 if (compile.generated_llvm_ir != null) try zig_args.append("-femit-llvm-ir");
490 if (compile.generated_h != null) try zig_args.append("-femit-h");
491
492 try addFlag(&zig_args, "formatted-panics", compile.formatted_panics);
493
494 switch (compile.compress_debug_sections) {
495 .none => {},
496 .zlib => try zig_args.append("--compress-debug-sections=zlib"),
497 .zstd => try zig_args.append("--compress-debug-sections=zstd"),
498 }
499
500 if (compile.link_eh_frame_hdr) {
501 try zig_args.append("--eh-frame-hdr");
502 }
503 if (compile.link_emit_relocs) {
504 try zig_args.append("--emit-relocs");
505 }
506 if (compile.link_function_sections) {
507 try zig_args.append("-ffunction-sections");
508 }
509 if (compile.link_data_sections) {
510 try zig_args.append("-fdata-sections");
511 }
512 if (compile.link_gc_sections) |x| {
513 try zig_args.append(if (x) "--gc-sections" else "--no-gc-sections");
514 }
515 if (!compile.linker_dynamicbase) {
516 try zig_args.append("--no-dynamicbase");
517 }
518 if (compile.linker_allow_shlib_undefined) |x| {
519 try zig_args.append(if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined");
520 }
521 if (compile.link_z_notext) {
522 try zig_args.append("-z");
523 try zig_args.append("notext");
524 }
525 if (!compile.link_z_relro) {
526 try zig_args.append("-z");
527 try zig_args.append("norelro");
528 }
529 if (compile.link_z_lazy) {
530 try zig_args.append("-z");
531 try zig_args.append("lazy");
532 }
533 if (compile.link_z_common_page_size) |size| {
534 try zig_args.append("-z");
535 try zig_args.append(b.fmt("common-page-size={d}", .{size}));
536 }
537 if (compile.link_z_max_page_size) |size| {
538 try zig_args.append("-z");
539 try zig_args.append(b.fmt("max-page-size={d}", .{size}));
540 }
541 if (compile.link_z_defs) {
542 try zig_args.append("-z");
543 try zig_args.append("defs");
544 }
545
546 if (compile.libc_file) |libc_file| {
547 try zig_args.append("--libc");
548 try zig_args.append(libc_file.getPath2(b, step));
549 } else if (b.libc_file) |libc_file| {
550 try zig_args.append("--libc");
551 try zig_args.append(libc_file);
552 }
553
554 try zig_args.append("--cache-dir");
555 try zig_args.append(b.cache_root.path orelse ".");
556
557 try zig_args.append("--global-cache-dir");
558 try zig_args.append(b.graph.global_cache_root.path orelse ".");
559
560 if (b.graph.debug_compiler_runtime_libs) |mode|
561 try zig_args.append(b.fmt("--debug-rt={t}", .{mode}));
562
563 try zig_args.append("--name");
564 try zig_args.append(compile.name);
565
566 if (compile.linkage) |some| switch (some) {
567 .dynamic => try zig_args.append("-dynamic"),
568 .static => try zig_args.append("-static"),
569 };
570 if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) {
571 if (compile.version) |version| {
572 try zig_args.append("--version");
573 try zig_args.append(b.fmt("{f}", .{version}));
574 }
575
576 if (compile.rootModuleTarget().os.tag.isDarwin()) {
577 const install_name = compile.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{
578 compile.rootModuleTarget().libPrefix(),
579 compile.name,
580 compile.rootModuleTarget().dynamicLibSuffix(),
581 });
582 try zig_args.append("-install_name");
583 try zig_args.append(install_name);
584 }
585 }
586
587 if (compile.entitlements) |entitlements| {
588 try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements });
589 }
590 if (compile.pagezero_size) |pagezero_size| {
591 const size = try std.fmt.allocPrint(arena, "{x}", .{pagezero_size});
592 try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size });
593 }
594 if (compile.headerpad_size) |headerpad_size| {
595 const size = try std.fmt.allocPrint(arena, "{x}", .{headerpad_size});
596 try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size });
597 }
598 if (compile.headerpad_max_install_names) {
599 try zig_args.append("-headerpad_max_install_names");
600 }
601 if (compile.dead_strip_dylibs) {
602 try zig_args.append("-dead_strip_dylibs");
603 }
604 if (compile.force_load_objc) {
605 try zig_args.append("-ObjC");
606 }
607 if (compile.discard_local_symbols) {
608 try zig_args.append("--discard-all");
609 }
610
611 try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt);
612 try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt);
613 try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns);
614 if (compile.rdynamic) {
615 try zig_args.append("-rdynamic");
616 }
617 if (compile.import_memory) {
618 try zig_args.append("--import-memory");
619 }
620 if (compile.export_memory) {
621 try zig_args.append("--export-memory");
622 }
623 if (compile.import_symbols) {
624 try zig_args.append("--import-symbols");
625 }
626 if (compile.import_table) {
627 try zig_args.append("--import-table");
628 }
629 if (compile.export_table) {
630 try zig_args.append("--export-table");
631 }
632 if (compile.initial_memory) |initial_memory| {
633 try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory}));
634 }
635 if (compile.max_memory) |max_memory| {
636 try zig_args.append(b.fmt("--max-memory={d}", .{max_memory}));
637 }
638 if (compile.shared_memory) {
639 try zig_args.append("--shared-memory");
640 }
641 if (compile.global_base) |global_base| {
642 try zig_args.append(b.fmt("--global-base={d}", .{global_base}));
643 }
644
645 if (compile.wasi_exec_model) |model| {
646 try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)}));
647 }
648 if (compile.linker_script) |linker_script| {
649 try zig_args.append("--script");
650 try zig_args.append(linker_script.getPath2(b, step));
651 }
652
653 if (compile.version_script) |version_script| {
654 try zig_args.append("--version-script");
655 try zig_args.append(version_script.getPath2(b, step));
656 }
657 if (compile.linker_allow_undefined_version) |x| {
658 try zig_args.append(if (x) "--undefined-version" else "--no-undefined-version");
659 }
660
661 if (compile.linker_enable_new_dtags) |enabled| {
662 try zig_args.append(if (enabled) "--enable-new-dtags" else "--disable-new-dtags");
663 }
664
665 if (compile.kind == .@"test") {
666 if (compile.exec_cmd_args) |exec_cmd_args| {
667 for (exec_cmd_args) |cmd_arg| {
668 if (cmd_arg) |arg| {
669 try zig_args.append("--test-cmd");
670 try zig_args.append(arg);
671 } else {
672 try zig_args.append("--test-cmd-bin");
673 }
674 }
675 }
676 }
677
678 if (b.sysroot) |sysroot| {
679 try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot });
680 }
681
682 // -I and -L arguments that appear after the last --mod argument apply to all modules.
683 const cwd: Io.Dir = .cwd();
684 const io = b.graph.io;
685
686 for (b.search_prefixes.items) |search_prefix| {
687 var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| {
688 return step.fail("unable to open prefix directory '{s}': {s}", .{
689 search_prefix, @errorName(err),
690 });
691 };
692 defer prefix_dir.close(io);
693
694 // Avoid passing -L and -I flags for nonexistent directories.
695 // This prevents a warning, that should probably be upgraded to an error in Zig's
696 // CLI parsing code, when the linker sees an -L directory that does not exist.
697
698 if (prefix_dir.access(io, "lib", .{})) |_| {
699 try zig_args.appendSlice(&.{
700 "-L", b.pathJoin(&.{ search_prefix, "lib" }),
701 });
702 } else |err| switch (err) {
703 error.FileNotFound => {},
704 else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{
705 search_prefix, @errorName(e),
706 }),
707 }
708
709 if (prefix_dir.access(io, "include", .{})) |_| {
710 try zig_args.appendSlice(&.{
711 "-I", b.pathJoin(&.{ search_prefix, "include" }),
712 });
713 } else |err| switch (err) {
714 error.FileNotFound => {},
715 else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{
716 search_prefix, @errorName(e),
717 }),
718 }
719 }
720
721 if (compile.rc_includes != .any) {
722 try zig_args.append("-rcincludes");
723 try zig_args.append(@tagName(compile.rc_includes));
724 }
725
726 try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath);
727
728 if (compile.build_id orelse b.build_id) |build_id| {
729 try zig_args.append(switch (build_id) {
730 .hexstring => |hs| b.fmt("--build-id=0x{x}", .{hs.toSlice()}),
731 .none, .fast, .uuid, .sha1, .md5 => b.fmt("--build-id={s}", .{@tagName(build_id)}),
732 });
733 }
734
735 const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir|
736 dir.getPath2(b, step)
737 else if (b.graph.zig_lib_directory.path) |_|
738 b.fmt("{f}", .{b.graph.zig_lib_directory})
739 else
740 null;
741
742 if (opt_zig_lib_dir) |zig_lib_dir| {
743 try zig_args.append("--zig-lib-dir");
744 try zig_args.append(zig_lib_dir);
745 }
746
747 try addFlag(&zig_args, "PIE", compile.pie);
748
749 if (compile.lto) |lto| {
750 try zig_args.append(switch (lto) {
751 .full => "-flto=full",
752 .thin => "-flto=thin",
753 .none => "-fno-lto",
754 });
755 }
756
757 try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard);
758
759 if (compile.subsystem) |subsystem| {
760 try zig_args.append("--subsystem");
761 try zig_args.append(@tagName(subsystem));
762 }
763
764 if (compile.mingw_unicode_entry_point) {
765 try zig_args.append("-municode");
766 }
767
768 if (compile.error_limit) |err_limit| try zig_args.appendSlice(&.{
769 "--error-limit", b.fmt("{d}", .{err_limit}),
770 });
771
772 try addFlag(&zig_args, "incremental", b.graph.incremental);
773
774 try zig_args.append("--listen=-");
775
776 // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux
777 // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and
778 // pass that to zig, e.g. via 'zig build-lib @args.rsp'
779 // See @file syntax here: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html
780 var args_length: usize = 0;
781 for (zig_args.items) |arg| {
782 args_length += arg.len + 1; // +1 to account for null terminator
783 }
784 if (args_length >= 30 * 1024) {
785 try b.cache_root.handle.createDirPath(io, "args");
786
787 const args_to_escape = zig_args.items[2..];
788 var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len);
789 arg_blk: for (args_to_escape) |arg| {
790 for (arg, 0..) |c, arg_idx| {
791 if (c == '\\' or c == '"') {
792 // Slow path for arguments that need to be escaped. We'll need to allocate and copy
793 var escaped: std.ArrayList(u8) = .empty;
794 try escaped.ensureTotalCapacityPrecise(arena, arg.len + 1);
795 try escaped.appendSlice(arena, arg[0..arg_idx]);
796 for (arg[arg_idx..]) |to_escape| {
797 if (to_escape == '\\' or to_escape == '"') try escaped.append(arena, '\\');
798 try escaped.append(arena, to_escape);
799 }
800 escaped_args.appendAssumeCapacity(escaped.items);
801 continue :arg_blk;
802 }
803 }
804 escaped_args.appendAssumeCapacity(arg); // no escaping needed so just use original argument
805 }
806
807 // Write the args to zig-cache/args/<SHA256 hash of args> to avoid conflicts with
808 // other zig build commands running in parallel.
809 const partially_quoted = try std.mem.join(arena, "\" \"", escaped_args.items);
810 const args = try std.mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" });
811
812 var args_hash: [Sha256.digest_length]u8 = undefined;
813 Sha256.hash(args, &args_hash, .{});
814 var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined;
815 _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash});
816
817 const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash;
818 if (b.cache_root.handle.access(io, args_file, .{})) |_| {
819 // The args file is already present from a previous run.
820 } else |err| switch (err) {
821 error.FileNotFound => {
822 var af = b.cache_root.handle.createFileAtomic(io, args_file, .{
823 .replace = false,
824 .make_path = true,
825 }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{
826 b.cache_root, args_file, e,
827 });
828 defer af.deinit(io);
829
830 af.file.writeStreamingAll(io, args) catch |e| {
831 return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{
832 b.cache_root, args_file, e,
833 });
834 };
835 // Note we can't clean up this file, not even after build
836 // success, because that might interfere with another build
837 // process that needs the same file.
838 af.link(io) catch |e| switch (e) {
839 error.PathAlreadyExists => {
840 // The args file was created by another concurrent build process.
841 },
842 else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{
843 b.cache_root, args_file, other_err,
844 }),
845 };
846 },
847 else => |other_err| return other_err,
848 }
849
850 const resolved_args_file = try mem.concat(arena, u8, &.{
851 "@",
852 try b.cache_root.join(arena, &.{args_file}),
853 });
854
855 zig_args.shrinkRetainingCapacity(2);
856 try zig_args.append(resolved_args_file);
857 }
858
859 return try zig_args.toOwnedSlice();
860}
861
862pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path {
863 c.step.result_error_msgs.clearRetainingCapacity();
864 c.step.result_stderr = "";
865
866 c.step.result_error_bundle.deinit(gpa);
867 c.step.result_error_bundle = std.zig.ErrorBundle.empty;
868
869 if (c.step.result_failed_command) |cmd| {
870 gpa.free(cmd);
871 c.step.result_failed_command = null;
872 }
873
874 const zig_args = try getZigArgs(c, true);
875 const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa);
876 return maybe_output_bin_path.?;
877}
878
879pub fn doAtomicSymLinks(
880 step: *Step,
881 output_path: []const u8,
882 filename_major_only: []const u8,
883 filename_name_only: []const u8,
884) !void {
885 const b = step.owner;
886 const io = b.graph.io;
887 const out_dir = fs.path.dirname(output_path) orelse ".";
888 const out_basename = fs.path.basename(output_path);
889 // sym link for libfoo.so.1 to libfoo.so.1.2.3
890 const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only });
891 const cwd: Io.Dir = .cwd();
892 cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| {
893 return step.fail("unable to symlink {s} -> {s}: {s}", .{
894 major_only_path, out_basename, @errorName(err),
895 });
896 };
897 // sym link for libfoo.so to libfoo.so.1
898 const name_only_path = b.pathJoin(&.{ out_dir, filename_name_only });
899 cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| {
900 return step.fail("Unable to symlink {s} -> {s}: {s}", .{
901 name_only_path, filename_major_only, @errorName(err),
902 });
903 };
904}
905
906fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg {
907 const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config";
908 const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore);
909 var list = std.array_list.Managed(PkgConfigPkg).init(b.allocator);
910 errdefer list.deinit();
911 var line_it = mem.tokenizeAny(u8, stdout, "\r\n");
912 while (line_it.next()) |line| {
913 if (mem.trim(u8, line, " \t").len == 0) continue;
914 var tok_it = mem.tokenizeAny(u8, line, " \t");
915 try list.append(PkgConfigPkg{
916 .name = tok_it.next() orelse return error.PkgConfigInvalidOutput,
917 .desc = tok_it.rest(),
918 });
919 }
920 return list.toOwnedSlice();
921}
922
923fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg {
924 if (b.pkg_config_pkg_list) |res| {
925 return res;
926 }
927 var code: u8 = undefined;
928 if (execPkgConfigList(b, &code)) |list| {
929 b.pkg_config_pkg_list = list;
930 return list;
931 } else |err| {
932 const result = switch (err) {
933 error.ProcessTerminated => error.PkgConfigCrashed,
934 error.ExecNotSupported => error.PkgConfigFailed,
935 error.ExitCodeFailure => error.PkgConfigFailed,
936 error.FileNotFound => error.PkgConfigNotInstalled,
937 error.InvalidName => error.PkgConfigNotInstalled,
938 error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput,
939 else => return err,
940 };
941 b.pkg_config_pkg_list = result;
942 return result;
943 }
944}
945
946fn addFlag(args: *std.array_list.Managed([]const u8), comptime name: []const u8, opt: ?bool) !void {
947 const cond = opt orelse return;
948 try args.ensureUnusedCapacity(1);
949 if (cond) {
950 args.appendAssumeCapacity("-f" ++ name);
951 } else {
952 args.appendAssumeCapacity("-fno-" ++ name);
953 }
954}
955
956const PkgConfigResult = struct {
957 cflags: []const []const u8,
958 libs: []const []const u8,
959};
960
961/// Run pkg-config for the given library name and parse the output, returning the arguments
962/// that should be passed to zig to link the given library.
963fn runPkgConfig(compile: *Compile, lib_name: []const u8) !PkgConfigResult {
964 const wl_rpath_prefix = "-Wl,-rpath,";
965
966 const b = compile.step.owner;
967 const arena = b.allocator;
968 const pkg_name = match: {
969 // First we have to map the library name to pkg config name. Unfortunately,
970 // there are several examples where this is not straightforward:
971 // -lSDL2 -> pkg-config sdl2
972 // -lgdk-3 -> pkg-config gdk-3.0
973 // -latk-1.0 -> pkg-config atk
974 // -lpulse -> pkg-config libpulse
975 const pkgs = try getPkgConfigList(b);
976
977 // Exact match means instant winner.
978 for (pkgs) |pkg| {
979 if (mem.eql(u8, pkg.name, lib_name)) {
980 break :match pkg.name;
981 }
982 }
983
984 // Next we'll try ignoring case.
985 for (pkgs) |pkg| {
986 if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) {
987 break :match pkg.name;
988 }
989 }
990
991 // Prefixed "lib" or suffixed ".0".
992 for (pkgs) |pkg| {
993 if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| {
994 const prefix = pkg.name[0..pos];
995 const suffix = pkg.name[pos + lib_name.len ..];
996 if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue;
997 if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue;
998 break :match pkg.name;
999 }
1000 }
1001
1002 // Trimming "-1.0".
1003 if (mem.endsWith(u8, lib_name, "-1.0")) {
1004 const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len];
1005 for (pkgs) |pkg| {
1006 if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) {
1007 break :match pkg.name;
1008 }
1009 }
1010 }
1011
1012 return error.PackageNotFound;
1013 };
1014
1015 var code: u8 = undefined;
1016 const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config";
1017 const stdout = if (b.runAllowFail(&[_][]const u8{
1018 pkg_config_exe,
1019 pkg_name,
1020 "--cflags",
1021 "--libs",
1022 }, &code, .ignore)) |stdout| stdout else |err| switch (err) {
1023 error.ProcessTerminated => return error.PkgConfigCrashed,
1024 error.ExecNotSupported => return error.PkgConfigFailed,
1025 error.ExitCodeFailure => return error.PkgConfigFailed,
1026 error.FileNotFound => return error.PkgConfigNotInstalled,
1027 else => return err,
1028 };
1029
1030 var zig_cflags: std.ArrayList([]const u8) = .empty;
1031 defer zig_cflags.deinit(arena);
1032 var zig_libs: std.ArrayList([]const u8) = .empty;
1033 defer zig_libs.deinit(arena);
1034
1035 var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t");
1036 while (arg_it.next()) |arg| {
1037 if (mem.eql(u8, arg, "-I")) {
1038 const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput;
1039 try zig_cflags.appendSlice(arena, &.{ "-I", dir });
1040 } else if (mem.startsWith(u8, arg, "-I")) {
1041 try zig_cflags.append(arena, arg);
1042 } else if (mem.eql(u8, arg, "-L")) {
1043 const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput;
1044 try zig_libs.appendSlice(arena, &.{ "-L", dir });
1045 } else if (mem.startsWith(u8, arg, "-L")) {
1046 try zig_libs.append(arena, arg);
1047 } else if (mem.eql(u8, arg, "-l")) {
1048 const lib = arg_it.next() orelse return error.PkgConfigInvalidOutput;
1049 try zig_libs.appendSlice(arena, &.{ "-l", lib });
1050 } else if (mem.startsWith(u8, arg, "-l")) {
1051 try zig_libs.append(arena, arg);
1052 } else if (mem.eql(u8, arg, "-D")) {
1053 const macro = arg_it.next() orelse return error.PkgConfigInvalidOutput;
1054 try zig_cflags.appendSlice(arena, &.{ "-D", macro });
1055 } else if (mem.startsWith(u8, arg, "-D")) {
1056 try zig_cflags.append(arena, arg);
1057 } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) {
1058 try zig_cflags.appendSlice(arena, &.{ "-rpath", arg[wl_rpath_prefix.len..] });
1059 } else if (b.debug_pkg_config) {
1060 return compile.step.fail("unknown pkg-config flag '{s}'", .{arg});
1061 }
1062 }
1063
1064 try zig_cflags.shrinkToLen(arena);
1065 try zig_libs.shrinkToLen(arena);
1066
1067 return .{
1068 .cflags = zig_cflags.toOwnedSliceAssert(),
1069 .libs = zig_libs.toOwnedSliceAssert(),
1070 };
1071}
1072
1073fn checkCompileErrors(compile: *Compile) !void {
1074 // Clear this field so that it does not get printed by the build runner.
1075 const actual_eb = compile.step.result_error_bundle;
1076 compile.step.result_error_bundle = .empty;
1077
1078 const arena = compile.step.owner.allocator;
1079
1080 const actual_errors = ae: {
1081 var aw: std.Io.Writer.Allocating = .init(arena);
1082 defer aw.deinit();
1083 try actual_eb.renderToWriter(.{
1084 .include_reference_trace = false,
1085 .include_source_line = false,
1086 }, &aw.writer);
1087 break :ae try aw.toOwnedSlice();
1088 };
1089
1090 // Render the expected lines into a string that we can compare verbatim.
1091 var expected_generated: std.ArrayList(u8) = .empty;
1092 const expect_errors = compile.expect_errors.?;
1093
1094 var actual_line_it = mem.splitScalar(u8, actual_errors, '\n');
1095
1096 // TODO merge this with the testing.expectEqualStrings logic, and also CheckFile
1097 switch (expect_errors) {
1098 .starts_with => |expect_starts_with| {
1099 if (std.mem.startsWith(u8, actual_errors, expect_starts_with)) return;
1100 return compile.step.fail(
1101 \\
1102 \\========= should start with: ============
1103 \\{s}
1104 \\========= but not found: ================
1105 \\{s}
1106 \\=========================================
1107 , .{ expect_starts_with, actual_errors });
1108 },
1109 .contains => |expect_line| {
1110 while (actual_line_it.next()) |actual_line| {
1111 if (!matchCompileError(actual_line, expect_line)) continue;
1112 return;
1113 }
1114
1115 return compile.step.fail(
1116 \\
1117 \\========= should contain: ===============
1118 \\{s}
1119 \\========= but not found: ================
1120 \\{s}
1121 \\=========================================
1122 , .{ expect_line, actual_errors });
1123 },
1124 .stderr_contains => |expect_line| {
1125 const actual_stderr: []const u8 = if (compile.step.result_error_msgs.items.len > 0)
1126 compile.step.result_error_msgs.items[0]
1127 else
1128 &.{};
1129 compile.step.result_error_msgs.clearRetainingCapacity();
1130
1131 var stderr_line_it = mem.splitScalar(u8, actual_stderr, '\n');
1132
1133 while (stderr_line_it.next()) |actual_line| {
1134 if (!matchCompileError(actual_line, expect_line)) continue;
1135 return;
1136 }
1137
1138 return compile.step.fail(
1139 \\
1140 \\========= should contain: ===============
1141 \\{s}
1142 \\========= but not found: ================
1143 \\{s}
1144 \\=========================================
1145 , .{ expect_line, actual_stderr });
1146 },
1147 .exact => |expect_lines| {
1148 for (expect_lines) |expect_line| {
1149 const actual_line = actual_line_it.next() orelse {
1150 try expected_generated.appendSlice(arena, expect_line);
1151 try expected_generated.append(arena, '\n');
1152 continue;
1153 };
1154 if (matchCompileError(actual_line, expect_line)) {
1155 try expected_generated.appendSlice(arena, actual_line);
1156 try expected_generated.append(arena, '\n');
1157 continue;
1158 }
1159 try expected_generated.appendSlice(arena, expect_line);
1160 try expected_generated.append(arena, '\n');
1161 }
1162
1163 if (mem.eql(u8, expected_generated.items, actual_errors)) return;
1164
1165 return compile.step.fail(
1166 \\
1167 \\========= expected: =====================
1168 \\{s}
1169 \\========= but found: ====================
1170 \\{s}
1171 \\=========================================
1172 , .{ expected_generated.items, actual_errors });
1173 },
1174 }
1175}
1176
1177fn matchCompileError(actual: []const u8, expected: []const u8) bool {
1178 if (mem.endsWith(u8, actual, expected)) return true;
1179 if (mem.startsWith(u8, expected, ":?:?: ")) {
1180 if (mem.endsWith(u8, actual, expected[":?:?: ".len..])) return true;
1181 }
1182 // We scan for /?/ in expected line and if there is a match, we match everything
1183 // up to and after /?/.
1184 const expected_trim = mem.trim(u8, expected, " ");
1185 if (mem.find(u8, expected_trim, "/?/")) |index| {
1186 const actual_trim = mem.trim(u8, actual, " ");
1187 const lhs = expected_trim[0..index];
1188 const rhs = expected_trim[index + "/?/".len ..];
1189 if (mem.startsWith(u8, actual_trim, lhs) and mem.endsWith(u8, actual_trim, rhs)) return true;
1190 }
1191 return false;
1192}
1193
1194fn moduleNeedsCliArg(mod: *const Module) bool {
1195 return for (mod.link_objects.items) |o| switch (o) {
1196 .c_source_file, .c_source_files, .assembly_file, .win32_resource_file => break true,
1197 else => continue,
1198 } else false;
1199}
1200
lib/compiler/maker/Step/InstallArtifact.zig deleted-96
...@@ -1,96 +0,0 @@
1
2fn make(step: *Step, options: Step.MakeOptions) !void {
3 _ = options;
4 const install_artifact: *InstallArtifact = @fieldParentPtr("step", step);
5 const b = step.owner;
6 const io = b.graph.io;
7
8 var all_cached = true;
9
10 if (install_artifact.dest_dir) |dest_dir| {
11 const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path);
12 const p = try step.installFile(install_artifact.emitted_bin.?, full_dest_path);
13 all_cached = all_cached and p == .fresh;
14
15 if (install_artifact.dylib_symlinks) |dls| {
16 try Step.Compile.doAtomicSymLinks(step, full_dest_path, dls.major_only_filename, dls.name_only_filename);
17 }
18
19 install_artifact.artifact.installed_path = full_dest_path;
20 }
21
22 if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| {
23 const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step));
24 const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path);
25 all_cached = all_cached and p == .fresh;
26 }
27
28 if (install_artifact.implib_dir) |implib_dir| {
29 const full_implib_path = b.getInstallPath(implib_dir, install_artifact.emitted_implib.?.basename(b, step));
30 const p = try step.installFile(install_artifact.emitted_implib.?, full_implib_path);
31 all_cached = all_cached and p == .fresh;
32 }
33
34 if (install_artifact.pdb_dir) |pdb_dir| {
35 const full_pdb_path = b.getInstallPath(pdb_dir, install_artifact.emitted_pdb.?.basename(b, step));
36 const p = try step.installFile(install_artifact.emitted_pdb.?, full_pdb_path);
37 all_cached = all_cached and p == .fresh;
38 }
39
40 if (install_artifact.h_dir) |h_dir| {
41 if (install_artifact.emitted_h) |emitted_h| {
42 const full_h_path = b.getInstallPath(h_dir, emitted_h.basename(b, step));
43 const p = try step.installFile(emitted_h, full_h_path);
44 all_cached = all_cached and p == .fresh;
45 }
46
47 for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) {
48 .file => |file| {
49 const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path);
50 const p = try step.installFile(file.source, full_h_path);
51 all_cached = all_cached and p == .fresh;
52 },
53 .directory => |dir| {
54 const src_dir_path = dir.source.getPath3(b, step);
55 const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path);
56
57 var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| {
58 return step.fail("unable to open source directory '{f}': {s}", .{
59 src_dir_path, @errorName(err),
60 });
61 };
62 defer src_dir.close(io);
63
64 var it = try src_dir.walk(b.allocator);
65 next_entry: while (try it.next(io)) |entry| {
66 for (dir.options.exclude_extensions) |ext| {
67 if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry;
68 }
69 if (dir.options.include_extensions) |incs| {
70 for (incs) |inc| {
71 if (std.mem.endsWith(u8, entry.path, inc)) break;
72 } else {
73 continue :next_entry;
74 }
75 }
76
77 const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path });
78 switch (entry.kind) {
79 .directory => {
80 try Step.handleVerbose(b, .inherit, &.{ "install", "-d", full_dest_path });
81 const p = try step.installDir(full_dest_path);
82 all_cached = all_cached and p == .existed;
83 },
84 .file => {
85 const p = try step.installFile(try dir.source.join(b.allocator, entry.path), full_dest_path);
86 all_cached = all_cached and p == .fresh;
87 },
88 else => continue,
89 }
90 }
91 },
92 };
93 }
94
95 step.result_cached = all_cached;
96}
lib/compiler/maker/Step/Run.zig deleted-2130
...@@ -1,2130 +0,0 @@
1const Run = @This();
2
3const builtin = @import("builtin");
4
5const std = @import("std");
6const Io = std.Io;
7const Dir = std.Io.Dir;
8const mem = std.mem;
9const process = std.process;
10const EnvMap = std.process.Environ.Map;
11const assert = std.debug.assert;
12const Cache = std.Build.Cache;
13const Path = std.Build.Cache.Path;
14
15const Step = @import("../Step.zig");
16
17/// If this is a Zig unit test binary, this tracks the names of the unit
18/// tests that are also fuzz tests. Indexes cannot be used as they may
19/// change between reruns.
20fuzz_tests: std.ArrayList([]const u8),
21cached_test_metadata: ?CachedTestMetadata = null,
22
23/// Populated during the fuzz phase if this run step corresponds to a unit test
24/// executable that contains fuzz tests.
25rebuilt_executable: ?Path,
26
27fn make(step: *Step, options: Step.MakeOptions) !void {
28 const b = step.owner;
29 const io = b.graph.io;
30 const arena = b.allocator;
31 const run: *Run = @fieldParentPtr("step", step);
32 const has_side_effects = run.hasSideEffects();
33
34 var argv_list = std.array_list.Managed([]const u8).init(arena);
35 var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena);
36
37 var man = b.graph.cache.obtain();
38 defer man.deinit();
39
40 if (run.environ_map) |environ_map| {
41 for (environ_map.keys(), environ_map.values()) |key, value| {
42 man.hash.addBytes(key);
43 man.hash.addBytes(value);
44 }
45 }
46
47 man.hash.add(run.color);
48 man.hash.add(run.disable_zig_progress);
49
50 for (run.argv.items) |arg| {
51 switch (arg) {
52 .bytes => |bytes| {
53 try argv_list.append(bytes);
54 man.hash.addBytes(bytes);
55 },
56 .lazy_path => |file| {
57 const file_path = file.lazy_path.getPath3(b, step);
58 try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) }));
59 man.hash.addBytes(file.prefix);
60 _ = try man.addFilePath(file_path, null);
61 },
62 .decorated_directory => |dd| {
63 const file_path = dd.lazy_path.getPath3(b, step);
64 const resolved_arg = b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix });
65 try argv_list.append(resolved_arg);
66 man.hash.addBytes(resolved_arg);
67 },
68 .file_content => |file_plp| {
69 const file_path = file_plp.lazy_path.getPath3(b, step);
70
71 var result: std.Io.Writer.Allocating = .init(arena);
72 errdefer result.deinit();
73 result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory;
74
75 const file = file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}) catch |err| {
76 return step.fail(
77 "unable to open input file '{f}': {t}",
78 .{ file_path, err },
79 );
80 };
81 defer file.close(io);
82
83 var buf: [1024]u8 = undefined;
84 var file_reader = file.reader(io, &buf);
85 _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) {
86 error.ReadFailed => return step.fail(
87 "failed to read from '{f}': {t}",
88 .{ file_path, file_reader.err.? },
89 ),
90 error.WriteFailed => return error.OutOfMemory,
91 };
92
93 try argv_list.append(result.written());
94 man.hash.addBytes(file_plp.prefix);
95 _ = try man.addFilePath(file_path, null);
96 },
97 .artifact => |pa| {
98 const artifact = pa.artifact;
99
100 if (artifact.rootModuleTarget().os.tag == .windows) {
101 // On Windows we don't have rpaths so we have to add .dll search paths to PATH
102 run.addPathForDynLibs(artifact);
103 }
104 const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?;
105
106 try argv_list.append(b.fmt("{s}{s}", .{
107 pa.prefix,
108 run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }),
109 }));
110
111 _ = try man.addFile(file_path, null);
112 },
113 .output_file, .output_directory => |output| {
114 man.hash.addBytes(output.prefix);
115 man.hash.addBytes(output.basename);
116 // Add a placeholder into the argument list because we need the
117 // manifest hash to be updated with all arguments before the
118 // object directory is computed.
119 try output_placeholders.append(.{
120 .index = argv_list.items.len,
121 .tag = arg,
122 .output = output,
123 });
124 _ = try argv_list.addOne();
125 },
126 }
127 }
128
129 switch (run.stdin) {
130 .bytes => |bytes| {
131 man.hash.addBytes(bytes);
132 },
133 .lazy_path => |lazy_path| {
134 const file_path = lazy_path.getPath2(b, step);
135 _ = try man.addFile(file_path, null);
136 },
137 .none => {},
138 }
139
140 if (run.captured_stdout) |captured| {
141 man.hash.addBytes(captured.output.basename);
142 man.hash.add(captured.trim_whitespace);
143 }
144
145 if (run.captured_stderr) |captured| {
146 man.hash.addBytes(captured.output.basename);
147 man.hash.add(captured.trim_whitespace);
148 }
149
150 hashStdIo(&man.hash, run.stdio);
151
152 for (run.file_inputs.items) |lazy_path| {
153 _ = try man.addFile(lazy_path.getPath2(b, step), null);
154 }
155
156 if (run.cwd) |cwd| {
157 const cwd_path = cwd.getPath3(b, step);
158 _ = man.hash.addBytes(try cwd_path.toString(arena));
159 }
160
161 if (!has_side_effects and try step.cacheHitAndWatch(&man)) {
162 // cache hit, skip running command
163 const digest = man.final();
164
165 try populateGeneratedPaths(
166 arena,
167 output_placeholders.items,
168 run.captured_stdout,
169 run.captured_stderr,
170 b.cache_root,
171 &digest,
172 );
173
174 step.result_cached = true;
175 return;
176 }
177
178 const dep_output_file = run.dep_output_file orelse {
179 // We already know the final output paths, use them directly.
180 const digest = if (has_side_effects)
181 man.hash.final()
182 else
183 man.final();
184
185 try populateGeneratedPaths(
186 arena,
187 output_placeholders.items,
188 run.captured_stdout,
189 run.captured_stderr,
190 b.cache_root,
191 &digest,
192 );
193
194 const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest;
195 for (output_placeholders.items) |placeholder| {
196 const output_sub_path = b.pathJoin(&.{ output_dir_path, placeholder.output.basename });
197 const output_sub_dir_path = switch (placeholder.tag) {
198 .output_file => Dir.path.dirname(output_sub_path).?,
199 .output_directory => output_sub_path,
200 else => unreachable,
201 };
202 b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| {
203 return step.fail("unable to make path '{f}{s}': {s}", .{
204 b.cache_root, output_sub_dir_path, @errorName(err),
205 });
206 };
207 const arg_output_path = run.convertPathArg(.{
208 .root_dir = .cwd(),
209 .sub_path = placeholder.output.generated_file.getPath(),
210 });
211 argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0)
212 arg_output_path
213 else
214 b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path });
215 }
216
217 try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null);
218 if (!has_side_effects) try step.writeManifestAndWatch(&man);
219 return;
220 };
221
222 // We do not know the final output paths yet, use temp paths to run the command.
223 var rand_int: u64 = undefined;
224 io.random(@ptrCast(&rand_int));
225 const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int);
226
227 for (output_placeholders.items) |placeholder| {
228 const output_components = .{ tmp_dir_path, placeholder.output.basename };
229 const output_sub_path = b.pathJoin(&output_components);
230 const output_sub_dir_path = switch (placeholder.tag) {
231 .output_file => Dir.path.dirname(output_sub_path).?,
232 .output_directory => output_sub_path,
233 else => unreachable,
234 };
235 b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| {
236 return step.fail("unable to make path '{f}{s}': {s}", .{
237 b.cache_root, output_sub_dir_path, @errorName(err),
238 });
239 };
240 const raw_output_path: Cache.Path = .{
241 .root_dir = b.cache_root,
242 .sub_path = b.pathJoin(&output_components),
243 };
244 placeholder.output.generated_file.path = raw_output_path.toString(b.graph.arena) catch @panic("OOM");
245 argv_list.items[placeholder.index] = b.fmt("{s}{s}", .{
246 placeholder.output.prefix,
247 run.convertPathArg(raw_output_path),
248 });
249 }
250
251 try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null);
252
253 const dep_file_dir = Dir.cwd();
254 const dep_file_basename = dep_output_file.generated_file.getPath2(b, step);
255 if (has_side_effects)
256 try man.addDepFile(dep_file_dir, dep_file_basename)
257 else
258 try man.addDepFilePost(dep_file_dir, dep_file_basename);
259
260 const digest = if (has_side_effects)
261 man.hash.final()
262 else
263 man.final();
264
265 const any_output = output_placeholders.items.len > 0 or
266 run.captured_stdout != null or run.captured_stderr != null;
267
268 // Rename into place
269 if (any_output) {
270 const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest;
271
272 b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |err| switch (err) {
273 Dir.RenameError.DirNotEmpty => {
274 b.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| {
275 return step.fail("unable to remove dir '{f}'{s}: {t}", .{
276 b.cache_root, tmp_dir_path, del_err,
277 });
278 };
279 b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |retry_err| {
280 return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{
281 b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, retry_err,
282 });
283 };
284 },
285 else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{
286 b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, err,
287 }),
288 };
289 }
290
291 if (!has_side_effects) try step.writeManifestAndWatch(&man);
292
293 try populateGeneratedPaths(
294 arena,
295 output_placeholders.items,
296 run.captured_stdout,
297 run.captured_stderr,
298 b.cache_root,
299 &digest,
300 );
301}
302
303/// Reads stdout of a Zig test process until a termination condition is reached:
304/// * A write fails, indicating the child unexpectedly closed stdin
305/// * A test (or a response from the test runner) times out
306/// * The wait fails, indicating the child closed stdout and stderr
307fn waitZigTest(
308 run: *Run,
309 child: *process.Child,
310 options: Step.MakeOptions,
311 multi_reader: *Io.File.MultiReader,
312 opt_metadata: *?TestMetadata,
313 results: *Step.TestResults,
314) !union(enum) {
315 write_failed: anyerror,
316 no_poll: struct {
317 active_test_index: ?u32,
318 ns_elapsed: u64,
319 },
320 timeout: struct {
321 active_test_index: ?u32,
322 ns_elapsed: u64,
323 },
324} {
325 const gpa = run.step.owner.allocator;
326 const arena = run.step.owner.allocator;
327 const io = run.step.owner.graph.io;
328
329 var sub_prog_node: ?std.Progress.Node = null;
330 defer if (sub_prog_node) |n| n.end();
331
332 if (opt_metadata.*) |*md| {
333 // Previous unit test process died or was killed; we're continuing where it left off
334 requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err };
335 } else {
336 // Running unit tests normally
337 run.fuzz_tests.clearRetainingCapacity();
338 sendMessage(io, child.stdin.?, .query_test_metadata) catch |err| return .{ .write_failed = err };
339 }
340
341 var active_test_index: ?u32 = null;
342
343 var last_update: Io.Clock.Timestamp = .now(io, .awake);
344
345 // This timeout is used when we're waiting on the test runner itself rather than a user-specified
346 // test. For instance, if the test runner leaves this much time between us requesting a test to
347 // start and it acknowledging the test starting, we terminate the child and raise an error. This
348 // *should* never happen, but could in theory be caused by some very unlucky IB in a test.
349 const response_timeout: Io.Clock.Duration = t: {
350 if (fuzz_context != null) break :t null; // don't timeout fuzz tests
351 const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s);
352 break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) };
353 };
354 const test_timeout: ?Io.Clock.Duration = if (options.unit_test_timeout_ns) |ns| .{
355 .clock = .awake,
356 .raw = .fromNanoseconds(ns),
357 } else null;
358
359 const stdout = multi_reader.reader(0);
360 const stderr = multi_reader.reader(1);
361 const Header = std.zig.Server.Message.Header;
362
363 while (true) {
364 const timeout: Io.Timeout = t: {
365 const opt_duration = if (active_test_index == null) response_timeout else test_timeout;
366 const duration = opt_duration orelse break :t .none;
367 break :t .{ .deadline = last_update.addDuration(duration) };
368 };
369
370 // This block is exited when `stdout` contains enough bytes for a `Header`.
371 header_ready: {
372 if (stdout.buffered().len >= @sizeOf(Header)) {
373 // We already have one, no need to poll!
374 break :header_ready;
375 }
376
377 multi_reader.fill(64, timeout) catch |err| switch (err) {
378 error.Timeout => return .{ .timeout = .{
379 .active_test_index = active_test_index,
380 .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
381 } },
382 error.EndOfStream => return .{ .no_poll = .{
383 .active_test_index = active_test_index,
384 .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
385 } },
386 else => |e| return e,
387 };
388
389 continue;
390 }
391 // There is definitely a header available now -- read it.
392 const header = stdout.takeStruct(Header, .little) catch unreachable;
393
394 while (stdout.buffered().len < header.bytes_len) {
395 multi_reader.fill(64, timeout) catch |err| switch (err) {
396 error.Timeout => return .{ .timeout = .{
397 .active_test_index = active_test_index,
398 .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
399 } },
400 error.EndOfStream => return .{ .no_poll = .{
401 .active_test_index = active_test_index,
402 .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
403 } },
404 else => |e| return e,
405 };
406 }
407
408 const body = stdout.take(header.bytes_len) catch unreachable;
409 var body_r: std.Io.Reader = .fixed(body);
410 switch (header.tag) {
411 .zig_version => {
412 if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail(
413 "zig version mismatch build runner vs compiler: '{s}' vs '{s}'",
414 .{ builtin.zig_version_string, body },
415 );
416 },
417 .test_metadata => {
418 // `metadata` would only be populated if we'd already seen a `test_metadata`, but we
419 // only request it once (and importantly, we don't re-request it if we kill and
420 // restart the test runner).
421 assert(opt_metadata.* == null);
422
423 const tm_hdr = body_r.takeStruct(std.zig.Server.Message.TestMetadata, .little) catch unreachable;
424 results.test_count = tm_hdr.tests_len;
425
426 const names = try arena.alloc(u32, results.test_count);
427 for (names) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable;
428
429 const expected_panic_msgs = try arena.alloc(u32, results.test_count);
430 for (expected_panic_msgs) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable;
431
432 const string_bytes = body_r.take(tm_hdr.string_bytes_len) catch unreachable;
433
434 options.progress_node.setEstimatedTotalItems(names.len);
435 opt_metadata.* = .{
436 .string_bytes = try arena.dupe(u8, string_bytes),
437 .ns_per_test = try arena.alloc(u64, results.test_count),
438 .names = names,
439 .expected_panic_msgs = expected_panic_msgs,
440 .next_index = 0,
441 .prog_node = options.progress_node,
442 };
443 @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64));
444
445 active_test_index = null;
446 last_update = .now(io, .awake);
447
448 requestNextTest(io, child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch |err| return .{ .write_failed = err };
449 },
450 .test_started => {
451 active_test_index = opt_metadata.*.?.next_index - 1;
452 last_update = .now(io, .awake);
453 },
454 .test_results => {
455 const md = &opt_metadata.*.?;
456
457 const tr_hdr = body_r.takeStruct(std.zig.Server.Message.TestResults, .little) catch unreachable;
458 assert(tr_hdr.index == active_test_index);
459
460 switch (tr_hdr.flags.status) {
461 .pass => {},
462 .skip => results.skip_count +|= 1,
463 .fail => results.fail_count +|= 1,
464 }
465 const leak_count = tr_hdr.flags.leak_count;
466 const log_err_count = tr_hdr.flags.log_err_count;
467 results.leak_count +|= leak_count;
468 results.log_err_count +|= log_err_count;
469
470 if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index));
471
472 if (tr_hdr.flags.status == .fail) {
473 const name = md.testName(tr_hdr.index);
474 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
475 stderr.tossBuffered();
476 if (stderr_bytes.len == 0) {
477 try run.step.addError("'{s}' failed without output", .{name});
478 } else {
479 try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes });
480 }
481 } else if (leak_count > 0) {
482 const name = md.testName(tr_hdr.index);
483 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
484 stderr.tossBuffered();
485 try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes });
486 } else if (log_err_count > 0) {
487 const name = md.testName(tr_hdr.index);
488 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
489 stderr.tossBuffered();
490 try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes });
491 }
492
493 active_test_index = null;
494
495 const now: Io.Clock.Timestamp = .now(io, .awake);
496 md.ns_per_test[tr_hdr.index] = @intCast(last_update.durationTo(now).raw.nanoseconds);
497 last_update = now;
498
499 requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err };
500 },
501 else => {}, // ignore other messages
502 }
503 }
504}
505
506const FuzzTestRunner = struct {
507 run: *Run,
508 ctx: FuzzContext,
509 coverage_id: ?u64,
510
511 instances: []Instance,
512 /// The indexes of this are layed out such that it is effectively an array
513 /// of `[instances.len][3]Io.Operation.Storage` of stdin, stdout, stderr.
514 batch: Io.Batch,
515 /// LIFO. Stream of message bodies trailed by PendingBroadcastFooter.
516 pending_broadcasts: std.ArrayList(u8),
517 broadcast: std.ArrayList(u8),
518 broadcast_undelivered: u32,
519
520 const Instance = struct {
521 child: process.Child,
522 message: std.ArrayListAligned(u8, .@"4"),
523 broadcast_written: usize,
524 stderr: std.ArrayList(u8),
525 stdin_vec: [1][]u8,
526 stdout_vec: [1][]u8,
527 stderr_vec: [1][]u8,
528 progress_node: std.Progress.Node,
529
530 fn messageHeader(instance: *Instance) InHeader {
531 assert(instance.message.items.len >= @sizeOf(InHeader));
532 const header_ptr: *InHeader = @ptrCast(instance.message.items);
533 var header = header_ptr.*;
534 if (std.builtin.Endian.native != .little) {
535 std.mem.byteSwapAllFields(InHeader, &header);
536 }
537 return header;
538 }
539 };
540
541 const PendingBroadcastFooter = struct {
542 from_id: u32,
543 body_len: u32,
544 };
545
546 const InHeader = std.zig.Server.Message.Header;
547 const OutHeader = std.zig.Client.Message.Header;
548
549 const stdin_i = 0;
550 const stdout_i = 1;
551 const stderr_i = 2;
552
553 fn init(
554 run: *Run,
555 ctx: FuzzContext,
556 progress_node: std.Progress.Node,
557 spawn_options: process.SpawnOptions,
558 ) !FuzzTestRunner {
559 const step_owner = run.step.owner;
560 const gpa = step_owner.allocator;
561 const io = step_owner.graph.io;
562
563 const n_instances = switch (ctx.fuzz.mode) {
564 .forever => step_owner.graph.max_jobs orelse @min(
565 std.Thread.getCpuCount() catch 1,
566 (std.math.maxInt(u32) - 2) / 3,
567 ),
568 .limit => 1,
569 };
570 const instances = try gpa.alloc(Instance, n_instances);
571 errdefer gpa.free(instances);
572 const batch_storage = try gpa.alloc(Io.Operation.Storage, instances.len * 3);
573 errdefer gpa.free(batch_storage);
574
575 @memset(instances, .{
576 .child = undefined,
577 .message = .empty,
578 .broadcast_written = undefined,
579 .stderr = .empty,
580 .stdin_vec = undefined,
581 .stdout_vec = undefined,
582 .stderr_vec = undefined,
583 .progress_node = undefined,
584 });
585 for (0.., instances) |id, *instance| {
586 errdefer for (instances[0..id]) |*spawned| {
587 spawned.child.kill(io);
588 spawned.progress_node.end();
589 };
590 instance.child = try process.spawn(io, spawn_options);
591 instance.progress_node = progress_node.start("starting fuzzer", 0);
592 }
593
594 return .{
595 .run = run,
596 .ctx = ctx,
597 .coverage_id = null,
598
599 .instances = instances,
600 .batch = .init(batch_storage),
601 .pending_broadcasts = .empty,
602 .broadcast = .empty,
603 .broadcast_undelivered = 0,
604 };
605 }
606
607 fn deinit(f: *FuzzTestRunner) void {
608 const step_owner = f.run.step.owner;
609 const gpa = step_owner.allocator;
610 const io = step_owner.graph.io;
611
612 f.batch.cancel(io);
613 gpa.free(f.batch.storage);
614 var total_rss: usize = 0;
615 for (f.instances) |*instance| {
616 instance.child.kill(io);
617 instance.message.deinit(gpa);
618 instance.stderr.deinit(gpa);
619 instance.progress_node.end();
620 total_rss += instance.child.resource_usage_statistics.getMaxRss() orelse 0;
621 }
622 f.run.step.result_peak_rss = @max(f.run.step.result_peak_rss, total_rss);
623 gpa.free(f.instances);
624 }
625
626 fn startInstances(f: *FuzzTestRunner) !void {
627 const step_owner = f.run.step.owner;
628 const io = step_owner.graph.io;
629
630 for (0.., f.instances) |id, *instance| {
631 const id32: u32 = @intCast(id);
632 (switch (f.ctx.fuzz.mode) {
633 .forever => sendRunFuzzTestMessage(
634 io,
635 instance.child.stdin.?,
636 f.run.fuzz_tests.items,
637 .forever,
638 id32,
639 ),
640 .limit => |limit| sendRunFuzzTestMessage(
641 io,
642 instance.child.stdin.?,
643 f.run.fuzz_tests.items,
644 .iterations,
645 limit.amount,
646 ),
647 }) catch |write_err| {
648 // The runner unexpectedly closed stdin, which means it crashed during initialization.
649 // Clean up everything and wait for the child to exit.
650 instance.child.stdin.?.close(io);
651 instance.child.stdin = null;
652 const term = try instance.child.wait(io);
653 return f.run.step.fail(
654 "unable to write stdin ({t}); test process unexpectedly {f}",
655 .{ write_err, fmtTerm(term) },
656 );
657 };
658
659 try f.addStdoutRead(id32, @sizeOf(InHeader));
660 try f.addStderrRead(id32);
661 }
662 }
663
664 fn listen(f: *FuzzTestRunner) !void {
665 const step_owner = f.run.step.owner;
666 const io = step_owner.graph.io;
667
668 while (true) {
669 try f.batch.awaitConcurrent(io, .none);
670 while (f.batch.next()) |completion| {
671 const id = completion.index / 3;
672 const result = completion.result;
673 switch (completion.index % 3) {
674 0 => try f.completeStdinWrite(id, result.file_write_streaming catch |e| switch (e) {
675 // Avoid calling `instanceEos` until EndOfStream is seen with stderr so
676 // that all stderr is collected.
677 error.BrokenPipe => continue,
678 else => |write_e| return write_e,
679 }),
680 1 => try f.completeStdoutRead(id, result.file_read_streaming catch |e| switch (e) {
681 // Avoid calling `instanceEos` until EndOfStream is seen with stderr so
682 // that all stderr is collected.
683 error.EndOfStream => continue,
684 else => |read_e| return read_e,
685 }),
686 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) {
687 error.EndOfStream => return f.instanceEos(id),
688 else => |read_e| return read_e,
689 }),
690 else => unreachable,
691 }
692 }
693 }
694 }
695
696 fn completeStdoutRead(f: *FuzzTestRunner, id: u32, n: usize) !void {
697 const step_owner = f.run.step.owner;
698 const gpa = step_owner.allocator;
699 const io = step_owner.graph.io;
700 const instance = &f.instances[id];
701
702 instance.message.items.len += n;
703 const total_read = instance.message.items.len;
704 if (total_read < @sizeOf(InHeader)) {
705 try f.addStdoutRead(id, @sizeOf(InHeader));
706 return;
707 }
708
709 const header = instance.messageHeader();
710 const body = instance.message.items[@sizeOf(InHeader)..];
711 if (body.len != header.bytes_len) {
712 try f.addStdoutRead(id, @sizeOf(InHeader) + header.bytes_len);
713 return;
714 }
715
716 switch (header.tag) {
717 .zig_version => {
718 if (!std.mem.eql(u8, builtin.zig_version_string, body)) return f.run.step.fail(
719 "zig version mismatch build runner vs compiler: '{s}' vs '{s}'",
720 .{ builtin.zig_version_string, body },
721 );
722 },
723 .coverage_id => {
724 var body_r: Io.Reader = .fixed(body);
725 f.coverage_id = body_r.takeInt(u64, .little) catch unreachable;
726 const cumulative_runs = body_r.takeInt(u64, .little) catch unreachable;
727 const cumulative_unique = body_r.takeInt(u64, .little) catch unreachable;
728 const cumulative_coverage = body_r.takeInt(u64, .little) catch unreachable;
729
730 const fuzz = f.ctx.fuzz;
731 fuzz.queue_mutex.lockUncancelable(io);
732 defer fuzz.queue_mutex.unlock(io);
733 try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{
734 .id = f.coverage_id.?,
735 .cumulative = .{
736 .runs = cumulative_runs,
737 .unique = cumulative_unique,
738 .coverage = cumulative_coverage,
739 },
740 .run = f.run,
741 } });
742 fuzz.queue_cond.signal(io);
743 },
744 .fuzz_start_addr => {
745 var body_r: Io.Reader = .fixed(body);
746 const fuzz = f.ctx.fuzz;
747 const addr = body_r.takeInt(u64, .little) catch unreachable;
748
749 fuzz.queue_mutex.lockUncancelable(io);
750 defer fuzz.queue_mutex.unlock(io);
751 try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{
752 .addr = addr,
753 .coverage_id = f.coverage_id.?,
754 } });
755 fuzz.queue_cond.signal(io);
756 },
757 .fuzz_test_change => {
758 const test_i = std.mem.readInt(u32, body[0..4], .little);
759 instance.progress_node.setName(f.run.fuzz_tests.items[test_i]);
760 },
761 .broadcast_fuzz_input => {
762 if (f.instances.len == 1) {
763 // No other processes to broadcast to.
764 } else if (f.broadcast_undelivered == 0) {
765 try f.instanceBroadcast(id, body);
766 } else {
767 const footer: PendingBroadcastFooter = .{
768 .from_id = id,
769 .body_len = @intCast(body.len),
770 };
771 // There is another broadcast in progress so add this one to the queue.
772 const size = @sizeOf(PendingBroadcastFooter) + body.len;
773 try f.pending_broadcasts.ensureUnusedCapacity(gpa, size);
774 f.pending_broadcasts.appendSliceAssumeCapacity(body);
775 f.pending_broadcasts.appendSliceAssumeCapacity(@ptrCast(&footer));
776 }
777 },
778 else => {}, // ignore other messages
779 }
780
781 instance.message.clearRetainingCapacity();
782 try f.addStdoutRead(id, @sizeOf(InHeader));
783 }
784
785 fn completeStderrRead(f: *FuzzTestRunner, id: u32, n: usize) !void {
786 const instance = &f.instances[id];
787 instance.stderr.items.len += n;
788 try f.addStderrRead(id);
789 }
790
791 fn completeStdinWrite(f: *FuzzTestRunner, id: u32, n: usize) !void {
792 const instance = &f.instances[id];
793
794 instance.broadcast_written += n;
795 if (instance.broadcast_written == f.broadcast.items.len) {
796 f.broadcast_undelivered -= 1;
797 if (f.broadcast_undelivered == 0) {
798 try f.broadcastComplete();
799 }
800 } else {
801 f.addStdinWrite(id);
802 }
803 }
804
805 fn addStdoutRead(f: *FuzzTestRunner, id: u32, end: usize) !void {
806 const step_owner = f.run.step.owner;
807 const gpa = step_owner.allocator;
808 const instance = &f.instances[id];
809
810 try instance.message.ensureTotalCapacity(gpa, end);
811 const start = instance.message.items.len;
812 instance.stdout_vec = .{instance.message.allocatedSlice()[start..end]};
813 f.batch.addAt(id * 3 + stdout_i, .{ .file_read_streaming = .{
814 .file = instance.child.stdout.?,
815 .data = &instance.stdout_vec,
816 } });
817 }
818
819 fn addStderrRead(f: *FuzzTestRunner, id: u32) !void {
820 const step_owner = f.run.step.owner;
821 const gpa = step_owner.allocator;
822 const instance = &f.instances[id];
823
824 try instance.stderr.ensureUnusedCapacity(gpa, 1);
825 instance.stderr_vec = .{instance.stderr.unusedCapacitySlice()};
826 f.batch.addAt(id * 3 + stderr_i, .{ .file_read_streaming = .{
827 .file = instance.child.stderr.?,
828 .data = &instance.stderr_vec,
829 } });
830 }
831
832 fn addStdinWrite(f: *FuzzTestRunner, id: u32) void {
833 const instance = &f.instances[id];
834
835 assert(f.broadcast.items.len != instance.broadcast_written);
836 instance.stdin_vec = .{f.broadcast.items[instance.broadcast_written..]};
837 f.batch.addAt(id * 3 + stdin_i, .{ .file_write_streaming = .{
838 .file = instance.child.stdin.?,
839 .data = &instance.stdin_vec,
840 } });
841 }
842
843 fn instanceEos(f: *FuzzTestRunner, id: u32) !void {
844 const step_owner = f.run.step.owner;
845 const io = step_owner.graph.io;
846 const instance = &f.instances[id];
847
848 instance.child.stdin.?.close(io);
849 instance.child.stdin = null;
850 const term = try instance.child.wait(io);
851 if (!termMatches(.{ .exited = 0 }, term)) {
852 f.run.step.result_stderr = try f.mergedStderr();
853 try f.saveCrash(id, term);
854 return f.run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)});
855 }
856 }
857
858 fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void {
859 const step = &f.run.step;
860 const b = step.owner;
861 const io = b.graph.io;
862
863 if (f.coverage_id == null) return;
864
865 // Search for the input file corresponding to the instance
866 const InputHeader = Build.abi.fuzz.MmapInputHeader;
867 var in_r_buf: [@sizeOf(InputHeader)]u8 = undefined;
868 var in_r: Io.File.Reader = undefined;
869 var in_f: Io.File = undefined;
870 var in_name_buf: [12]u8 = undefined;
871 var in_name: []const u8 = undefined;
872 var i: u32 = 0;
873 const header: InputHeader = while (true) : ({
874 if (i == std.math.maxInt(u32)) return;
875 i += 1;
876 }) {
877 const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in";
878 in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable;
879 in_f = b.cache_root.handle.openFile(io, in_name, .{
880 .lock = .exclusive,
881 .lock_nonblocking = true,
882 }) catch |e| switch (e) {
883 error.FileNotFound => return,
884 error.WouldBlock => continue, // Can not be from
885 // the crashed instance since it is still locked.
886 else => return step.fail("failed to open file '{f}{s}': {t}", .{
887 b.cache_root, in_name, e,
888 }),
889 };
890
891 in_r = in_f.readerStreaming(io, &in_r_buf);
892 const header = in_r.interface.takeStruct(InputHeader, .little) catch |e| {
893 in_f.close(io);
894 switch (e) {
895 error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{
896 b.cache_root, in_name, in_r.err.?,
897 }),
898 error.EndOfStream => continue,
899 }
900 };
901
902 if (header.pc_digest == f.coverage_id.? and
903 header.instance_id == id and
904 header.test_i < f.run.fuzz_tests.items.len)
905 {
906 break header;
907 }
908
909 in_f.close(io);
910 };
911 defer in_f.close(io);
912
913 // Save it to a seperate file
914 const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash";
915 const out = b.cache_root.handle.createFile(io, crash_name, .{
916 .lock = .exclusive, // Multiple run steps could have found a crash at the same time
917 }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{
918 b.cache_root, crash_name, e,
919 });
920 defer out.close(io);
921
922 var out_w_buf: [512]u8 = undefined;
923 var out_w = out.writerStreaming(io, &out_w_buf);
924 _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) {
925 error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{
926 b.cache_root, in_name, in_r.err.?,
927 }),
928 error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{
929 b.cache_root, crash_name, out_w.err.?,
930 }),
931 };
932
933 return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{
934 f.run.fuzz_tests.items[header.test_i],
935 fmtTerm(term),
936 b.cache_root,
937 crash_name,
938 });
939 }
940
941 fn instanceBroadcast(f: *FuzzTestRunner, from_id: u32, bytes: []const u8) !void {
942 assert(f.instances.len > 1);
943 assert(f.broadcast_undelivered == 0); // no other broadcast is progress
944 assert(f.broadcast.items.len == 0);
945 assert(from_id < f.instances.len);
946
947 const step_owner = f.run.step.owner;
948 const gpa = step_owner.allocator;
949
950 var out_header: OutHeader = .{
951 .tag = .new_fuzz_input,
952 .bytes_len = @intCast(bytes.len),
953 };
954 if (std.builtin.Endian.native != .little) {
955 std.mem.byteSwapAllFields(OutHeader, &out_header);
956 }
957 try f.broadcast.ensureTotalCapacity(gpa, @sizeOf(OutHeader) + bytes.len);
958 f.broadcast.appendSliceAssumeCapacity(@ptrCast(&out_header));
959 f.broadcast.appendSliceAssumeCapacity(bytes);
960
961 f.broadcast_undelivered = @intCast(f.instances.len - 1);
962 for (0.., f.instances) |to_id, *instance| {
963 if (to_id == from_id) continue;
964 instance.broadcast_written = 0;
965 f.addStdinWrite(@intCast(to_id));
966 }
967 }
968
969 fn broadcastComplete(f: *FuzzTestRunner) !void {
970 assert(f.instances.len > 1);
971 assert(f.broadcast_undelivered == 0);
972 f.broadcast.clearRetainingCapacity();
973
974 const pending = &f.pending_broadcasts;
975 if (pending.items.len != 0) {
976 // Another broadcast is pending; copy it over to `broadcast`
977
978 const footer_len = @sizeOf(PendingBroadcastFooter);
979 const footer_bytes = pending.items[pending.items.len - footer_len ..];
980 const footer: *align(1) PendingBroadcastFooter = @ptrCast(footer_bytes);
981 pending.items.len -= footer_len;
982
983 const body = pending.items[pending.items.len - footer.body_len ..];
984 try f.instanceBroadcast(footer.from_id, body);
985 pending.items.len -= body.len;
986 }
987 }
988
989 fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 {
990 const step_owner = f.run.step.owner;
991 const arena = step_owner.allocator;
992
993 // Collect any available stderr
994 while (f.batch.next()) |completion| {
995 if (completion.index % 3 != 2) continue;
996 const len = completion.result.file_read_streaming catch continue;
997 f.instances[completion.index / 3].stderr.items.len += len;
998 }
999
1000 var stderr_len: usize = 0;
1001 for (f.instances) |*instance| stderr_len += instance.stderr.items.len;
1002 const stderr = try arena.alloc(u8, stderr_len);
1003
1004 stderr_len = 0;
1005 for (f.instances) |*instance| {
1006 @memcpy(stderr[stderr_len..][0..instance.stderr.items.len], instance.stderr.items);
1007 stderr_len += instance.stderr.items.len;
1008 }
1009 return stderr;
1010 }
1011};
1012
1013fn evalFuzzTest(
1014 run: *Run,
1015 spawn_options: process.SpawnOptions,
1016 options: Step.MakeOptions,
1017 fuzz_context: FuzzContext,
1018) !void {
1019 var f: FuzzTestRunner = try .init(run, fuzz_context, options.progress_node, spawn_options);
1020 defer f.deinit();
1021 try f.startInstances();
1022 try f.listen();
1023}
1024
1025const StdioPollEnum = enum { stdout, stderr };
1026
1027fn evalZigTest(
1028 run: *Run,
1029 spawn_options: process.SpawnOptions,
1030 options: Step.MakeOptions,
1031 fuzz_context: ?FuzzContext,
1032) !void {
1033 if (fuzz_context != null) {
1034 try evalFuzzTest(run, spawn_options, options, fuzz_context.?);
1035 return;
1036 }
1037
1038 const step_owner = run.step.owner;
1039 const gpa = step_owner.allocator;
1040 const arena = step_owner.allocator;
1041 const io = step_owner.graph.io;
1042
1043 // We will update this every time a child runs.
1044 run.step.result_peak_rss = 0;
1045
1046 var test_results: Step.TestResults = .{
1047 .test_count = 0,
1048 .skip_count = 0,
1049 .fail_count = 0,
1050 .crash_count = 0,
1051 .timeout_count = 0,
1052 .leak_count = 0,
1053 .log_err_count = 0,
1054 };
1055 var test_metadata: ?TestMetadata = null;
1056
1057 while (true) {
1058 var child = try process.spawn(io, spawn_options);
1059 var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined;
1060 var multi_reader: Io.File.MultiReader = undefined;
1061 multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? });
1062 var child_killed = false;
1063 defer if (!child_killed) {
1064 child.kill(io);
1065 multi_reader.deinit();
1066 run.step.result_peak_rss = @max(
1067 run.step.result_peak_rss,
1068 child.resource_usage_statistics.getMaxRss() orelse 0,
1069 );
1070 };
1071
1072 switch (try waitZigTest(
1073 run,
1074 &child,
1075 options,
1076 &multi_reader,
1077 &test_metadata,
1078 &test_results,
1079 )) {
1080 .write_failed => |err| {
1081 // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured
1082 // all available stderr to make our error output as useful as possible.
1083 const stderr_fr = multi_reader.fileReader(1);
1084 while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) {
1085 error.ReadFailed => return stderr_fr.err.?,
1086 error.EndOfStream => {},
1087 }
1088 run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered());
1089
1090 // Clean up everything and wait for the child to exit.
1091 child.stdin.?.close(io);
1092 child.stdin = null;
1093 multi_reader.deinit();
1094 child_killed = true;
1095 const term = try child.wait(io);
1096 run.step.result_peak_rss = @max(
1097 run.step.result_peak_rss,
1098 child.resource_usage_statistics.getMaxRss() orelse 0,
1099 );
1100
1101 // The individual unit test results are irrelevant: the test runner itself broke!
1102 // Fail immediately without populating `s.test_results`.
1103 return run.step.fail("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) });
1104 },
1105 .no_poll => |no_poll| {
1106 // This might be a success (we requested exit and the child dutifully closed stdout) or
1107 // a crash of some kind. Either way, the child will terminate by itself -- wait for it.
1108 const stderr_reader = multi_reader.reader(1);
1109 const stderr_owned = try arena.dupe(u8, stderr_reader.buffered());
1110
1111 // Clean up everything and wait for the child to exit.
1112 child.stdin.?.close(io);
1113 child.stdin = null;
1114 multi_reader.deinit();
1115 child_killed = true;
1116 const term = try child.wait(io);
1117 run.step.result_peak_rss = @max(
1118 run.step.result_peak_rss,
1119 child.resource_usage_statistics.getMaxRss() orelse 0,
1120 );
1121
1122 if (no_poll.active_test_index) |test_index| {
1123 // A test was running, so this is definitely a crash. Report it against that
1124 // test, and continue to the next test.
1125 test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed;
1126 test_results.crash_count += 1;
1127 try run.step.addError("'{s}' {f}{s}{s}", .{
1128 test_metadata.?.testName(test_index),
1129 fmtTerm(term),
1130 if (stderr_owned.len != 0) " with stderr:\n" else "",
1131 std.mem.trim(u8, stderr_owned, "\n"),
1132 });
1133 continue;
1134 }
1135
1136 // Report an error if the child terminated uncleanly or if we were still trying to run more tests.
1137 run.step.result_stderr = stderr_owned;
1138 const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32);
1139 if (!tests_done or !termMatches(.{ .exited = 0 }, term)) {
1140 // The individual unit test results are irrelevant: the test runner itself broke!
1141 // Fail immediately without populating `s.test_results`.
1142 return run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)});
1143 }
1144
1145 // We're done with all of the tests! Commit the test results and return.
1146 run.step.test_results = test_results;
1147 if (test_metadata) |tm| {
1148 run.cached_test_metadata = tm.toCachedTestMetadata();
1149 if (options.web_server) |ws| {
1150 if (run.step.owner.graph.time_report) {
1151 ws.updateTimeReportRunTest(
1152 run,
1153 &run.cached_test_metadata.?,
1154 tm.ns_per_test,
1155 );
1156 }
1157 }
1158 }
1159 return;
1160 },
1161 .timeout => |timeout| {
1162 const stderr_reader = multi_reader.reader(1);
1163 const stderr = stderr_reader.buffered();
1164 stderr_reader.tossBuffered();
1165 if (timeout.active_test_index) |test_index| {
1166 // A test was running. Report the timeout against that test, and continue on to
1167 // the next test.
1168 test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed;
1169 test_results.timeout_count += 1;
1170 try run.step.addError("'{s}' timed out after {f}{s}{s}", .{
1171 test_metadata.?.testName(test_index),
1172 Io.Duration{ .nanoseconds = timeout.ns_elapsed },
1173 if (stderr.len != 0) " with stderr:\n" else "",
1174 std.mem.trim(u8, stderr, "\n"),
1175 });
1176 continue;
1177 }
1178 // Just log an error and let the child be killed.
1179 run.step.result_stderr = try arena.dupe(u8, stderr);
1180 // The individual unit test results in `results` are irrelevant: the test runner
1181 // is broken! Fail immediately without populating `s.test_results`.
1182 return run.step.fail("test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }});
1183 },
1184 }
1185 comptime unreachable;
1186 }
1187}
1188
1189const TestMetadata = struct {
1190 names: []const u32,
1191 ns_per_test: []u64,
1192 expected_panic_msgs: []const u32,
1193 string_bytes: []const u8,
1194 next_index: u32,
1195 prog_node: std.Progress.Node,
1196
1197 fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata {
1198 return .{
1199 .names = tm.names,
1200 .string_bytes = tm.string_bytes,
1201 };
1202 }
1203
1204 fn testName(tm: TestMetadata, index: u32) []const u8 {
1205 return tm.toCachedTestMetadata().testName(index);
1206 }
1207};
1208
1209pub const CachedTestMetadata = struct {
1210 names: []const u32,
1211 string_bytes: []const u8,
1212
1213 pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 {
1214 return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0);
1215 }
1216};
1217
1218fn requestNextTest(io: Io, in: Io.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void {
1219 while (metadata.next_index < metadata.names.len) {
1220 const i = metadata.next_index;
1221 metadata.next_index += 1;
1222
1223 if (metadata.expected_panic_msgs[i] != 0) continue;
1224
1225 const name = metadata.testName(i);
1226 if (sub_prog_node.*) |n| n.end();
1227 sub_prog_node.* = metadata.prog_node.start(name, 0);
1228
1229 try sendRunTestMessage(io, in, .run_test, i);
1230 return;
1231 } else {
1232 metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done
1233 try sendMessage(io, in, .exit);
1234 }
1235}
1236
1237fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void {
1238 const header: std.zig.Client.Message.Header = .{
1239 .tag = tag,
1240 .bytes_len = 0,
1241 };
1242 var w = file.writerStreaming(io, &.{});
1243 w.interface.writeStruct(header, .little) catch |err| switch (err) {
1244 error.WriteFailed => return w.err.?,
1245 };
1246}
1247
1248fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, index: u32) !void {
1249 const header: std.zig.Client.Message.Header = .{
1250 .tag = tag,
1251 .bytes_len = 4,
1252 };
1253 var w = file.writerStreaming(io, &.{});
1254 w.interface.writeStruct(header, .little) catch |err| switch (err) {
1255 error.WriteFailed => return w.err.?,
1256 };
1257 w.interface.writeInt(u32, index, .little) catch |err| switch (err) {
1258 error.WriteFailed => return w.err.?,
1259 };
1260}
1261
1262fn sendRunFuzzTestMessage(
1263 io: Io,
1264 file: Io.File,
1265 test_names: []const []const u8,
1266 kind: std.Build.abi.fuzz.LimitKind,
1267 amount_or_instance: u64,
1268) !void {
1269 const header: std.zig.Client.Message.Header = .{
1270 .tag = .start_fuzzing,
1271 .bytes_len = 1 + 8 + 4 + count: {
1272 var c: u32 = @intCast(test_names.len * 4);
1273 for (test_names) |name| {
1274 c += @intCast(name.len);
1275 }
1276 break :count c;
1277 },
1278 };
1279 var w = file.writerStreaming(io, &.{});
1280 w.interface.writeStruct(header, .little) catch |err| switch (err) {
1281 error.WriteFailed => return w.err.?,
1282 };
1283 w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) {
1284 error.WriteFailed => return w.err.?,
1285 };
1286 w.interface.writeInt(u64, amount_or_instance, .little) catch |err| switch (err) {
1287 error.WriteFailed => return w.err.?,
1288 };
1289 w.interface.writeInt(u32, @intCast(test_names.len), .little) catch |err| switch (err) {
1290 error.WriteFailed => return w.err.?,
1291 };
1292 for (test_names) |test_name| {
1293 w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) {
1294 error.WriteFailed => return w.err.?,
1295 };
1296 w.interface.writeAll(test_name) catch |err| switch (err) {
1297 error.WriteFailed => return w.err.?,
1298 };
1299 }
1300}
1301
1302fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResult {
1303 const b = run.step.owner;
1304 const io = b.graph.io;
1305 const arena = b.allocator;
1306 const gpa = b.allocator;
1307
1308 var child = try process.spawn(io, spawn_options);
1309 defer child.kill(io);
1310
1311 switch (run.stdin) {
1312 .bytes => |bytes| {
1313 child.stdin.?.writeStreamingAll(io, bytes) catch |err| {
1314 return run.step.fail("unable to write stdin: {t}", .{err});
1315 };
1316 child.stdin.?.close(io);
1317 child.stdin = null;
1318 },
1319 .lazy_path => |lazy_path| {
1320 const path = lazy_path.getPath3(b, &run.step);
1321 const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| {
1322 return run.step.fail("unable to open stdin file: {t}", .{err});
1323 };
1324 defer file.close(io);
1325 // TODO https://github.com/ziglang/zig/issues/23955
1326 var read_buffer: [1024]u8 = undefined;
1327 var file_reader = file.reader(io, &read_buffer);
1328 var write_buffer: [1024]u8 = undefined;
1329 var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer);
1330 _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) {
1331 error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{
1332 path, file_reader.err.?,
1333 }),
1334 error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{
1335 stdin_writer.err.?,
1336 }),
1337 };
1338 stdin_writer.interface.flush() catch |err| switch (err) {
1339 error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{
1340 stdin_writer.err.?,
1341 }),
1342 };
1343 child.stdin.?.close(io);
1344 child.stdin = null;
1345 },
1346 .none => {},
1347 }
1348
1349 var stdout_bytes: ?[]const u8 = null;
1350 var stderr_bytes: ?[]const u8 = null;
1351
1352 if (child.stdout) |stdout| {
1353 if (child.stderr) |stderr| {
1354 var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined;
1355 var multi_reader: Io.File.MultiReader = undefined;
1356 multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr });
1357 defer multi_reader.deinit();
1358
1359 const stdout_reader = multi_reader.reader(0);
1360 const stderr_reader = multi_reader.reader(1);
1361
1362 while (multi_reader.fill(64, .none)) |_| {
1363 if (run.stdio_limit.toInt()) |limit| {
1364 if (stdout_reader.buffered().len > limit)
1365 return error.StdoutStreamTooLong;
1366 if (stderr_reader.buffered().len > limit)
1367 return error.StderrStreamTooLong;
1368 }
1369 } else |err| switch (err) {
1370 error.Timeout => unreachable,
1371 error.EndOfStream => {},
1372 else => |e| return e,
1373 }
1374
1375 try multi_reader.checkAnyError();
1376
1377 // TODO: this string can leak since alloc below can return error.
1378 stdout_bytes = try multi_reader.toOwnedSlice(0);
1379 // TODO: this string can leak since its allocated using gpa and `try child.wait(io)` below can fail.
1380 stderr_bytes = try multi_reader.toOwnedSlice(1);
1381 } else {
1382 var stdout_reader = stdout.readerStreaming(io, &.{});
1383 stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) {
1384 error.OutOfMemory => |e| return e,
1385 error.ReadFailed => return stdout_reader.err.?,
1386 error.StreamTooLong => return error.StdoutStreamTooLong,
1387 };
1388 }
1389 } else if (child.stderr) |stderr| {
1390 var stderr_reader = stderr.readerStreaming(io, &.{});
1391 stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) {
1392 error.OutOfMemory => |e| return e,
1393 error.ReadFailed => return stderr_reader.err.?,
1394 error.StreamTooLong => return error.StderrStreamTooLong,
1395 };
1396 }
1397
1398 if (stderr_bytes) |bytes| if (bytes.len > 0) {
1399 // Treat stderr as an error message.
1400 const stderr_is_diagnostic = run.captured_stderr == null and switch (run.stdio) {
1401 .check => |checks| !checksContainStderr(checks.items),
1402 else => true,
1403 };
1404 if (stderr_is_diagnostic) {
1405 run.step.result_stderr = bytes;
1406 }
1407 };
1408
1409 run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0;
1410
1411 return .{
1412 .term = try child.wait(io),
1413 .stdout = stdout_bytes,
1414 .stderr = stderr_bytes,
1415 };
1416}
1417
1418const IndexedOutput = struct {
1419 index: usize,
1420 tag: @typeInfo(Arg).@"union".tag_type.?,
1421 output: *Output,
1422};
1423
1424pub fn rerunInFuzzMode(
1425 run: *Run,
1426 fuzz: *std.Build.Fuzz,
1427 prog_node: std.Progress.Node,
1428) !void {
1429 const step = &run.step;
1430 const b = step.owner;
1431 const io = b.graph.io;
1432 const arena = b.allocator;
1433 var argv_list: std.ArrayList([]const u8) = .empty;
1434 for (run.argv.items) |arg| {
1435 switch (arg) {
1436 .bytes => |bytes| {
1437 try argv_list.append(arena, bytes);
1438 },
1439 .lazy_path => |file| {
1440 const file_path = file.lazy_path.getPath3(b, step);
1441 try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) }));
1442 },
1443 .decorated_directory => |dd| {
1444 const file_path = dd.lazy_path.getPath3(b, step);
1445 try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix }));
1446 },
1447 .file_content => |file_plp| {
1448 const file_path = file_plp.lazy_path.getPath3(b, step);
1449
1450 var result: std.Io.Writer.Allocating = .init(arena);
1451 errdefer result.deinit();
1452 result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory;
1453
1454 const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{});
1455 defer file.close(io);
1456
1457 var buf: [1024]u8 = undefined;
1458 var file_reader = file.reader(io, &buf);
1459 _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) {
1460 error.ReadFailed => return file_reader.err.?,
1461 error.WriteFailed => return error.OutOfMemory,
1462 };
1463
1464 try argv_list.append(arena, result.written());
1465 },
1466 .artifact => |pa| {
1467 const artifact = pa.artifact;
1468 const file_path: []const u8 = p: {
1469 if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?});
1470 break :p artifact.installed_path orelse artifact.generated_bin.?.path.?;
1471 };
1472 try argv_list.append(arena, b.fmt("{s}{s}", .{
1473 pa.prefix,
1474 run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }),
1475 }));
1476 },
1477 .output_file, .output_directory => unreachable,
1478 }
1479 }
1480
1481 if (run.step.result_failed_command) |cmd| {
1482 fuzz.gpa.free(cmd);
1483 run.step.result_failed_command = null;
1484 }
1485
1486 const has_side_effects = false;
1487 var rand_int: u64 = undefined;
1488 io.random(@ptrCast(&rand_int));
1489 const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int);
1490 try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{
1491 .progress_node = prog_node,
1492 .watch = undefined, // not used by `runCommand`
1493 .web_server = null, // only needed for time reports
1494 .unit_test_timeout_ns = null, // don't time out fuzz tests for now
1495 .gpa = fuzz.gpa,
1496 }, .{
1497 .fuzz = fuzz,
1498 });
1499}
1500
1501fn populateGeneratedPaths(
1502 arena: std.mem.Allocator,
1503 output_placeholders: []const IndexedOutput,
1504 captured_stdout: ?*CapturedStdIo,
1505 captured_stderr: ?*CapturedStdIo,
1506 cache_root: Cache.Directory,
1507 digest: *const Cache.HexDigest,
1508) !void {
1509 for (output_placeholders) |placeholder| {
1510 placeholder.output.generated_file.path = try cache_root.join(arena, &.{
1511 "o", digest, placeholder.output.basename,
1512 });
1513 }
1514
1515 if (captured_stdout) |captured| {
1516 captured.output.generated_file.path = try cache_root.join(arena, &.{
1517 "o", digest, captured.output.basename,
1518 });
1519 }
1520
1521 if (captured_stderr) |captured| {
1522 captured.output.generated_file.path = try cache_root.join(arena, &.{
1523 "o", digest, captured.output.basename,
1524 });
1525 }
1526}
1527
1528fn formatTerm(term: ?process.Child.Term, w: *std.Io.Writer) std.Io.Writer.Error!void {
1529 if (term) |t| switch (t) {
1530 .exited => |code| try w.print("exited with code {d}", .{code}),
1531 .signal => |sig| try w.print("terminated with signal {t}", .{sig}),
1532 .stopped => |sig| try w.print("stopped with signal {t}", .{sig}),
1533 .unknown => |code| try w.print("terminated for unknown reason with code {d}", .{code}),
1534 } else {
1535 try w.writeAll("exited with any code");
1536 }
1537}
1538fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTerm) {
1539 return .{ .data = term };
1540}
1541
1542const FuzzContext = struct {
1543 fuzz: *std.Build.Fuzz,
1544};
1545
1546fn runCommand(
1547 run: *Run,
1548 argv: []const []const u8,
1549 has_side_effects: bool,
1550 output_dir_path: []const u8,
1551 options: Step.MakeOptions,
1552 fuzz_context: ?FuzzContext,
1553) !void {
1554 const step = &run.step;
1555 const b = step.owner;
1556 const arena = b.allocator;
1557 const gpa = options.gpa;
1558 const io = b.graph.io;
1559
1560 const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit;
1561
1562 try step.handleChildProcUnsupported();
1563 try Step.handleVerbose2(step.owner, cwd, run.environ_map, argv);
1564
1565 const allow_skip = switch (run.stdio) {
1566 .check, .zig_test => run.skip_foreign_checks,
1567 else => false,
1568 };
1569
1570 var interp_argv = std.array_list.Managed([]const u8).init(b.allocator);
1571 defer interp_argv.deinit();
1572
1573 var environ_map: EnvMap = env: {
1574 const orig = run.environ_map orelse &b.graph.environ_map;
1575 break :env try orig.clone(gpa);
1576 };
1577 defer environ_map.deinit();
1578
1579 const opt_generic_result = spawnChildAndCollect(run, argv, &environ_map, has_side_effects, options, fuzz_context) catch |err| term: {
1580 // InvalidExe: cpu arch mismatch
1581 // FileNotFound: can happen with a wrong dynamic linker path
1582 if (err == error.InvalidExe or err == error.FileNotFound) interpret: {
1583 // TODO: learn the target from the binary directly rather than from
1584 // relying on it being a Compile step. This will make this logic
1585 // work even for the edge case that the binary was produced by a
1586 // third party.
1587 const exe = switch (run.argv.items[0]) {
1588 .artifact => |exe| exe.artifact,
1589 else => break :interpret,
1590 };
1591 switch (exe.kind) {
1592 .exe, .@"test" => {},
1593 else => break :interpret,
1594 }
1595
1596 const root_target = exe.rootModuleTarget();
1597 const need_cross_libc = exe.is_linking_libc and
1598 (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic));
1599 const other_target = exe.root_module.resolved_target.?.result;
1600 switch (std.zig.system.getExternalExecutor(io, &b.graph.host.result, &other_target, .{
1601 .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null,
1602 .link_libc = exe.is_linking_libc,
1603 })) {
1604 .native, .rosetta => {
1605 if (allow_skip) return error.MakeSkipped;
1606 break :interpret;
1607 },
1608 .wine => |bin_name| {
1609 if (b.enable_wine) {
1610 try interp_argv.append(bin_name);
1611 try interp_argv.appendSlice(argv);
1612
1613 // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but
1614 // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired.
1615 if (environ_map.get("WINEDEBUG") == null) {
1616 try environ_map.put("WINEDEBUG", "-all");
1617 }
1618 } else {
1619 return failForeign(run, "-fwine", argv[0], exe);
1620 }
1621 },
1622 .qemu => |bin_name| {
1623 if (b.enable_qemu) {
1624 try interp_argv.append(bin_name);
1625
1626 if (need_cross_libc) {
1627 if (b.libc_runtimes_dir) |dir| {
1628 try interp_argv.append("-L");
1629 try interp_argv.append(b.pathJoin(&.{
1630 dir,
1631 try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple(
1632 b.allocator,
1633 root_target.cpu.arch,
1634 root_target.os.tag,
1635 root_target.abi,
1636 ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple(
1637 b.allocator,
1638 root_target.cpu.arch,
1639 root_target.abi,
1640 ) else unreachable,
1641 }));
1642 } else return failForeign(run, "--libc-runtimes", argv[0], exe);
1643 }
1644
1645 try interp_argv.appendSlice(argv);
1646 } else return failForeign(run, "-fqemu", argv[0], exe);
1647 },
1648 .darling => |bin_name| {
1649 if (b.enable_darling) {
1650 try interp_argv.append(bin_name);
1651 try interp_argv.appendSlice(argv);
1652 } else {
1653 return failForeign(run, "-fdarling", argv[0], exe);
1654 }
1655 },
1656 .wasmtime => |bin_name| {
1657 if (b.enable_wasmtime) {
1658 try interp_argv.append(bin_name);
1659 try interp_argv.append("--dir=.");
1660 // Wasmtime doeesn't inherit environment variables from the parent process
1661 // by default. '-S inherit-env' was added in Wasmtime version 20.
1662 try interp_argv.append("-Sinherit-env");
1663 try interp_argv.append(argv[0]);
1664 try interp_argv.appendSlice(argv[1..]);
1665 } else {
1666 return failForeign(run, "-fwasmtime", argv[0], exe);
1667 }
1668 },
1669 .bad_dl => |foreign_dl| {
1670 if (allow_skip) return error.MakeSkipped;
1671
1672 const host_dl = b.graph.host.result.dynamic_linker.get() orelse "(none)";
1673
1674 return step.fail(
1675 \\the host system is unable to execute binaries from the target
1676 \\ because the host dynamic linker is '{s}',
1677 \\ while the target dynamic linker is '{s}'.
1678 \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step
1679 , .{ host_dl, foreign_dl });
1680 },
1681 .bad_os_or_cpu => {
1682 if (allow_skip) return error.MakeSkipped;
1683
1684 const host_name = try b.graph.host.result.zigTriple(b.allocator);
1685 const foreign_name = try root_target.zigTriple(b.allocator);
1686
1687 return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{
1688 host_name, foreign_name,
1689 });
1690 },
1691 }
1692
1693 if (root_target.os.tag == .windows) {
1694 // On Windows we don't have rpaths so we have to add .dll search paths to PATH
1695 run.addPathForDynLibs(exe);
1696 }
1697
1698 gpa.free(step.result_failed_command.?);
1699 step.result_failed_command = null;
1700 try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items);
1701
1702 break :term spawnChildAndCollect(run, interp_argv.items, &environ_map, has_side_effects, options, fuzz_context) catch |e| {
1703 if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped;
1704 if (e == error.MakeFailed) return error.MakeFailed; // error already reported
1705 return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e });
1706 };
1707 }
1708 if (err == error.MakeFailed) return error.MakeFailed; // error already reported
1709
1710 return step.fail("failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err });
1711 };
1712
1713 const generic_result = opt_generic_result orelse {
1714 assert(run.stdio == .zig_test);
1715 // Specific errors have already been reported, and test results are populated. All we need
1716 // to do is report step failure if any test failed.
1717 if (!step.test_results.isSuccess()) return error.MakeFailed;
1718 return;
1719 };
1720
1721 assert(fuzz_context == null);
1722 assert(run.stdio != .zig_test);
1723
1724 // Capture stdout and stderr to GeneratedFile objects.
1725 const Stream = struct {
1726 captured: ?*CapturedStdIo,
1727 bytes: ?[]const u8,
1728 };
1729 for ([_]Stream{
1730 .{
1731 .captured = run.captured_stdout,
1732 .bytes = generic_result.stdout,
1733 },
1734 .{
1735 .captured = run.captured_stderr,
1736 .bytes = generic_result.stderr,
1737 },
1738 }) |stream| {
1739 if (stream.captured) |captured| {
1740 const output_components = .{ output_dir_path, captured.output.basename };
1741 const output_path = try b.cache_root.join(arena, &output_components);
1742 captured.output.generated_file.path = output_path;
1743
1744 const sub_path = b.pathJoin(&output_components);
1745 const sub_path_dirname = Dir.path.dirname(sub_path).?;
1746 b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| {
1747 return step.fail("unable to make path '{f}{s}': {s}", .{
1748 b.cache_root, sub_path_dirname, @errorName(err),
1749 });
1750 };
1751 const data = switch (captured.trim_whitespace) {
1752 .none => stream.bytes.?,
1753 .all => mem.trim(u8, stream.bytes.?, &std.ascii.whitespace),
1754 .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace),
1755 .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace),
1756 };
1757 b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| {
1758 return step.fail("unable to write file '{f}{s}': {s}", .{
1759 b.cache_root, sub_path, @errorName(err),
1760 });
1761 };
1762 }
1763 }
1764
1765 switch (run.stdio) {
1766 .zig_test => unreachable,
1767 .check => |checks| for (checks.items) |check| switch (check) {
1768 .expect_stderr_exact => |expected_bytes| {
1769 if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) {
1770 return step.fail(
1771 \\========= expected this stderr: =========
1772 \\{s}
1773 \\========= but found: ====================
1774 \\{s}
1775 , .{
1776 expected_bytes,
1777 generic_result.stderr.?,
1778 });
1779 }
1780 },
1781 .expect_stderr_match => |match| {
1782 if (mem.find(u8, generic_result.stderr.?, match) == null) {
1783 return step.fail(
1784 \\========= expected to find in stderr: =========
1785 \\{s}
1786 \\========= but stderr does not contain it: =====
1787 \\{s}
1788 , .{
1789 match,
1790 generic_result.stderr.?,
1791 });
1792 }
1793 },
1794 .expect_stdout_exact => |expected_bytes| {
1795 if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) {
1796 return step.fail(
1797 \\========= expected this stdout: =========
1798 \\{s}
1799 \\========= but found: ====================
1800 \\{s}
1801 , .{
1802 expected_bytes,
1803 generic_result.stdout.?,
1804 });
1805 }
1806 },
1807 .expect_stdout_match => |match| {
1808 if (mem.find(u8, generic_result.stdout.?, match) == null) {
1809 return step.fail(
1810 \\========= expected to find in stdout: =========
1811 \\{s}
1812 \\========= but stdout does not contain it: =====
1813 \\{s}
1814 , .{
1815 match,
1816 generic_result.stdout.?,
1817 });
1818 }
1819 },
1820 .expect_term => |expected_term| {
1821 if (!termMatches(expected_term, generic_result.term)) {
1822 return step.fail("process {f} (expected {f})", .{
1823 fmtTerm(generic_result.term),
1824 fmtTerm(expected_term),
1825 });
1826 }
1827 },
1828 },
1829 else => {
1830 // On failure, report captured stderr like normal standard error output.
1831 const bad_exit = switch (generic_result.term) {
1832 .exited => |code| code != 0,
1833 .signal, .stopped, .unknown => true,
1834 };
1835 if (bad_exit) {
1836 if (generic_result.stderr) |bytes| {
1837 run.step.result_stderr = bytes;
1838 }
1839 }
1840
1841 try step.handleChildProcessTerm(generic_result.term);
1842 },
1843 }
1844}
1845
1846const EvalGenericResult = struct {
1847 term: process.Child.Term,
1848 stdout: ?[]const u8,
1849 stderr: ?[]const u8,
1850};
1851
1852fn spawnChildAndCollect(
1853 run: *Run,
1854 argv: []const []const u8,
1855 environ_map: *EnvMap,
1856 has_side_effects: bool,
1857 options: Step.MakeOptions,
1858 fuzz_context: ?FuzzContext,
1859) !?EvalGenericResult {
1860 const b = run.step.owner;
1861 const graph = b.graph;
1862 const io = graph.io;
1863
1864 if (fuzz_context != null) {
1865 assert(!has_side_effects);
1866 assert(run.stdio == .zig_test);
1867 }
1868
1869 const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit;
1870
1871 // If an error occurs, it's caused by this command:
1872 assert(run.step.result_failed_command == null);
1873 run.step.result_failed_command = try Step.allocPrintCmd(options.gpa, child_cwd, .{
1874 .child = environ_map,
1875 .parent = &graph.environ_map,
1876 }, argv);
1877
1878 var spawn_options: process.SpawnOptions = .{
1879 .argv = argv,
1880 .cwd = child_cwd,
1881 .environ_map = environ_map,
1882 .request_resource_usage_statistics = true,
1883 .stdin = if (run.stdin != .none) s: {
1884 assert(run.stdio != .inherit);
1885 break :s .pipe;
1886 } else switch (run.stdio) {
1887 .infer_from_args => if (has_side_effects) .inherit else .ignore,
1888 .inherit => .inherit,
1889 .check => .ignore,
1890 .zig_test => .pipe,
1891 },
1892 .stdout = if (run.captured_stdout != null) .pipe else switch (run.stdio) {
1893 .infer_from_args => if (has_side_effects) .inherit else .ignore,
1894 .inherit => .inherit,
1895 .check => |checks| if (checksContainStdout(checks.items)) .pipe else .ignore,
1896 .zig_test => .pipe,
1897 },
1898 .stderr = if (run.captured_stderr != null) .pipe else switch (run.stdio) {
1899 .infer_from_args => if (has_side_effects) .inherit else .pipe,
1900 .inherit => .inherit,
1901 .check => .pipe,
1902 .zig_test => .pipe,
1903 },
1904 };
1905
1906 if (run.stdio == .zig_test) {
1907 const started: Io.Clock.Timestamp = .now(io, .awake);
1908 const result = evalZigTest(run, spawn_options, options, fuzz_context) catch |err| switch (err) {
1909 error.Canceled => |e| return e,
1910 else => |e| e,
1911 };
1912 run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds);
1913 try result;
1914 return null;
1915 } else {
1916 const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit;
1917 if (!run.disable_zig_progress and !inherit) {
1918 spawn_options.progress_node = options.progress_node;
1919 }
1920 const terminal_mode: Io.Terminal.Mode = if (inherit) m: {
1921 const stderr = try io.lockStderr(&.{}, graph.stderr_mode);
1922 break :m stderr.terminal_mode;
1923 } else .no_color;
1924 defer if (inherit) io.unlockStderr();
1925 try setColorEnvironmentVariables(run, environ_map, terminal_mode);
1926
1927 const started: Io.Clock.Timestamp = .now(io, .awake);
1928 const result = evalGeneric(run, spawn_options) catch |err| switch (err) {
1929 error.Canceled => |e| return e,
1930 else => |e| e,
1931 };
1932 run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds);
1933 return try result;
1934 }
1935}
1936
1937fn hashStdIo(hh: *Cache.HashHelper, stdio: StdIo) void {
1938 switch (stdio) {
1939 .infer_from_args, .inherit, .zig_test => {},
1940 .check => |checks| for (checks.items) |check| {
1941 hh.add(@as(std.meta.Tag(StdIo.Check), check));
1942 switch (check) {
1943 .expect_stderr_exact,
1944 .expect_stderr_match,
1945 .expect_stdout_exact,
1946 .expect_stdout_match,
1947 => |s| hh.addBytes(s),
1948
1949 .expect_term => |term| {
1950 hh.add(@as(std.meta.Tag(process.Child.Term), term));
1951 switch (term) {
1952 inline .exited, .signal, .stopped => |x| hh.add(x),
1953 .unknown => |x| hh.add(x),
1954 }
1955 },
1956 }
1957 },
1958 }
1959}
1960fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool {
1961 return if (expected) |e| switch (e) {
1962 .exited => |expected_code| switch (actual) {
1963 .exited => |actual_code| expected_code == actual_code,
1964 else => false,
1965 },
1966 .signal => |expected_sig| switch (actual) {
1967 .signal => |actual_sig| expected_sig == actual_sig,
1968 else => false,
1969 },
1970 .stopped => |expected_sig| switch (actual) {
1971 .stopped => |actual_sig| expected_sig == actual_sig,
1972 else => false,
1973 },
1974 .unknown => |expected_code| switch (actual) {
1975 .unknown => |actual_code| expected_code == actual_code,
1976 else => false,
1977 },
1978 } else switch (actual) {
1979 .exited => true,
1980 else => false,
1981 };
1982}
1983
1984fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: Io.Terminal.Mode) !void {
1985 color: switch (run.color) {
1986 .manual => {},
1987 .enable => {
1988 try environ_map.put("CLICOLOR_FORCE", "1");
1989 _ = environ_map.swapRemove("NO_COLOR");
1990 },
1991 .disable => {
1992 try environ_map.put("NO_COLOR", "1");
1993 _ = environ_map.swapRemove("CLICOLOR_FORCE");
1994 },
1995 .inherit => switch (terminal_mode) {
1996 .no_color, .windows_api => continue :color .disable,
1997 .escape_codes => continue :color .enable,
1998 },
1999 .auto => {
2000 const capture_stderr = run.captured_stderr != null or switch (run.stdio) {
2001 .check => |checks| checksContainStderr(checks.items),
2002 .infer_from_args, .inherit, .zig_test => false,
2003 };
2004 if (capture_stderr) {
2005 continue :color .disable;
2006 } else {
2007 continue :color .inherit;
2008 }
2009 },
2010 }
2011}
2012
2013fn checksContainStdout(checks: []const StdIo.Check) bool {
2014 for (checks) |check| switch (check) {
2015 .expect_stderr_exact,
2016 .expect_stderr_match,
2017 .expect_term,
2018 => continue,
2019
2020 .expect_stdout_exact,
2021 .expect_stdout_match,
2022 => return true,
2023 };
2024 return false;
2025}
2026
2027fn checksContainStderr(checks: []const StdIo.Check) bool {
2028 for (checks) |check| switch (check) {
2029 .expect_stdout_exact,
2030 .expect_stdout_match,
2031 .expect_term,
2032 => continue,
2033
2034 .expect_stderr_exact,
2035 .expect_stderr_match,
2036 => return true,
2037 };
2038 return false;
2039}
2040
2041/// Returns whether the Run step has side effects *other than* updating the output arguments.
2042fn hasSideEffects(run: Run) bool {
2043 if (run.has_side_effects) return true;
2044 return switch (run.stdio) {
2045 .infer_from_args => !run.hasAnyOutputArgs(),
2046 .inherit => true,
2047 .check => false,
2048 .zig_test => false,
2049 };
2050}
2051
2052fn hasAnyOutputArgs(run: Run) bool {
2053 if (run.captured_stdout != null) return true;
2054 if (run.captured_stderr != null) return true;
2055 for (run.argv.items) |arg| switch (arg) {
2056 .output_file, .output_directory => return true,
2057 else => continue,
2058 };
2059 return false;
2060}
2061
2062/// If `path` is cwd-relative, make it relative to the cwd of the child instead.
2063///
2064/// Whenever a path is included in the argv of a child, it should be put through this function first
2065/// to make sure the child doesn't see paths relative to a cwd other than its own.
2066fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 {
2067 const b = run.step.owner;
2068 const graph = b.graph;
2069 const arena = graph.arena;
2070
2071 const path_str = path.toString(arena) catch @panic("OOM");
2072 if (Dir.path.isAbsolute(path_str)) {
2073 // Absolute paths don't need changing.
2074 return path_str;
2075 }
2076 const child_cwd_rel: []const u8 = rel: {
2077 const child_lazy_cwd = run.cwd orelse break :rel path_str;
2078 const child_cwd = child_lazy_cwd.getPath3(b, &run.step).toString(arena) catch @panic("OOM");
2079 // Convert it from relative to *our* cwd, to relative to the *child's* cwd.
2080 break :rel Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str) catch @panic("OOM");
2081 };
2082 // Not every path can be made relative, e.g. if the path and the child cwd are on different
2083 // disk designators on Windows. In that case, `relative` will return an absolute path which we can
2084 // just return.
2085 if (Dir.path.isAbsolute(child_cwd_rel)) return child_cwd_rel;
2086
2087 // We're not done yet. In some cases this path must be prefixed with './':
2088 // * On POSIX, the executable name cannot be a single component like 'foo'
2089 // * Some executables might treat a leading '-' like a flag, which we must avoid
2090 // There's no harm in it, so just *always* apply this prefix.
2091 return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM");
2092}
2093
2094fn addPathForDynLibs(run: *Run, artifact: *Step.Compile) void {
2095 const b = run.step.owner;
2096 const compiles = artifact.getCompileDependencies(true);
2097 for (compiles) |compile| {
2098 if (compile.root_module.resolved_target.?.result.os.tag == .windows and
2099 compile.isDynamicLibrary())
2100 {
2101 addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?);
2102 }
2103 }
2104}
2105
2106fn failForeign(
2107 run: *Run,
2108 suggested_flag: []const u8,
2109 argv0: []const u8,
2110 exe: *Step.Compile,
2111) error{ MakeFailed, MakeSkipped, OutOfMemory } {
2112 switch (run.stdio) {
2113 .check, .zig_test => {
2114 if (run.skip_foreign_checks)
2115 return error.MakeSkipped;
2116
2117 const b = run.step.owner;
2118 const host_name = try b.graph.host.result.zigTriple(b.allocator);
2119 const foreign_name = try exe.rootModuleTarget().zigTriple(b.allocator);
2120
2121 return run.step.fail(
2122 \\unable to spawn foreign binary '{s}' ({s}) on host system ({s})
2123 \\ consider using {s} or enabling skip_foreign_checks in the Run step
2124 , .{ argv0, foreign_name, host_name, suggested_flag });
2125 },
2126 else => {
2127 return run.step.fail("unable to spawn foreign binary '{s}'", .{argv0});
2128 },
2129 }
2130}
lib/compiler/maker/Step/WriteFile.zig deleted-206
...@@ -1,206 +0,0 @@
1
2fn make(step: *Step, options: Step.MakeOptions) !void {
3 _ = options;
4 const b = step.owner;
5 const graph = b.graph;
6 const io = graph.io;
7 const arena = b.allocator;
8 const gpa = graph.cache.gpa;
9 const write_file: *WriteFile = @fieldParentPtr("step", step);
10
11 const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len);
12 var open_dirs_count: usize = 0;
13 defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]);
14
15 switch (write_file.mode) {
16 .whole_cached => {
17 step.clearWatchInputs();
18
19 // The cache is used here not really as a way to speed things up - because writing
20 // the data to a file would probably be very fast - but as a way to find a canonical
21 // location to put build artifacts.
22
23 // If, for example, a hard-coded path was used as the location to put WriteFile
24 // files, then two WriteFiles executing in parallel might clobber each other.
25
26 var man = b.graph.cache.obtain();
27 defer man.deinit();
28
29 for (write_file.files.items) |file| {
30 man.hash.addBytes(file.sub_path);
31
32 switch (file.contents) {
33 .bytes => |bytes| {
34 man.hash.addBytes(bytes);
35 },
36 .copy => |lazy_path| {
37 const path = lazy_path.getPath3(b, step);
38 _ = try man.addFilePath(path, null);
39 try step.addWatchInput(lazy_path);
40 },
41 }
42 }
43
44 for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| {
45 man.hash.addBytes(dir.sub_path);
46 for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext);
47 if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc);
48
49 const need_derived_inputs = try step.addDirectoryWatchInput(dir.source);
50 const src_dir_path = dir.source.getPath3(b, step);
51
52 var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| {
53 return step.fail("unable to open source directory '{f}': {s}", .{
54 src_dir_path, @errorName(err),
55 });
56 };
57 open_dir_cache_elem.* = src_dir;
58 open_dirs_count += 1;
59
60 var it = try src_dir.walk(gpa);
61 defer it.deinit();
62 while (try it.next(io)) |entry| {
63 if (!dir.options.pathIncluded(entry.path)) continue;
64
65 switch (entry.kind) {
66 .directory => {
67 if (need_derived_inputs) {
68 const entry_path = try src_dir_path.join(arena, entry.path);
69 try step.addDirectoryWatchInputFromPath(entry_path);
70 }
71 },
72 .file => {
73 const entry_path = try src_dir_path.join(arena, entry.path);
74 _ = try man.addFilePath(entry_path, null);
75 },
76 else => continue,
77 }
78 }
79 }
80
81 if (try step.cacheHit(&man)) {
82 const digest = man.final();
83 write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest });
84 assert(step.result_cached);
85 return;
86 }
87
88 const digest = man.final();
89 const cache_path = "o" ++ Dir.path.sep_str ++ digest;
90
91 write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path});
92
93 try operate(write_file, open_dir_cache, .{
94 .root_dir = b.cache_root,
95 .sub_path = cache_path,
96 });
97
98 try step.writeManifest(&man);
99 },
100 .tmp => {
101 step.result_cached = false;
102
103 var rand_int: u64 = undefined;
104 io.random(@ptrCast(&rand_int));
105 const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int);
106
107 write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path});
108
109 try operate(write_file, open_dir_cache, .{
110 .root_dir = b.cache_root,
111 .sub_path = tmp_dir_sub_path,
112 });
113 },
114 .mutate => |lp| {
115 step.result_cached = false;
116 const root_path = try lp.getPath4(b, step);
117 write_file.generated_directory.path = try root_path.toString(arena);
118 try operate(write_file, open_dir_cache, root_path);
119 },
120 }
121}
122
123fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void {
124 const step = &write_file.step;
125 const b = step.owner;
126 const io = b.graph.io;
127 const gpa = b.graph.cache.gpa;
128 const arena = b.allocator;
129
130 var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err|
131 return step.fail("unable to make path {f}: {t}", .{ root_path, err });
132 defer cache_dir.close(io);
133
134 for (write_file.files.items) |file| {
135 if (Dir.path.dirname(file.sub_path)) |dirname| {
136 cache_dir.createDirPath(io, dirname) catch |err| {
137 return step.fail("unable to make path '{f}{c}{s}': {t}", .{
138 root_path, Dir.path.sep, dirname, err,
139 });
140 };
141 }
142 switch (file.contents) {
143 .bytes => |bytes| {
144 cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| {
145 return step.fail("unable to write file '{f}{c}{s}': {t}", .{
146 root_path, Dir.path.sep, file.sub_path, err,
147 });
148 };
149 },
150 .copy => |file_source| {
151 const source_path = file_source.getPath2(b, step);
152 const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| {
153 return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{
154 source_path, root_path, Dir.path.sep, file.sub_path, err,
155 });
156 };
157 // At this point we already will mark the step as a cache miss.
158 // But this is kind of a partial cache hit since individual
159 // file copies may be avoided. Oh well, this information is
160 // discarded.
161 _ = prev_status;
162 },
163 }
164 }
165
166 for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| {
167 const src_dir_path = dir.source.getPath3(b, step);
168 const dest_dirname = dir.sub_path;
169
170 if (dest_dirname.len != 0) {
171 cache_dir.createDirPath(io, dest_dirname) catch |err| {
172 return step.fail("unable to make path '{f}{c}{s}': {t}", .{
173 root_path, Dir.path.sep, dest_dirname, err,
174 });
175 };
176 }
177
178 var it = try already_open_dir.walk(gpa);
179 defer it.deinit();
180 while (try it.next(io)) |entry| {
181 if (!dir.options.pathIncluded(entry.path)) continue;
182
183 const src_entry_path = try src_dir_path.join(arena, entry.path);
184 const dest_path = b.pathJoin(&.{ dest_dirname, entry.path });
185 switch (entry.kind) {
186 .directory => try cache_dir.createDirPath(io, dest_path),
187 .file => {
188 const prev_status = Io.Dir.updateFile(
189 src_entry_path.root_dir.handle,
190 io,
191 src_entry_path.sub_path,
192 cache_dir,
193 dest_path,
194 .{},
195 ) catch |err| {
196 return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{
197 src_entry_path, root_path, Dir.path.sep, dest_path, err,
198 });
199 };
200 _ = prev_status;
201 },
202 else => continue,
203 }
204 }
205 }
206}
lib/compiler/maker/Watch.zig deleted-976
...@@ -1,976 +0,0 @@
1const Watch = @This();
2const builtin = @import("builtin");
3
4const std = @import("std");
5const Io = std.Io;
6const Allocator = std.mem.Allocator;
7const assert = std.debug.assert;
8const fatal = std.process.fatal;
9const Configuration = std.Build.Configuration;
10
11const FsEvents = @import("Watch/FsEvents.zig");
12const Step = @import("Step.zig");
13
14os: Os,
15/// The number to show as the number of directories being watched.
16dir_count: usize,
17// These fields are common to most implementations so are kept here for simplicity.
18// They are `undefined` on implementations which do not utilize then.
19dir_table: DirTable,
20generation: Generation,
21configuration: *const Configuration,
22make_steps: []Step,
23
24pub const have_impl = Os != void;
25
26/// Key is the directory to watch which contains one or more files we are
27/// interested in noticing changes to.
28///
29/// Value is generation.
30const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAdapter, false);
31
32/// Special key of "." means any changes in this directory trigger the steps.
33const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet);
34const StepSet = std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, Generation);
35
36const Generation = u8;
37
38const Hash = std.hash.Wyhash;
39const Cache = std.Build.Cache;
40
41const Os = switch (builtin.os.tag) {
42 .linux => struct {
43 const posix = std.posix;
44
45 /// Keyed differently but indexes correspond 1:1 with `dir_table`.
46 handle_table: HandleTable,
47 /// fanotify file descriptors are keyed by mount id since marks
48 /// are limited to a single filesystem.
49 poll_fds: std.AutoArrayHashMapUnmanaged(MountId, posix.pollfd),
50
51 const MountId = i32;
52 const HandleTable = std.ArrayHashMapUnmanaged(FileHandle, struct { mount_id: MountId, reaction_set: ReactionSet }, FileHandle.Adapter, false);
53
54 const fan_mask: std.os.linux.fanotify.MarkMask = .{
55 .CLOSE_WRITE = true,
56 .CREATE = true,
57 .DELETE = true,
58 .DELETE_SELF = true,
59 .EVENT_ON_CHILD = true,
60 .MOVED_FROM = true,
61 .MOVED_TO = true,
62 .MOVE_SELF = true,
63 .ONDIR = true,
64 };
65
66 const FileHandle = struct {
67 handle: *align(1) std.os.linux.file_handle,
68
69 fn clone(lfh: FileHandle, gpa: Allocator) Allocator.Error!FileHandle {
70 const bytes = lfh.slice();
71 const new_ptr = try gpa.alignedAlloc(
72 u8,
73 .of(std.os.linux.file_handle),
74 @sizeOf(std.os.linux.file_handle) + bytes.len,
75 );
76 const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr);
77 new_header.* = lfh.handle.*;
78 const new: FileHandle = .{ .handle = new_header };
79 @memcpy(new.slice(), lfh.slice());
80 return new;
81 }
82
83 fn destroy(lfh: FileHandle, gpa: Allocator) void {
84 const ptr: [*]u8 = @ptrCast(lfh.handle);
85 const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes];
86 return gpa.free(allocated_slice);
87 }
88
89 fn slice(lfh: FileHandle) []u8 {
90 const ptr: [*]u8 = &lfh.handle.f_handle;
91 return ptr[0..lfh.handle.handle_bytes];
92 }
93
94 const Adapter = struct {
95 pub fn hash(self: Adapter, a: FileHandle) u32 {
96 _ = self;
97 const unsigned_type: u32 = @bitCast(a.handle.handle_type);
98 return @truncate(Hash.hash(unsigned_type, a.slice()));
99 }
100 pub fn eql(self: Adapter, a: FileHandle, b: FileHandle, b_index: usize) bool {
101 _ = self;
102 _ = b_index;
103 return a.handle.handle_type == b.handle.handle_type and std.mem.eql(u8, a.slice(), b.slice());
104 }
105 };
106 };
107
108 fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch {
109 _ = cwd_path;
110 return .{
111 .dir_table = .{},
112 .dir_count = 0,
113 .os = switch (builtin.os.tag) {
114 .linux => .{
115 .handle_table = .{},
116 .poll_fds = .{},
117 },
118 else => {},
119 },
120 .generation = 0,
121 .make_steps = make_steps,
122 .configuration = configuration,
123 };
124 }
125
126 fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path, mount_id: *MountId) !FileHandle {
127 var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined;
128 var buf: [std.fs.max_path_bytes]u8 = undefined;
129 const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{
130 path.sub_path,
131 }) catch return error.NameTooLong;
132 const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer);
133 stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle);
134 try posix.name_to_handle_at(path.root_dir.handle.handle, adjusted_path, stack_ptr, mount_id, std.os.linux.AT.HANDLE_FID);
135 const stack_lfh: FileHandle = .{ .handle = stack_ptr };
136 return stack_lfh.clone(gpa);
137 }
138
139 fn markDirtySteps(w: *Watch, gpa: Allocator, fan_fd: posix.fd_t) !bool {
140 const fanotify = std.os.linux.fanotify;
141 const M = fanotify.event_metadata;
142 var events_buf: [256 + 4096]u8 = undefined;
143 var any_dirty = false;
144 while (true) {
145 var len = posix.read(fan_fd, &events_buf) catch |err| switch (err) {
146 error.WouldBlock => return any_dirty,
147 else => |e| return e,
148 };
149 var meta: [*]align(1) M = @ptrCast(&events_buf);
150 while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({
151 len -= meta[0].event_len;
152 meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len);
153 }) {
154 assert(meta[0].vers == M.VERSION);
155 if (meta[0].mask.Q_OVERFLOW) {
156 any_dirty = true;
157 std.log.warn("file system watch queue overflowed; falling back to fstat", .{});
158 markAllFilesDirty(w, gpa);
159 return true;
160 }
161 const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1);
162 switch (fid.hdr.info_type) {
163 .DFID_NAME => {
164 const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
165 const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes);
166 const file_name = std.mem.span(file_name_z);
167 const lfh: FileHandle = .{ .handle = file_handle };
168 if (w.os.handle_table.getPtr(lfh)) |value| {
169 if (value.reaction_set.getPtr(".")) |glob_set|
170 any_dirty = markStepSetDirty(gpa, w.make_steps, glob_set, any_dirty);
171 if (value.reaction_set.getPtr(file_name)) |step_set|
172 any_dirty = markStepSetDirty(gpa, w.make_steps, step_set, any_dirty);
173 }
174 },
175 else => |t| std.log.warn("unexpected fanotify event '{t}'", .{t}),
176 }
177 }
178 }
179 }
180
181 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
182 // Add missing marks and note persisted ones.
183 for (steps) |step_index| {
184 const step = &w.make_steps[@intFromEnum(step_index)];
185 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
186 const reaction_set = rs: {
187 const gop = try w.dir_table.getOrPut(gpa, path);
188 if (!gop.found_existing) {
189 var mount_id: MountId = undefined;
190 const dir_handle = getDirHandle(gpa, path, &mount_id) catch |err| switch (err) {
191 error.FileNotFound => {
192 std.debug.assert(w.dir_table.swapRemove(path));
193 continue;
194 },
195 else => return err,
196 };
197 const fan_fd = blk: {
198 const fd_gop = try w.os.poll_fds.getOrPut(gpa, mount_id);
199 if (!fd_gop.found_existing) {
200 const fan_fd = std.posix.fanotify_init(.{
201 .CLASS = .NOTIF,
202 .CLOEXEC = true,
203 .NONBLOCK = true,
204 .REPORT_NAME = true,
205 .REPORT_DIR_FID = true,
206 .REPORT_FID = true,
207 .REPORT_TARGET_FID = true,
208 }, 0) catch |err| switch (err) {
209 error.UnsupportedFlags => fatal("fanotify_init failed due to old kernel; requires 5.17+", .{}),
210 else => |e| return e,
211 };
212 fd_gop.value_ptr.* = .{
213 .fd = fan_fd,
214 .events = std.posix.POLL.IN,
215 .revents = undefined,
216 };
217 }
218 break :blk fd_gop.value_ptr.*.fd;
219 };
220 // `dir_handle` may already be present in the table in
221 // the case that we have multiple Cache.Path instances
222 // that compare inequal but ultimately point to the same
223 // directory on the file system.
224 // In such case, we must revert adding this directory, but keep
225 // the additions to the step set.
226 const dh_gop = try w.os.handle_table.getOrPut(gpa, dir_handle);
227 if (dh_gop.found_existing) {
228 _ = w.dir_table.pop();
229 } else {
230 assert(dh_gop.index == gop.index);
231 dh_gop.value_ptr.* = .{ .mount_id = mount_id, .reaction_set = .{} };
232 posix.fanotify_mark(fan_fd, .{
233 .ADD = true,
234 .ONLYDIR = true,
235 }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| {
236 fatal("unable to watch {f}: {s}", .{ path, @errorName(err) });
237 };
238 }
239 break :rs &dh_gop.value_ptr.reaction_set;
240 }
241 break :rs &w.os.handle_table.values()[gop.index].reaction_set;
242 };
243 for (files.items) |basename| {
244 const gop = try reaction_set.getOrPut(gpa, basename);
245 if (!gop.found_existing) gop.value_ptr.* = .{};
246 try gop.value_ptr.put(gpa, step_index, w.generation);
247 }
248 }
249 }
250
251 {
252 // Remove marks for files that are no longer inputs.
253 var i: usize = 0;
254 while (i < w.os.handle_table.entries.len) {
255 {
256 const reaction_set = &w.os.handle_table.values()[i].reaction_set;
257 var step_set_i: usize = 0;
258 while (step_set_i < reaction_set.entries.len) {
259 const step_set = &reaction_set.values()[step_set_i];
260 var dirent_i: usize = 0;
261 while (dirent_i < step_set.entries.len) {
262 const generations = step_set.values();
263 if (generations[dirent_i] == w.generation) {
264 dirent_i += 1;
265 continue;
266 }
267 step_set.swapRemoveAt(dirent_i);
268 }
269 if (step_set.entries.len > 0) {
270 step_set_i += 1;
271 continue;
272 }
273 reaction_set.swapRemoveAt(step_set_i);
274 }
275 if (reaction_set.entries.len > 0) {
276 i += 1;
277 continue;
278 }
279 }
280
281 const path = w.dir_table.keys()[i];
282
283 const mount_id = w.os.handle_table.values()[i].mount_id;
284 const fan_fd = w.os.poll_fds.getEntry(mount_id).?.value_ptr.fd;
285 posix.fanotify_mark(fan_fd, .{
286 .REMOVE = true,
287 .ONLYDIR = true,
288 }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| switch (err) {
289 error.FileNotFound => {}, // Expected, harmless.
290 else => |e| std.log.warn("unable to unwatch '{f}': {s}", .{ path, @errorName(e) }),
291 };
292
293 w.dir_table.swapRemoveAt(i);
294 w.os.handle_table.swapRemoveAt(i);
295 }
296 w.generation +%= 1;
297 }
298 w.dir_count = w.dir_table.count();
299 }
300
301 fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult {
302 _ = io;
303 const events_len = try std.posix.poll(w.os.poll_fds.values(), timeout.to_i32_ms());
304 if (events_len == 0)
305 return .timeout;
306 for (w.os.poll_fds.values()) |poll_fd| {
307 if (poll_fd.revents & std.posix.POLL.IN == std.posix.POLL.IN and try markDirtySteps(w, gpa, poll_fd.fd))
308 return .dirty;
309 }
310 return .clean;
311 }
312 },
313 .windows => struct {
314 const windows = std.os.windows;
315
316 /// Keyed differently but indexes correspond 1:1 with `dir_table`.
317 handle_table: std.ArrayHashMapUnmanaged(*Directory, void, Directory.TableAdapter, false),
318 ready_dirs: std.DoublyLinkedList,
319
320 const FileId = struct {
321 volumeSerialNumber: windows.ULONG,
322 indexNumber: windows.LARGE_INTEGER,
323 };
324
325 const Directory = struct {
326 reaction_set: ReactionSet,
327 id: FileId,
328 file: Io.File,
329 state: enum { idle, listening, ready },
330 iosb: windows.IO_STATUS_BLOCK,
331 // 64 KB is the packet size limit when monitoring over a network.
332 // https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-readdirectorychangesw#remarks
333 buffer: [64 * 1024]u8 align(@alignOf(windows.FILE.NOTIFY.INFORMATION)),
334 ready_node: std.DoublyLinkedList.Node,
335
336 /// Start listening for events, buffer field will be overwritten eventually.
337 fn startListening(dir: *Directory, w: *Watch) !void {
338 assert(dir.file.flags.nonblocking);
339 assert(dir.state == .idle);
340 switch (windows.ntdll.NtNotifyChangeDirectoryFileEx(
341 dir.file.handle,
342 null,
343 &notifyApc,
344 w,
345 &dir.iosb,
346 &dir.buffer,
347 dir.buffer.len,
348 .{
349 .FILE_NAME = true,
350 .DIR_NAME = true,
351 .SIZE = true,
352 .LAST_WRITE = true,
353 .CREATION = true,
354 },
355 .FALSE,
356 .Notify,
357 )) {
358 .SUCCESS, .PENDING => dir.state = .listening,
359 .ILLEGAL_FUNCTION => return error.ReadDirectoryChangesUnsupported,
360 else => |status| return windows.unexpectedStatus(status),
361 }
362 }
363
364 fn notifyApc(apc_context: ?*anyopaque, iosb: *windows.IO_STATUS_BLOCK, _: windows.ULONG) align(std.Io.Threaded.apc_align) callconv(.winapi) void {
365 const w: *Watch = @ptrCast(@alignCast(apc_context));
366 const dir: *Directory = @fieldParentPtr("iosb", iosb);
367 assert(iosb.u.Status != .PENDING);
368 assert(dir.state == .listening);
369 w.os.ready_dirs.append(&dir.ready_node);
370 dir.state = .ready;
371 }
372
373 fn init(gpa: Allocator, path: Cache.Path) !*Directory {
374 // The following code is a drawn out NtCreateFile call. (mostly adapted from Io.Dir.makeOpenDirAccessMaskW)
375 // It's necessary in order to get the specific flags that are required when calling ReadDirectoryChangesW.
376 var dir_handle: windows.HANDLE = undefined;
377 const root_fd = path.root_dir.handle.handle;
378 const sub_path = path.subPathOrDot();
379 const sub_path_w = try Io.Threaded.sliceToPrefixedFileW(root_fd, sub_path, .{}); // TODO eliminate this call
380 var iosb: windows.IO_STATUS_BLOCK = undefined;
381 switch (windows.ntdll.NtCreateFile(
382 &dir_handle,
383 .{
384 .SPECIFIC = .{ .FILE_DIRECTORY = .{
385 .LIST = true,
386 } },
387 .STANDARD = .{ .SYNCHRONIZE = true },
388 .GENERIC = .{ .READ = true },
389 },
390 &.{
391 .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(sub_path_w.span())) null else root_fd,
392 .ObjectName = @constCast(&sub_path_w.string()),
393 },
394 &iosb,
395 null,
396 .{},
397 .VALID_FLAGS,
398 .OPEN,
399 .{
400 .DIRECTORY_FILE = true,
401 .IO = .ASYNCHRONOUS,
402 .OPEN_FOR_BACKUP_INTENT = true,
403 },
404 null,
405 0,
406 )) {
407 .SUCCESS => {},
408 .OBJECT_NAME_INVALID => return error.BadPathName,
409 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
410 .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
411 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
412 .NOT_A_DIRECTORY => return error.NotDir,
413 // This can happen if the directory has 'List folder contents' permission set to 'Deny'
414 .ACCESS_DENIED => return error.AccessDenied,
415 .INVALID_PARAMETER => unreachable,
416 else => |rc| return windows.unexpectedStatus(rc),
417 }
418 assert(dir_handle != windows.INVALID_HANDLE_VALUE);
419 errdefer windows.CloseHandle(dir_handle);
420
421 const dir_id = try getFileId(dir_handle);
422
423 const dir = try gpa.create(Directory);
424 dir.* = .{
425 .reaction_set = .empty,
426 .id = dir_id,
427 .file = .{ .handle = dir_handle, .flags = .{ .nonblocking = true } },
428 .state = .idle,
429 .iosb = undefined,
430 .buffer = undefined,
431 .ready_node = undefined,
432 };
433 return dir;
434 }
435
436 fn deinit(dir: *Directory, gpa: Allocator, w: *Watch) void {
437 state: switch (dir.state) {
438 .idle => {},
439 .listening => {
440 var cancel_iosb: windows.IO_STATUS_BLOCK = undefined;
441 _ = windows.ntdll.NtCancelIoFileEx(dir.file.handle, &dir.iosb, &cancel_iosb);
442 while (switch (dir.state) {
443 .idle => unreachable,
444 .listening => true,
445 .ready => false,
446 }) Io.Threaded.waitForApcOrAlert();
447 continue :state .ready;
448 },
449 .ready => w.os.ready_dirs.remove(&dir.ready_node),
450 }
451 windows.CloseHandle(dir.file.handle);
452 gpa.destroy(dir);
453 }
454
455 /// Useful to make `*Directory` a key in `std.ArrayHashMap`.
456 const TableAdapter = struct {
457 pub fn hash(_: TableAdapter, lhs_dir: *Directory) u32 {
458 return @truncate(Hash.hash(lhs_dir.id.volumeSerialNumber, @ptrCast(&lhs_dir.id.indexNumber)));
459 }
460 pub fn eql(_: TableAdapter, lhs_dir: *Directory, rhs_dir: *Directory, rhs_index: usize) bool {
461 _ = rhs_index;
462 return lhs_dir.id.volumeSerialNumber == rhs_dir.id.volumeSerialNumber and
463 lhs_dir.id.indexNumber == rhs_dir.id.indexNumber;
464 }
465 };
466 };
467
468 fn init(cwd_path: []const u8) !Watch {
469 _ = cwd_path;
470 return .{
471 .dir_table = .{},
472 .dir_count = 0,
473 .os = switch (builtin.os.tag) {
474 .windows => .{
475 .handle_table = .empty,
476 .ready_dirs = .{},
477 },
478 else => {},
479 },
480 .generation = 0,
481 };
482 }
483
484 fn getFileId(handle: windows.HANDLE) !FileId {
485 var file_id: FileId = undefined;
486 var io_status: windows.IO_STATUS_BLOCK = undefined;
487 var volume_info: windows.FILE.FS_VOLUME_INFORMATION = undefined;
488 switch (windows.ntdll.NtQueryVolumeInformationFile(
489 handle,
490 &io_status,
491 &volume_info,
492 @sizeOf(windows.FILE.FS_VOLUME_INFORMATION),
493 .Volume,
494 )) {
495 .SUCCESS => {},
496 // Buffer overflow here indicates that there is more information available than was able to be stored in the buffer
497 // size provided. This is treated as success because the type of variable-length information that this would be relevant for
498 // (name, volume name, etc) we don't care about.
499 .BUFFER_OVERFLOW => {},
500 else => |rc| return windows.unexpectedStatus(rc),
501 }
502 file_id.volumeSerialNumber = volume_info.VolumeSerialNumber;
503 var internal_info: windows.FILE.INTERNAL_INFORMATION = undefined;
504 switch (windows.ntdll.NtQueryInformationFile(
505 handle,
506 &io_status,
507 &internal_info,
508 @sizeOf(windows.FILE.INTERNAL_INFORMATION),
509 .Internal,
510 )) {
511 .SUCCESS => {},
512 else => |rc| return windows.unexpectedStatus(rc),
513 }
514 file_id.indexNumber = internal_info.IndexNumber;
515 return file_id;
516 }
517
518 fn markDirtySteps(w: *Watch, gpa: Allocator, dir: *Directory) !bool {
519 var any_dirty = false;
520 const bytes_returned = dir.iosb.Information;
521 if (bytes_returned == 0) {
522 std.log.warn("file system watch queue overflowed; falling back to fstat", .{});
523 markAllFilesDirty(w, gpa);
524 try dir.startListening(w);
525 return true;
526 }
527 var file_name_buf: [std.fs.max_path_bytes]u8 = undefined;
528 var offset: usize = 0;
529 while (true) {
530 const notify: *windows.FILE.NOTIFY.INFORMATION = @ptrCast(@alignCast(&dir.buffer[offset]));
531 const file_name = file_name_buf[0..std.unicode.wtf16LeToWtf8(&file_name_buf, notify.fileName())];
532 if (dir.reaction_set.getPtr(".")) |glob_set|
533 any_dirty = markStepSetDirty(gpa, glob_set, any_dirty);
534 if (dir.reaction_set.getPtr(file_name)) |step_set|
535 any_dirty = markStepSetDirty(gpa, step_set, any_dirty);
536 if (notify.NextEntryOffset == 0)
537 break;
538
539 offset += notify.NextEntryOffset;
540 }
541
542 // We call this now since at this point we have finished reading dir.buffer.
543 try dir.startListening(w);
544 return any_dirty;
545 }
546
547 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
548 // Add missing marks and note persisted ones.
549 for (steps) |step| {
550 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
551 const dir = dir: {
552 const gop = try w.dir_table.getOrPut(gpa, path);
553 if (!gop.found_existing) {
554 const dir: *Directory = try .init(gpa, path);
555 errdefer dir.deinit(gpa, w);
556 // `dir.id` may already be present in the table in
557 // the case that we have multiple Cache.Path instances
558 // that compare inequal but ultimately point to the same
559 // directory on the file system.
560 // In such case, we must revert adding this directory, but keep
561 // the additions to the step set.
562 const dh_gop = try w.os.handle_table.getOrPut(gpa, dir);
563 if (dh_gop.found_existing) {
564 dir.deinit(gpa, w);
565 _ = w.dir_table.pop();
566 break :dir w.os.handle_table.keys()[dh_gop.index];
567 } else {
568 assert(dh_gop.index == gop.index);
569 try dir.startListening(w);
570 break :dir dir;
571 }
572 }
573 break :dir w.os.handle_table.keys()[gop.index];
574 };
575 for (files.items) |basename| {
576 const gop = try dir.reaction_set.getOrPut(gpa, basename);
577 if (!gop.found_existing) gop.value_ptr.* = .{};
578 try gop.value_ptr.put(gpa, step, w.generation);
579 }
580 }
581 }
582
583 {
584 // Remove marks for files that are no longer inputs.
585 var i: usize = 0;
586 while (i < w.os.handle_table.entries.len) {
587 const dir = w.os.handle_table.keys()[i];
588 {
589 var step_set_i: usize = 0;
590 while (step_set_i < dir.reaction_set.entries.len) {
591 const step_set = &dir.reaction_set.values()[step_set_i];
592 var dirent_i: usize = 0;
593 while (dirent_i < step_set.entries.len) {
594 const generations = step_set.values();
595 if (generations[dirent_i] == w.generation) {
596 dirent_i += 1;
597 continue;
598 }
599 step_set.swapRemoveAt(dirent_i);
600 }
601 if (step_set.entries.len > 0) {
602 step_set_i += 1;
603 continue;
604 }
605 dir.reaction_set.swapRemoveAt(step_set_i);
606 }
607 if (dir.reaction_set.entries.len > 0) {
608 i += 1;
609 continue;
610 }
611 }
612
613 w.dir_table.swapRemoveAt(i);
614 w.os.handle_table.swapRemoveAt(i);
615 dir.deinit(gpa, w);
616 }
617 w.generation +%= 1;
618 }
619 w.dir_count = w.dir_table.count();
620 }
621
622 fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult {
623 for (0..2) |attempt| {
624 while (w.os.ready_dirs.popFirst()) |ready_node| {
625 const dir: *Directory = @fieldParentPtr("ready_node", ready_node);
626 assert(dir.state == .ready);
627 dir.state = .idle;
628 switch (dir.iosb.u.Status) {
629 .SUCCESS => return if (try markDirtySteps(w, gpa, dir)) .dirty else .clean,
630 .PENDING => unreachable,
631 .CANCELLED => {},
632 else => |status| return windows.unexpectedStatus(status),
633 }
634 try dir.startListening(w);
635 }
636 try io.checkCancel();
637 if (attempt == 1) return .timeout;
638 const delay_interval: windows.LARGE_INTEGER = switch (timeout) {
639 .none => std.math.minInt(windows.LARGE_INTEGER),
640 .ms => |ms| -@as(windows.LARGE_INTEGER, ms) * (std.time.ns_per_ms / 100),
641 };
642 _ = windows.ntdll.NtDelayExecution(.TRUE, &delay_interval);
643 } else unreachable;
644 }
645 },
646 .dragonfly, .freebsd, .netbsd, .openbsd, .ios, .tvos, .visionos, .watchos => struct {
647 const posix = std.posix;
648
649 kq_fd: i32,
650 /// Indexes correspond 1:1 with `dir_table`.
651 handles: std.MultiArrayList(struct {
652 rs: ReactionSet,
653 /// If the corresponding dir_table Path has sub_path == "", then it
654 /// suffices as the open directory handle, and this value will be
655 /// -1. Otherwise, it needs to be opened in update(), and will be
656 /// stored here.
657 dir_fd: i32,
658 }),
659
660 const dir_open_flags: posix.O = f: {
661 var f: posix.O = .{
662 .ACCMODE = .RDONLY,
663 .NOFOLLOW = false,
664 .DIRECTORY = true,
665 .CLOEXEC = true,
666 };
667 if (@hasField(posix.O, "EVTONLY")) f.EVTONLY = true;
668 if (@hasField(posix.O, "PATH")) f.PATH = true;
669 break :f f;
670 };
671
672 const EV = std.c.EV;
673 const NOTE = std.c.NOTE;
674
675 fn init(cwd_path: []const u8) !Watch {
676 _ = cwd_path;
677 return .{
678 .dir_table = .{},
679 .dir_count = 0,
680 .os = .{
681 .kq_fd = try Io.Kqueue.createFileDescriptor(),
682 .handles = .empty,
683 },
684 .generation = 0,
685 };
686 }
687
688 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
689 const handles = &w.os.handles;
690 for (steps) |step| {
691 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
692 const reaction_set = rs: {
693 const gop = try w.dir_table.getOrPut(gpa, path);
694 if (!gop.found_existing) {
695 const skip_open_dir = path.sub_path.len == 0;
696 const dir_fd = if (skip_open_dir)
697 path.root_dir.handle.handle
698 else
699 posix.openat(path.root_dir.handle.handle, path.sub_path, dir_open_flags, 0) catch |err| {
700 fatal("failed to open directory {f}: {t}", .{ path, err });
701 };
702 // Empirically the dir has to stay open or else no events are triggered.
703 errdefer if (!skip_open_dir) std.Io.Threaded.closeFd(dir_fd);
704 const changes = [1]posix.Kevent{.{
705 .ident = @bitCast(@as(isize, dir_fd)),
706 .filter = std.c.EVFILT.VNODE,
707 .flags = EV.ADD | EV.ENABLE | EV.CLEAR,
708 .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE,
709 .data = 0,
710 .udata = gop.index,
711 }};
712 _ = try Io.Kqueue.kevent(w.os.kq_fd, &changes, &.{}, null);
713 assert(handles.len == gop.index);
714 try handles.append(gpa, .{
715 .rs = .{},
716 .dir_fd = if (skip_open_dir) -1 else dir_fd,
717 });
718 }
719
720 break :rs &handles.items(.rs)[gop.index];
721 };
722 for (files.items) |basename| {
723 const gop = try reaction_set.getOrPut(gpa, basename);
724 if (!gop.found_existing) gop.value_ptr.* = .{};
725 try gop.value_ptr.put(gpa, step, w.generation);
726 }
727 }
728 }
729
730 {
731 // Remove marks for files that are no longer inputs.
732 var i: usize = 0;
733 while (i < handles.len) {
734 {
735 const reaction_set = &handles.items(.rs)[i];
736 var step_set_i: usize = 0;
737 while (step_set_i < reaction_set.entries.len) {
738 const step_set = &reaction_set.values()[step_set_i];
739 var dirent_i: usize = 0;
740 while (dirent_i < step_set.entries.len) {
741 const generations = step_set.values();
742 if (generations[dirent_i] == w.generation) {
743 dirent_i += 1;
744 continue;
745 }
746 step_set.swapRemoveAt(dirent_i);
747 }
748 if (step_set.entries.len > 0) {
749 step_set_i += 1;
750 continue;
751 }
752 reaction_set.swapRemoveAt(step_set_i);
753 }
754 if (reaction_set.entries.len > 0) {
755 i += 1;
756 continue;
757 }
758 }
759
760 // If the sub_path == "" then this patch has already the
761 // dir fd that we need to use as the ident to remove the
762 // event. If it was opened above with openat() then we need
763 // to access that data via the dir_fd field.
764 const path = w.dir_table.keys()[i];
765 const dir_fd = if (path.sub_path.len == 0)
766 path.root_dir.handle.handle
767 else
768 handles.items(.dir_fd)[i];
769 assert(dir_fd != -1);
770
771 // The changelist also needs to update the udata field of the last
772 // event, since we are doing a swap remove, and we store the dir_table
773 // index in the udata field.
774 const last_dir_fd = fd: {
775 const last_path = w.dir_table.keys()[handles.len - 1];
776 const last_dir_fd = if (last_path.sub_path.len == 0)
777 last_path.root_dir.handle.handle
778 else
779 handles.items(.dir_fd)[handles.len - 1];
780 assert(last_dir_fd != -1);
781 break :fd last_dir_fd;
782 };
783 const changes = [_]posix.Kevent{
784 .{
785 .ident = @bitCast(@as(isize, dir_fd)),
786 .filter = std.c.EVFILT.VNODE,
787 .flags = EV.DELETE,
788 .fflags = 0,
789 .data = 0,
790 .udata = i,
791 },
792 .{
793 .ident = @bitCast(@as(isize, last_dir_fd)),
794 .filter = std.c.EVFILT.VNODE,
795 .flags = EV.ADD,
796 .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE,
797 .data = 0,
798 .udata = i,
799 },
800 };
801 const filtered_changes = if (i == handles.len - 1) changes[0..1] else &changes;
802 _ = try Io.Kqueue.kevent(w.os.kq_fd, filtered_changes, &.{}, null);
803 if (path.sub_path.len != 0) std.Io.Threaded.closeFd(dir_fd);
804
805 w.dir_table.swapRemoveAt(i);
806 handles.swapRemove(i);
807 }
808 w.generation +%= 1;
809 }
810 w.dir_count = w.dir_table.count();
811 }
812
813 fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult {
814 _ = io;
815 var timespec_buffer: posix.timespec = undefined;
816 var event_buffer: [100]posix.Kevent = undefined;
817 var n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, timeout.toTimespec(&timespec_buffer));
818 if (n == 0) return .timeout;
819 const reaction_sets = w.os.handles.items(.rs);
820 var any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], false);
821 timespec_buffer = .{ .sec = 0, .nsec = 0 };
822 while (n == event_buffer.len) {
823 n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, &timespec_buffer);
824 if (n == 0) break;
825 any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], any_dirty);
826 }
827 return if (any_dirty) .dirty else .clean;
828 }
829
830 fn markDirtySteps(
831 gpa: Allocator,
832 reaction_sets: []ReactionSet,
833 events: []const std.c.Kevent,
834 start_any_dirty: bool,
835 ) bool {
836 var any_dirty = start_any_dirty;
837 for (events) |event| {
838 const index: usize = @intCast(event.udata);
839 const reaction_set = &reaction_sets[index];
840 // If we knew the basename of the changed file, here we would
841 // mark only the step set dirty, and possibly the glob set:
842 //if (reaction_set.getPtr(".")) |glob_set|
843 // any_dirty = markStepSetDirty(gpa, glob_set, any_dirty);
844 //if (reaction_set.getPtr(file_name)) |step_set|
845 // any_dirty = markStepSetDirty(gpa, step_set, any_dirty);
846 // However we don't know the file name so just mark all the
847 // sets dirty for this directory.
848 for (reaction_set.values()) |*step_set| {
849 any_dirty = markStepSetDirty(gpa, step_set, any_dirty);
850 }
851 }
852 return any_dirty;
853 }
854 },
855 .macos => struct {
856 fse: FsEvents,
857
858 fn init(cwd_path: []const u8) !Watch {
859 return .{
860 .os = .{ .fse = try .init(cwd_path) },
861 .dir_count = 0,
862 .dir_table = undefined,
863 .generation = undefined,
864 };
865 }
866 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
867 try w.os.fse.setPaths(gpa, steps);
868 w.dir_count = w.os.fse.watch_roots.len;
869 }
870 fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult {
871 _ = io;
872 return w.os.fse.wait(gpa, switch (timeout) {
873 .none => null,
874 .ms => |ms| @as(u64, ms) * std.time.ns_per_ms,
875 });
876 }
877 },
878 else => void,
879};
880
881pub fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch {
882 return Os.init(cwd_path, configuration, make_steps);
883}
884
885pub const Match = struct {
886 /// Relative to the watched directory, the file path that triggers this
887 /// match.
888 basename: []const u8,
889 /// The step to re-run when file corresponding to `basename` is changed.
890 step_index: Configuration.Step.Index,
891
892 pub const Context = struct {
893 pub fn hash(self: Context, a: Match) u32 {
894 _ = self;
895 var hasher = Hash.init(@intFromEnum(a.step_index));
896 hasher.update(a.basename);
897 return @truncate(hasher.final());
898 }
899 pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool {
900 _ = self;
901 _ = b_index;
902 return a.step_index == b.step_index and std.mem.eql(u8, a.basename, b.basename);
903 }
904 };
905};
906
907fn markAllFilesDirty(w: *Watch, gpa: Allocator) void {
908 for (switch (builtin.os.tag) {
909 .windows => w.os.handle_table.keys(),
910 else => w.os.handle_table.values(),
911 }) |item| {
912 const reaction_set = switch (builtin.os.tag) {
913 .linux, .windows => item.reaction_set,
914 else => item,
915 };
916 for (reaction_set.values()) |step_set| {
917 for (step_set.keys()) |step_index| {
918 const step = &w.make_steps[@intFromEnum(step_index)];
919 _ = step.invalidateResult(gpa);
920 }
921 }
922 }
923}
924
925fn markStepSetDirty(gpa: Allocator, make_steps: []Step, step_set: *StepSet, any_dirty: bool) bool {
926 var this_any_dirty = false;
927 for (step_set.keys()) |step_index| {
928 const step = &make_steps[@intFromEnum(step_index)];
929 if (step.invalidateResult(gpa)) this_any_dirty = true;
930 }
931 return any_dirty or this_any_dirty;
932}
933
934pub fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
935 return Os.update(w, gpa, steps);
936}
937
938pub const Timeout = union(enum) {
939 none,
940 ms: u16,
941
942 pub fn to_i32_ms(t: Timeout) i32 {
943 return switch (t) {
944 .none => -1,
945 .ms => |ms| ms,
946 };
947 }
948
949 pub fn toTimespec(t: Timeout, buf: *std.posix.timespec) ?*std.posix.timespec {
950 return switch (t) {
951 .none => null,
952 .ms => |ms_u16| {
953 const ms: isize = ms_u16;
954 buf.* = .{
955 .sec = @divTrunc(ms, std.time.ms_per_s),
956 .nsec = @rem(ms, std.time.ms_per_s) * std.time.ns_per_ms,
957 };
958 return buf;
959 },
960 };
961 }
962};
963
964pub const WaitResult = enum {
965 timeout,
966 /// File system watching triggered on files that were marked as inputs to at least one Step.
967 /// Relevant steps have been marked dirty.
968 dirty,
969 /// File system watching triggered but none of the events were relevant to
970 /// what we are listening to. There is nothing to do.
971 clean,
972};
973
974pub fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult {
975 return Os.wait(w, gpa, io, timeout);
976}
lib/compiler/maker/Watch/FsEvents.zig deleted-479
...@@ -1,479 +0,0 @@
1//! An implementation of file-system watching based on the `FSEventStream` API in macOS.
2//! While macOS supports kqueue, it does not allow detecting changes to files without
3//! placing watches on each individual file, meaning FD limits are reached incredibly
4//! quickly. The File System Events API works differently: it implements *recursive*
5//! directory watches, managed by a system service. Rather than being in libc, the API is
6//! exposed by the CoreServices framework. To avoid a compile dependency on the framework
7//! bundle, we dynamically load CoreServices with `std.DynLib`.
8//!
9//! While the logic in this file *is* specialized to `std.Build.Watch`, efforts have been
10//! made to keep that specialization to a minimum. Other use cases could be served with
11//! relatively minimal modifications to the `watch_paths` field and its usages (in
12//! particular the `setPaths` function). We avoid using the global GCD dispatch queue in
13//! favour of creating our own and synchronizing with an explicit semaphore, meaning this
14//! logic is thread-safe and does not affect process-global state.
15//!
16//! In theory, this API is quite good at avoiding filesystem race conditions. In practice,
17//! the logic that would avoid them is currently disabled, because the build system kind
18//! of relies on them at the time of writing to avoid redundant work -- see the comment at
19//! the top of `wait` for details.
20
21const enable_debug_logs = false;
22
23core_services: std.DynLib,
24resolved_symbols: ResolvedSymbols,
25
26paths_arena: std.heap.ArenaAllocator.State,
27/// The roots of the recursive watches. FSEvents has relatively small limits on the number
28/// of watched paths, so this slice must not be too long. The paths themselves are allocated
29/// into `paths_arena`, but this slice is allocated into the GPA.
30watch_roots: [][:0]const u8,
31/// All of the paths being watched. Value is the set of steps which depend on the file/directory.
32/// Keys and values are in `paths_arena`, but this map is allocated into the GPA.
33watch_paths: std.StringArrayHashMapUnmanaged([]const *std.Build.Step),
34
35/// The semaphore we use to block the thread calling `wait` until the callback determines a relevant
36/// event has occurred. This is retained across `wait` calls for simplicity and efficiency.
37waiting_semaphore: dispatch.semaphore_t,
38/// This dispatch queue is created by us and executes serially. It exists exclusively to trigger the
39/// callbacks of the FSEventStream we create. This is not in use outside of `wait`, but is retained
40/// across `wait` calls for simplicity and efficiency.
41dispatch_queue: dispatch.queue_t,
42/// In theory, this field avoids race conditions. In practice, it is essentially unused at the time
43/// of writing. See the comment at the start of `wait` for details.
44since_event: FSEventStreamEventId,
45
46cwd_path: []const u8,
47
48/// All of the symbols we pull from the `dlopen`ed CoreServices framework. If any of these symbols
49/// is not present, `init` will close the framework and return an error.
50const ResolvedSymbols = struct {
51 FSEventStreamCreate: *const fn (
52 allocator: CFAllocatorRef,
53 callback: FSEventStreamCallback,
54 ctx: ?*const FSEventStreamContext,
55 paths_to_watch: CFArrayRef,
56 since_when: FSEventStreamEventId,
57 latency: CFTimeInterval,
58 flags: FSEventStreamCreateFlags,
59 ) callconv(.c) FSEventStreamRef,
60 FSEventStreamSetDispatchQueue: *const fn (stream: FSEventStreamRef, queue: dispatch.queue_t) callconv(.c) void,
61 FSEventStreamStart: *const fn (stream: FSEventStreamRef) callconv(.c) bool,
62 FSEventStreamStop: *const fn (stream: FSEventStreamRef) callconv(.c) void,
63 FSEventStreamInvalidate: *const fn (stream: FSEventStreamRef) callconv(.c) void,
64 FSEventStreamRelease: *const fn (stream: FSEventStreamRef) callconv(.c) void,
65 FSEventStreamGetLatestEventId: *const fn (stream: ConstFSEventStreamRef) callconv(.c) FSEventStreamEventId,
66 FSEventsGetCurrentEventId: *const fn () callconv(.c) FSEventStreamEventId,
67 CFRelease: *const fn (cf: *const anyopaque) callconv(.c) void,
68 CFArrayCreate: *const fn (
69 allocator: CFAllocatorRef,
70 values: [*]const usize,
71 num_values: CFIndex,
72 call_backs: ?*const CFArrayCallBacks,
73 ) callconv(.c) CFArrayRef,
74 CFStringCreateWithCString: *const fn (
75 alloc: CFAllocatorRef,
76 c_str: [*:0]const u8,
77 encoding: CFStringEncoding,
78 ) callconv(.c) CFStringRef,
79 CFAllocatorCreate: *const fn (allocator: CFAllocatorRef, context: *const CFAllocatorContext) callconv(.c) CFAllocatorRef,
80 kCFAllocatorUseContext: *const CFAllocatorRef,
81};
82
83pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreServicesSymbol, SystemResources }!FsEvents {
84 var core_services = std.DynLib.open("/System/Library/Frameworks/CoreServices.framework/CoreServices") catch
85 return error.OpenFrameworkFailed;
86 errdefer core_services.close();
87
88 var resolved_symbols: ResolvedSymbols = undefined;
89 inline for (@typeInfo(ResolvedSymbols).@"struct".fields) |f| {
90 @field(resolved_symbols, f.name) = core_services.lookup(f.type, f.name) orelse return error.MissingCoreServicesSymbol;
91 }
92
93 return .{
94 .core_services = core_services,
95 .resolved_symbols = resolved_symbols,
96 .paths_arena = .{},
97 .watch_roots = &.{},
98 .watch_paths = .empty,
99 .waiting_semaphore = dispatch.semaphore_create(0) orelse return error.SystemResources,
100 .dispatch_queue = dispatch.queue_create("zig-watch", .SERIAL()) orelse return error.SystemResources,
101 // Not `.since_now`, because this means we can init `FsEvents` *before* we do work in order
102 // to notice any changes which happened during said work.
103 .since_event = resolved_symbols.FSEventsGetCurrentEventId(),
104 .cwd_path = cwd_path,
105 };
106}
107
108pub fn deinit(fse: *FsEvents, gpa: Allocator, io: Io) void {
109 fse.waiting_semaphore.as_object().release();
110 fse.dispatch_queue.as_object().release();
111 fse.core_services.close(io);
112
113 gpa.free(fse.watch_roots);
114 fse.watch_paths.deinit(gpa);
115 {
116 var paths_arena = fse.paths_arena.promote(gpa);
117 paths_arena.deinit();
118 }
119}
120
121pub fn setPaths(fse: *FsEvents, gpa: Allocator, steps: []const *std.Build.Step) !void {
122 var paths_arena_instance = fse.paths_arena.promote(gpa);
123 defer fse.paths_arena = paths_arena_instance.state;
124 const paths_arena = paths_arena_instance.allocator();
125
126 var need_dirs: std.StringArrayHashMapUnmanaged(void) = .empty;
127 defer need_dirs.deinit(gpa);
128
129 fse.watch_paths.clearRetainingCapacity();
130
131 // We take `step` by pointer for a slight memory optimization in a moment.
132 for (steps) |*step| {
133 for (step.*.inputs.table.keys(), step.*.inputs.table.values()) |path, *files| {
134 const resolved_dir = try std.fs.path.resolvePosix(paths_arena, &.{
135 fse.cwd_path, path.root_dir.path orelse ".", path.sub_path,
136 });
137 try need_dirs.put(gpa, resolved_dir, {});
138 for (files.items) |file_name| {
139 const watch_path = if (std.mem.eql(u8, file_name, "."))
140 resolved_dir
141 else
142 try std.fs.path.join(paths_arena, &.{ resolved_dir, file_name });
143 const gop = try fse.watch_paths.getOrPut(gpa, watch_path);
144 if (gop.found_existing) {
145 const old_steps = gop.value_ptr.*;
146 const new_steps = try paths_arena.alloc(*std.Build.Step, old_steps.len + 1);
147 @memcpy(new_steps[0..old_steps.len], old_steps);
148 new_steps[old_steps.len] = step.*;
149 gop.value_ptr.* = new_steps;
150 } else {
151 // This is why we captured `step` by pointer! We can avoid allocating a slice of one
152 // step in the arena in the common case where a file is referenced by only one step.
153 gop.value_ptr.* = step[0..1];
154 }
155 }
156 }
157 }
158
159 {
160 // There's no point looking at directories inside other ones (e.g. "/foo" and "/foo/bar").
161 // To eliminate these, we'll re-add directories in order of path length with a redundancy check.
162 const old_dirs = try gpa.dupe([]const u8, need_dirs.keys());
163 defer gpa.free(old_dirs);
164 std.mem.sort([]const u8, old_dirs, {}, struct {
165 fn lessThan(ctx: void, a: []const u8, b: []const u8) bool {
166 ctx;
167 return std.mem.lessThan(u8, a, b);
168 }
169 }.lessThan);
170 need_dirs.clearRetainingCapacity();
171 for (old_dirs) |dir_path| {
172 var it: std.fs.path.ComponentIterator(.posix, u8) = .init(dir_path);
173 while (it.next()) |component| {
174 if (need_dirs.contains(component.path)) {
175 // this path is '/foo/bar/qux', but '/foo' or '/foo/bar' was already added
176 break;
177 }
178 } else {
179 need_dirs.putAssumeCapacityNoClobber(dir_path, {});
180 }
181 }
182 }
183
184 // `need_dirs` is now a set of directories to watch with no redundancy. In practice, this is very
185 // likely to have reduced it to a quite small set (e.g. it'll typically coalesce a full `src/`
186 // directory into one entry). However, the FSEventStream API has a fairly low undocumented limit
187 // on total watches (supposedly 4096), so we should handle the case where we exceed it. To be
188 // safe, because this API can be a little unpredictable, we'll cap ourselves a little *below*
189 // that known limit.
190 if (need_dirs.count() > 2048) {
191 // Fallback: watch the whole filesystem. This is excessive, but... it *works* :P
192 if (enable_debug_logs) watch_log.debug("too many dirs; recursively watching root", .{});
193 fse.watch_roots = try gpa.realloc(fse.watch_roots, 1);
194 fse.watch_roots[0] = "/";
195 } else {
196 fse.watch_roots = try gpa.realloc(fse.watch_roots, need_dirs.count());
197 for (fse.watch_roots, need_dirs.keys()) |*out, in| {
198 out.* = try paths_arena.dupeSentinel(u8, in, 0);
199 }
200 }
201 if (enable_debug_logs) {
202 watch_log.debug("watching {d} paths using {d} recursive watches:", .{ fse.watch_paths.count(), fse.watch_roots.len });
203 for (fse.watch_roots) |dir_path| {
204 watch_log.debug("- '{s}'", .{dir_path});
205 }
206 }
207}
208
209pub fn wait(fse: *FsEvents, gpa: Allocator, timeout_ns: ?u64) error{ OutOfMemory, StartFailed }!std.Build.Watch.WaitResult {
210 if (fse.watch_roots.len == 0) @panic("nothing to watch");
211
212 const rs = fse.resolved_symbols;
213
214 // At the time of writing, using `since_event` in the obvious way causes redundant rebuilds
215 // to occur, because one step modifies a file which is an input to another step. The solution
216 // to this problem will probably be either:
217 //
218 // a) Don't include the output of one step as a watch input of another; only mark external
219 // files as watch inputs. Or...
220 //
221 // b) Note the current event ID when a step begins, and disregard events preceding that ID
222 // when considering whether to dirty that step in `eventCallback`.
223 //
224 // For now, to avoid the redundant rebuilds, we bypass this `since_event` mechanism. This does
225 // introduce race conditions, but the other `std.Build.Watch` implementations suffer from those
226 // too at the time of writing, so this is kind of expected.
227 fse.since_event = .since_now;
228
229 const cf_allocator = rs.CFAllocatorCreate(rs.kCFAllocatorUseContext.*, &.{
230 .version = 0,
231 .info = @constCast(&gpa),
232 .retain = null,
233 .release = null,
234 .copy_description = null,
235 .allocate = &cf_alloc_callbacks.allocate,
236 .reallocate = &cf_alloc_callbacks.reallocate,
237 .deallocate = &cf_alloc_callbacks.deallocate,
238 .preferred_size = null,
239 }) orelse return error.OutOfMemory;
240 defer rs.CFRelease(cf_allocator);
241
242 const cf_paths = try gpa.alloc(?CFStringRef, fse.watch_roots.len);
243 @memset(cf_paths, null);
244 defer {
245 for (cf_paths) |o| if (o) |p| rs.CFRelease(p);
246 gpa.free(cf_paths);
247 }
248 for (fse.watch_roots, cf_paths) |raw_path, *cf_path| {
249 cf_path.* = rs.CFStringCreateWithCString(cf_allocator, raw_path, .utf8);
250 }
251 const cf_paths_array = rs.CFArrayCreate(cf_allocator, @ptrCast(cf_paths), @intCast(cf_paths.len), null);
252 defer rs.CFRelease(cf_paths_array);
253
254 const callback_ctx: EventCallbackCtx = .{
255 .fse = fse,
256 .gpa = gpa,
257 };
258 const event_stream = rs.FSEventStreamCreate(
259 null,
260 &eventCallback,
261 &.{
262 .version = 0,
263 .info = @constCast(&callback_ctx),
264 .retain = null,
265 .release = null,
266 .copy_description = null,
267 },
268 cf_paths_array,
269 fse.since_event,
270 0.05, // 0.05s latency; higher values increase efficiency by coalescing more events
271 .{ .watch_root = true, .file_events = true },
272 );
273 defer rs.FSEventStreamRelease(event_stream);
274 rs.FSEventStreamSetDispatchQueue(event_stream, fse.dispatch_queue);
275 defer rs.FSEventStreamInvalidate(event_stream);
276 if (!rs.FSEventStreamStart(event_stream)) return error.StartFailed;
277 defer rs.FSEventStreamStop(event_stream);
278 const result = fse.waiting_semaphore.wait(timeout: {
279 const ns = timeout_ns orelse break :timeout .FOREVER;
280 break :timeout .time(.NOW, @intCast(ns));
281 });
282 return switch (result) {
283 0 => .dirty,
284 else => .timeout,
285 };
286}
287
288const cf_alloc_callbacks = struct {
289 const log = std.log.scoped(.cf_alloc);
290 fn allocate(size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque {
291 if (enable_debug_logs) log.debug("allocate {d}", .{size});
292 _ = hint;
293 const gpa: *const Allocator = @ptrCast(@alignCast(info));
294 const mem = gpa.alignedAlloc(u8, .of(usize), @intCast(size + @sizeOf(usize))) catch return null;
295 const metadata: *usize = @ptrCast(mem);
296 metadata.* = @intCast(size);
297 return mem[@sizeOf(usize)..].ptr;
298 }
299 fn reallocate(ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque {
300 if (enable_debug_logs) log.debug("reallocate @{*} {d}", .{ ptr, new_size });
301 _ = hint;
302 if (ptr == null or new_size == 0) return null; // not a bug: documentation explicitly states that realloc on NULL should return NULL
303 const gpa: *const Allocator = @ptrCast(@alignCast(info));
304 const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize));
305 const old_size = @as(*const usize, @ptrCast(old_base)).*;
306 const old_mem = old_base[0 .. old_size + @sizeOf(usize)];
307 const new_mem = gpa.realloc(old_mem, @intCast(new_size + @sizeOf(usize))) catch return null;
308 const metadata: *usize = @ptrCast(new_mem);
309 metadata.* = @intCast(new_size);
310 return new_mem[@sizeOf(usize)..].ptr;
311 }
312 fn deallocate(ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void {
313 if (enable_debug_logs) log.debug("deallocate @{*}", .{ptr});
314 const gpa: *const Allocator = @ptrCast(@alignCast(info));
315 const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize));
316 const old_size = @as(*const usize, @ptrCast(old_base)).*;
317 const old_mem = old_base[0 .. old_size + @sizeOf(usize)];
318 gpa.free(old_mem);
319 }
320};
321
322const EventCallbackCtx = struct {
323 fse: *FsEvents,
324 gpa: Allocator,
325};
326
327fn eventCallback(
328 stream: ConstFSEventStreamRef,
329 client_callback_info: ?*anyopaque,
330 num_events: usize,
331 events_paths_ptr: *anyopaque,
332 events_flags_ptr: [*]const FSEventStreamEventFlags,
333 events_ids_ptr: [*]const FSEventStreamEventId,
334) callconv(.c) void {
335 const ctx: *const EventCallbackCtx = @ptrCast(@alignCast(client_callback_info));
336 const fse = ctx.fse;
337 const gpa = ctx.gpa;
338 const rs = fse.resolved_symbols;
339 const events_paths_ptr_casted: [*]const [*:0]const u8 = @ptrCast(@alignCast(events_paths_ptr));
340 const events_paths = events_paths_ptr_casted[0..num_events];
341 const events_ids = events_ids_ptr[0..num_events];
342 const events_flags = events_flags_ptr[0..num_events];
343 var any_dirty = false;
344 for (events_paths, events_ids, events_flags) |event_path_nts, event_id, event_flags| {
345 _ = event_id;
346 if (event_flags.history_done) continue; // sentinel
347 const event_path = std.mem.span(event_path_nts);
348 switch (event_flags.must_scan_sub_dirs) {
349 false => {
350 if (fse.watch_paths.get(event_path)) |steps| {
351 assert(steps.len > 0);
352 for (steps) |s| {
353 if (s.invalidateResult(gpa)) any_dirty = true;
354 }
355 }
356 if (std.fs.path.dirname(event_path)) |event_dirname| {
357 // Modifying '/foo/bar' triggers the watch on '/foo'.
358 if (fse.watch_paths.get(event_dirname)) |steps| {
359 assert(steps.len > 0);
360 for (steps) |s| {
361 if (s.invalidateResult(gpa)) any_dirty = true;
362 }
363 }
364 }
365 },
366 true => {
367 // This is unlikely, but can occasionally happen when bottlenecked: events have been
368 // coalesced into one. We want to see if any of these events are actually relevant
369 // to us. The only way we can reasonably do that in this rare edge case is iterate
370 // the watch paths and see if any is under this directory. That's acceptable because
371 // we would otherwise kick off a rebuild which would be clearing those paths anyway.
372 const changed_path = std.fs.path.dirname(event_path) orelse event_path;
373 for (fse.watch_paths.keys(), fse.watch_paths.values()) |watching_path, steps| {
374 if (dirStartsWith(watching_path, changed_path)) {
375 for (steps) |s| {
376 if (s.invalidateResult(gpa)) any_dirty = true;
377 }
378 }
379 }
380 },
381 }
382 }
383 if (any_dirty) {
384 fse.since_event = rs.FSEventStreamGetLatestEventId(stream);
385 _ = fse.waiting_semaphore.signal();
386 }
387}
388fn dirStartsWith(path: []const u8, prefix: []const u8) bool {
389 if (std.mem.eql(u8, path, prefix)) return true;
390 if (!std.mem.startsWith(u8, path, prefix)) return false;
391 if (path[prefix.len] != '/') return false; // `path` is `/foo/barx`, `prefix` is `/foo/bar`
392 return true; // `path` is `/foo/bar/...`, `prefix` is `/foo/bar`
393}
394
395const CFAllocatorRef = ?*const opaque {};
396const CFArrayRef = *const opaque {};
397const CFStringRef = *const opaque {};
398const CFTimeInterval = f64;
399const CFIndex = i32;
400const CFOptionFlags = enum(u32) { _ };
401const CFAllocatorRetainCallBack = *const fn (info: ?*const anyopaque) callconv(.c) *const anyopaque;
402const CFAllocatorReleaseCallBack = *const fn (info: ?*const anyopaque) callconv(.c) void;
403const CFAllocatorCopyDescriptionCallBack = *const fn (info: ?*const anyopaque) callconv(.c) CFStringRef;
404const CFAllocatorAllocateCallBack = *const fn (alloc_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque;
405const CFAllocatorReallocateCallBack = *const fn (ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque;
406const CFAllocatorDeallocateCallBack = *const fn (ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void;
407const CFAllocatorPreferredSizeCallBack = *const fn (size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) CFIndex;
408const CFAllocatorContext = extern struct {
409 version: CFIndex,
410 info: ?*anyopaque,
411 retain: ?CFAllocatorRetainCallBack,
412 release: ?CFAllocatorReleaseCallBack,
413 copy_description: ?CFAllocatorCopyDescriptionCallBack,
414 allocate: CFAllocatorAllocateCallBack,
415 reallocate: ?CFAllocatorReallocateCallBack,
416 deallocate: ?CFAllocatorDeallocateCallBack,
417 preferred_size: ?CFAllocatorPreferredSizeCallBack,
418};
419const CFArrayCallBacks = opaque {};
420const CFStringEncoding = enum(u32) {
421 invalid_id = std.math.maxInt(u32),
422 mac_roman = 0,
423 windows_latin_1 = 0x500,
424 iso_latin_1 = 0x201,
425 next_step_latin = 0xB01,
426 ascii = 0x600,
427 unicode = 0x100,
428 utf8 = 0x8000100,
429 non_lossy_ascii = 0xBFF,
430};
431
432const FSEventStreamRef = *opaque {};
433const ConstFSEventStreamRef = *const @typeInfo(FSEventStreamRef).pointer.child;
434const FSEventStreamCallback = *const fn (
435 stream: ConstFSEventStreamRef,
436 client_callback_info: ?*anyopaque,
437 num_events: usize,
438 event_paths: *anyopaque,
439 event_flags: [*]const FSEventStreamEventFlags,
440 event_ids: [*]const FSEventStreamEventId,
441) callconv(.c) void;
442const FSEventStreamContext = extern struct {
443 version: CFIndex,
444 info: ?*anyopaque,
445 retain: ?CFAllocatorRetainCallBack,
446 release: ?CFAllocatorReleaseCallBack,
447 copy_description: ?CFAllocatorCopyDescriptionCallBack,
448};
449const FSEventStreamEventId = enum(u64) {
450 since_now = std.math.maxInt(u64),
451 _,
452};
453const FSEventStreamCreateFlags = packed struct(u32) {
454 use_cf_types: bool = false,
455 no_defer: bool = false,
456 watch_root: bool = false,
457 ignore_self: bool = false,
458 file_events: bool = false,
459 _: u27 = 0,
460};
461const FSEventStreamEventFlags = packed struct(u32) {
462 must_scan_sub_dirs: bool,
463 user_dropped: bool,
464 kernel_dropped: bool,
465 event_ids_wrapped: bool,
466 history_done: bool,
467 root_changed: bool,
468 mount: bool,
469 unmount: bool,
470 _: u24 = 0,
471};
472
473const dispatch = std.c.dispatch;
474const std = @import("std");
475const Io = std.Io;
476const assert = std.debug.assert;
477const Allocator = std.mem.Allocator;
478const watch_log = std.log.scoped(.watch);
479const FsEvents = @This();
lib/compiler/maker/WebServer.zig deleted-940
...@@ -1,940 +0,0 @@
1const WebServer = @This();
2
3const builtin = @import("builtin");
4
5const std = @import("std");
6const Allocator = std.mem.Allocator;
7const Cache = std.Build.Cache;
8const Configuration = std.Build.Configuration;
9const Io = std.Io;
10const abi = std.Build.abi;
11const assert = std.debug.assert;
12const http = std.http;
13const log = std.log.scoped(.web_server);
14const mem = std.mem;
15const net = std.Io.net;
16
17const Fuzz = @import("Fuzz.zig");
18const Graph = @import("Graph.zig");
19const Step = @import("Step.zig");
20
21gpa: Allocator,
22graph: *const Graph,
23all_steps: []const Configuration.Step.Index,
24listen_address: net.IpAddress,
25root_prog_node: std.Progress.Node,
26watch: bool,
27
28tcp_server: ?net.Server,
29serve_task: ?Io.Future(Io.Cancelable!void),
30
31/// Uses `Io.Clock.awake`.
32base_timestamp: Io.Timestamp,
33/// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`.
34step_names_trailing: []u8,
35
36/// The bit-packed "step status" data. Values are `abi.StepUpdate.Status`. LSBs are earlier steps.
37/// Accessed atomically.
38step_status_bits: []u8,
39
40fuzz: ?Fuzz,
41time_report_mutex: Io.Mutex,
42time_report_msgs: [][]u8,
43time_report_update_times: []i64,
44
45build_status: std.atomic.Value(abi.BuildStatus),
46/// When an event occurs which means WebSocket clients should be sent updates, call `notifyUpdate`
47/// to increment this value. Each client thread waits for this increment with `Io.futexWaitTimeout`, so
48/// `notifyUpdate` will wake those threads. Updates are sent on a short interval regardless, so it
49/// is recommended to only use `notifyUpdate` for changes which the user should see immediately. For
50/// instance, we do not call `notifyUpdate` when the number of "unique runs" in the fuzzer changes,
51/// because this value changes quickly so this would result in constantly spamming all clients with
52/// an unreasonable number of packets.
53update_id: std.atomic.Value(u32),
54
55runner_request_mutex: Io.Mutex,
56runner_request_ready_cond: Io.Condition,
57runner_request_empty_cond: Io.Condition,
58runner_request: ?RunnerRequest,
59
60/// If a client is not explicitly notified of changes with `notifyUpdate`, it will be sent updates
61/// on a fixed interval of this many milliseconds.
62const default_update_interval_ms = 500;
63
64pub const base_clock: Io.Clock = .awake;
65
66/// Thread-safe. Triggers updates to be sent to connected WebSocket clients; see `update_id`.
67pub fn notifyUpdate(ws: *WebServer) void {
68 _ = ws.update_id.rmw(.Add, 1, .release);
69 ws.graph.io.futexWake(u32, &ws.update_id.raw, 16);
70}
71
72pub const Options = struct {
73 gpa: Allocator,
74 graph: *const Graph,
75 all_steps: []const Configuration.Step.Index,
76 root_prog_node: std.Progress.Node,
77 watch: bool,
78 listen_address: net.IpAddress,
79 base_timestamp: Io.Clock.Timestamp,
80 configuration: *const Configuration,
81};
82pub fn init(opts: Options) WebServer {
83 // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent`
84 // instead of threads, so that the web server can function in single-threaded builds.
85 comptime assert(!builtin.single_threaded);
86 assert(opts.base_timestamp.clock == base_clock);
87
88 const all_steps = opts.all_steps;
89 const c = opts.configuration;
90
91 const step_names_trailing = opts.gpa.alloc(u8, len: {
92 var name_bytes: usize = 0;
93 for (all_steps) |step_index| name_bytes += step_index.ptr(c).name.slice(c).len;
94 break :len name_bytes + all_steps.len * 4;
95 }) catch @panic("out of memory");
96 {
97 const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]);
98 var idx: usize = all_steps.len * 4;
99 for (all_steps, step_name_lens) |step_index, *name_len| {
100 const step_name = step_index.ptr(c).name.slice(c);
101 name_len.* = @intCast(step_name.len);
102 @memcpy(step_names_trailing[idx..][0..step_name.len], step_name);
103 idx += step_name.len;
104 }
105 assert(idx == step_names_trailing.len);
106 }
107
108 const step_status_bits = opts.gpa.alloc(
109 u8,
110 std.math.divCeil(usize, all_steps.len, 4) catch unreachable,
111 ) catch @panic("out of memory");
112 @memset(step_status_bits, 0);
113
114 const time_reports_len: usize = if (opts.graph.time_report) all_steps.len else 0;
115 const time_report_msgs = opts.gpa.alloc([]u8, time_reports_len) catch @panic("out of memory");
116 const time_report_update_times = opts.gpa.alloc(i64, time_reports_len) catch @panic("out of memory");
117 @memset(time_report_msgs, &.{});
118 @memset(time_report_update_times, std.math.minInt(i64));
119
120 return .{
121 .gpa = opts.gpa,
122 .graph = opts.graph,
123 .all_steps = all_steps,
124 .listen_address = opts.listen_address,
125 .root_prog_node = opts.root_prog_node,
126 .watch = opts.watch,
127
128 .tcp_server = null,
129 .serve_task = null,
130
131 .base_timestamp = opts.base_timestamp.raw,
132 .step_names_trailing = step_names_trailing,
133
134 .step_status_bits = step_status_bits,
135
136 .fuzz = null,
137 .time_report_mutex = .init,
138 .time_report_msgs = time_report_msgs,
139 .time_report_update_times = time_report_update_times,
140
141 .build_status = .init(.idle),
142 .update_id = .init(0),
143
144 .runner_request_mutex = .init,
145 .runner_request_ready_cond = .init,
146 .runner_request_empty_cond = .init,
147 .runner_request = null,
148 };
149}
150pub fn deinit(ws: *WebServer) void {
151 const gpa = ws.gpa;
152 const io = ws.graph.io;
153
154 gpa.free(ws.step_names_trailing);
155 gpa.free(ws.step_status_bits);
156
157 if (ws.fuzz) |*f| f.deinit();
158 for (ws.time_report_msgs) |msg| gpa.free(msg);
159 gpa.free(ws.time_report_msgs);
160 gpa.free(ws.time_report_update_times);
161
162 if (ws.serve_task) |t| {
163 if (ws.tcp_server) |*s| s.stream.close(io);
164 t.await();
165 }
166 if (ws.tcp_server) |*s| s.deinit();
167
168 gpa.free(ws.step_names_trailing);
169}
170pub fn start(ws: *WebServer) error{AlreadyReported}!void {
171 assert(ws.tcp_server == null);
172 assert(ws.serve_task == null);
173 const io = ws.graph.io;
174
175 ws.tcp_server = ws.listen_address.listen(io, .{ .reuse_address = true }) catch |err| {
176 log.err("failed to listen to port {d}: {t}", .{ ws.listen_address.getPort(), err });
177 return error.AlreadyReported;
178 };
179 ws.serve_task = io.concurrent(serve, .{ws}) catch |err| {
180 log.err("unable to spawn web server thread: {t}", .{err});
181 ws.tcp_server.?.deinit(io);
182 ws.tcp_server = null;
183 return error.AlreadyReported;
184 };
185
186 log.info("web interface listening at http://{f}/", .{ws.tcp_server.?.socket.address});
187 if (ws.listen_address.getPort() == 0) {
188 log.info("hint: pass '--webui={f}' to use the same port next time", .{ws.tcp_server.?.socket.address});
189 }
190}
191fn serve(ws: *WebServer) Io.Cancelable!void {
192 const io = ws.graph.io;
193 var group: Io.Group = .init;
194 defer group.cancel(io);
195 while (true) {
196 var stream = ws.tcp_server.?.accept(io) catch |err| switch (err) {
197 error.Canceled => |e| return e,
198 else => |e| {
199 log.err("failed to accept connection: {t}", .{e});
200 return;
201 },
202 };
203 group.concurrent(io, accept, .{ ws, stream }) catch |err| {
204 log.err("unable to spawn connection thread: {t}", .{err});
205 stream.close(io);
206 continue;
207 };
208 }
209}
210
211pub fn startBuild(ws: *WebServer) void {
212 if (ws.fuzz) |*fuzz| {
213 fuzz.deinit();
214 ws.fuzz = null;
215 }
216 for (ws.step_status_bits) |*bits| @atomicStore(u8, bits, 0, .monotonic);
217 ws.build_status.store(.running, .monotonic);
218 ws.notifyUpdate();
219}
220
221pub fn updateStepStatus(
222 ws: *WebServer,
223 step_index: Configuration.Step.Index,
224 new_status: abi.StepUpdate.Status,
225) void {
226 // TODO don't do linear search, especially in a hot loop like this
227 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
228 if (s == step_index) break @intCast(i);
229 } else unreachable;
230 const ptr = &ws.step_status_bits[step_idx / 4];
231 const bit_offset: u3 = @intCast((step_idx % 4) * 2);
232 const old_bits: u2 = @truncate(@atomicLoad(u8, ptr, .monotonic) >> bit_offset);
233 const mask = @as(u8, @intFromEnum(new_status) ^ old_bits) << bit_offset;
234 _ = @atomicRmw(u8, ptr, .Xor, mask, .monotonic);
235 ws.notifyUpdate();
236}
237
238pub fn finishBuild(ws: *WebServer, opts: struct {
239 fuzz: bool,
240}) void {
241 if (opts.fuzz) {
242 switch (builtin.os.tag) {
243 // Current implementation depends on two things that need to be ported to Windows:
244 // * Memory-mapping to share data between the fuzzer and build runner.
245 // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
246 // many addresses to source locations).
247 .windows => std.process.fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}),
248 else => {},
249 }
250 if (@bitSizeOf(usize) != 64) {
251 // Current implementation depends on posix.mmap()'s second
252 // parameter, `length: usize`, being compatible with file system's
253 // u64 return value. This is not the case on 32-bit platforms.
254 // Affects or affected by issues #5185, #22523, and #22464.
255 std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
256 }
257
258 assert(ws.fuzz == null);
259
260 ws.build_status.store(.fuzz_init, .monotonic);
261 ws.notifyUpdate();
262
263 ws.fuzz = Fuzz.init(
264 ws.gpa,
265 ws.graph.io,
266 ws.all_steps,
267 ws.root_prog_node,
268 .{ .forever = .{ .ws = ws } },
269 ) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)});
270 ws.fuzz.?.start();
271 }
272
273 ws.build_status.store(if (ws.watch) .watching else .idle, .monotonic);
274 ws.notifyUpdate();
275}
276
277pub fn now(s: *const WebServer) i64 {
278 const io = s.graph.io;
279 const ts = base_clock.now(io);
280 return @intCast(s.base_timestamp.durationTo(ts).toNanoseconds());
281}
282
283fn accept(ws: *WebServer, stream: net.Stream) void {
284 const io = ws.graph.io;
285 defer {
286 // `net.Stream.close` wants to helpfully overwrite `stream` with
287 // `undefined`, but it cannot do so since it is an immutable parameter.
288 var copy = stream;
289 copy.close(io);
290 }
291 var send_buffer: [4096]u8 = undefined;
292 var recv_buffer: [4096]u8 = undefined;
293 var connection_reader = stream.reader(io, &recv_buffer);
294 var connection_writer = stream.writer(io, &send_buffer);
295 var server: http.Server = .init(&connection_reader.interface, &connection_writer.interface);
296
297 while (true) {
298 var request = server.receiveHead() catch |err| switch (err) {
299 error.HttpConnectionClosing => return,
300 else => return log.err("failed to receive http request: {t}", .{err}),
301 };
302 switch (request.upgradeRequested()) {
303 .websocket => |opt_key| {
304 const key = opt_key orelse return log.err("missing websocket key", .{});
305 var web_socket = request.respondWebSocket(.{ .key = key }) catch {
306 return log.err("failed to respond web socket: {t}", .{connection_writer.err.?});
307 };
308 ws.serveWebSocket(&web_socket) catch |err| {
309 log.err("failed to serve websocket: {t}", .{err});
310 return;
311 };
312 comptime unreachable;
313 },
314 .other => |name| return log.err("unknown upgrade request: {s}", .{name}),
315 .none => {
316 ws.serveRequest(&request) catch |err| switch (err) {
317 error.AlreadyReported => return,
318 else => {
319 log.err("failed to serve '{s}': {t}", .{ request.head.target, err });
320 return;
321 },
322 };
323 },
324 }
325 }
326}
327
328fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn {
329 const io = ws.graph.io;
330
331 var prev_build_status = ws.build_status.load(.monotonic);
332
333 const prev_step_status_bits = try ws.gpa.alloc(u8, ws.step_status_bits.len);
334 defer ws.gpa.free(prev_step_status_bits);
335 for (prev_step_status_bits, ws.step_status_bits) |*copy, *shared| {
336 copy.* = @atomicLoad(u8, shared, .monotonic);
337 }
338
339 var recv_thread = try io.concurrent(recvWebSocketMessages, .{ ws, sock });
340 defer recv_thread.cancel(io);
341
342 {
343 const hello_header: abi.Hello = .{
344 .status = prev_build_status,
345 .flags = .{
346 .time_report = ws.graph.time_report,
347 },
348 .timestamp = ws.now(),
349 .steps_len = @intCast(ws.all_steps.len),
350 };
351 var bufs: [3][]const u8 = .{ @ptrCast(&hello_header), ws.step_names_trailing, prev_step_status_bits };
352 try sock.writeMessageVec(&bufs, .binary);
353 }
354
355 var prev_fuzz: Fuzz.Previous = .init;
356 var prev_time: i64 = std.math.minInt(i64);
357 while (true) {
358 const start_time = ws.now();
359 const start_update_id = ws.update_id.load(.acquire);
360
361 if (ws.fuzz) |*fuzz| {
362 try fuzz.sendUpdate(sock, &prev_fuzz);
363 }
364
365 {
366 try ws.time_report_mutex.lock(io);
367 defer ws.time_report_mutex.unlock(io);
368 for (ws.time_report_msgs, ws.time_report_update_times) |msg, update_time| {
369 if (update_time <= prev_time) continue;
370 // We want to send `msg`, but shouldn't block `ws.time_report_mutex` while we do, so
371 // that we don't hold up the build system on the client accepting this packet.
372 const owned_msg = try ws.gpa.dupe(u8, msg);
373 defer ws.gpa.free(owned_msg);
374 // Temporarily unlock, then re-lock after the message is sent.
375 ws.time_report_mutex.unlock(io);
376 defer ws.time_report_mutex.lockUncancelable(io);
377 try sock.writeMessage(owned_msg, .binary);
378 }
379 }
380
381 {
382 const build_status = ws.build_status.load(.monotonic);
383 if (build_status != prev_build_status) {
384 prev_build_status = build_status;
385 const msg: abi.StatusUpdate = .{ .new = build_status };
386 try sock.writeMessage(@ptrCast(&msg), .binary);
387 }
388 }
389
390 for (prev_step_status_bits, ws.step_status_bits, 0..) |*prev_byte, *shared, byte_idx| {
391 const cur_byte = @atomicLoad(u8, shared, .monotonic);
392 if (prev_byte.* == cur_byte) continue;
393 const cur: [4]abi.StepUpdate.Status = .{
394 @enumFromInt(@as(u2, @truncate(cur_byte >> 0))),
395 @enumFromInt(@as(u2, @truncate(cur_byte >> 2))),
396 @enumFromInt(@as(u2, @truncate(cur_byte >> 4))),
397 @enumFromInt(@as(u2, @truncate(cur_byte >> 6))),
398 };
399 const prev: [4]abi.StepUpdate.Status = .{
400 @enumFromInt(@as(u2, @truncate(prev_byte.* >> 0))),
401 @enumFromInt(@as(u2, @truncate(prev_byte.* >> 2))),
402 @enumFromInt(@as(u2, @truncate(prev_byte.* >> 4))),
403 @enumFromInt(@as(u2, @truncate(prev_byte.* >> 6))),
404 };
405 for (cur, prev, byte_idx * 4..) |cur_status, prev_status, step_idx| {
406 const msg: abi.StepUpdate = .{ .step_idx = @intCast(step_idx), .bits = .{ .status = cur_status } };
407 if (cur_status != prev_status) try sock.writeMessage(@ptrCast(&msg), .binary);
408 }
409 prev_byte.* = cur_byte;
410 }
411
412 prev_time = start_time;
413
414 const old_cp = io.swapCancelProtection(.blocked);
415 defer _ = io.swapCancelProtection(old_cp);
416 io.futexWaitTimeout(
417 u32,
418 &ws.update_id.raw,
419 start_update_id,
420 .{ .duration = .{
421 .clock = .awake,
422 .raw = .fromMilliseconds(default_update_interval_ms),
423 } },
424 ) catch |err| switch (err) {
425 error.Canceled => unreachable,
426 };
427 }
428}
429fn recvWebSocketMessages(ws: *WebServer, sock: *http.Server.WebSocket) void {
430 const io = ws.graph.io;
431
432 while (true) {
433 const msg = sock.readSmallMessage() catch return;
434 if (msg.opcode != .binary) continue;
435 if (msg.data.len == 0) continue;
436 const tag: abi.ToServerTag = @enumFromInt(msg.data[0]);
437 switch (tag) {
438 _ => continue,
439 .rebuild => while (true) {
440 ws.runner_request_mutex.lock(io) catch |err| switch (err) {
441 error.Canceled => return,
442 };
443 defer ws.runner_request_mutex.unlock(io);
444 if (ws.runner_request == null) {
445 ws.runner_request = .rebuild;
446 ws.runner_request_ready_cond.signal(io);
447 break;
448 }
449 ws.runner_request_empty_cond.wait(io, &ws.runner_request_mutex) catch return;
450 },
451 }
452 }
453}
454
455fn serveRequest(ws: *WebServer, req: *http.Server.Request) !void {
456 // Strip an optional leading '/debug' component from the request.
457 const target: []const u8, const debug: bool = target: {
458 if (mem.eql(u8, req.head.target, "/debug")) break :target .{ "/", true };
459 if (mem.eql(u8, req.head.target, "/debug/")) break :target .{ "/", true };
460 if (mem.startsWith(u8, req.head.target, "/debug/")) break :target .{ req.head.target["/debug".len..], true };
461 break :target .{ req.head.target, false };
462 };
463
464 if (mem.eql(u8, target, "/")) return serveLibFile(ws, req, "build-web/index.html", "text/html");
465 if (mem.eql(u8, target, "/main.js")) return serveLibFile(ws, req, "build-web/main.js", "application/javascript");
466 if (mem.eql(u8, target, "/style.css")) return serveLibFile(ws, req, "build-web/style.css", "text/css");
467 if (mem.eql(u8, target, "/time_report.css")) return serveLibFile(ws, req, "build-web/time_report.css", "text/css");
468 if (mem.eql(u8, target, "/main.wasm")) return serveClientWasm(ws, req, if (debug) .Debug else .ReleaseFast);
469
470 if (ws.fuzz) |*fuzz| {
471 if (mem.eql(u8, target, "/sources.tar")) return fuzz.serveSourcesTar(req);
472 }
473
474 try req.respond("not found", .{
475 .status = .not_found,
476 .extra_headers = &.{
477 .{ .name = "Content-Type", .value = "text/plain" },
478 },
479 });
480}
481
482fn serveLibFile(
483 ws: *WebServer,
484 request: *http.Server.Request,
485 sub_path: []const u8,
486 content_type: []const u8,
487) !void {
488 return serveFile(ws, request, .{
489 .root_dir = ws.graph.zig_lib_directory,
490 .sub_path = sub_path,
491 }, content_type);
492}
493fn serveClientWasm(
494 ws: *WebServer,
495 req: *http.Server.Request,
496 optimize_mode: std.builtin.OptimizeMode,
497) !void {
498 var arena_state: std.heap.ArenaAllocator = .init(ws.gpa);
499 defer arena_state.deinit();
500 const arena = arena_state.allocator();
501
502 // We always rebuild the wasm on-the-fly, so that if it is edited the user can just refresh the page.
503 const bin_path = try buildClientWasm(ws, arena, optimize_mode);
504 return serveFile(ws, req, bin_path, "application/wasm");
505}
506
507pub fn serveFile(
508 ws: *WebServer,
509 request: *http.Server.Request,
510 path: Cache.Path,
511 content_type: []const u8,
512) !void {
513 const gpa = ws.gpa;
514 const io = ws.graph.io;
515 // The desired API is actually sendfile, which will require enhancing http.Server.
516 // We load the file with every request so that the user can make changes to the file
517 // and refresh the HTML page without restarting this server.
518 const file_contents = path.root_dir.handle.readFileAlloc(io, path.sub_path, gpa, .limited(10 * 1024 * 1024)) catch |err| {
519 log.err("failed to read '{f}': {t}", .{ path, err });
520 return error.AlreadyReported;
521 };
522 defer gpa.free(file_contents);
523 try request.respond(file_contents, .{
524 .extra_headers = &.{
525 .{ .name = "Content-Type", .value = content_type },
526 cache_control_header,
527 },
528 });
529}
530pub fn serveTarFile(ws: *WebServer, request: *http.Server.Request, paths: []const Cache.Path) !void {
531 const graph = ws.graph;
532 const io = graph.io;
533
534 var send_buffer: [0x4000]u8 = undefined;
535 var response = try request.respondStreaming(&send_buffer, .{
536 .respond_options = .{
537 .extra_headers = &.{
538 .{ .name = "Content-Type", .value = "application/x-tar" },
539 cache_control_header,
540 },
541 },
542 });
543
544 var archiver: std.tar.Writer = .{ .underlying_writer = &response.writer };
545
546 for (paths) |path| {
547 var file = path.root_dir.handle.openFile(io, path.sub_path, .{}) catch |err| {
548 log.err("failed to open '{f}': {s}", .{ path, @errorName(err) });
549 continue;
550 };
551 defer file.close(io);
552 const stat = try file.stat(io);
553 var read_buffer: [1024]u8 = undefined;
554 var file_reader: Io.File.Reader = .initSize(file, io, &read_buffer, stat.size);
555
556 // TODO: this logic is completely bogus -- obviously so, because `path.root_dir.path` can
557 // be cwd-relative. This is also related to why linkification doesn't work in the fuzzer UI:
558 // it turns out the WASM treats the first path component as the module name, typically
559 // resulting in modules named "" and "src". The compiler needs to tell the build system
560 // about the module graph so that the build system can correctly encode this information in
561 // the tar file.
562 //
563 // Additionally, this needs to ensure that all path separators for both prefix and
564 // sub_path are using the POSIX-style `/` on platforms that don't use it as their native
565 // path separator.
566 archiver.prefix = path.root_dir.path orelse graph.cache.cwd;
567 try archiver.writeFile(path.sub_path, &file_reader, @intCast(stat.mtime.toSeconds()));
568 }
569
570 // intentionally not calling `archiver.finishPedantically`
571 try response.end();
572}
573
574fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.OptimizeMode) !Cache.Path {
575 const root_name = "build-web";
576 const arch_os_abi = "wasm32-freestanding";
577 const cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext";
578
579 const gpa = ws.gpa;
580 const graph = ws.graph;
581 const io = graph.io;
582
583 const main_src_path: Cache.Path = .{
584 .root_dir = graph.zig_lib_directory,
585 .sub_path = "build-web/main.zig",
586 };
587 const walk_src_path: Cache.Path = .{
588 .root_dir = graph.zig_lib_directory,
589 .sub_path = "docs/wasm/Walk.zig",
590 };
591 const html_render_src_path: Cache.Path = .{
592 .root_dir = graph.zig_lib_directory,
593 .sub_path = "docs/wasm/html_render.zig",
594 };
595
596 var argv: std.ArrayList([]const u8) = .empty;
597
598 try argv.appendSlice(arena, &.{
599 graph.zig_exe, "build-exe", //
600 "-fno-entry", //
601 "-O", @tagName(optimize), //
602 "-target", arch_os_abi, //
603 "-mcpu", cpu_features, //
604 "--cache-dir", graph.global_cache_root.path orelse ".", //
605 "--global-cache-dir", graph.global_cache_root.path orelse ".", //
606 "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", //
607 "--name", root_name, //
608 "-rdynamic", //
609 "-fsingle-threaded", //
610 "--dep", "Walk", //
611 "--dep", "html_render", //
612 try std.fmt.allocPrint(arena, "-Mroot={f}", .{main_src_path}), //
613 try std.fmt.allocPrint(arena, "-MWalk={f}", .{walk_src_path}), //
614 "--dep", "Walk", //
615 try std.fmt.allocPrint(arena, "-Mhtml_render={f}", .{html_render_src_path}), //
616 "--listen=-",
617 });
618
619 var child = try std.process.spawn(io, .{
620 .argv = argv.items,
621 .environ_map = &graph.environ_map,
622 .stdin = .pipe,
623 .stdout = .pipe,
624 .stderr = .pipe,
625 });
626 defer child.kill(io);
627
628 var stderr_task = try io.concurrent(readStreamAlloc, .{ gpa, io, child.stderr.?, .unlimited });
629 defer if (stderr_task.cancel(io)) |slice| gpa.free(slice) else |_| {};
630
631 var stdout_buffer: [512]u8 = undefined;
632 var stdout_reader: Io.File.Reader = .initStreaming(child.stdout.?, io, &stdout_buffer);
633 const stdout = &stdout_reader.interface;
634
635 {
636 var w = child.stdin.?.writer(io, &.{});
637 w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .update, .bytes_len = 0 }, .little) catch |err| switch (err) {
638 error.WriteFailed => return w.err.?,
639 };
640 w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .exit, .bytes_len = 0 }, .little) catch |err| switch (err) {
641 error.WriteFailed => return w.err.?,
642 };
643 }
644
645 const Header = std.zig.Server.Message.Header;
646
647 var result: ?Cache.Path = null;
648 var result_error_bundle = std.zig.ErrorBundle.empty;
649 var body_buffer: std.ArrayList(u8) = .empty;
650 defer body_buffer.deinit(gpa);
651
652 while (true) {
653 const header = stdout.takeStruct(Header, .little) catch |err| switch (err) {
654 error.ReadFailed => |e| return e,
655 error.EndOfStream => break,
656 };
657 body_buffer.clearRetainingCapacity();
658 try stdout.appendExact(gpa, &body_buffer, header.bytes_len);
659 const body = body_buffer.items;
660
661 switch (header.tag) {
662 .zig_version => {
663 if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
664 return error.ZigProtocolVersionMismatch;
665 }
666 },
667 .error_bundle => {
668 result_error_bundle = try std.zig.Server.allocErrorBundle(arena, body);
669 },
670 .emit_digest => {
671 const EmitDigest = std.zig.Server.Message.EmitDigest;
672 const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body);
673 if (!ebp_hdr.flags.cache_hit) {
674 log.info("source changes detected; rebuilt wasm component", .{});
675 }
676 const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len];
677 result = .{
678 .root_dir = graph.global_cache_root,
679 .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)),
680 };
681 },
682 else => {}, // ignore other messages
683 }
684 }
685
686 const stderr_contents = try stderr_task.await(io);
687 if (stderr_contents.len > 0) {
688 std.debug.print("{s}", .{stderr_contents});
689 }
690
691 // Send EOF to stdin.
692 child.stdin.?.close(io);
693 child.stdin = null;
694
695 switch (try child.wait(io)) {
696 .exited => |code| {
697 if (code != 0) {
698 log.err(
699 "the following command exited with error code {d}:\n{s}",
700 .{ code, try Step.allocPrintCmd(arena, .inherit, null, argv.items) },
701 );
702 return error.WasmCompilationFailed;
703 }
704 },
705 .signal => |sig| {
706 log.err(
707 "the following command terminated with signal {t}:\n{s}",
708 .{ sig, try Step.allocPrintCmd(arena, .inherit, null, argv.items) },
709 );
710 return error.WasmCompilationFailed;
711 },
712 .stopped => |sig| {
713 log.err(
714 "the following command stopped unexpectedly with signal {t}:\n{s}",
715 .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) },
716 );
717 return error.WasmCompilationFailed;
718 },
719 .unknown => {
720 log.err(
721 "the following command terminated unexpectedly:\n{s}",
722 .{try Step.allocPrintCmd(arena, .inherit, null, argv.items)},
723 );
724 return error.WasmCompilationFailed;
725 },
726 }
727
728 if (result_error_bundle.errorMessageCount() > 0) {
729 try result_error_bundle.renderToStderr(io, .{}, .auto);
730 log.err("the following command failed with {d} compilation errors:\n{s}", .{
731 result_error_bundle.errorMessageCount(),
732 try Step.allocPrintCmd(arena, .inherit, null, argv.items),
733 });
734 return error.WasmCompilationFailed;
735 }
736
737 const base_path = result orelse {
738 log.err("child process failed to report result\n{s}", .{
739 try Step.allocPrintCmd(arena, .inherit, null, argv.items),
740 });
741 return error.WasmCompilationFailed;
742 };
743 const bin_name = try std.zig.binNameAlloc(arena, .{
744 .root_name = root_name,
745 .target = &(std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{
746 .arch_os_abi = arch_os_abi,
747 .cpu_features = cpu_features,
748 }) catch unreachable) catch unreachable),
749 .output_mode = .Exe,
750 });
751 return base_path.join(arena, bin_name);
752}
753
754fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 {
755 var file_reader: Io.File.Reader = .initStreaming(file, io, &.{});
756 return file_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) {
757 error.ReadFailed => return file_reader.err.?,
758 else => |e| return e,
759 };
760}
761
762pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
763 compile_step: Configuration.Step.Index,
764
765 use_llvm: bool,
766 stats: abi.time_report.CompileResult.Stats,
767 ns_total: u64,
768
769 llvm_pass_timings_len: u32,
770 files_len: u32,
771 decls_len: u32,
772
773 /// The trailing data of `abi.time_report.CompileResult`, except the step name.
774 trailing: []const u8,
775}) void {
776 const gpa = ws.gpa;
777 const io = ws.graph.io;
778
779 // TODO don't do linear search
780 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
781 if (s == opts.compile_step) break @intCast(i);
782 } else unreachable;
783
784 const old_buf = old: {
785 ws.time_report_mutex.lock(io) catch return;
786 defer ws.time_report_mutex.unlock(io);
787 const old = ws.time_report_msgs[step_idx];
788 ws.time_report_msgs[step_idx] = &.{};
789 break :old old;
790 };
791 const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.CompileResult) + opts.trailing.len) catch @panic("out of memory");
792
793 const out_header: *align(1) abi.time_report.CompileResult = @ptrCast(buf[0..@sizeOf(abi.time_report.CompileResult)]);
794 out_header.* = .{
795 .step_idx = step_idx,
796 .flags = .{
797 .use_llvm = opts.use_llvm,
798 },
799 .stats = opts.stats,
800 .ns_total = opts.ns_total,
801 .llvm_pass_timings_len = opts.llvm_pass_timings_len,
802 .files_len = opts.files_len,
803 .decls_len = opts.decls_len,
804 };
805 @memcpy(buf[@sizeOf(abi.time_report.CompileResult)..], opts.trailing);
806
807 {
808 ws.time_report_mutex.lock(io) catch return;
809 defer ws.time_report_mutex.unlock(io);
810 assert(ws.time_report_msgs[step_idx].len == 0);
811 ws.time_report_msgs[step_idx] = buf;
812 ws.time_report_update_times[step_idx] = ws.now();
813 }
814 ws.notifyUpdate();
815}
816
817pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.Index, duration: Io.Duration) void {
818 const gpa = ws.gpa;
819 const io = ws.graph.io;
820
821 // TODO don't do linear search
822 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
823 if (s == step_index) break @intCast(i);
824 } else unreachable;
825
826 const old_buf = old: {
827 ws.time_report_mutex.lock(io) catch return;
828 defer ws.time_report_mutex.unlock(io);
829 const old = ws.time_report_msgs[step_idx];
830 ws.time_report_msgs[step_idx] = &.{};
831 break :old old;
832 };
833 const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.GenericResult)) catch @panic("out of memory");
834 const out: *align(1) abi.time_report.GenericResult = @ptrCast(buf);
835 out.* = .{
836 .step_idx = step_idx,
837 .ns_total = @intCast(duration.toNanoseconds()),
838 };
839 {
840 ws.time_report_mutex.lock(io) catch return;
841 defer ws.time_report_mutex.unlock(io);
842 assert(ws.time_report_msgs[step_idx].len == 0);
843 ws.time_report_msgs[step_idx] = buf;
844 ws.time_report_update_times[step_idx] = ws.now();
845 }
846 ws.notifyUpdate();
847}
848
849pub fn updateTimeReportRunTest(
850 ws: *WebServer,
851 run_step_index: Configuration.Step.Index,
852 tests: *const Step.Run.CachedTestMetadata,
853 ns_per_test: []const u64,
854) void {
855 const gpa = ws.gpa;
856 const io = ws.graph.io;
857
858 // TODO don't do linear search
859 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
860 if (s == run_step_index) break @intCast(i);
861 } else unreachable;
862
863 assert(tests.names.len == ns_per_test.len);
864 const tests_len: u32 = @intCast(tests.names.len);
865
866 const new_len: u64 = len: {
867 var names_len: u64 = 0;
868 for (0..tests_len) |i| {
869 names_len += tests.testName(@intCast(i)).len + 1;
870 }
871 break :len @sizeOf(abi.time_report.RunTestResult) + names_len + 8 * tests_len;
872 };
873 const old_buf = old: {
874 ws.time_report_mutex.lock(io) catch return;
875 defer ws.time_report_mutex.unlock(io);
876 const old = ws.time_report_msgs[step_idx];
877 ws.time_report_msgs[step_idx] = &.{};
878 break :old old;
879 };
880 const buf = gpa.realloc(old_buf, new_len) catch @panic("out of memory");
881
882 const out_header: *align(1) abi.time_report.RunTestResult = @ptrCast(buf[0..@sizeOf(abi.time_report.RunTestResult)]);
883 out_header.* = .{
884 .step_idx = step_idx,
885 .tests_len = tests_len,
886 };
887 var offset: usize = @sizeOf(abi.time_report.RunTestResult);
888 const ns_per_test_out: []align(1) u64 = @ptrCast(buf[offset..][0 .. tests_len * 8]);
889 @memcpy(ns_per_test_out, ns_per_test);
890 offset += tests_len * 8;
891 for (0..tests_len) |i| {
892 const name = tests.testName(@intCast(i));
893 @memcpy(buf[offset..][0..name.len], name);
894 buf[offset..][name.len] = 0;
895 offset += name.len + 1;
896 }
897 assert(offset == buf.len);
898
899 {
900 ws.time_report_mutex.lock(io) catch return;
901 defer ws.time_report_mutex.unlock(io);
902 assert(ws.time_report_msgs[step_idx].len == 0);
903 ws.time_report_msgs[step_idx] = buf;
904 ws.time_report_update_times[step_idx] = ws.now();
905 }
906 ws.notifyUpdate();
907}
908
909const RunnerRequest = union(enum) {
910 rebuild,
911};
912pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest {
913 const io = ws.graph.io;
914 ws.runner_request_mutex.lock(io) catch return;
915 defer ws.runner_request_mutex.unlock(io);
916 if (ws.runner_request) |req| {
917 ws.runner_request = null;
918 ws.runner_request_empty_cond.signal();
919 return req;
920 }
921 return null;
922}
923pub fn wait(ws: *WebServer) Io.Cancelable!RunnerRequest {
924 const io = ws.graph.io;
925 try ws.runner_request_mutex.lock(io);
926 defer ws.runner_request_mutex.unlock(io);
927 while (true) {
928 if (ws.runner_request) |req| {
929 ws.runner_request = null;
930 ws.runner_request_empty_cond.signal(io);
931 return req;
932 }
933 try ws.runner_request_ready_cond.wait(io, &ws.runner_request_mutex);
934 }
935}
936
937const cache_control_header: http.Header = .{
938 .name = "Cache-Control",
939 .value = "max-age=0, must-revalidate",
940};
src/main.zig+1-1
...@@ -5799,7 +5799,7 @@ fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunn...@@ -5799,7 +5799,7 @@ fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunn
57995799
5800 const main_mod_paths: Package.Module.CreateOptions.Paths = .{5800 const main_mod_paths: Package.Module.CreateOptions.Paths = .{
5801 .root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"),5801 .root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"),
5802 .root_src_path = "maker.zig",5802 .root_src_path = "Maker.zig",
5803 };5803 };
58045804
5805 const config = try Compilation.Config.resolve(.{5805 const config = try Compilation.Config.resolve(.{