authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-23 20:49:00-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-25 18:52:20-07:00
log047640383e5e635ffe52ab360e03dbe08e73d025
tree47b7062d2f80b34655481bf91f06bef4d132cfe7
parent6f3767862d6886d5fde7e3734455a30f168ba80b

add `--fuzz` CLI argument to `zig build`

This flag makes the build runner rebuild unit tests after the pipeline finishes, if it finds any unit tests. I did not make this integrate with file system watching yet. The test runner is updated to detect which tests are fuzz tests. Run step is updated to track which test indexes are fuzz tests.

5 files changed, 97 insertions(+), 17 deletions(-)

lib/compiler/build_runner.zig+56-5
......@@ -10,7 +10,8 @@ const File = std.fs.File;
1010const Step = std.Build.Step;
1111const Watch = std.Build.Watch;
1212const Allocator = std.mem.Allocator;
13const fatal = std.zig.fatal;
13const fatal = std.process.fatal;
14const runner = @This();
1415
1516pub const root = @import("@build");
1617pub const dependencies = @import("@dependencies");
......@@ -102,6 +103,7 @@ pub fn main() !void {
102103 var steps_menu = false;
103104 var output_tmp_nonce: ?[16]u8 = null;
104105 var watch = false;
106 var fuzz = false;
105107 var debounce_interval_ms: u16 = 50;
106108
107109 while (nextArg(args, &arg_idx)) |arg| {
......@@ -234,6 +236,8 @@ pub fn main() !void {
234236 prominent_compile_errors = true;
235237 } else if (mem.eql(u8, arg, "--watch")) {
236238 watch = true;
239 } else if (mem.eql(u8, arg, "--fuzz")) {
240 fuzz = true;
237241 } else if (mem.eql(u8, arg, "-fincremental")) {
238242 graph.incremental = true;
239243 } else if (mem.eql(u8, arg, "-fno-incremental")) {
......@@ -353,6 +357,7 @@ pub fn main() !void {
353357 .max_rss_mutex = .{},
354358 .skip_oom_steps = skip_oom_steps,
355359 .watch = watch,
360 .fuzz = fuzz,
356361 .memory_blocked_steps = std.ArrayList(*Step).init(arena),
357362 .step_stack = .{},
358363 .prominent_compile_errors = prominent_compile_errors,
......@@ -394,6 +399,10 @@ pub fn main() !void {
394399 },
395400 else => return err,
396401 };
402 if (fuzz) {
403 startFuzzing(&run.thread_pool, run.step_stack.keys(), main_progress_node);
404 }
405
397406 if (!watch) return cleanExit();
398407
399408 switch (builtin.os.tag) {
......@@ -430,6 +439,43 @@ pub fn main() !void {
430439 }
431440}
432441
442fn startFuzzing(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void {
443 {
444 const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0);
445 defer rebuild_node.end();
446 var count: usize = 0;
447 var wait_group: std.Thread.WaitGroup = .{};
448 defer wait_group.wait();
449 for (all_steps) |step| {
450 const run = step.cast(Step.Run) orelse continue;
451 if (run.fuzz_tests.items.len > 0 and run.producer != null) {
452 thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node });
453 count += 1;
454 }
455 }
456 if (count == 0) {
457 std.debug.lockStdErr();
458 std.debug.print("no fuzz tests found\n", .{});
459 process.exit(2);
460 }
461 rebuild_node.setEstimatedTotalItems(count);
462 }
463 @panic("TODO do something with the rebuilt unit tests");
464}
465
466fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void {
467 const compile_step = run.producer.?;
468 const prog_node = parent_prog_node.start(compile_step.step.name, 0);
469 defer prog_node.end();
470 const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| {
471 std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{
472 compile_step.step.name, @errorName(err),
473 });
474 return;
475 };
476 std.debug.print("rebuilt binary: '{s}'\n", .{rebuilt_bin_path});
477}
478
433479fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void {
434480 for (all_steps) |step| switch (step.state) {
435481 .dependency_failure, .failure, .skipped => step.recursiveReset(gpa),
......@@ -457,6 +503,7 @@ const Run = struct {
457503 max_rss_mutex: std.Thread.Mutex,
458504 skip_oom_steps: bool,
459505 watch: bool,
506 fuzz: bool,
460507 memory_blocked_steps: std.ArrayList(*Step),
461508 step_stack: std.AutoArrayHashMapUnmanaged(*Step, void),
462509 prominent_compile_errors: bool,
......@@ -466,6 +513,11 @@ const Run = struct {
466513 summary: Summary,
467514 ttyconf: std.io.tty.Config,
468515 stderr: File,
516
517 fn cleanExit(run: Run) void {
518 if (run.watch or run.fuzz) return;
519 return runner.cleanExit();
520 }
469521};
470522
471523fn prepare(
......@@ -614,8 +666,7 @@ fn runStepNames(
614666 else => false,
615667 };
616668 if (failure_count == 0 and failures_only) {
617 if (!run.watch) cleanExit();
618 return;
669 return run.cleanExit();
619670 }
620671
621672 const ttyconf = run.ttyconf;
......@@ -672,8 +723,7 @@ fn runStepNames(
672723 }
673724
674725 if (failure_count == 0) {
675 if (!run.watch) cleanExit();
676 return;
726 return run.cleanExit();
677727 }
678728
679729 // Finally, render compile errors at the bottom of the terminal.
......@@ -1226,6 +1276,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void {
12261276 \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss
12271277 \\ --fetch Exit after fetching dependency tree
12281278 \\ --watch Continuously rebuild when source files are modified
1279 \\ --fuzz Continuously search for unit test failures
12291280 \\ --debounce <ms> Delay before rebuilding after changed file detected
12301281 \\ -fincremental Enable incremental compilation
12311282 \\ -fno-incremental Disable incremental compilation
lib/compiler/test_runner.zig+6-1
......@@ -143,6 +143,7 @@ fn mainTerminal() void {
143143 var ok_count: usize = 0;
144144 var skip_count: usize = 0;
145145 var fail_count: usize = 0;
146 var fuzz_count: usize = 0;
146147 const root_node = std.Progress.start(.{
147148 .root_name = "Test",
148149 .estimated_total_items = test_fn_list.len,
......@@ -168,7 +169,7 @@ fn mainTerminal() void {
168169 if (!have_tty) {
169170 std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name });
170171 }
171 // Track in a global variable so that `fuzzInput` can see it.
172 is_fuzz_test = false;
172173 if (test_fn.func()) |_| {
173174 ok_count += 1;
174175 test_node.end();
......@@ -198,6 +199,7 @@ fn mainTerminal() void {
198199 test_node.end();
199200 },
200201 }
202 fuzz_count += @intFromBool(is_fuzz_test);
201203 }
202204 root_node.end();
203205 if (ok_count == test_fn_list.len) {
......@@ -211,6 +213,9 @@ fn mainTerminal() void {
211213 if (leaks != 0) {
212214 std.debug.print("{d} tests leaked memory.\n", .{leaks});
213215 }
216 if (fuzz_count != 0) {
217 std.debug.print("{d} fuzz tests found.\n", .{fuzz_count});
218 }
214219 if (leaks != 0 or log_err_count != 0 or fail_count != 0) {
215220 std.process.exit(1);
216221 }
lib/std/Build.zig+1
......@@ -977,6 +977,7 @@ pub fn addRunArtifact(b: *Build, exe: *Step.Compile) *Step.Run {
977977 // Consider that this is declarative; the run step may not be run unless a user
978978 // option is supplied.
979979 const run_step = Step.Run.create(b, b.fmt("run {s}", .{exe.name}));
980 run_step.producer = exe;
980981 if (exe.kind == .@"test") {
981982 if (exe.exec_cmd_args) |exec_cmd_args| {
982983 for (exec_cmd_args) |cmd_arg| {
lib/std/Build/Step/Compile.zig+21-11
......@@ -1004,7 +1004,7 @@ fn getGeneratedFilePath(compile: *Compile, comptime tag_name: []const u8, asking
10041004 return path;
10051005}
10061006
1007fn getZigArgs(compile: *Compile) ![][]const u8 {
1007fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
10081008 const step = &compile.step;
10091009 const b = step.owner;
10101010 const arena = b.allocator;
......@@ -1055,6 +1055,10 @@ fn getZigArgs(compile: *Compile) ![][]const u8 {
10551055 try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size}));
10561056 }
10571057
1058 if (fuzz) {
1059 try zig_args.append("-ffuzz");
1060 }
1061
10581062 {
10591063 // Stores system libraries that have already been seen for at least one
10601064 // module, along with any arguments that need to be passed to the
......@@ -1757,7 +1761,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
17571761 const b = step.owner;
17581762 const compile: *Compile = @fieldParentPtr("step", step);
17591763
1760 const zig_args = try getZigArgs(compile);
1764 const zig_args = try getZigArgs(compile, false);
17611765
17621766 const maybe_output_bin_path = step.evalZigProcess(
17631767 zig_args,
......@@ -1835,6 +1839,12 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
18351839 }
18361840}
18371841
1842pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 {
1843 const zig_args = try getZigArgs(c, true);
1844 const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false);
1845 return maybe_output_bin_path.?;
1846}
1847
18381848pub fn doAtomicSymLinks(
18391849 step: *Step,
18401850 output_path: []const u8,
......@@ -1861,10 +1871,10 @@ pub fn doAtomicSymLinks(
18611871 };
18621872}
18631873
1864fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg {
1865 const pkg_config_exe = compile.graph.env_map.get("PKG_CONFIG") orelse "pkg-config";
1866 const stdout = try compile.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore);
1867 var list = ArrayList(PkgConfigPkg).init(compile.allocator);
1874fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg {
1875 const pkg_config_exe = b.graph.env_map.get("PKG_CONFIG") orelse "pkg-config";
1876 const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore);
1877 var list = ArrayList(PkgConfigPkg).init(b.allocator);
18681878 errdefer list.deinit();
18691879 var line_it = mem.tokenizeAny(u8, stdout, "\r\n");
18701880 while (line_it.next()) |line| {
......@@ -1878,13 +1888,13 @@ fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunE
18781888 return list.toOwnedSlice();
18791889}
18801890
1881fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg {
1882 if (compile.pkg_config_pkg_list) |res| {
1891fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg {
1892 if (b.pkg_config_pkg_list) |res| {
18831893 return res;
18841894 }
18851895 var code: u8 = undefined;
1886 if (execPkgConfigList(compile, &code)) |list| {
1887 compile.pkg_config_pkg_list = list;
1896 if (execPkgConfigList(b, &code)) |list| {
1897 b.pkg_config_pkg_list = list;
18881898 return list;
18891899 } else |err| {
18901900 const result = switch (err) {
......@@ -1896,7 +1906,7 @@ fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg {
18961906 error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput,
18971907 else => return err,
18981908 };
1899 compile.pkg_config_pkg_list = result;
1909 b.pkg_config_pkg_list = result;
19001910 return result;
19011911 }
19021912}
lib/std/Build/Step/Run.zig+13
......@@ -86,6 +86,13 @@ dep_output_file: ?*Output,
8686
8787has_side_effects: bool,
8888
89/// If this is a Zig unit test binary, this tracks the indexes of the unit
90/// tests that are also fuzz tests.
91fuzz_tests: std.ArrayListUnmanaged(u32),
92
93/// If this Run step was produced by a Compile step, it is tracked here.
94producer: ?*Step.Compile,
95
8996pub const StdIn = union(enum) {
9097 none,
9198 bytes: []const u8,
......@@ -175,6 +182,8 @@ pub fn create(owner: *std.Build, name: []const u8) *Run {
175182 .captured_stderr = null,
176183 .dep_output_file = null,
177184 .has_side_effects = false,
185 .fuzz_tests = .{},
186 .producer = null,
178187 };
179188 return run;
180189}
......@@ -1347,6 +1356,8 @@ fn evalZigTest(
13471356 var sub_prog_node: ?std.Progress.Node = null;
13481357 defer if (sub_prog_node) |n| n.end();
13491358
1359 run.fuzz_tests.clearRetainingCapacity();
1360
13501361 poll: while (true) {
13511362 while (stdout.readableLength() < @sizeOf(Header)) {
13521363 if (!(try poller.poll())) break :poll;
......@@ -1404,6 +1415,8 @@ fn evalZigTest(
14041415 leak_count +|= @intFromBool(tr_hdr.flags.leak);
14051416 log_err_count +|= tr_hdr.flags.log_err_count;
14061417
1418 if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index);
1419
14071420 if (tr_hdr.flags.fail or tr_hdr.flags.leak or tr_hdr.flags.log_err_count > 0) {
14081421 const name = std.mem.sliceTo(md.string_bytes[md.names[tr_hdr.index]..], 0);
14091422 const orig_msg = stderr.readableSlice(0);