Commit 047640383e5e635ffe52ab360e03dbe08e73d025 | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-07-23 20:49:00-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-07-25 18:52:20-07:00
log	047640383e5e635ffe52ab360e03dbe08e73d025
tree	47b7062d2f80b34655481bf91f06bef4d132cfe7
parent	6f3767862d6886d5fde7e3734455a30f168ba80b

add `--fuzz` CLI argument to `zig build`

This flag makes the build runner rebuild unit tests after the pipeline
finishes, if it finds any unit tests.

I did not make this integrate with file system watching yet.

The test runner is updated to detect which tests are fuzz tests.

Run step is updated to track which test indexes are fuzz tests.

5 files changed, 97 insertions(+), 17 deletions(-)

lib
- compiler
  - build_runner.zig
  - test_runner.zig
- std
  - Build.zig
  - Build
    Step
    Compile.zig
    Run.zig

lib/compiler/build_runner.zig+56-5

 @@ -10,7 +10,8 @@ const File = std.fs.File;
 const Step = std.Build.Step;
 const Watch = std.Build.Watch;
 const Allocator = std.mem.Allocator;
-const fatal = std.zig.fatal;
+const fatal = std.process.fatal;
+const runner = @This();
 pub const root = @import("@build");
 pub const dependencies = @import("@dependencies");
 @@ -102,6 +103,7 @@ pub fn main() !void {
     var steps_menu = false;
     var output_tmp_nonce: ?[16]u8 = null;
     var watch = false;
+    var fuzz = false;
     var debounce_interval_ms: u16 = 50;
     while (nextArg(args, &arg_idx)) |arg| {
 @@ -234,6 +236,8 @@ pub fn main() !void {
                 prominent_compile_errors = true;
             } else if (mem.eql(u8, arg, "--watch")) {
                 watch = true;
+            } else if (mem.eql(u8, arg, "--fuzz")) {
+                fuzz = true;
             } else if (mem.eql(u8, arg, "-fincremental")) {
                 graph.incremental = true;
             } else if (mem.eql(u8, arg, "-fno-incremental")) {
 @@ -353,6 +357,7 @@ pub fn main() !void {
         .max_rss_mutex = .{},
         .skip_oom_steps = skip_oom_steps,
         .watch = watch,
+        .fuzz = fuzz,
         .memory_blocked_steps = std.ArrayList(*Step).init(arena),
         .step_stack = .{},
         .prominent_compile_errors = prominent_compile_errors,
 @@ -394,6 +399,10 @@ pub fn main() !void {
             },
             else => return err,
         };
+        if (fuzz) {
+            startFuzzing(&run.thread_pool, run.step_stack.keys(), main_progress_node);
+        }
         if (!watch) return cleanExit();
         switch (builtin.os.tag) {
 @@ -430,6 +439,43 @@ pub fn main() !void {
     }
 }
+fn startFuzzing(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void {
+    {
+        const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0);
+        defer rebuild_node.end();
+        var count: usize = 0;
+        var wait_group: std.Thread.WaitGroup = .{};
+        defer wait_group.wait();
+        for (all_steps) |step| {
+            const run = step.cast(Step.Run) orelse continue;
+            if (run.fuzz_tests.items.len > 0 and run.producer != null) {
+                thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node });
+                count += 1;
+            }
+        }
+        if (count == 0) {
+            std.debug.lockStdErr();
+            std.debug.print("no fuzz tests found\n", .{});
+            process.exit(2);
+        }
+        rebuild_node.setEstimatedTotalItems(count);
+    }
+    @panic("TODO do something with the rebuilt unit tests");
+}
+fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void {
+    const compile_step = run.producer.?;
+    const prog_node = parent_prog_node.start(compile_step.step.name, 0);
+    defer prog_node.end();
+    const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| {
+        std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{
+            compile_step.step.name, @errorName(err),
+        });
+        return;
+    };
+    std.debug.print("rebuilt binary: '{s}'\n", .{rebuilt_bin_path});
+}
 fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void {
     for (all_steps) |step| switch (step.state) {
         .dependency_failure, .failure, .skipped => step.recursiveReset(gpa),
 @@ -457,6 +503,7 @@ const Run = struct {
     max_rss_mutex: std.Thread.Mutex,
     skip_oom_steps: bool,
     watch: bool,
+    fuzz: bool,
     memory_blocked_steps: std.ArrayList(*Step),
     step_stack: std.AutoArrayHashMapUnmanaged(*Step, void),
     prominent_compile_errors: bool,
 @@ -466,6 +513,11 @@ const Run = struct {
     summary: Summary,
     ttyconf: std.io.tty.Config,
     stderr: File,
+    fn cleanExit(run: Run) void {
+        if (run.watch or run.fuzz) return;
+        return runner.cleanExit();
+    }
 };
 fn prepare(
 @@ -614,8 +666,7 @@ fn runStepNames(
         else => false,
     };
     if (failure_count == 0 and failures_only) {
-        if (!run.watch) cleanExit();
+        return run.cleanExit();
-        return;
     }
     const ttyconf = run.ttyconf;
 @@ -672,8 +723,7 @@ fn runStepNames(
     }
     if (failure_count == 0) {
-        if (!run.watch) cleanExit();
+        return run.cleanExit();
-        return;
     }
     // Finally, render compile errors at the bottom of the terminal.
 @@ -1226,6 +1276,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void {
         \\  --skip-oom-steps             Instead of failing, skip steps that would exceed --maxrss
         \\  --fetch                      Exit after fetching dependency tree
         \\  --watch                      Continuously rebuild when source files are modified
+        \\  --fuzz                       Continuously search for unit test failures
         \\  --debounce <ms>              Delay before rebuilding after changed file detected
         \\     -fincremental             Enable incremental compilation
         \\  -fno-incremental             Disable incremental compilation

lib/compiler/test_runner.zig+6-1

...	@@ -143,6 +143,7 @@ fn mainTerminal() void {	...	@@ -143,6 +143,7 @@ fn mainTerminal() void {
143	var ok_count: usize = 0;	143	var ok_count: usize = 0;
144	var skip_count: usize = 0;	144	var skip_count: usize = 0;
145	var fail_count: usize = 0;	145	var fail_count: usize = 0;
		146	var fuzz_count: usize = 0;
146	const root_node = std.Progress.start(.{	147	const root_node = std.Progress.start(.{
147	.root_name = "Test",	148	.root_name = "Test",
148	.estimated_total_items = test_fn_list.len,	149	.estimated_total_items = test_fn_list.len,
...	@@ -168,7 +169,7 @@ fn mainTerminal() void {	...	@@ -168,7 +169,7 @@ fn mainTerminal() void {
168	if (!have_tty) {	169	if (!have_tty) {
169	std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name });	170	std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name });
170	}	171	}
171	// Track in a global variable so that `fuzzInput` can see it.	172	is_fuzz_test = false;
172	if (test_fn.func()) \|_\| {	173	if (test_fn.func()) \|_\| {
173	ok_count += 1;	174	ok_count += 1;
174	test_node.end();	175	test_node.end();
...	@@ -198,6 +199,7 @@ fn mainTerminal() void {	...	@@ -198,6 +199,7 @@ fn mainTerminal() void {
198	test_node.end();	199	test_node.end();
199	},	200	},
200	}	201	}
		202	fuzz_count += @intFromBool(is_fuzz_test);
201	}	203	}
202	root_node.end();	204	root_node.end();
203	if (ok_count == test_fn_list.len) {	205	if (ok_count == test_fn_list.len) {
...	@@ -211,6 +213,9 @@ fn mainTerminal() void {	...	@@ -211,6 +213,9 @@ fn mainTerminal() void {
211	if (leaks != 0) {	213	if (leaks != 0) {
212	std.debug.print("{d} tests leaked memory.\n", .{leaks});	214	std.debug.print("{d} tests leaked memory.\n", .{leaks});
213	}	215	}
		216	if (fuzz_count != 0) {
		217	std.debug.print("{d} fuzz tests found.\n", .{fuzz_count});
		218	}
214	if (leaks != 0 or log_err_count != 0 or fail_count != 0) {	219	if (leaks != 0 or log_err_count != 0 or fail_count != 0) {
215	std.process.exit(1);	220	std.process.exit(1);
216	}	221	}

lib/std/Build.zig+1

...	@@ -977,6 +977,7 @@ pub fn addRunArtifact(b: Build, exe: Step.Compile) *Step.Run {	...	@@ -977,6 +977,7 @@ pub fn addRunArtifact(b: Build, exe: Step.Compile) *Step.Run {
977	// Consider that this is declarative; the run step may not be run unless a user	977	// Consider that this is declarative; the run step may not be run unless a user
978	// option is supplied.	978	// option is supplied.
979	const run_step = Step.Run.create(b, b.fmt("run {s}", .{exe.name}));	979	const run_step = Step.Run.create(b, b.fmt("run {s}", .{exe.name}));
		980	run_step.producer = exe;
980	if (exe.kind == .@"test") {	981	if (exe.kind == .@"test") {
981	if (exe.exec_cmd_args) \|exec_cmd_args\| {	982	if (exe.exec_cmd_args) \|exec_cmd_args\| {
982	for (exec_cmd_args) \|cmd_arg\| {	983	for (exec_cmd_args) \|cmd_arg\| {

lib/std/Build/Step/Compile.zig+21-11

 @@ -1004,7 +1004,7 @@ fn getGeneratedFilePath(compile: *Compile, comptime tag_name: []const u8, asking
     return path;
 }
-fn getZigArgs(compile: *Compile) ![][]const u8 {
+fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
     const step = &compile.step;
     const b = step.owner;
     const arena = b.allocator;
 @@ -1055,6 +1055,10 @@ fn getZigArgs(compile: *Compile) ![][]const u8 {
         try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size}));
     }
+    if (fuzz) {
+        try zig_args.append("-ffuzz");
+    }
     {
         // Stores system libraries that have already been seen for at least one
         // module, along with any arguments that need to be passed to the
 @@ -1757,7 +1761,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
     const b = step.owner;
     const compile: *Compile = @fieldParentPtr("step", step);
-    const zig_args = try getZigArgs(compile);
+    const zig_args = try getZigArgs(compile, false);
     const maybe_output_bin_path = step.evalZigProcess(
         zig_args,
 @@ -1835,6 +1839,12 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
     }
 }
+pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 {
+    const zig_args = try getZigArgs(c, true);
+    const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false);
+    return maybe_output_bin_path.?;
+}
 pub fn doAtomicSymLinks(
     step: *Step,
     output_path: []const u8,
 @@ -1861,10 +1871,10 @@ pub fn doAtomicSymLinks(
     };
 }
-fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg {
+fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg {
-    const pkg_config_exe = compile.graph.env_map.get("PKG_CONFIG") orelse "pkg-config";
+    const pkg_config_exe = b.graph.env_map.get("PKG_CONFIG") orelse "pkg-config";
-    const stdout = try compile.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore);
+    const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore);
-    var list = ArrayList(PkgConfigPkg).init(compile.allocator);
+    var list = ArrayList(PkgConfigPkg).init(b.allocator);
     errdefer list.deinit();
     var line_it = mem.tokenizeAny(u8, stdout, "\r\n");
     while (line_it.next()) |line| {
 @@ -1878,13 +1888,13 @@ fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunE
     return list.toOwnedSlice();
 }
-fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg {
+fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg {
-    if (compile.pkg_config_pkg_list) |res| {
+    if (b.pkg_config_pkg_list) |res| {
         return res;
     }
     var code: u8 = undefined;
-    if (execPkgConfigList(compile, &code)) |list| {
+    if (execPkgConfigList(b, &code)) |list| {
-        compile.pkg_config_pkg_list = list;
+        b.pkg_config_pkg_list = list;
         return list;
     } else |err| {
         const result = switch (err) {
 @@ -1896,7 +1906,7 @@ fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg {
             error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput,
             else => return err,
         };
-        compile.pkg_config_pkg_list = result;
+        b.pkg_config_pkg_list = result;
         return result;
     }
 }

lib/std/Build/Step/Run.zig+13

 @@ -86,6 +86,13 @@ dep_output_file: ?*Output,
 has_side_effects: bool,
+/// If this is a Zig unit test binary, this tracks the indexes of the unit
+/// tests that are also fuzz tests.
+fuzz_tests: std.ArrayListUnmanaged(u32),
+/// If this Run step was produced by a Compile step, it is tracked here.
+producer: ?*Step.Compile,
 pub const StdIn = union(enum) {
     none,
     bytes: []const u8,
 @@ -175,6 +182,8 @@ pub fn create(owner: *std.Build, name: []const u8) *Run {
         .captured_stderr = null,
         .dep_output_file = null,
         .has_side_effects = false,
+        .fuzz_tests = .{},
+        .producer = null,
     };
     return run;
 }
 @@ -1347,6 +1356,8 @@ fn evalZigTest(
     var sub_prog_node: ?std.Progress.Node = null;
     defer if (sub_prog_node) |n| n.end();
+    run.fuzz_tests.clearRetainingCapacity();
     poll: while (true) {
         while (stdout.readableLength() < @sizeOf(Header)) {
             if (!(try poller.poll())) break :poll;
 @@ -1404,6 +1415,8 @@ fn evalZigTest(
                 leak_count +|= @intFromBool(tr_hdr.flags.leak);
                 log_err_count +|= tr_hdr.flags.log_err_count;
+                if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index);
                 if (tr_hdr.flags.fail or tr_hdr.flags.leak or tr_hdr.flags.log_err_count > 0) {
                     const name = std.mem.sliceTo(md.string_bytes[md.names[tr_hdr.index]..], 0);
                     const orig_msg = stderr.readableSlice(0);