From 047640383e5e635ffe52ab360e03dbe08e73d025 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 23 Jul 2024 20:49:00 -0700 Subject: [PATCH] add `--fuzz` CLI argument to `zig build` This flag makes the build runner rebuild unit tests after the pipeline finishes, if it finds any unit tests. I did not make this integrate with file system watching yet. The test runner is updated to detect which tests are fuzz tests. Run step is updated to track which test indexes are fuzz tests. --- lib/compiler/build_runner.zig | 61 +++++++++++++++++++++++++++++++--- lib/compiler/test_runner.zig | 7 +++- lib/std/Build.zig | 1 + lib/std/Build/Step/Compile.zig | 32 ++++++++++++------ lib/std/Build/Step/Run.zig | 13 ++++++++ 5 files changed, 97 insertions(+), 17 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index f06ae7d4b9cb407eb5cb7a065563fbe1d62bc3a4..5f7c12fe927451c013c012da606e3a239f36c675 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -10,7 +10,8 @@ const File = std.fs.File; const Step = std.Build.Step; const Watch = std.Build.Watch; const Allocator = std.mem.Allocator; -const fatal = std.zig.fatal; +const fatal = std.process.fatal; +const runner = @This(); pub const root = @import("@build"); pub const dependencies = @import("@dependencies"); @@ -102,6 +103,7 @@ pub fn main() !void { var steps_menu = false; var output_tmp_nonce: ?[16]u8 = null; var watch = false; + var fuzz = false; var debounce_interval_ms: u16 = 50; while (nextArg(args, &arg_idx)) |arg| { @@ -234,6 +236,8 @@ pub fn main() !void { prominent_compile_errors = true; } else if (mem.eql(u8, arg, "--watch")) { watch = true; + } else if (mem.eql(u8, arg, "--fuzz")) { + fuzz = true; } else if (mem.eql(u8, arg, "-fincremental")) { graph.incremental = true; } else if (mem.eql(u8, arg, "-fno-incremental")) { @@ -353,6 +357,7 @@ pub fn main() !void { .max_rss_mutex = .{}, .skip_oom_steps = skip_oom_steps, .watch = watch, + .fuzz = fuzz, .memory_blocked_steps = std.ArrayList(*Step).init(arena), .step_stack = .{}, .prominent_compile_errors = prominent_compile_errors, @@ -394,6 +399,10 @@ pub fn main() !void { }, else => return err, }; + if (fuzz) { + startFuzzing(&run.thread_pool, run.step_stack.keys(), main_progress_node); + } + if (!watch) return cleanExit(); switch (builtin.os.tag) { @@ -430,6 +439,43 @@ pub fn main() !void { } } +fn startFuzzing(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void { + { + const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0); + defer rebuild_node.end(); + var count: usize = 0; + var wait_group: std.Thread.WaitGroup = .{}; + defer wait_group.wait(); + for (all_steps) |step| { + const run = step.cast(Step.Run) orelse continue; + if (run.fuzz_tests.items.len > 0 and run.producer != null) { + thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node }); + count += 1; + } + } + if (count == 0) { + std.debug.lockStdErr(); + std.debug.print("no fuzz tests found\n", .{}); + process.exit(2); + } + rebuild_node.setEstimatedTotalItems(count); + } + @panic("TODO do something with the rebuilt unit tests"); +} + +fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void { + const compile_step = run.producer.?; + const prog_node = parent_prog_node.start(compile_step.step.name, 0); + defer prog_node.end(); + const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| { + std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{ + compile_step.step.name, @errorName(err), + }); + return; + }; + std.debug.print("rebuilt binary: '{s}'\n", .{rebuilt_bin_path}); +} + fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { for (all_steps) |step| switch (step.state) { .dependency_failure, .failure, .skipped => step.recursiveReset(gpa), @@ -457,6 +503,7 @@ const Run = struct { max_rss_mutex: std.Thread.Mutex, skip_oom_steps: bool, watch: bool, + fuzz: bool, memory_blocked_steps: std.ArrayList(*Step), step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), prominent_compile_errors: bool, @@ -466,6 +513,11 @@ const Run = struct { summary: Summary, ttyconf: std.io.tty.Config, stderr: File, + + fn cleanExit(run: Run) void { + if (run.watch or run.fuzz) return; + return runner.cleanExit(); + } }; fn prepare( @@ -614,8 +666,7 @@ fn runStepNames( else => false, }; if (failure_count == 0 and failures_only) { - if (!run.watch) cleanExit(); - return; + return run.cleanExit(); } const ttyconf = run.ttyconf; @@ -672,8 +723,7 @@ fn runStepNames( } if (failure_count == 0) { - if (!run.watch) cleanExit(); - return; + return run.cleanExit(); } // Finally, render compile errors at the bottom of the terminal. @@ -1226,6 +1276,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void { \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss \\ --fetch Exit after fetching dependency tree \\ --watch Continuously rebuild when source files are modified + \\ --fuzz Continuously search for unit test failures \\ --debounce Delay before rebuilding after changed file detected \\ -fincremental Enable incremental compilation \\ -fno-incremental Disable incremental compilation diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 15905ca7f9718c19613e2354de2288b7dcbc998a..81eb156e73ba943458e35411a7b4a165aca8a67e 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -143,6 +143,7 @@ fn mainTerminal() void { var ok_count: usize = 0; var skip_count: usize = 0; var fail_count: usize = 0; + var fuzz_count: usize = 0; const root_node = std.Progress.start(.{ .root_name = "Test", .estimated_total_items = test_fn_list.len, @@ -168,7 +169,7 @@ fn mainTerminal() void { if (!have_tty) { std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); } - // Track in a global variable so that `fuzzInput` can see it. + is_fuzz_test = false; if (test_fn.func()) |_| { ok_count += 1; test_node.end(); @@ -198,6 +199,7 @@ fn mainTerminal() void { test_node.end(); }, } + fuzz_count += @intFromBool(is_fuzz_test); } root_node.end(); if (ok_count == test_fn_list.len) { @@ -211,6 +213,9 @@ fn mainTerminal() void { if (leaks != 0) { std.debug.print("{d} tests leaked memory.\n", .{leaks}); } + if (fuzz_count != 0) { + std.debug.print("{d} fuzz tests found.\n", .{fuzz_count}); + } if (leaks != 0 or log_err_count != 0 or fail_count != 0) { std.process.exit(1); } diff --git a/lib/std/Build.zig b/lib/std/Build.zig index f76e3263cb3a6fd08f0a691ceb3de70f856270eb..1ad2e0ee51721cdcf838f0e1c8ebe73fe8a703fc 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -977,6 +977,7 @@ pub fn addRunArtifact(b: *Build, exe: *Step.Compile) *Step.Run { // Consider that this is declarative; the run step may not be run unless a user // option is supplied. const run_step = Step.Run.create(b, b.fmt("run {s}", .{exe.name})); + run_step.producer = exe; if (exe.kind == .@"test") { if (exe.exec_cmd_args) |exec_cmd_args| { for (exec_cmd_args) |cmd_arg| { diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 4f504151c02ca3404469d3399a365b7cf6f182eb..ffb2337ac527824bbddf037f425f0ecf7d9200f1 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -1004,7 +1004,7 @@ fn getGeneratedFilePath(compile: *Compile, comptime tag_name: []const u8, asking return path; } -fn getZigArgs(compile: *Compile) ![][]const u8 { +fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { const step = &compile.step; const b = step.owner; const arena = b.allocator; @@ -1055,6 +1055,10 @@ fn getZigArgs(compile: *Compile) ![][]const u8 { try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); } + if (fuzz) { + try zig_args.append("-ffuzz"); + } + { // Stores system libraries that have already been seen for at least one // module, along with any arguments that need to be passed to the @@ -1757,7 +1761,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { const b = step.owner; const compile: *Compile = @fieldParentPtr("step", step); - const zig_args = try getZigArgs(compile); + const zig_args = try getZigArgs(compile, false); const maybe_output_bin_path = step.evalZigProcess( zig_args, @@ -1835,6 +1839,12 @@ fn make(step: *Step, options: Step.MakeOptions) !void { } } +pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 { + const zig_args = try getZigArgs(c, true); + const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false); + return maybe_output_bin_path.?; +} + pub fn doAtomicSymLinks( step: *Step, output_path: []const u8, @@ -1861,10 +1871,10 @@ pub fn doAtomicSymLinks( }; } -fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { - const pkg_config_exe = compile.graph.env_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = try compile.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore); - var list = ArrayList(PkgConfigPkg).init(compile.allocator); +fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { + const pkg_config_exe = b.graph.env_map.get("PKG_CONFIG") orelse "pkg-config"; + const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore); + var list = ArrayList(PkgConfigPkg).init(b.allocator); errdefer list.deinit(); var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); while (line_it.next()) |line| { @@ -1878,13 +1888,13 @@ fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunE return list.toOwnedSlice(); } -fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg { - if (compile.pkg_config_pkg_list) |res| { +fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { + if (b.pkg_config_pkg_list) |res| { return res; } var code: u8 = undefined; - if (execPkgConfigList(compile, &code)) |list| { - compile.pkg_config_pkg_list = list; + if (execPkgConfigList(b, &code)) |list| { + b.pkg_config_pkg_list = list; return list; } else |err| { const result = switch (err) { @@ -1896,7 +1906,7 @@ fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg { error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, else => return err, }; - compile.pkg_config_pkg_list = result; + b.pkg_config_pkg_list = result; return result; } } diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index dbb865047be766d276588646ac3fccc3ec7d5601..7927ac9479a1e595db4e8944654f1ce350020943 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -86,6 +86,13 @@ dep_output_file: ?*Output, has_side_effects: bool, +/// If this is a Zig unit test binary, this tracks the indexes of the unit +/// tests that are also fuzz tests. +fuzz_tests: std.ArrayListUnmanaged(u32), + +/// If this Run step was produced by a Compile step, it is tracked here. +producer: ?*Step.Compile, + pub const StdIn = union(enum) { none, bytes: []const u8, @@ -175,6 +182,8 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { .captured_stderr = null, .dep_output_file = null, .has_side_effects = false, + .fuzz_tests = .{}, + .producer = null, }; return run; } @@ -1347,6 +1356,8 @@ fn evalZigTest( var sub_prog_node: ?std.Progress.Node = null; defer if (sub_prog_node) |n| n.end(); + run.fuzz_tests.clearRetainingCapacity(); + poll: while (true) { while (stdout.readableLength() < @sizeOf(Header)) { if (!(try poller.poll())) break :poll; @@ -1404,6 +1415,8 @@ fn evalZigTest( leak_count +|= @intFromBool(tr_hdr.flags.leak); log_err_count +|= tr_hdr.flags.log_err_count; + if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index); + if (tr_hdr.flags.fail or tr_hdr.flags.leak or tr_hdr.flags.log_err_count > 0) { const name = std.mem.sliceTo(md.string_bytes[md.names[tr_hdr.index]..], 0); const orig_msg = stderr.readableSlice(0); -- 2.54.0