From 5058beb1793aafaaf63cb5b9ef4c751fba08a62c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 22 Jul 2024 16:45:19 -0700 Subject: [PATCH 01/12] implement std.testing.fuzzInput for the -fno-fuzz case. The other case will take more work in libfuzzer. --- lib/std/io/test.zig | 2 +- lib/std/testing.zig | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/lib/std/io/test.zig b/lib/std/io/test.zig index 2f9464eef4af7f23f393fac9b1ab6e29b1f7cb19..5ac4bb65d2b58a7367d07bc568591712b680627d 100644 --- a/lib/std/io/test.zig +++ b/lib/std/io/test.zig @@ -16,7 +16,7 @@ test "write a file, read it, then delete it" { defer tmp.cleanup(); var data: [1024]u8 = undefined; - var prng = DefaultPrng.init(1234); + var prng = DefaultPrng.init(std.testing.random_seed); const random = prng.random(); random.bytes(data[0..]); const tmp_file_name = "temp_test_file.txt"; diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 6f29fbd613d0f77d8f8ab1c71afa6f1628a06174..35f3c5a6dda606228dd9ceeee920842c8c336b43 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -1136,3 +1136,33 @@ pub fn refAllDeclsRecursive(comptime T: type) void { _ = &@field(T, decl.name); } } + +const FuzzerSlice = extern struct { + ptr: [*]const u8, + len: usize, + + fn toSlice(s: FuzzerSlice) []const u8 { + return s.ptr[0..s.len]; + } +}; + +extern fn fuzzer_next() FuzzerSlice; + +pub const FuzzInputOptions = struct { + corpus: []const []const u8 = &.{}, +}; + +pub fn fuzzInput(options: FuzzInputOptions) []const u8 { + @disableInstrumentation(); + if (builtin.fuzz) { + return fuzzer_next().toSlice(); + } else { + if (options.corpus.len == 0) { + return ""; + } else { + var prng = std.Random.DefaultPrng.init(std.testing.random_seed); + const random = prng.random(); + return options.corpus[random.uintLessThan(usize, options.corpus.len)]; + } + } +} -- 2.54.0 From 3256df2ff80be565095993b03f8e1edfd5072367 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 22 Jul 2024 18:02:24 -0700 Subject: [PATCH 02/12] llvm: always include instrumentation function attributes --- src/codegen/llvm.zig | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index eec0412492c934459ea2faf748f090bbbfffb04b..f83d115b1804d47bd2bf403495cd1d1cfc109650 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -1392,16 +1392,12 @@ pub const Object = struct { } if (owner_mod.fuzz and !func_analysis.disable_instrumentation) { try attributes.addFnAttr(.optforfuzzing, &o.builder); - if (comp.config.any_fuzz) { - _ = try attributes.removeFnAttr(.skipprofile); - _ = try attributes.removeFnAttr(.nosanitize_coverage); - } + _ = try attributes.removeFnAttr(.skipprofile); + _ = try attributes.removeFnAttr(.nosanitize_coverage); } else { _ = try attributes.removeFnAttr(.optforfuzzing); - if (comp.config.any_fuzz) { - try attributes.addFnAttr(.skipprofile, &o.builder); - try attributes.addFnAttr(.nosanitize_coverage, &o.builder); - } + try attributes.addFnAttr(.skipprofile, &o.builder); + try attributes.addFnAttr(.nosanitize_coverage, &o.builder); } // TODO: disable this if safety is off for the function scope -- 2.54.0 From 6f3767862d6886d5fde7e3734455a30f168ba80b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 22 Jul 2024 18:06:58 -0700 Subject: [PATCH 03/12] implement std.testing.fuzzInput For now this returns a dummy fuzz input. --- lib/compiler/test_runner.zig | 91 +++++++++++++++++++++++++++--------- lib/fuzzer.zig | 56 ++++++++++++++++++++-- lib/std/testing.zig | 26 +---------- lib/std/zig/Server.zig | 5 +- 4 files changed, 125 insertions(+), 53 deletions(-) diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 478728455b340173e0493e4d2024eef51ae99710..15905ca7f9718c19613e2354de2288b7dcbc998a 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -1,18 +1,26 @@ //! Default test runner for unit tests. +const builtin = @import("builtin"); const std = @import("std"); const io = std.io; -const builtin = @import("builtin"); +const testing = std.testing; pub const std_options = .{ .logFn = log, }; var log_err_count: usize = 0; -var cmdline_buffer: [4096]u8 = undefined; -var fba = std.heap.FixedBufferAllocator.init(&cmdline_buffer); +var fba_buffer: [8192]u8 = undefined; +var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); + +const crippled = switch (builtin.zig_backend) { + .stage2_riscv64 => true, + else => false, +}; pub fn main() void { - if (builtin.zig_backend == .stage2_riscv64) { + @disableInstrumentation(); + + if (crippled) { return mainSimple() catch @panic("test failure\n"); } @@ -25,13 +33,15 @@ pub fn main() void { if (std.mem.eql(u8, arg, "--listen=-")) { listen = true; } else if (std.mem.startsWith(u8, arg, "--seed=")) { - std.testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch + testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch @panic("unable to parse --seed command line argument"); } else { @panic("unrecognized command line argument"); } } + fba.reset(); + if (listen) { return mainServer() catch @panic("internal test runner failure"); } else { @@ -40,6 +50,7 @@ pub fn main() void { } fn mainServer() !void { + @disableInstrumentation(); var server = try std.zig.Server.init(.{ .gpa = fba.allocator(), .in = std.io.getStdIn(), @@ -55,24 +66,24 @@ fn mainServer() !void { return std.process.exit(0); }, .query_test_metadata => { - std.testing.allocator_instance = .{}; - defer if (std.testing.allocator_instance.deinit() == .leak) { + testing.allocator_instance = .{}; + defer if (testing.allocator_instance.deinit() == .leak) { @panic("internal test runner memory leak"); }; var string_bytes: std.ArrayListUnmanaged(u8) = .{}; - defer string_bytes.deinit(std.testing.allocator); - try string_bytes.append(std.testing.allocator, 0); // Reserve 0 for null. + defer string_bytes.deinit(testing.allocator); + try string_bytes.append(testing.allocator, 0); // Reserve 0 for null. const test_fns = builtin.test_functions; - const names = try std.testing.allocator.alloc(u32, test_fns.len); - defer std.testing.allocator.free(names); - const expected_panic_msgs = try std.testing.allocator.alloc(u32, test_fns.len); - defer std.testing.allocator.free(expected_panic_msgs); + const names = try testing.allocator.alloc(u32, test_fns.len); + defer testing.allocator.free(names); + const expected_panic_msgs = try testing.allocator.alloc(u32, test_fns.len); + defer testing.allocator.free(expected_panic_msgs); for (test_fns, names, expected_panic_msgs) |test_fn, *name, *expected_panic_msg| { name.* = @as(u32, @intCast(string_bytes.items.len)); - try string_bytes.ensureUnusedCapacity(std.testing.allocator, test_fn.name.len + 1); + try string_bytes.ensureUnusedCapacity(testing.allocator, test_fn.name.len + 1); string_bytes.appendSliceAssumeCapacity(test_fn.name); string_bytes.appendAssumeCapacity(0); expected_panic_msg.* = 0; @@ -86,13 +97,13 @@ fn mainServer() !void { }, .run_test => { - std.testing.allocator_instance = .{}; + testing.allocator_instance = .{}; log_err_count = 0; const index = try server.receiveBody_u32(); const test_fn = builtin.test_functions[index]; var fail = false; var skip = false; - var leak = false; + is_fuzz_test = false; test_fn.func() catch |err| switch (err) { error.SkipZigTest => skip = true, else => { @@ -102,13 +113,14 @@ fn mainServer() !void { } }, }; - leak = std.testing.allocator_instance.deinit() == .leak; + const leak = testing.allocator_instance.deinit() == .leak; try server.serveTestResults(.{ .index = index, .flags = .{ .fail = fail, .skip = skip, .leak = leak, + .fuzz = is_fuzz_test, .log_err_count = std.math.lossyCast( @TypeOf(@as(std.zig.Server.Message.TestResults.Flags, undefined).log_err_count), log_err_count, @@ -118,7 +130,7 @@ fn mainServer() !void { }, else => { - std.debug.print("unsupported message: {x}", .{@intFromEnum(hdr.tag)}); + std.debug.print("unsupported message: {x}\n", .{@intFromEnum(hdr.tag)}); std.process.exit(1); }, } @@ -126,6 +138,7 @@ fn mainServer() !void { } fn mainTerminal() void { + @disableInstrumentation(); const test_fn_list = builtin.test_functions; var ok_count: usize = 0; var skip_count: usize = 0; @@ -143,18 +156,19 @@ fn mainTerminal() void { var leaks: usize = 0; for (test_fn_list, 0..) |test_fn, i| { - std.testing.allocator_instance = .{}; + testing.allocator_instance = .{}; defer { - if (std.testing.allocator_instance.deinit() == .leak) { + if (testing.allocator_instance.deinit() == .leak) { leaks += 1; } } - std.testing.log_level = .warn; + testing.log_level = .warn; const test_node = root_node.start(test_fn.name, 0); if (!have_tty) { std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); } + // Track in a global variable so that `fuzzInput` can see it. if (test_fn.func()) |_| { ok_count += 1; test_node.end(); @@ -208,10 +222,11 @@ pub fn log( comptime format: []const u8, args: anytype, ) void { + @disableInstrumentation(); if (@intFromEnum(message_level) <= @intFromEnum(std.log.Level.err)) { log_err_count +|= 1; } - if (@intFromEnum(message_level) <= @intFromEnum(std.testing.log_level)) { + if (@intFromEnum(message_level) <= @intFromEnum(testing.log_level)) { std.debug.print( "[" ++ @tagName(scope) ++ "] (" ++ @tagName(message_level) ++ "): " ++ format ++ "\n", args, @@ -222,6 +237,7 @@ pub fn log( /// Simpler main(), exercising fewer language features, so that /// work-in-progress backends can handle it. pub fn mainSimple() anyerror!void { + @disableInstrumentation(); // is the backend capable of printing to stderr? const enable_print = switch (builtin.zig_backend) { else => false, @@ -266,3 +282,34 @@ pub fn mainSimple() anyerror!void { } if (failed != 0) std.process.exit(1); } + +const FuzzerSlice = extern struct { + ptr: [*]const u8, + len: usize, + + inline fn toSlice(s: FuzzerSlice) []const u8 { + return s.ptr[0..s.len]; + } +}; + +var is_fuzz_test: bool = undefined; + +extern fn fuzzer_next() FuzzerSlice; + +pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 { + @disableInstrumentation(); + if (crippled) { + return ""; + } else if (builtin.fuzz) { + return fuzzer_next().toSlice(); + } else { + is_fuzz_test = true; + if (options.corpus.len == 0) { + return ""; + } else { + var prng = std.Random.DefaultPrng.init(testing.random_seed); + const random = prng.random(); + return options.corpus[random.uintLessThan(usize, options.corpus.len)]; + } + } +} diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 49dd33894b302babed543077b646acdfd918d46d..8329d1eb2e98485050e996fc749dcb7d8ede1e81 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -1,13 +1,14 @@ const std = @import("std"); +const Allocator = std.mem.Allocator; export threadlocal var __sancov_lowest_stack: usize = 0; export fn __sanitizer_cov_8bit_counters_init(start: [*]u8, stop: [*]u8) void { - std.debug.print("__sanitizer_cov_8bit_counters_init start={*}, stop={*}\n", .{ start, stop }); + std.log.debug("__sanitizer_cov_8bit_counters_init start={*}, stop={*}", .{ start, stop }); } export fn __sanitizer_cov_pcs_init(pcs_beg: [*]const usize, pcs_end: [*]const usize) void { - std.debug.print("__sanitizer_cov_pcs_init pcs_beg={*}, pcs_end={*}\n", .{ pcs_beg, pcs_end }); + std.log.debug("__sanitizer_cov_pcs_init pcs_beg={*}, pcs_end={*}", .{ pcs_beg, pcs_end }); } export fn __sanitizer_cov_trace_const_cmp1(arg1: u8, arg2: u8) void { @@ -47,16 +48,61 @@ export fn __sanitizer_cov_trace_switch(val: u64, cases_ptr: [*]u64) void { const len = cases_ptr[0]; const val_size_in_bits = cases_ptr[1]; const cases = cases_ptr[2..][0..len]; - std.debug.print("0x{x}: switch on value {d} ({d} bits) with {d} cases\n", .{ + std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{ pc, val, val_size_in_bits, cases.len, }); } export fn __sanitizer_cov_trace_pc_indir(callee: usize) void { const pc = @returnAddress(); - std.debug.print("0x{x}: indirect call to 0x{x}\n", .{ pc, callee }); + std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee }); } fn handleCmp(pc: usize, arg1: u64, arg2: u64) void { - std.debug.print("0x{x}: comparison of {d} and {d}\n", .{ pc, arg1, arg2 }); + std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 }); +} + +const Fuzzer = struct { + gpa: Allocator, + rng: std.Random.DefaultPrng, + input: std.ArrayListUnmanaged(u8), + + const Slice = extern struct { + ptr: [*]const u8, + len: usize, + + fn toSlice(s: Slice) []const u8 { + return s.ptr[0..s.len]; + } + + fn fromSlice(s: []const u8) Slice { + return .{ + .ptr = s.ptr, + .len = s.len, + }; + } + }; + + fn next(f: *Fuzzer) ![]const u8 { + const gpa = f.gpa; + const rng = fuzzer.rng.random(); + const len = rng.uintLessThan(usize, 64); + try f.input.resize(gpa, len); + rng.bytes(f.input.items); + return f.input.items; + } +}; + +var general_purpose_allocator: std.heap.GeneralPurposeAllocator(.{}) = .{}; + +var fuzzer: Fuzzer = .{ + .gpa = general_purpose_allocator.allocator(), + .rng = std.Random.DefaultPrng.init(0), + .input = .{}, +}; + +export fn fuzzer_next() Fuzzer.Slice { + return Fuzzer.Slice.fromSlice(fuzzer.next() catch |err| switch (err) { + error.OutOfMemory => @panic("out of memory"), + }); } diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 35f3c5a6dda606228dd9ceeee920842c8c336b43..80e8ab13bbff719b64dc7e6bdb770371829901ea 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -1137,32 +1137,10 @@ pub fn refAllDeclsRecursive(comptime T: type) void { } } -const FuzzerSlice = extern struct { - ptr: [*]const u8, - len: usize, - - fn toSlice(s: FuzzerSlice) []const u8 { - return s.ptr[0..s.len]; - } -}; - -extern fn fuzzer_next() FuzzerSlice; - pub const FuzzInputOptions = struct { corpus: []const []const u8 = &.{}, }; -pub fn fuzzInput(options: FuzzInputOptions) []const u8 { - @disableInstrumentation(); - if (builtin.fuzz) { - return fuzzer_next().toSlice(); - } else { - if (options.corpus.len == 0) { - return ""; - } else { - var prng = std.Random.DefaultPrng.init(std.testing.random_seed); - const random = prng.random(); - return options.corpus[random.uintLessThan(usize, options.corpus.len)]; - } - } +pub inline fn fuzzInput(options: FuzzInputOptions) []const u8 { + return @import("root").fuzzInput(options); } diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 4046fe4014ec2a70401f103237ed6b2fe18a42a3..f1e564d43e02af6b8aa5d5624e7eb48aed0223b5 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -53,7 +53,7 @@ pub const Message = struct { /// - null-terminated string_bytes index /// * expected_panic_msg: [tests_len]u32, /// - null-terminated string_bytes index - /// - 0 means does not expect pani + /// - 0 means does not expect panic /// * string_bytes: [string_bytes_len]u8, pub const TestMetadata = extern struct { string_bytes_len: u32, @@ -68,7 +68,8 @@ pub const Message = struct { fail: bool, skip: bool, leak: bool, - log_err_count: u29 = 0, + fuzz: bool, + log_err_count: u28 = 0, }; }; -- 2.54.0 From 047640383e5e635ffe52ab360e03dbe08e73d025 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 23 Jul 2024 20:49:00 -0700 Subject: [PATCH 04/12] add `--fuzz` CLI argument to `zig build` This flag makes the build runner rebuild unit tests after the pipeline finishes, if it finds any unit tests. I did not make this integrate with file system watching yet. The test runner is updated to detect which tests are fuzz tests. Run step is updated to track which test indexes are fuzz tests. --- lib/compiler/build_runner.zig | 61 +++++++++++++++++++++++++++++++--- lib/compiler/test_runner.zig | 7 +++- lib/std/Build.zig | 1 + lib/std/Build/Step/Compile.zig | 32 ++++++++++++------ lib/std/Build/Step/Run.zig | 13 ++++++++ 5 files changed, 97 insertions(+), 17 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index f06ae7d4b9cb407eb5cb7a065563fbe1d62bc3a4..5f7c12fe927451c013c012da606e3a239f36c675 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -10,7 +10,8 @@ const File = std.fs.File; const Step = std.Build.Step; const Watch = std.Build.Watch; const Allocator = std.mem.Allocator; -const fatal = std.zig.fatal; +const fatal = std.process.fatal; +const runner = @This(); pub const root = @import("@build"); pub const dependencies = @import("@dependencies"); @@ -102,6 +103,7 @@ pub fn main() !void { var steps_menu = false; var output_tmp_nonce: ?[16]u8 = null; var watch = false; + var fuzz = false; var debounce_interval_ms: u16 = 50; while (nextArg(args, &arg_idx)) |arg| { @@ -234,6 +236,8 @@ pub fn main() !void { prominent_compile_errors = true; } else if (mem.eql(u8, arg, "--watch")) { watch = true; + } else if (mem.eql(u8, arg, "--fuzz")) { + fuzz = true; } else if (mem.eql(u8, arg, "-fincremental")) { graph.incremental = true; } else if (mem.eql(u8, arg, "-fno-incremental")) { @@ -353,6 +357,7 @@ pub fn main() !void { .max_rss_mutex = .{}, .skip_oom_steps = skip_oom_steps, .watch = watch, + .fuzz = fuzz, .memory_blocked_steps = std.ArrayList(*Step).init(arena), .step_stack = .{}, .prominent_compile_errors = prominent_compile_errors, @@ -394,6 +399,10 @@ pub fn main() !void { }, else => return err, }; + if (fuzz) { + startFuzzing(&run.thread_pool, run.step_stack.keys(), main_progress_node); + } + if (!watch) return cleanExit(); switch (builtin.os.tag) { @@ -430,6 +439,43 @@ pub fn main() !void { } } +fn startFuzzing(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void { + { + const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0); + defer rebuild_node.end(); + var count: usize = 0; + var wait_group: std.Thread.WaitGroup = .{}; + defer wait_group.wait(); + for (all_steps) |step| { + const run = step.cast(Step.Run) orelse continue; + if (run.fuzz_tests.items.len > 0 and run.producer != null) { + thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node }); + count += 1; + } + } + if (count == 0) { + std.debug.lockStdErr(); + std.debug.print("no fuzz tests found\n", .{}); + process.exit(2); + } + rebuild_node.setEstimatedTotalItems(count); + } + @panic("TODO do something with the rebuilt unit tests"); +} + +fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void { + const compile_step = run.producer.?; + const prog_node = parent_prog_node.start(compile_step.step.name, 0); + defer prog_node.end(); + const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| { + std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{ + compile_step.step.name, @errorName(err), + }); + return; + }; + std.debug.print("rebuilt binary: '{s}'\n", .{rebuilt_bin_path}); +} + fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { for (all_steps) |step| switch (step.state) { .dependency_failure, .failure, .skipped => step.recursiveReset(gpa), @@ -457,6 +503,7 @@ const Run = struct { max_rss_mutex: std.Thread.Mutex, skip_oom_steps: bool, watch: bool, + fuzz: bool, memory_blocked_steps: std.ArrayList(*Step), step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), prominent_compile_errors: bool, @@ -466,6 +513,11 @@ const Run = struct { summary: Summary, ttyconf: std.io.tty.Config, stderr: File, + + fn cleanExit(run: Run) void { + if (run.watch or run.fuzz) return; + return runner.cleanExit(); + } }; fn prepare( @@ -614,8 +666,7 @@ fn runStepNames( else => false, }; if (failure_count == 0 and failures_only) { - if (!run.watch) cleanExit(); - return; + return run.cleanExit(); } const ttyconf = run.ttyconf; @@ -672,8 +723,7 @@ fn runStepNames( } if (failure_count == 0) { - if (!run.watch) cleanExit(); - return; + return run.cleanExit(); } // Finally, render compile errors at the bottom of the terminal. @@ -1226,6 +1276,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void { \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss \\ --fetch Exit after fetching dependency tree \\ --watch Continuously rebuild when source files are modified + \\ --fuzz Continuously search for unit test failures \\ --debounce Delay before rebuilding after changed file detected \\ -fincremental Enable incremental compilation \\ -fno-incremental Disable incremental compilation diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 15905ca7f9718c19613e2354de2288b7dcbc998a..81eb156e73ba943458e35411a7b4a165aca8a67e 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -143,6 +143,7 @@ fn mainTerminal() void { var ok_count: usize = 0; var skip_count: usize = 0; var fail_count: usize = 0; + var fuzz_count: usize = 0; const root_node = std.Progress.start(.{ .root_name = "Test", .estimated_total_items = test_fn_list.len, @@ -168,7 +169,7 @@ fn mainTerminal() void { if (!have_tty) { std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); } - // Track in a global variable so that `fuzzInput` can see it. + is_fuzz_test = false; if (test_fn.func()) |_| { ok_count += 1; test_node.end(); @@ -198,6 +199,7 @@ fn mainTerminal() void { test_node.end(); }, } + fuzz_count += @intFromBool(is_fuzz_test); } root_node.end(); if (ok_count == test_fn_list.len) { @@ -211,6 +213,9 @@ fn mainTerminal() void { if (leaks != 0) { std.debug.print("{d} tests leaked memory.\n", .{leaks}); } + if (fuzz_count != 0) { + std.debug.print("{d} fuzz tests found.\n", .{fuzz_count}); + } if (leaks != 0 or log_err_count != 0 or fail_count != 0) { std.process.exit(1); } diff --git a/lib/std/Build.zig b/lib/std/Build.zig index f76e3263cb3a6fd08f0a691ceb3de70f856270eb..1ad2e0ee51721cdcf838f0e1c8ebe73fe8a703fc 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -977,6 +977,7 @@ pub fn addRunArtifact(b: *Build, exe: *Step.Compile) *Step.Run { // Consider that this is declarative; the run step may not be run unless a user // option is supplied. const run_step = Step.Run.create(b, b.fmt("run {s}", .{exe.name})); + run_step.producer = exe; if (exe.kind == .@"test") { if (exe.exec_cmd_args) |exec_cmd_args| { for (exec_cmd_args) |cmd_arg| { diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 4f504151c02ca3404469d3399a365b7cf6f182eb..ffb2337ac527824bbddf037f425f0ecf7d9200f1 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -1004,7 +1004,7 @@ fn getGeneratedFilePath(compile: *Compile, comptime tag_name: []const u8, asking return path; } -fn getZigArgs(compile: *Compile) ![][]const u8 { +fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { const step = &compile.step; const b = step.owner; const arena = b.allocator; @@ -1055,6 +1055,10 @@ fn getZigArgs(compile: *Compile) ![][]const u8 { try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); } + if (fuzz) { + try zig_args.append("-ffuzz"); + } + { // Stores system libraries that have already been seen for at least one // module, along with any arguments that need to be passed to the @@ -1757,7 +1761,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { const b = step.owner; const compile: *Compile = @fieldParentPtr("step", step); - const zig_args = try getZigArgs(compile); + const zig_args = try getZigArgs(compile, false); const maybe_output_bin_path = step.evalZigProcess( zig_args, @@ -1835,6 +1839,12 @@ fn make(step: *Step, options: Step.MakeOptions) !void { } } +pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 { + const zig_args = try getZigArgs(c, true); + const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false); + return maybe_output_bin_path.?; +} + pub fn doAtomicSymLinks( step: *Step, output_path: []const u8, @@ -1861,10 +1871,10 @@ pub fn doAtomicSymLinks( }; } -fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { - const pkg_config_exe = compile.graph.env_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = try compile.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore); - var list = ArrayList(PkgConfigPkg).init(compile.allocator); +fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { + const pkg_config_exe = b.graph.env_map.get("PKG_CONFIG") orelse "pkg-config"; + const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore); + var list = ArrayList(PkgConfigPkg).init(b.allocator); errdefer list.deinit(); var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); while (line_it.next()) |line| { @@ -1878,13 +1888,13 @@ fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunE return list.toOwnedSlice(); } -fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg { - if (compile.pkg_config_pkg_list) |res| { +fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { + if (b.pkg_config_pkg_list) |res| { return res; } var code: u8 = undefined; - if (execPkgConfigList(compile, &code)) |list| { - compile.pkg_config_pkg_list = list; + if (execPkgConfigList(b, &code)) |list| { + b.pkg_config_pkg_list = list; return list; } else |err| { const result = switch (err) { @@ -1896,7 +1906,7 @@ fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg { error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, else => return err, }; - compile.pkg_config_pkg_list = result; + b.pkg_config_pkg_list = result; return result; } } diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index dbb865047be766d276588646ac3fccc3ec7d5601..7927ac9479a1e595db4e8944654f1ce350020943 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -86,6 +86,13 @@ dep_output_file: ?*Output, has_side_effects: bool, +/// If this is a Zig unit test binary, this tracks the indexes of the unit +/// tests that are also fuzz tests. +fuzz_tests: std.ArrayListUnmanaged(u32), + +/// If this Run step was produced by a Compile step, it is tracked here. +producer: ?*Step.Compile, + pub const StdIn = union(enum) { none, bytes: []const u8, @@ -175,6 +182,8 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { .captured_stderr = null, .dep_output_file = null, .has_side_effects = false, + .fuzz_tests = .{}, + .producer = null, }; return run; } @@ -1347,6 +1356,8 @@ fn evalZigTest( var sub_prog_node: ?std.Progress.Node = null; defer if (sub_prog_node) |n| n.end(); + run.fuzz_tests.clearRetainingCapacity(); + poll: while (true) { while (stdout.readableLength() < @sizeOf(Header)) { if (!(try poller.poll())) break :poll; @@ -1404,6 +1415,8 @@ fn evalZigTest( leak_count +|= @intFromBool(tr_hdr.flags.leak); log_err_count +|= tr_hdr.flags.log_err_count; + if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index); + if (tr_hdr.flags.fail or tr_hdr.flags.leak or tr_hdr.flags.log_err_count > 0) { const name = std.mem.sliceTo(md.string_bytes[md.names[tr_hdr.index]..], 0); const orig_msg = stderr.readableSlice(0); -- 2.54.0 From 711ed56ce361fd9051fcf6039de48022b8dbc2d1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 23 Jul 2024 21:17:14 -0700 Subject: [PATCH 05/12] build runner: extract logic to std.Build.Fuzz --- lib/compiler/build_runner.zig | 40 ++---------------------------- lib/std/Build.zig | 1 + lib/std/Build/Fuzz.zig | 46 +++++++++++++++++++++++++++++++++++ lib/std/Build/Step/Run.zig | 4 +++ 4 files changed, 53 insertions(+), 38 deletions(-) create mode 100644 lib/std/Build/Fuzz.zig diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 5f7c12fe927451c013c012da606e3a239f36c675..e27b4e87620b5db94257bd6bdac22a61887353c0 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -9,6 +9,7 @@ const ArrayList = std.ArrayList; const File = std.fs.File; const Step = std.Build.Step; const Watch = std.Build.Watch; +const Fuzz = std.Build.Fuzz; const Allocator = std.mem.Allocator; const fatal = std.process.fatal; const runner = @This(); @@ -400,7 +401,7 @@ pub fn main() !void { else => return err, }; if (fuzz) { - startFuzzing(&run.thread_pool, run.step_stack.keys(), main_progress_node); + Fuzz.start(&run.thread_pool, run.step_stack.keys(), main_progress_node); } if (!watch) return cleanExit(); @@ -439,43 +440,6 @@ pub fn main() !void { } } -fn startFuzzing(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void { - { - const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0); - defer rebuild_node.end(); - var count: usize = 0; - var wait_group: std.Thread.WaitGroup = .{}; - defer wait_group.wait(); - for (all_steps) |step| { - const run = step.cast(Step.Run) orelse continue; - if (run.fuzz_tests.items.len > 0 and run.producer != null) { - thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node }); - count += 1; - } - } - if (count == 0) { - std.debug.lockStdErr(); - std.debug.print("no fuzz tests found\n", .{}); - process.exit(2); - } - rebuild_node.setEstimatedTotalItems(count); - } - @panic("TODO do something with the rebuilt unit tests"); -} - -fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void { - const compile_step = run.producer.?; - const prog_node = parent_prog_node.start(compile_step.step.name, 0); - defer prog_node.end(); - const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| { - std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{ - compile_step.step.name, @errorName(err), - }); - return; - }; - std.debug.print("rebuilt binary: '{s}'\n", .{rebuilt_bin_path}); -} - fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { for (all_steps) |step| switch (step.state) { .dependency_failure, .failure, .skipped => step.recursiveReset(gpa), diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 1ad2e0ee51721cdcf838f0e1c8ebe73fe8a703fc..8a4f3e87dd29caa89779f0d8b6c8234ae85b9db1 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -21,6 +21,7 @@ pub const Cache = @import("Build/Cache.zig"); pub const Step = @import("Build/Step.zig"); pub const Module = @import("Build/Module.zig"); pub const Watch = @import("Build/Watch.zig"); +pub const Fuzz = @import("Build/Fuzz.zig"); /// Shared state among all Build instances. graph: *Graph, diff --git a/lib/std/Build/Fuzz.zig b/lib/std/Build/Fuzz.zig new file mode 100644 index 0000000000000000000000000000000000000000..a0e754fab10547a18d6841cd8e96d6341289014d --- /dev/null +++ b/lib/std/Build/Fuzz.zig @@ -0,0 +1,46 @@ +const std = @import("../std.zig"); +const Fuzz = @This(); +const Step = std.Build.Step; +const assert = std.debug.assert; +const fatal = std.process.fatal; + +pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void { + { + const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0); + defer rebuild_node.end(); + var count: usize = 0; + var wait_group: std.Thread.WaitGroup = .{}; + defer wait_group.wait(); + for (all_steps) |step| { + const run = step.cast(Step.Run) orelse continue; + if (run.fuzz_tests.items.len > 0 and run.producer != null) { + thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node }); + count += 1; + } + } + if (count == 0) fatal("no fuzz tests found", .{}); + rebuild_node.setEstimatedTotalItems(count); + } + + // Detect failure. + for (all_steps) |step| { + const run = step.cast(Step.Run) orelse continue; + if (run.fuzz_tests.items.len > 0 and run.rebuilt_executable == null) + fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); + } + + @panic("TODO do something with the rebuilt unit tests"); +} + +fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void { + const compile_step = run.producer.?; + const prog_node = parent_prog_node.start(compile_step.step.name, 0); + defer prog_node.end(); + const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| { + std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{ + compile_step.step.name, @errorName(err), + }); + return; + }; + run.rebuilt_executable = rebuilt_bin_path; +} diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 7927ac9479a1e595db4e8944654f1ce350020943..9b7997306f29890c57c79b952d4b88b4003f5c82 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -89,6 +89,9 @@ has_side_effects: bool, /// If this is a Zig unit test binary, this tracks the indexes of the unit /// tests that are also fuzz tests. fuzz_tests: std.ArrayListUnmanaged(u32), +/// Populated during the fuzz phase if this run step corresponds to a unit test +/// executable that contains fuzz tests. +rebuilt_executable: ?[]const u8, /// If this Run step was produced by a Compile step, it is tracked here. producer: ?*Step.Compile, @@ -183,6 +186,7 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { .dep_output_file = null, .has_side_effects = false, .fuzz_tests = .{}, + .rebuilt_executable = null, .producer = null, }; return run; -- 2.54.0 From bce3b1efb0879ba2f0da4d215c3190f3e8a4345b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 23 Jul 2024 22:59:25 -0700 Subject: [PATCH 06/12] build runner sends a start_fuzzing message to test runner --- lib/compiler/build_runner.zig | 16 +++--- lib/std/Build/Fuzz.zig | 82 +++++++++++++++++++++++++----- lib/std/Build/Step/Run.zig | 96 ++++++++++++++++++++++++++++++----- lib/std/zig/Client.zig | 3 ++ 4 files changed, 166 insertions(+), 31 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index e27b4e87620b5db94257bd6bdac22a61887353c0..8c22e6d29167031353d2d9e9400683a16db62196 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -401,7 +401,7 @@ pub fn main() !void { else => return err, }; if (fuzz) { - Fuzz.start(&run.thread_pool, run.step_stack.keys(), main_progress_node); + Fuzz.start(&run.thread_pool, run.step_stack.keys(), run.ttyconf, main_progress_node); } if (!watch) return cleanExit(); @@ -1072,7 +1072,7 @@ fn workerMakeOneStep( std.debug.lockStdErr(); defer std.debug.unlockStdErr(); - printErrorMessages(b, s, run) catch {}; + printErrorMessages(b, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {}; } handle_result: { @@ -1125,10 +1125,14 @@ fn workerMakeOneStep( } } -fn printErrorMessages(b: *std.Build, failing_step: *Step, run: *const Run) !void { +pub fn printErrorMessages( + b: *std.Build, + failing_step: *Step, + ttyconf: std.io.tty.Config, + stderr: File, + prominent_compile_errors: bool, +) !void { const gpa = b.allocator; - const stderr = run.stderr; - const ttyconf = run.ttyconf; // Provide context for where these error messages are coming from by // printing the corresponding Step subtree. @@ -1166,7 +1170,7 @@ fn printErrorMessages(b: *std.Build, failing_step: *Step, run: *const Run) !void } } - if (!run.prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0) + if (!prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0) try failing_step.result_error_bundle.renderToWriter(renderOptions(ttyconf), stderr.writer()); for (failing_step.result_error_msgs.items) |msg| { diff --git a/lib/std/Build/Fuzz.zig b/lib/std/Build/Fuzz.zig index a0e754fab10547a18d6841cd8e96d6341289014d..458d7e5d8b0aadd5cf3acfcacefe750b2fbd34b9 100644 --- a/lib/std/Build/Fuzz.zig +++ b/lib/std/Build/Fuzz.zig @@ -3,9 +3,15 @@ const Fuzz = @This(); const Step = std.Build.Step; const assert = std.debug.assert; const fatal = std.process.fatal; +const build_runner = @import("root"); -pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void { - { +pub fn start( + thread_pool: *std.Thread.Pool, + all_steps: []const *Step, + ttyconf: std.io.tty.Config, + prog_node: std.Progress.Node, +) void { + const count = block: { const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0); defer rebuild_node.end(); var count: usize = 0; @@ -14,13 +20,14 @@ pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: for (all_steps) |step| { const run = step.cast(Step.Run) orelse continue; if (run.fuzz_tests.items.len > 0 and run.producer != null) { - thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node }); + thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, ttyconf, rebuild_node }); count += 1; } } if (count == 0) fatal("no fuzz tests found", .{}); rebuild_node.setEstimatedTotalItems(count); - } + break :block count; + }; // Detect failure. for (all_steps) |step| { @@ -29,18 +36,69 @@ pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); } - @panic("TODO do something with the rebuilt unit tests"); + { + const rebuild_node = prog_node.start("Fuzzing", count); + defer rebuild_node.end(); + var wait_group: std.Thread.WaitGroup = .{}; + defer wait_group.wait(); + + for (all_steps) |step| { + const run = step.cast(Step.Run) orelse continue; + for (run.fuzz_tests.items) |unit_test_index| { + assert(run.rebuilt_executable != null); + thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{ run, unit_test_index, ttyconf, prog_node }); + } + } + } + + fatal("all fuzz workers crashed", .{}); } -fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void { +fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void { const compile_step = run.producer.?; const prog_node = parent_prog_node.start(compile_step.step.name, 0); defer prog_node.end(); - const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| { - std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{ - compile_step.step.name, @errorName(err), - }); - return; + if (compile_step.rebuildInFuzzMode(prog_node)) |rebuilt_bin_path| { + run.rebuilt_executable = rebuilt_bin_path; + } else |err| switch (err) { + error.MakeFailed => { + const b = run.step.owner; + const stderr = std.io.getStdErr(); + std.debug.lockStdErr(); + defer std.debug.unlockStdErr(); + build_runner.printErrorMessages(b, &compile_step.step, ttyconf, stderr, false) catch {}; + }, + else => { + std.debug.print("step '{s}': failed to rebuild in fuzz mode: {s}\n", .{ + compile_step.step.name, @errorName(err), + }); + }, + } +} + +fn fuzzWorkerRun( + run: *Step.Run, + unit_test_index: u32, + ttyconf: std.io.tty.Config, + parent_prog_node: std.Progress.Node, +) void { + const test_name = run.cached_test_metadata.?.testName(unit_test_index); + + const prog_node = parent_prog_node.start(test_name, 0); + defer prog_node.end(); + + run.rerunInFuzzMode(unit_test_index, prog_node) catch |err| switch (err) { + error.MakeFailed => { + const b = run.step.owner; + const stderr = std.io.getStdErr(); + std.debug.lockStdErr(); + defer std.debug.unlockStdErr(); + build_runner.printErrorMessages(b, &run.step, ttyconf, stderr, false) catch {}; + }, + else => { + std.debug.print("step '{s}': failed to rebuild '{s}' in fuzz mode: {s}\n", .{ + run.step.name, test_name, @errorName(err), + }); + }, }; - run.rebuilt_executable = rebuilt_bin_path; } diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 9b7997306f29890c57c79b952d4b88b4003f5c82..43e3ddedfde5fc2bd39d17619fe62f4817ff94d2 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -89,6 +89,8 @@ has_side_effects: bool, /// If this is a Zig unit test binary, this tracks the indexes of the unit /// tests that are also fuzz tests. fuzz_tests: std.ArrayListUnmanaged(u32), +cached_test_metadata: ?CachedTestMetadata = null, + /// Populated during the fuzz phase if this run step corresponds to a unit test /// executable that contains fuzz tests. rebuilt_executable: ?[]const u8, @@ -754,7 +756,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { b.fmt("{s}{s}", .{ placeholder.output.prefix, output_path }); } - try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node); + try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node, null); if (!has_side_effects) try step.writeManifestAndWatch(&man); return; }; @@ -784,7 +786,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { b.fmt("{s}{s}", .{ placeholder.output.prefix, output_path }); } - try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node); + try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, null); const dep_file_dir = std.fs.cwd(); const dep_file_basename = dep_output_file.generated_file.getPath(); @@ -843,6 +845,38 @@ fn make(step: *Step, options: Step.MakeOptions) !void { ); } +pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress.Node) !void { + const step = &run.step; + const b = step.owner; + const arena = b.allocator; + var argv_list: std.ArrayListUnmanaged([]const u8) = .{}; + for (run.argv.items) |arg| { + switch (arg) { + .bytes => |bytes| { + try argv_list.append(arena, bytes); + }, + .lazy_path => |file| { + const file_path = file.lazy_path.getPath2(b, step); + try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, file_path })); + }, + .directory_source => |file| { + const file_path = file.lazy_path.getPath2(b, step); + try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, file_path })); + }, + .artifact => |pa| { + const artifact = pa.artifact; + const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; + try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path })); + }, + .output_file, .output_directory => unreachable, + } + } + const has_side_effects = false; + const rand_int = std.crypto.random.int(u64); + const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); + try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, unit_test_index); +} + fn populateGeneratedPaths( arena: std.mem.Allocator, output_placeholders: []const IndexedOutput, @@ -921,6 +955,7 @@ fn runCommand( has_side_effects: bool, output_dir_path: []const u8, prog_node: std.Progress.Node, + fuzz_unit_test_index: ?u32, ) !void { const step = &run.step; const b = step.owner; @@ -939,7 +974,7 @@ fn runCommand( var interp_argv = std.ArrayList([]const u8).init(b.allocator); defer interp_argv.deinit(); - const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node) catch |err| term: { + const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node, fuzz_unit_test_index) catch |err| term: { // InvalidExe: cpu arch mismatch // FileNotFound: can happen with a wrong dynamic linker path if (err == error.InvalidExe or err == error.FileNotFound) interpret: { @@ -1075,7 +1110,7 @@ fn runCommand( try Step.handleVerbose2(step.owner, cwd, run.env_map, interp_argv.items); - break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node) catch |e| { + break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node, fuzz_unit_test_index) catch |e| { if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; return step.fail("unable to spawn interpreter {s}: {s}", .{ @@ -1090,6 +1125,15 @@ fn runCommand( step.result_duration_ns = result.elapsed_ns; step.result_peak_rss = result.peak_rss; step.test_results = result.stdio.test_results; + if (result.stdio.test_metadata) |tm| + run.cached_test_metadata = tm.toCachedTestMetadata(); + + const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items; + + if (fuzz_unit_test_index != null) { + try step.handleChildProcessTerm(result.term, cwd, final_argv); + return; + } // Capture stdout and stderr to GeneratedFile objects. const Stream = struct { @@ -1126,8 +1170,6 @@ fn runCommand( } } - const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items; - switch (run.stdio) { .check => |checks| for (checks.items) |check| switch (check) { .expect_stderr_exact => |expected_bytes| { @@ -1253,10 +1295,16 @@ fn spawnChildAndCollect( argv: []const []const u8, has_side_effects: bool, prog_node: std.Progress.Node, + fuzz_unit_test_index: ?u32, ) !ChildProcResult { const b = run.step.owner; const arena = b.allocator; + if (fuzz_unit_test_index != null) { + assert(!has_side_effects); + assert(run.stdio == .zig_test); + } + var child = std.process.Child.init(argv, arena); if (run.cwd) |lazy_cwd| { child.cwd = lazy_cwd.getPath2(b, &run.step); @@ -1306,7 +1354,7 @@ fn spawnChildAndCollect( var timer = try std.time.Timer.start(); const result = if (run.stdio == .zig_test) - evalZigTest(run, &child, prog_node) + evalZigTest(run, &child, prog_node, fuzz_unit_test_index) else evalGeneric(run, &child); @@ -1332,6 +1380,7 @@ fn evalZigTest( run: *Run, child: *std.process.Child, prog_node: std.Progress.Node, + fuzz_unit_test_index: ?u32, ) !StdIoResult { const gpa = run.step.owner.allocator; const arena = run.step.owner.allocator; @@ -1342,7 +1391,12 @@ fn evalZigTest( }); defer poller.deinit(); - try sendMessage(child.stdin.?, .query_test_metadata); + if (fuzz_unit_test_index) |index| { + try sendRunTestMessage(child.stdin.?, .start_fuzzing, index); + } else { + run.fuzz_tests.clearRetainingCapacity(); + try sendMessage(child.stdin.?, .query_test_metadata); + } const Header = std.zig.Server.Message.Header; @@ -1360,8 +1414,6 @@ fn evalZigTest( var sub_prog_node: ?std.Progress.Node = null; defer if (sub_prog_node) |n| n.end(); - run.fuzz_tests.clearRetainingCapacity(); - poll: while (true) { while (stdout.readableLength() < @sizeOf(Header)) { if (!(try poller.poll())) break :poll; @@ -1382,6 +1434,7 @@ fn evalZigTest( } }, .test_metadata => { + assert(fuzz_unit_test_index == null); const TmHdr = std.zig.Server.Message.TestMetadata; const tm_hdr = @as(*align(1) const TmHdr, @ptrCast(body)); test_count = tm_hdr.tests_len; @@ -1410,6 +1463,7 @@ fn evalZigTest( try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node); }, .test_results => { + assert(fuzz_unit_test_index == null); const md = metadata.?; const TrHdr = std.zig.Server.Message.TestResults; @@ -1479,7 +1533,23 @@ const TestMetadata = struct { next_index: u32, prog_node: std.Progress.Node, + fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata { + return .{ + .names = tm.names, + .string_bytes = tm.string_bytes, + }; + } + fn testName(tm: TestMetadata, index: u32) []const u8 { + return tm.toCachedTestMetadata().testName(index); + } +}; + +pub const CachedTestMetadata = struct { + names: []const u32, + string_bytes: []const u8, + + pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 { return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); } }; @@ -1495,7 +1565,7 @@ fn requestNextTest(in: fs.File, metadata: *TestMetadata, sub_prog_node: *?std.Pr if (sub_prog_node.*) |n| n.end(); sub_prog_node.* = metadata.prog_node.start(name, 0); - try sendRunTestMessage(in, i); + try sendRunTestMessage(in, .run_test, i); return; } else { try sendMessage(in, .exit); @@ -1510,9 +1580,9 @@ fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { try file.writeAll(std.mem.asBytes(&header)); } -fn sendRunTestMessage(file: std.fs.File, index: u32) !void { +fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index: u32) !void { const header: std.zig.Client.Message.Header = .{ - .tag = .run_test, + .tag = tag, .bytes_len = 4, }; const full_msg = std.mem.asBytes(&header) ++ std.mem.asBytes(&index); diff --git a/lib/std/zig/Client.zig b/lib/std/zig/Client.zig index af4c29d37d6d36a84cf3b0a08c9ebe350fdb5f67..345b9f97973b68bb3a271bb867d35da69f0b2a58 100644 --- a/lib/std/zig/Client.zig +++ b/lib/std/zig/Client.zig @@ -33,6 +33,9 @@ pub const Message = struct { /// Ask the test runner to run a particular test. /// The message body is a u32 test index. run_test, + /// Ask the test runner to start fuzzing a particular test. + /// The message body is a u32 test index. + start_fuzzing, _, }; -- 2.54.0 From 7366b4b9e2b9495ab79fd7edbf3d88dcbc8d6593 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 23 Jul 2024 23:23:54 -0700 Subject: [PATCH 07/12] test runner: handle start_fuzzing message --- lib/compiler/test_runner.zig | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 81eb156e73ba943458e35411a7b4a165aca8a67e..d2e0ce27493344d39bfd5c954ba7def8e2320841 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -128,6 +128,28 @@ fn mainServer() !void { }, }); }, + .start_fuzzing => { + const index = try server.receiveBody_u32(); + const test_fn = builtin.test_functions[index]; + while (true) { + testing.allocator_instance = .{}; + defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); + log_err_count = 0; + is_fuzz_test = false; + test_fn.func() catch |err| switch (err) { + error.SkipZigTest => continue, + else => { + if (@errorReturnTrace()) |trace| { + std.debug.dumpStackTrace(trace.*); + } + std.debug.print("failed with error.{s}\n", .{@errorName(err)}); + std.process.exit(1); + }, + }; + if (!is_fuzz_test) @panic("missed call to std.testing.fuzzInput"); + if (log_err_count != 0) @panic("error logs detected"); + } + }, else => { std.debug.print("unsupported message: {x}\n", .{@intFromEnum(hdr.tag)}); -- 2.54.0 From b501adccbe7603bbd35b5babb545bbedc5de2f09 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 24 Jul 2024 01:02:43 -0700 Subject: [PATCH 08/12] std.Build.Fuzz: fix progress node hierarchy --- lib/std/Build/Fuzz.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/std/Build/Fuzz.zig b/lib/std/Build/Fuzz.zig index 458d7e5d8b0aadd5cf3acfcacefe750b2fbd34b9..7a86e3857b9667c56a707f7e84c631bd5636abce 100644 --- a/lib/std/Build/Fuzz.zig +++ b/lib/std/Build/Fuzz.zig @@ -37,8 +37,8 @@ pub fn start( } { - const rebuild_node = prog_node.start("Fuzzing", count); - defer rebuild_node.end(); + const fuzz_node = prog_node.start("Fuzzing", count); + defer fuzz_node.end(); var wait_group: std.Thread.WaitGroup = .{}; defer wait_group.wait(); @@ -46,7 +46,7 @@ pub fn start( const run = step.cast(Step.Run) orelse continue; for (run.fuzz_tests.items) |unit_test_index| { assert(run.rebuilt_executable != null); - thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{ run, unit_test_index, ttyconf, prog_node }); + thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{ run, unit_test_index, ttyconf, fuzz_node }); } } } -- 2.54.0 From 90dfd86ebee1639e5455ace4e34157ff9b68ac0f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 24 Jul 2024 17:40:58 -0700 Subject: [PATCH 09/12] test runner: always report fuzz tests This way they can be smoke tested. --- lib/compiler/test_runner.zig | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index d2e0ce27493344d39bfd5c954ba7def8e2320841..25f029a183717a38ca4d0bc10517d3d248f49748 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -325,18 +325,11 @@ extern fn fuzzer_next() FuzzerSlice; pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 { @disableInstrumentation(); - if (crippled) { - return ""; - } else if (builtin.fuzz) { - return fuzzer_next().toSlice(); - } else { - is_fuzz_test = true; - if (options.corpus.len == 0) { - return ""; - } else { - var prng = std.Random.DefaultPrng.init(testing.random_seed); - const random = prng.random(); - return options.corpus[random.uintLessThan(usize, options.corpus.len)]; - } - } + if (crippled) return ""; + is_fuzz_test = true; + if (builtin.fuzz) return fuzzer_next().toSlice(); + if (options.corpus.len == 0) return ""; + var prng = std.Random.DefaultPrng.init(testing.random_seed); + const random = prng.random(); + return options.corpus[random.uintLessThan(usize, options.corpus.len)]; } -- 2.54.0 From a3c74aca99c7eaf719c745ec6e9ee7366cad1910 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 24 Jul 2024 17:41:44 -0700 Subject: [PATCH 10/12] add --debug-rt CLI arg to the compiler + bonus edits The flag makes compiler_rt and libfuzzer be in debug mode. Also: * fuzzer: override debug logs and disable debug logs for frequently called functions * std.Build.Fuzz: fix bug of rerunning the old unit test binary * report errors from rebuilding the unit tests better * link.Elf: additionally add tsan lib and fuzzer lib to the hash --- lib/compiler/build_runner.zig | 10 +++-- lib/fuzzer.zig | 72 +++++++++++++++++++++++++++++----- lib/std/Build.zig | 1 + lib/std/Build/Fuzz.zig | 36 +++++++++++------ lib/std/Build/Step/Compile.zig | 10 +++++ lib/std/Build/Step/Run.zig | 5 ++- src/Compilation.zig | 4 +- src/link/Elf.zig | 2 + src/main.zig | 5 +++ 9 files changed, 116 insertions(+), 29 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 8c22e6d29167031353d2d9e9400683a16db62196..91d7b8a043d412745404982b458a7b245d6ec60a 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -208,6 +208,8 @@ pub fn main() !void { try debug_log_scopes.append(next_arg); } else if (mem.eql(u8, arg, "--debug-pkg-config")) { builder.debug_pkg_config = true; + } else if (mem.eql(u8, arg, "--debug-rt")) { + graph.debug_compiler_runtime_libs = true; } else if (mem.eql(u8, arg, "--debug-compile-errors")) { builder.debug_compile_errors = true; } else if (mem.eql(u8, arg, "--system")) { @@ -1072,7 +1074,8 @@ fn workerMakeOneStep( std.debug.lockStdErr(); defer std.debug.unlockStdErr(); - printErrorMessages(b, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {}; + const gpa = b.allocator; + printErrorMessages(gpa, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {}; } handle_result: { @@ -1126,14 +1129,12 @@ fn workerMakeOneStep( } pub fn printErrorMessages( - b: *std.Build, + gpa: Allocator, failing_step: *Step, ttyconf: std.io.tty.Config, stderr: File, prominent_compile_errors: bool, ) !void { - const gpa = b.allocator; - // Provide context for where these error messages are coming from by // printing the corresponding Step subtree. @@ -1313,6 +1314,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void { \\ --seed [integer] For shuffling dependency traversal order (default: random) \\ --debug-log [scope] Enable debugging the compiler \\ --debug-pkg-config Fail if unknown pkg-config flags encountered + \\ --debug-rt Debug compiler runtime libraries \\ --verbose-link Enable compiler debug output for linking \\ --verbose-air Enable compiler debug output for Zig AIR \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 8329d1eb2e98485050e996fc749dcb7d8ede1e81..26ef66d1c13d38e54826f9f346a84e3c7d20504a 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -1,14 +1,40 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +pub const std_options = .{ + .logFn = logOverride, +}; + +var log_file: ?std.fs.File = null; + +fn logOverride( + comptime level: std.log.Level, + comptime scope: @TypeOf(.EnumLiteral), + comptime format: []const u8, + args: anytype, +) void { + const f = if (log_file) |f| f else f: { + const f = std.fs.cwd().createFile("libfuzzer.log", .{}) catch @panic("failed to open fuzzer log file"); + log_file = f; + break :f f; + }; + const prefix1 = comptime level.asText(); + const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; + f.writer().print(prefix1 ++ prefix2 ++ format ++ "\n", args) catch @panic("failed to write to fuzzer log"); +} + export threadlocal var __sancov_lowest_stack: usize = 0; export fn __sanitizer_cov_8bit_counters_init(start: [*]u8, stop: [*]u8) void { std.log.debug("__sanitizer_cov_8bit_counters_init start={*}, stop={*}", .{ start, stop }); } -export fn __sanitizer_cov_pcs_init(pcs_beg: [*]const usize, pcs_end: [*]const usize) void { - std.log.debug("__sanitizer_cov_pcs_init pcs_beg={*}, pcs_end={*}", .{ pcs_beg, pcs_end }); +export fn __sanitizer_cov_pcs_init(pc_start: [*]const usize, pc_end: [*]const usize) void { + std.log.debug("__sanitizer_cov_pcs_init pc_start={*}, pc_end={*}", .{ pc_start, pc_end }); + fuzzer.pc_range = .{ + .start = @intFromPtr(pc_start), + .end = @intFromPtr(pc_start), + }; } export fn __sanitizer_cov_trace_const_cmp1(arg1: u8, arg2: u8) void { @@ -48,34 +74,45 @@ export fn __sanitizer_cov_trace_switch(val: u64, cases_ptr: [*]u64) void { const len = cases_ptr[0]; const val_size_in_bits = cases_ptr[1]; const cases = cases_ptr[2..][0..len]; - std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{ - pc, val, val_size_in_bits, cases.len, - }); + _ = val; + _ = pc; + _ = val_size_in_bits; + _ = cases; + //std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{ + // pc, val, val_size_in_bits, cases.len, + //}); } export fn __sanitizer_cov_trace_pc_indir(callee: usize) void { const pc = @returnAddress(); - std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee }); + _ = callee; + _ = pc; + //std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee }); } fn handleCmp(pc: usize, arg1: u64, arg2: u64) void { - std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 }); + _ = pc; + _ = arg1; + _ = arg2; + //std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 }); } const Fuzzer = struct { gpa: Allocator, rng: std.Random.DefaultPrng, input: std.ArrayListUnmanaged(u8), + pc_range: PcRange, + count: usize, const Slice = extern struct { ptr: [*]const u8, len: usize, - fn toSlice(s: Slice) []const u8 { + fn toZig(s: Slice) []const u8 { return s.ptr[0..s.len]; } - fn fromSlice(s: []const u8) Slice { + fn fromZig(s: []const u8) Slice { return .{ .ptr = s.ptr, .len = s.len, @@ -83,14 +120,27 @@ const Fuzzer = struct { } }; + const PcRange = struct { + start: usize, + end: usize, + }; + fn next(f: *Fuzzer) ![]const u8 { const gpa = f.gpa; + + // Prepare next input. const rng = fuzzer.rng.random(); const len = rng.uintLessThan(usize, 64); try f.input.resize(gpa, len); rng.bytes(f.input.items); + f.resetCoverage(); + f.count += 1; return f.input.items; } + + fn resetCoverage(f: *Fuzzer) void { + _ = f; + } }; var general_purpose_allocator: std.heap.GeneralPurposeAllocator(.{}) = .{}; @@ -99,10 +149,12 @@ var fuzzer: Fuzzer = .{ .gpa = general_purpose_allocator.allocator(), .rng = std.Random.DefaultPrng.init(0), .input = .{}, + .pc_range = .{ .start = 0, .end = 0 }, + .count = 0, }; export fn fuzzer_next() Fuzzer.Slice { - return Fuzzer.Slice.fromSlice(fuzzer.next() catch |err| switch (err) { + return Fuzzer.Slice.fromZig(fuzzer.next() catch |err| switch (err) { error.OutOfMemory => @panic("out of memory"), }); } diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 8a4f3e87dd29caa89779f0d8b6c8234ae85b9db1..e2e417d178610670714eac9e5cbd1e941d542ff6 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -113,6 +113,7 @@ pub const Graph = struct { arena: Allocator, system_library_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .{}, system_package_mode: bool = false, + debug_compiler_runtime_libs: bool = false, cache: Cache, zig_exe: [:0]const u8, env_map: EnvMap, diff --git a/lib/std/Build/Fuzz.zig b/lib/std/Build/Fuzz.zig index 7a86e3857b9667c56a707f7e84c631bd5636abce..2628b9251621b0762f1f38b3181420a2c6bb170b 100644 --- a/lib/std/Build/Fuzz.zig +++ b/lib/std/Build/Fuzz.zig @@ -55,22 +55,32 @@ pub fn start( } fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void { - const compile_step = run.producer.?; - const prog_node = parent_prog_node.start(compile_step.step.name, 0); + const gpa = run.step.owner.allocator; + const stderr = std.io.getStdErr(); + + const compile = run.producer.?; + const prog_node = parent_prog_node.start(compile.step.name, 0); defer prog_node.end(); - if (compile_step.rebuildInFuzzMode(prog_node)) |rebuilt_bin_path| { + + const result = compile.rebuildInFuzzMode(prog_node); + + const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0; + const show_error_msgs = compile.step.result_error_msgs.items.len > 0; + const show_stderr = compile.step.result_stderr.len > 0; + + if (show_error_msgs or show_compile_errors or show_stderr) { + std.debug.lockStdErr(); + defer std.debug.unlockStdErr(); + build_runner.printErrorMessages(gpa, &compile.step, ttyconf, stderr, false) catch {}; + } + + if (result) |rebuilt_bin_path| { run.rebuilt_executable = rebuilt_bin_path; } else |err| switch (err) { - error.MakeFailed => { - const b = run.step.owner; - const stderr = std.io.getStdErr(); - std.debug.lockStdErr(); - defer std.debug.unlockStdErr(); - build_runner.printErrorMessages(b, &compile_step.step, ttyconf, stderr, false) catch {}; - }, + error.MakeFailed => {}, else => { std.debug.print("step '{s}': failed to rebuild in fuzz mode: {s}\n", .{ - compile_step.step.name, @errorName(err), + compile.step.name, @errorName(err), }); }, } @@ -82,6 +92,7 @@ fn fuzzWorkerRun( ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node, ) void { + const gpa = run.step.owner.allocator; const test_name = run.cached_test_metadata.?.testName(unit_test_index); const prog_node = parent_prog_node.start(test_name, 0); @@ -89,11 +100,10 @@ fn fuzzWorkerRun( run.rerunInFuzzMode(unit_test_index, prog_node) catch |err| switch (err) { error.MakeFailed => { - const b = run.step.owner; const stderr = std.io.getStdErr(); std.debug.lockStdErr(); defer std.debug.unlockStdErr(); - build_runner.printErrorMessages(b, &run.step, ttyconf, stderr, false) catch {}; + build_runner.printErrorMessages(gpa, &run.step, ttyconf, stderr, false) catch {}; }, else => { std.debug.print("step '{s}': failed to rebuild '{s}' in fuzz mode: {s}\n", .{ diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index ffb2337ac527824bbddf037f425f0ecf7d9200f1..8a418760c10f9708d723bcd2f412650cc35a121b 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -1483,6 +1483,8 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { try zig_args.append("--global-cache-dir"); try zig_args.append(b.graph.global_cache_root.path orelse "."); + if (b.graph.debug_compiler_runtime_libs) try zig_args.append("--debug-rt"); + try zig_args.append("--name"); try zig_args.append(compile.name); @@ -1840,6 +1842,14 @@ fn make(step: *Step, options: Step.MakeOptions) !void { } pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 { + const gpa = c.step.owner.allocator; + + c.step.result_error_msgs.clearRetainingCapacity(); + c.step.result_stderr = ""; + + c.step.result_error_bundle.deinit(gpa); + c.step.result_error_bundle = std.zig.ErrorBundle.empty; + const zig_args = try getZigArgs(c, true); const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false); return maybe_output_bin_path.?; diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 43e3ddedfde5fc2bd39d17619fe62f4817ff94d2..c2d25cd82cbbbdcc45608876d8a91a38252c6cea 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -865,7 +865,10 @@ pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress. }, .artifact => |pa| { const artifact = pa.artifact; - const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; + const file_path = if (artifact == run.producer.?) + run.rebuilt_executable.? + else + (artifact.installed_path orelse artifact.generated_bin.?.path.?); try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path })); }, .output_file, .output_directory => unreachable, diff --git a/src/Compilation.zig b/src/Compilation.zig index bc5a2bb4563392b4c015767fece0d4ddf04218da..8808e72e04f71d93f1f066f48c54bd63df6b0edb 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2180,7 +2180,9 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { comp.bin_file = try link.File.createEmpty(arena, comp, emit, whole.lf_open_opts); } }, - .incremental => {}, + .incremental => { + log.debug("Compilation.update for {s}, CacheMode.incremental", .{comp.root_name}); + }, } // From this point we add a preliminary set of file system inputs that diff --git a/src/link/Elf.zig b/src/link/Elf.zig index ecb38974ca87d9e2193720bb970125e96ec6b5b4..7c1d695bd94d941b8d04d77630a918fd0bccca02 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -2286,6 +2286,8 @@ fn linkWithLLD(self: *Elf, arena: Allocator, tid: Zcu.PerThread.Id, prog_node: s } try man.addOptionalFile(module_obj_path); try man.addOptionalFile(compiler_rt_path); + try man.addOptionalFile(if (comp.tsan_lib) |l| l.full_object_path else null); + try man.addOptionalFile(if (comp.fuzzer_lib) |l| l.full_object_path else null); // We can skip hashing libc and libc++ components that we are in charge of building from Zig // installation sources because they are always a product of the compiler version + target information. diff --git a/src/main.zig b/src/main.zig index 9940312bcda69477389055895dc5055e7cf62fe0..ddd2e79f44067a72b642cfabfcd156fe8371b3fb 100644 --- a/src/main.zig +++ b/src/main.zig @@ -655,6 +655,7 @@ const usage_build_generic = \\ --debug-log [scope] Enable printing debug/info log messages for scope \\ --debug-compile-errors Crash with helpful diagnostics at the first compile error \\ --debug-link-snapshot Enable dumping of the linker's state in JSON format + \\ --debug-rt Debug compiler runtime libraries \\ ; @@ -912,6 +913,7 @@ fn buildOutputType( var minor_subsystem_version: ?u16 = null; var mingw_unicode_entry_point: bool = false; var enable_link_snapshots: bool = false; + var debug_compiler_runtime_libs = false; var opt_incremental: ?bool = null; var install_name: ?[]const u8 = null; var hash_style: link.File.Elf.HashStyle = .both; @@ -1367,6 +1369,8 @@ fn buildOutputType( } else { enable_link_snapshots = true; } + } else if (mem.eql(u8, arg, "--debug-rt")) { + debug_compiler_runtime_libs = true; } else if (mem.eql(u8, arg, "-fincremental")) { dev.check(.incremental); opt_incremental = true; @@ -3408,6 +3412,7 @@ fn buildOutputType( // noise when --search-prefix and --mod are combined. .global_cc_argv = try cc_argv.toOwnedSlice(arena), .file_system_inputs = &file_system_inputs, + .debug_compiler_runtime_libs = debug_compiler_runtime_libs, }) catch |err| switch (err) { error.LibCUnavailable => { const triple_name = try target.zigTriple(arena); -- 2.54.0 From 6a6337205328168ca788d6ed0b8a011e95cbaca5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 24 Jul 2024 23:03:50 -0700 Subject: [PATCH 11/12] fuzzer: basic implementation just some experimentation. I didn't expect this to be effective so quickly but it already can find a comparison made with mem.eql --- lib/fuzzer.zig | 181 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 170 insertions(+), 11 deletions(-) diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 26ef66d1c13d38e54826f9f346a84e3c7d20504a..01cd359691bd10475ab3d6c7289bd39af5cd01b0 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -1,5 +1,7 @@ +const builtin = @import("builtin"); const std = @import("std"); const Allocator = std.mem.Allocator; +const assert = std.debug.assert; pub const std_options = .{ .logFn = logOverride, @@ -13,6 +15,7 @@ fn logOverride( comptime format: []const u8, args: anytype, ) void { + if (builtin.mode != .Debug) return; const f = if (log_file) |f| f else f: { const f = std.fs.cwd().createFile("libfuzzer.log", .{}) catch @panic("failed to open fuzzer log file"); log_file = f; @@ -75,7 +78,7 @@ export fn __sanitizer_cov_trace_switch(val: u64, cases_ptr: [*]u64) void { const val_size_in_bits = cases_ptr[1]; const cases = cases_ptr[2..][0..len]; _ = val; - _ = pc; + fuzzer.visitPc(pc); _ = val_size_in_bits; _ = cases; //std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{ @@ -86,14 +89,14 @@ export fn __sanitizer_cov_trace_switch(val: u64, cases_ptr: [*]u64) void { export fn __sanitizer_cov_trace_pc_indir(callee: usize) void { const pc = @returnAddress(); _ = callee; - _ = pc; + fuzzer.visitPc(pc); //std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee }); } fn handleCmp(pc: usize, arg1: u64, arg2: u64) void { - _ = pc; _ = arg1; _ = arg2; + fuzzer.visitPc(pc); //std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 }); } @@ -103,6 +106,46 @@ const Fuzzer = struct { input: std.ArrayListUnmanaged(u8), pc_range: PcRange, count: usize, + recent_cases: RunMap, + deduplicated_runs: usize, + coverage: Coverage, + + const RunMap = std.ArrayHashMapUnmanaged(Run, void, Run.HashContext, false); + + const Coverage = struct { + pc_table: std.AutoArrayHashMapUnmanaged(usize, void), + run_id_hasher: std.hash.Wyhash, + + fn reset(cov: *Coverage) void { + cov.pc_table.clearRetainingCapacity(); + cov.run_id_hasher = std.hash.Wyhash.init(0); + } + }; + + const Run = struct { + id: Id, + input: []const u8, + score: usize, + + const Id = u64; + + const HashContext = struct { + pub fn eql(ctx: HashContext, a: Run, b: Run, b_index: usize) bool { + _ = b_index; + _ = ctx; + return a.id == b.id; + } + pub fn hash(ctx: HashContext, a: Run) u32 { + _ = ctx; + return @truncate(a.id); + } + }; + + fn deinit(run: *Run, gpa: Allocator) void { + gpa.free(run.input); + run.* = undefined; + } + }; const Slice = extern struct { ptr: [*]const u8, @@ -125,24 +168,137 @@ const Fuzzer = struct { end: usize, }; + const Analysis = struct { + score: usize, + id: Run.Id, + }; + + fn analyzeLastRun(f: *Fuzzer) Analysis { + return .{ + .id = f.coverage.run_id_hasher.final(), + .score = f.coverage.pc_table.count(), + }; + } + fn next(f: *Fuzzer) ![]const u8 { const gpa = f.gpa; - - // Prepare next input. const rng = fuzzer.rng.random(); - const len = rng.uintLessThan(usize, 64); - try f.input.resize(gpa, len); - rng.bytes(f.input.items); - f.resetCoverage(); + + if (f.recent_cases.entries.len == 0) { + // Prepare initial input. + try f.recent_cases.ensureUnusedCapacity(gpa, 100); + const len = rng.uintLessThanBiased(usize, 80); + try f.input.resize(gpa, len); + rng.bytes(f.input.items); + f.recent_cases.putAssumeCapacity(.{ + .id = 0, + .input = try gpa.dupe(u8, f.input.items), + .score = 0, + }, {}); + } else { + if (f.count % 1000 == 0) f.dumpStats(); + + const analysis = f.analyzeLastRun(); + const gop = f.recent_cases.getOrPutAssumeCapacity(.{ + .id = analysis.id, + .input = undefined, + .score = undefined, + }); + if (gop.found_existing) { + //std.log.info("duplicate analysis: score={d} id={d}", .{ analysis.score, analysis.id }); + f.deduplicated_runs += 1; + if (f.input.items.len < gop.key_ptr.input.len or gop.key_ptr.score == 0) { + gpa.free(gop.key_ptr.input); + gop.key_ptr.input = try gpa.dupe(u8, f.input.items); + gop.key_ptr.score = analysis.score; + } + } else { + std.log.info("unique analysis: score={d} id={d}", .{ analysis.score, analysis.id }); + gop.key_ptr.* = .{ + .id = analysis.id, + .input = try gpa.dupe(u8, f.input.items), + .score = analysis.score, + }; + } + + if (f.recent_cases.entries.len >= 100) { + const Context = struct { + values: []const Run, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return ctx.values[b_index].score < ctx.values[a_index].score; + } + }; + f.recent_cases.sortUnstable(Context{ .values = f.recent_cases.keys() }); + const cap = 50; + // This has to be done before deinitializing the deleted items. + const doomed_runs = f.recent_cases.keys()[cap..]; + f.recent_cases.shrinkRetainingCapacity(cap); + for (doomed_runs) |*run| { + std.log.info("culling score={d} id={d}", .{ run.score, run.id }); + run.deinit(gpa); + } + } + } + + const chosen_index = rng.uintLessThanBiased(usize, f.recent_cases.entries.len); + const run = &f.recent_cases.keys()[chosen_index]; + f.input.clearRetainingCapacity(); + f.input.appendSliceAssumeCapacity(run.input); + try f.mutate(); + + f.coverage.reset(); f.count += 1; return f.input.items; } - fn resetCoverage(f: *Fuzzer) void { - _ = f; + fn visitPc(f: *Fuzzer, pc: usize) void { + errdefer |err| oom(err); + try f.coverage.pc_table.put(f.gpa, pc, {}); + f.coverage.run_id_hasher.update(std.mem.asBytes(&pc)); + } + + fn dumpStats(f: *Fuzzer) void { + std.log.info("stats: runs={d} deduplicated={d}", .{ + f.count, + f.deduplicated_runs, + }); + for (f.recent_cases.keys()[0..@min(f.recent_cases.entries.len, 5)], 0..) |run, i| { + std.log.info("best[{d}] id={x} score={d} input: '{}'", .{ + i, run.id, run.score, std.zig.fmtEscapes(run.input), + }); + } + } + + fn mutate(f: *Fuzzer) !void { + const gpa = f.gpa; + const rng = fuzzer.rng.random(); + + if (f.input.items.len == 0) { + const len = rng.uintLessThanBiased(usize, 80); + try f.input.resize(gpa, len); + rng.bytes(f.input.items); + return; + } + + const index = rng.uintLessThanBiased(usize, f.input.items.len * 3); + if (index < f.input.items.len) { + f.input.items[index] = rng.int(u8); + } else if (index < f.input.items.len * 2) { + _ = f.input.orderedRemove(index - f.input.items.len); + } else if (index < f.input.items.len * 3) { + try f.input.insert(gpa, index - f.input.items.len * 2, rng.int(u8)); + } else { + unreachable; + } } }; +fn oom(err: anytype) noreturn { + switch (err) { + error.OutOfMemory => @panic("out of memory"), + } +} + var general_purpose_allocator: std.heap.GeneralPurposeAllocator(.{}) = .{}; var fuzzer: Fuzzer = .{ @@ -151,6 +307,9 @@ var fuzzer: Fuzzer = .{ .input = .{}, .pc_range = .{ .start = 0, .end = 0 }, .count = 0, + .deduplicated_runs = 0, + .recent_cases = .{}, + .coverage = undefined, }; export fn fuzzer_next() Fuzzer.Slice { -- 2.54.0 From 688c2df6464bd10a2dcfdf49e89c313e01da9991 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 24 Jul 2024 23:17:06 -0700 Subject: [PATCH 12/12] fuzzer: use the cmp values seems to provide better scoring --- lib/fuzzer.zig | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 01cd359691bd10475ab3d6c7289bd39af5cd01b0..60876e0bfb25d2d4ff3521238a237ea033197425 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -94,9 +94,7 @@ export fn __sanitizer_cov_trace_pc_indir(callee: usize) void { } fn handleCmp(pc: usize, arg1: u64, arg2: u64) void { - _ = arg1; - _ = arg2; - fuzzer.visitPc(pc); + fuzzer.visitPc(pc ^ arg1 ^ arg2); //std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 }); } -- 2.54.0