authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-09-26 05:28:46-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-09-26 05:28:46-07:00
loge0dc2e4e3ffe72e5e637e30bdf1d2c59b56f3cb6
tree23cafd6ae8e80026c2f257d291b91bae932e5730
parent3b365a1f9b277dd2cf7f7dac51e71647e164ff3c
parent52a13f6a7fb0933c065348128ee3e9aecd64255b
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #25342 from ziglang/fuzz-limit

fuzzing: implement limited fuzzing

11 files changed, 445 insertions(+), 127 deletions(-)

lib/compiler/build_runner.zig+88-6
......@@ -112,7 +112,7 @@ pub fn main() !void {
112112 var steps_menu = false;
113113 var output_tmp_nonce: ?[16]u8 = null;
114114 var watch = false;
115 var fuzz = false;
115 var fuzz: ?std.Build.Fuzz.Mode = null;
116116 var debounce_interval_ms: u16 = 50;
117117 var webui_listen: ?std.net.Address = null;
118118
......@@ -274,10 +274,44 @@ pub fn main() !void {
274274 webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
275275 }
276276 } else if (mem.eql(u8, arg, "--fuzz")) {
277 fuzz = true;
277 fuzz = .{ .forever = undefined };
278278 if (webui_listen == null) {
279279 webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
280280 }
281 } else if (mem.startsWith(u8, arg, "--fuzz=")) {
282 const value = arg["--fuzz=".len..];
283 if (value.len == 0) fatal("missing argument to --fuzz", .{});
284
285 const unit: u8 = value[value.len - 1];
286 const digits = switch (unit) {
287 '0'...'9' => value,
288 'K', 'M', 'G' => value[0 .. value.len - 1],
289 else => fatal(
290 "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
291 .{},
292 ),
293 };
294
295 const amount = std.fmt.parseInt(u64, digits, 10) catch {
296 fatal(
297 "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
298 .{},
299 );
300 };
301
302 const normalized_amount = std.math.mul(u64, amount, switch (unit) {
303 else => unreachable,
304 '0'...'9' => 1,
305 'K' => 1000,
306 'M' => 1_000_000,
307 'G' => 1_000_000_000,
308 }) catch fatal("fuzzing limit amount overflows u64", .{});
309
310 fuzz = .{
311 .limit = .{
312 .amount = normalized_amount,
313 },
314 };
281315 } else if (mem.eql(u8, arg, "-fincremental")) {
282316 graph.incremental = true;
283317 } else if (mem.eql(u8, arg, "-fno-incremental")) {
......@@ -476,6 +510,7 @@ pub fn main() !void {
476510 targets.items,
477511 main_progress_node,
478512 &run,
513 fuzz,
479514 ) catch |err| switch (err) {
480515 error.UncleanExit => {
481516 assert(!run.watch and run.web_server == null);
......@@ -485,7 +520,12 @@ pub fn main() !void {
485520 };
486521
487522 if (run.web_server) |*web_server| {
488 web_server.finishBuild(.{ .fuzz = fuzz });
523 if (fuzz) |mode| if (mode != .forever) fatal(
524 "error: limited fuzzing is not implemented yet for --webui",
525 .{},
526 );
527
528 web_server.finishBuild(.{ .fuzz = fuzz != null });
489529 }
490530
491531 if (!watch and run.web_server == null) {
......@@ -651,6 +691,7 @@ fn runStepNames(
651691 step_names: []const []const u8,
652692 parent_prog_node: std.Progress.Node,
653693 run: *Run,
694 fuzz: ?std.Build.Fuzz.Mode,
654695) !void {
655696 const gpa = run.gpa;
656697 const step_stack = &run.step_stack;
......@@ -676,6 +717,7 @@ fn runStepNames(
676717 });
677718 }
678719 }
720
679721 assert(run.memory_blocked_steps.items.len == 0);
680722
681723 var test_skip_count: usize = 0;
......@@ -724,6 +766,45 @@ fn runStepNames(
724766 }
725767 }
726768
769 const ttyconf = run.ttyconf;
770
771 if (fuzz) |mode| blk: {
772 switch (builtin.os.tag) {
773 // Current implementation depends on two things that need to be ported to Windows:
774 // * Memory-mapping to share data between the fuzzer and build runner.
775 // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
776 // many addresses to source locations).
777 .windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}),
778 else => {},
779 }
780 if (@bitSizeOf(usize) != 64) {
781 // Current implementation depends on posix.mmap()'s second parameter, `length: usize`,
782 // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case
783 // on 32-bit platforms.
784 // Affects or affected by issues #5185, #22523, and #22464.
785 fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
786 }
787
788 switch (mode) {
789 .forever => break :blk,
790 .limit => {},
791 }
792
793 assert(mode == .limit);
794 var f = std.Build.Fuzz.init(
795 gpa,
796 thread_pool,
797 step_stack.keys(),
798 parent_prog_node,
799 ttyconf,
800 mode,
801 ) catch |err| fatal("failed to start fuzzer: {s}", .{@errorName(err)});
802 defer f.deinit();
803
804 f.start();
805 f.waitAndPrintReport();
806 }
807
727808 // A proper command line application defaults to silently succeeding.
728809 // The user may request verbose mode if they have a different preference.
729810 const failures_only = switch (run.summary) {
......@@ -737,8 +818,6 @@ fn runStepNames(
737818 std.Progress.setStatus(.failure);
738819 }
739820
740 const ttyconf = run.ttyconf;
741
742821 if (run.summary != .none) {
743822 const w = std.debug.lockStderrWriter(&stdio_buffer_allocation);
744823 defer std.debug.unlockStderrWriter();
......@@ -1366,7 +1445,10 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
13661445 \\ --watch Continuously rebuild when source files are modified
13671446 \\ --debounce <ms> Delay before rebuilding after changed file detected
13681447 \\ --webui[=ip] Enable the web interface on the given IP address
1369 \\ --fuzz Continuously search for unit test failures (implies '--webui')
1448 \\ --fuzz[=limit] Continuously search for unit test failures with an optional
1449 \\ limit to the max number of iterations. The argument supports
1450 \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies
1451 \\ '--webui' when no limit is specified.
13701452 \\ --time-report Force full rebuild and provide detailed information on
13711453 \\ compilation time of Zig source code (implies '--webui')
13721454 \\ -fincremental Enable incremental compilation
lib/compiler/test_runner.zig+27-9
......@@ -2,6 +2,7 @@
22const builtin = @import("builtin");
33
44const std = @import("std");
5const fatal = std.process.fatal;
56const testing = std.testing;
67const assert = std.debug.assert;
78const fuzz_abi = std.Build.abi.fuzz;
......@@ -55,12 +56,13 @@ pub fn main() void {
5556 }
5657 }
5758
58 fba.reset();
5959 if (builtin.fuzz) {
6060 const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument");
6161 fuzz_abi.fuzzer_init(.fromSlice(cache_dir));
6262 }
6363
64 fba.reset();
65
6466 if (listen) {
6567 return mainServer() catch @panic("internal test runner failure");
6668 } else {
......@@ -79,8 +81,13 @@ fn mainServer() !void {
7981 });
8082
8183 if (builtin.fuzz) {
82 const coverage_id = fuzz_abi.fuzzer_coverage_id();
83 try server.serveU64Message(.coverage_id, coverage_id);
84 const coverage = fuzz_abi.fuzzer_coverage();
85 try server.serveCoverageIdMessage(
86 coverage.id,
87 coverage.runs,
88 coverage.unique,
89 coverage.seen,
90 );
8491 }
8592
8693 while (true) {
......@@ -158,6 +165,9 @@ fn mainServer() !void {
158165 if (!builtin.fuzz) unreachable;
159166
160167 const index = try server.receiveBody_u32();
168 const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());
169 const amount_or_instance = try server.receiveBody_u64();
170
161171 const test_fn = builtin.test_functions[index];
162172 const entry_addr = @intFromPtr(test_fn.func);
163173
......@@ -165,6 +175,8 @@ fn mainServer() !void {
165175 defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
166176 is_fuzz_test = false;
167177 fuzz_test_index = index;
178 fuzz_mode = mode;
179 fuzz_amount_or_instance = amount_or_instance;
168180
169181 test_fn.func() catch |err| switch (err) {
170182 error.SkipZigTest => return,
......@@ -172,12 +184,14 @@ fn mainServer() !void {
172184 if (@errorReturnTrace()) |trace| {
173185 std.debug.dumpStackTrace(trace.*);
174186 }
175 std.debug.print("failed with error.{s}\n", .{@errorName(err)});
187 std.debug.print("failed with error.{t}\n", .{err});
176188 std.process.exit(1);
177189 },
178190 };
179191 if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");
180192 if (log_err_count != 0) @panic("error logs detected");
193 assert(mode != .forever);
194 std.process.exit(0);
181195 },
182196
183197 else => {
......@@ -240,11 +254,11 @@ fn mainTerminal() void {
240254 else => {
241255 fail_count += 1;
242256 if (have_tty) {
243 std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{
244 i + 1, test_fn_list.len, test_fn.name, @errorName(err),
257 std.debug.print("{d}/{d} {s}...FAIL ({t})\n", .{
258 i + 1, test_fn_list.len, test_fn.name, err,
245259 });
246260 } else {
247 std.debug.print("FAIL ({s})\n", .{@errorName(err)});
261 std.debug.print("FAIL ({t})\n", .{err});
248262 }
249263 if (@errorReturnTrace()) |trace| {
250264 std.debug.dumpStackTrace(trace.*);
......@@ -343,6 +357,8 @@ pub fn mainSimple() anyerror!void {
343357
344358var is_fuzz_test: bool = undefined;
345359var fuzz_test_index: u32 = undefined;
360var fuzz_mode: fuzz_abi.LimitKind = undefined;
361var fuzz_amount_or_instance: u64 = undefined;
346362
347363pub fn fuzz(
348364 context: anytype,
......@@ -383,7 +399,7 @@ pub fn fuzz(
383399 else => {
384400 std.debug.lockStdErr();
385401 if (@errorReturnTrace()) |trace| std.debug.dumpStackTrace(trace.*);
386 std.debug.print("failed with error.{s}\n", .{@errorName(err)});
402 std.debug.print("failed with error.{t}\n", .{err});
387403 std.process.exit(1);
388404 },
389405 };
......@@ -401,9 +417,11 @@ pub fn fuzz(
401417
402418 global.ctx = context;
403419 fuzz_abi.fuzzer_init_test(&global.test_one, .fromSlice(builtin.test_functions[fuzz_test_index].name));
420
404421 for (options.corpus) |elem|
405422 fuzz_abi.fuzzer_new_input(.fromSlice(elem));
406 fuzz_abi.fuzzer_main();
423
424 fuzz_abi.fuzzer_main(fuzz_mode, fuzz_amount_or_instance);
407425 return;
408426 }
409427
lib/fuzzer.zig+64-48
......@@ -1,5 +1,6 @@
11const builtin = @import("builtin");
22const std = @import("std");
3const fatal = std.process.fatal;
34const mem = std.mem;
45const math = std.math;
56const Allocator = mem.Allocator;
......@@ -105,6 +106,7 @@ const Executable = struct {
105106 const coverage_file_len = @sizeOf(abi.SeenPcsHeader) +
106107 pc_bitset_usizes * @sizeOf(usize) +
107108 pcs.len * @sizeOf(usize);
109
108110 if (populate) {
109111 defer coverage_file.lock(.shared) catch |e| panic(
110112 "failed to demote lock for coverage file '{s}': {t}",
......@@ -510,7 +512,7 @@ const Fuzzer = struct {
510512 self.corpus_pos = 0;
511513
512514 const rng = self.rng.random();
513 while (true) {
515 const m = while (true) {
514516 const m = self.mutations.items[rng.uintLessThanBiased(usize, self.mutations.items.len)];
515517 if (!m.mutate(
516518 rng,
......@@ -522,53 +524,53 @@ const Fuzzer = struct {
522524 inst.const_vals8.items,
523525 inst.const_vals16.items,
524526 )) continue;
527 break m;
528 };
525529
526 self.run();
527 if (inst.isFresh()) {
528 @branchHint(.unlikely);
529
530 const header = mem.bytesAsValue(
531 abi.SeenPcsHeader,
532 exec.shared_seen_pcs.items[0..@sizeOf(abi.SeenPcsHeader)],
533 );
534 _ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
530 self.run();
535531
536 inst.setFresh();
537 self.minimizeInput();
538 inst.updateSeen();
539
540 // An empty-input has always been tried, so if an empty input is fresh then the
541 // test has to be non-deterministic. This has to be checked as duplicate empty
542 // entries are not allowed.
543 if (self.input.items.len - 8 == 0) {
544 std.log.warn("non-deterministic test (empty input produces different hits)", .{});
545 _ = @atomicRmw(usize, &header.unique_runs, .Sub, 1, .monotonic);
546 return;
547 }
532 if (inst.isFresh()) {
533 @branchHint(.unlikely);
548534
549 const arena = self.arena_ctx.allocator();
550 const bytes = arena.dupe(u8, @volatileCast(self.input.items[8..])) catch @panic("OOM");
551
552 self.corpus.append(gpa, bytes) catch @panic("OOM");
553 self.mutations.appendNTimes(gpa, m, 6) catch @panic("OOM");
554
555 // Write new corpus to cache
556 var name_buf: [@sizeOf(usize) * 2]u8 = undefined;
557 self.corpus_dir.writeFile(.{
558 .sub_path = std.fmt.bufPrint(
559 &name_buf,
560 "{x}",
561 .{self.corpus_dir_idx},
562 ) catch unreachable,
563 .data = bytes,
564 }) catch |e| panic(
565 "failed to write corpus file '{x}': {t}",
566 .{ self.corpus_dir_idx, e },
567 );
568 self.corpus_dir_idx += 1;
535 const header = mem.bytesAsValue(
536 abi.SeenPcsHeader,
537 exec.shared_seen_pcs.items[0..@sizeOf(abi.SeenPcsHeader)],
538 );
539 _ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
540
541 inst.setFresh();
542 self.minimizeInput();
543 inst.updateSeen();
544
545 // An empty-input has always been tried, so if an empty input is fresh then the
546 // test has to be non-deterministic. This has to be checked as duplicate empty
547 // entries are not allowed.
548 if (self.input.items.len - 8 == 0) {
549 std.log.warn("non-deterministic test (empty input produces different hits)", .{});
550 _ = @atomicRmw(usize, &header.unique_runs, .Sub, 1, .monotonic);
551 return;
569552 }
570553
571 break;
554 const arena = self.arena_ctx.allocator();
555 const bytes = arena.dupe(u8, @volatileCast(self.input.items[8..])) catch @panic("OOM");
556
557 self.corpus.append(gpa, bytes) catch @panic("OOM");
558 self.mutations.appendNTimes(gpa, m, 6) catch @panic("OOM");
559
560 // Write new corpus to cache
561 var name_buf: [@sizeOf(usize) * 2]u8 = undefined;
562 self.corpus_dir.writeFile(.{
563 .sub_path = std.fmt.bufPrint(
564 &name_buf,
565 "{x}",
566 .{self.corpus_dir_idx},
567 ) catch unreachable,
568 .data = bytes,
569 }) catch |e| panic(
570 "failed to write corpus file '{x}': {t}",
571 .{ self.corpus_dir_idx, e },
572 );
573 self.corpus_dir_idx += 1;
572574 }
573575 }
574576};
......@@ -581,8 +583,21 @@ export fn fuzzer_init(cache_dir_path: abi.Slice) void {
581583}
582584
583585/// Invalid until `fuzzer_init` is called.
584export fn fuzzer_coverage_id() u64 {
585 return exec.pc_digest;
586export fn fuzzer_coverage() abi.Coverage {
587 const coverage_id = exec.pc_digest;
588 const header: *const abi.SeenPcsHeader = @ptrCast(@volatileCast(exec.shared_seen_pcs.items.ptr));
589
590 var seen_count: usize = 0;
591 for (header.seenBits()) |chunk| {
592 seen_count += @popCount(chunk);
593 }
594
595 return .{
596 .id = coverage_id,
597 .runs = header.n_runs,
598 .unique = header.unique_runs,
599 .seen = seen_count,
600 };
586601}
587602
588603/// fuzzer_init must be called beforehand
......@@ -600,9 +615,10 @@ export fn fuzzer_new_input(bytes: abi.Slice) void {
600615}
601616
602617/// fuzzer_init_test must be called first
603export fn fuzzer_main() void {
604 while (true) {
605 fuzzer.cycle();
618export fn fuzzer_main(limit_kind: abi.LimitKind, amount: u64) void {
619 switch (limit_kind) {
620 .forever => while (true) fuzzer.cycle(),
621 .iterations => for (0..amount) |_| fuzzer.cycle(),
606622 }
607623}
608624
lib/std/Build/Fuzz.zig+162-44
......@@ -8,17 +8,22 @@ const Allocator = std.mem.Allocator;
88const log = std.log;
99const Coverage = std.debug.Coverage;
1010const abi = Build.abi.fuzz;
11const tty = std.Io.tty;
1112
1213const Fuzz = @This();
1314const build_runner = @import("root");
1415
15ws: *Build.WebServer,
16gpa: Allocator,
17mode: Mode,
1618
17/// Allocated into `ws.gpa`.
19/// Allocated into `gpa`.
1820run_steps: []const *Step.Run,
1921
2022wait_group: std.Thread.WaitGroup,
23root_prog_node: std.Progress.Node,
2124prog_node: std.Progress.Node,
25thread_pool: *std.Thread.Pool,
26ttyconf: tty.Config,
2227
2328/// Protects `coverage_files`.
2429coverage_mutex: std.Thread.Mutex,
......@@ -28,9 +33,23 @@ queue_mutex: std.Thread.Mutex,
2833queue_cond: std.Thread.Condition,
2934msg_queue: std.ArrayListUnmanaged(Msg),
3035
36pub const Mode = union(enum) {
37 forever: struct { ws: *Build.WebServer },
38 limit: Limited,
39
40 pub const Limited = struct {
41 amount: u64,
42 };
43};
44
3145const Msg = union(enum) {
3246 coverage: struct {
3347 id: u64,
48 cumulative: struct {
49 runs: u64,
50 unique: u64,
51 coverage: u64,
52 },
3453 run: *Step.Run,
3554 },
3655 entry_point: struct {
......@@ -54,23 +73,28 @@ const CoverageMap = struct {
5473 }
5574};
5675
57pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {
58 const gpa = ws.gpa;
59
76pub fn init(
77 gpa: Allocator,
78 thread_pool: *std.Thread.Pool,
79 all_steps: []const *Build.Step,
80 root_prog_node: std.Progress.Node,
81 ttyconf: tty.Config,
82 mode: Mode,
83) Allocator.Error!Fuzz {
6084 const run_steps: []const *Step.Run = steps: {
6185 var steps: std.ArrayListUnmanaged(*Step.Run) = .empty;
6286 defer steps.deinit(gpa);
63 const rebuild_node = ws.root_prog_node.start("Rebuilding Unit Tests", 0);
87 const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);
6488 defer rebuild_node.end();
6589 var rebuild_wg: std.Thread.WaitGroup = .{};
6690 defer rebuild_wg.wait();
6791
68 for (ws.all_steps) |step| {
92 for (all_steps) |step| {
6993 const run = step.cast(Step.Run) orelse continue;
7094 if (run.producer == null) continue;
7195 if (run.fuzz_tests.items.len == 0) continue;
7296 try steps.append(gpa, run);
73 ws.thread_pool.spawnWg(&rebuild_wg, rebuildTestsWorkerRun, .{ run, gpa, ws.ttyconf, rebuild_node });
97 thread_pool.spawnWg(&rebuild_wg, rebuildTestsWorkerRun, .{ run, gpa, ttyconf, rebuild_node });
7498 }
7599
76100 if (steps.items.len == 0) fatal("no fuzz tests found", .{});
......@@ -86,9 +110,13 @@ pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {
86110 }
87111
88112 return .{
89 .ws = ws,
113 .gpa = gpa,
114 .mode = mode,
90115 .run_steps = run_steps,
91116 .wait_group = .{},
117 .thread_pool = thread_pool,
118 .ttyconf = ttyconf,
119 .root_prog_node = root_prog_node,
92120 .prog_node = .none,
93121 .coverage_files = .empty,
94122 .coverage_mutex = .{},
......@@ -99,32 +127,31 @@ pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {
99127}
100128
101129pub fn start(fuzz: *Fuzz) void {
102 const ws = fuzz.ws;
103 fuzz.prog_node = ws.root_prog_node.start("Fuzzing", fuzz.run_steps.len);
104
105 // For polling messages and sending updates to subscribers.
106 fuzz.wait_group.start();
107 _ = std.Thread.spawn(.{}, coverageRun, .{fuzz}) catch |err| {
108 fuzz.wait_group.finish();
109 fatal("unable to spawn coverage thread: {s}", .{@errorName(err)});
110 };
130 fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", fuzz.run_steps.len);
131
132 if (fuzz.mode == .forever) {
133 // For polling messages and sending updates to subscribers.
134 fuzz.wait_group.start();
135 _ = std.Thread.spawn(.{}, coverageRun, .{fuzz}) catch |err| {
136 fuzz.wait_group.finish();
137 fatal("unable to spawn coverage thread: {s}", .{@errorName(err)});
138 };
139 }
111140
112141 for (fuzz.run_steps) |run| {
113142 for (run.fuzz_tests.items) |unit_test_index| {
114143 assert(run.rebuilt_executable != null);
115 ws.thread_pool.spawnWg(&fuzz.wait_group, fuzzWorkerRun, .{
144 fuzz.thread_pool.spawnWg(&fuzz.wait_group, fuzzWorkerRun, .{
116145 fuzz, run, unit_test_index,
117146 });
118147 }
119148 }
120149}
150
121151pub fn deinit(fuzz: *Fuzz) void {
122 if (true) @panic("TODO: terminate the fuzzer processes");
123 fuzz.wait_group.wait();
152 if (!fuzz.wait_group.isDone()) @panic("TODO: terminate the fuzzer processes");
124153 fuzz.prog_node.end();
125
126 const gpa = fuzz.ws.gpa;
127 gpa.free(fuzz.run_steps);
154 fuzz.gpa.free(fuzz.run_steps);
128155}
129156
130157fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, ttyconf: std.Io.tty.Config, parent_prog_node: std.Progress.Node) void {
......@@ -177,7 +204,7 @@ fn fuzzWorkerRun(
177204 var buf: [256]u8 = undefined;
178205 const w = std.debug.lockStderrWriter(&buf);
179206 defer std.debug.unlockStderrWriter();
180 build_runner.printErrorMessages(gpa, &run.step, .{ .ttyconf = fuzz.ws.ttyconf }, w, false) catch {};
207 build_runner.printErrorMessages(gpa, &run.step, .{ .ttyconf = fuzz.ttyconf }, w, false) catch {};
181208 return;
182209 },
183210 else => {
......@@ -190,20 +217,20 @@ fn fuzzWorkerRun(
190217}
191218
192219pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
193 const gpa = fuzz.ws.gpa;
220 assert(fuzz.mode == .forever);
194221
195 var arena_state: std.heap.ArenaAllocator = .init(gpa);
222 var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa);
196223 defer arena_state.deinit();
197224 const arena = arena_state.allocator();
198225
199226 const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
200227 var dedup_table: DedupTable = .empty;
201 defer dedup_table.deinit(gpa);
228 defer dedup_table.deinit(fuzz.gpa);
202229
203230 for (fuzz.run_steps) |run_step| {
204231 const compile_inputs = run_step.producer.?.step.inputs.table;
205232 for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| {
206 try dedup_table.ensureUnusedCapacity(gpa, file_list.items.len);
233 try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len);
207234 for (file_list.items) |sub_path| {
208235 if (!std.mem.endsWith(u8, sub_path, ".zig")) continue;
209236 const joined_path = try dir_path.join(arena, sub_path);
......@@ -224,13 +251,18 @@ pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
224251 }
225252 };
226253 std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
227 return fuzz.ws.serveTarFile(req, deduped_paths);
254 return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths);
228255}
229256
230257pub const Previous = struct {
231258 unique_runs: usize,
232259 entry_points: usize,
233 pub const init: Previous = .{ .unique_runs = 0, .entry_points = 0 };
260 sent_source_index: bool,
261 pub const init: Previous = .{
262 .unique_runs = 0,
263 .entry_points = 0,
264 .sent_source_index = false,
265 };
234266};
235267pub fn sendUpdate(
236268 fuzz: *Fuzz,
......@@ -253,7 +285,8 @@ pub fn sendUpdate(
253285 const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic);
254286 const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic);
255287 {
256 if (unique_runs != 0 and prev.unique_runs == 0) {
288 if (!prev.sent_source_index) {
289 prev.sent_source_index = true;
257290 // We need to send initial context.
258291 const header: abi.SourceIndexHeader = .{
259292 .directories_len = @intCast(coverage_map.coverage.directories.entries.len),
......@@ -319,13 +352,13 @@ fn coverageRun(fuzz: *Fuzz) void {
319352 }
320353}
321354fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported }!void {
322 const ws = fuzz.ws;
323 const gpa = ws.gpa;
355 assert(fuzz.mode == .forever);
356 const ws = fuzz.mode.forever.ws;
324357
325358 fuzz.coverage_mutex.lock();
326359 defer fuzz.coverage_mutex.unlock();
327360
328 const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id);
361 const gop = try fuzz.coverage_files.getOrPut(fuzz.gpa, coverage_id);
329362 if (gop.found_existing) {
330363 // We are fuzzing the same executable with multiple threads.
331364 // Perhaps the same unit test; perhaps a different one. In any
......@@ -343,16 +376,16 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
343376 .entry_points = .{},
344377 .start_timestamp = ws.now(),
345378 };
346 errdefer gop.value_ptr.coverage.deinit(gpa);
379 errdefer gop.value_ptr.coverage.deinit(fuzz.gpa);
347380
348381 const rebuilt_exe_path = run_step.rebuilt_executable.?;
349 var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {
382 var debug_info = std.debug.Info.load(fuzz.gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {
350383 log.err("step '{s}': failed to load debug information for '{f}': {s}", .{
351384 run_step.step.name, rebuilt_exe_path, @errorName(err),
352385 });
353386 return error.AlreadyReported;
354387 };
355 defer debug_info.deinit(gpa);
388 defer debug_info.deinit(fuzz.gpa);
356389
357390 const coverage_file_path: Build.Cache.Path = .{
358391 .root_dir = run_step.step.owner.cache_root,
......@@ -386,14 +419,14 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
386419
387420 const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
388421 const pcs = header.pcAddrs();
389 const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len);
390 errdefer gpa.free(source_locations);
422 const source_locations = try fuzz.gpa.alloc(Coverage.SourceLocation, pcs.len);
423 errdefer fuzz.gpa.free(source_locations);
391424
392425 // Unfortunately the PCs array that LLVM gives us from the 8-bit PC
393426 // counters feature is not sorted.
394427 var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .{};
395 defer sorted_pcs.deinit(gpa);
396 try sorted_pcs.resize(gpa, pcs.len);
428 defer sorted_pcs.deinit(fuzz.gpa);
429 try sorted_pcs.resize(fuzz.gpa, pcs.len);
397430 @memcpy(sorted_pcs.items(.pc), pcs);
398431 for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i);
399432 sorted_pcs.sortUnstable(struct {
......@@ -404,7 +437,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
404437 }
405438 }{ .addrs = sorted_pcs.items(.pc) });
406439
407 debug_info.resolveAddresses(gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {
440 debug_info.resolveAddresses(fuzz.gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {
408441 log.err("failed to resolve addresses to source locations: {s}", .{@errorName(err)});
409442 return error.AlreadyReported;
410443 };
......@@ -414,6 +447,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
414447
415448 ws.notifyUpdate();
416449}
450
417451fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory }!void {
418452 fuzz.coverage_mutex.lock();
419453 defer fuzz.coverage_mutex.unlock();
......@@ -445,5 +479,89 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte
445479 addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1],
446480 });
447481 }
448 try coverage_map.entry_points.append(fuzz.ws.gpa, @intCast(index));
482 try coverage_map.entry_points.append(fuzz.gpa, @intCast(index));
483}
484
485pub fn waitAndPrintReport(fuzz: *Fuzz) void {
486 assert(fuzz.mode == .limit);
487
488 fuzz.wait_group.wait();
489 fuzz.wait_group.reset();
490
491 std.debug.print("======= FUZZING REPORT =======\n", .{});
492 for (fuzz.msg_queue.items) |msg| {
493 if (msg != .coverage) continue;
494
495 const cov = msg.coverage;
496 const coverage_file_path: std.Build.Cache.Path = .{
497 .root_dir = cov.run.step.owner.cache_root,
498 .sub_path = "v/" ++ std.fmt.hex(cov.id),
499 };
500 var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| {
501 fatal("step '{s}': failed to load coverage file '{f}': {s}", .{
502 cov.run.step.name, coverage_file_path, @errorName(err),
503 });
504 };
505 defer coverage_file.close();
506
507 const fuzz_abi = std.Build.abi.fuzz;
508 var rbuf: [0x1000]u8 = undefined;
509 var r = coverage_file.reader(&rbuf);
510
511 var header: fuzz_abi.SeenPcsHeader = undefined;
512 r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| {
513 fatal("step '{s}': failed to read from coverage file '{f}': {s}", .{
514 cov.run.step.name, coverage_file_path, @errorName(err),
515 });
516 };
517
518 if (header.pcs_len == 0) {
519 fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{
520 cov.run.step.name, coverage_file_path,
521 });
522 }
523
524 var seen_count: usize = 0;
525 const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len);
526 for (0..chunk_count) |_| {
527 const seen = r.interface.takeInt(usize, .little) catch |err| {
528 fatal("step '{s}': failed to read from coverage file '{f}': {s}", .{
529 cov.run.step.name, coverage_file_path, @errorName(err),
530 });
531 };
532 seen_count += @popCount(seen);
533 }
534
535 const seen_f: f64 = @floatFromInt(seen_count);
536 const total_f: f64 = @floatFromInt(header.pcs_len);
537 const ratio = seen_f / total_f;
538 std.debug.print(
539 \\Step: {s}
540 \\Fuzz test: "{s}" ({x})
541 \\Runs: {} -> {}
542 \\Unique runs: {} -> {}
543 \\Coverage: {}/{} -> {}/{} ({:.02}%)
544 \\
545 , .{
546 cov.run.step.name,
547 cov.run.cached_test_metadata.?.testName(cov.run.fuzz_tests.items[0]),
548 cov.id,
549 cov.cumulative.runs,
550 header.n_runs,
551 cov.cumulative.unique,
552 header.unique_runs,
553 cov.cumulative.coverage,
554 header.pcs_len,
555 seen_count,
556 header.pcs_len,
557 ratio * 100,
558 });
559
560 std.debug.print("------------------------------\n", .{});
561 }
562 std.debug.print(
563 \\Values are accumulated across multiple runs when preserving the cache.
564 \\==============================
565 \\
566 , .{});
449567}
lib/std/Build/Step/Run.zig+43-10
......@@ -1662,12 +1662,24 @@ fn evalZigTest(
16621662 // If this is `true`, we avoid ever entering the polling loop below, because the stdin pipe has
16631663 // somehow already closed; instead, we go straight to capturing stderr in case it has anything
16641664 // useful.
1665 const first_write_failed = if (fuzz_context) |fuzz| failed: {
1666 sendRunTestMessage(child.stdin.?, .start_fuzzing, fuzz.unit_test_index) catch |err| {
1667 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
1668 break :failed true;
1669 };
1670 break :failed false;
1665 const first_write_failed = if (fuzz_context) |fctx| failed: {
1666 switch (fctx.fuzz.mode) {
1667 .forever => {
1668 const instance_id = 0; // will be used by mutiprocess forever fuzzing
1669 sendRunFuzzTestMessage(child.stdin.?, fctx.unit_test_index, .forever, instance_id) catch |err| {
1670 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
1671 break :failed true;
1672 };
1673 break :failed false;
1674 },
1675 .limit => |limit| {
1676 sendRunFuzzTestMessage(child.stdin.?, fctx.unit_test_index, .iterations, limit.amount) catch |err| {
1677 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
1678 break :failed true;
1679 };
1680 break :failed false;
1681 },
1682 }
16711683 } else failed: {
16721684 run.fuzz_tests.clearRetainingCapacity();
16731685 sendMessage(child.stdin.?, .query_test_metadata) catch |err| {
......@@ -1778,13 +1790,18 @@ fn evalZigTest(
17781790 },
17791791 .coverage_id => {
17801792 const fuzz = fuzz_context.?.fuzz;
1781 const msg_ptr: *align(1) const u64 = @ptrCast(body);
1782 coverage_id = msg_ptr.*;
1793 const msg_ptr: *align(1) const [4]u64 = @ptrCast(body);
1794 coverage_id = msg_ptr[0];
17831795 {
17841796 fuzz.queue_mutex.lock();
17851797 defer fuzz.queue_mutex.unlock();
1786 try fuzz.msg_queue.append(fuzz.ws.gpa, .{ .coverage = .{
1798 try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{
17871799 .id = coverage_id.?,
1800 .cumulative = .{
1801 .runs = msg_ptr[1],
1802 .unique = msg_ptr[2],
1803 .coverage = msg_ptr[3],
1804 },
17881805 .run = run,
17891806 } });
17901807 fuzz.queue_cond.signal();
......@@ -1797,7 +1814,7 @@ fn evalZigTest(
17971814 {
17981815 fuzz.queue_mutex.lock();
17991816 defer fuzz.queue_mutex.unlock();
1800 try fuzz.msg_queue.append(fuzz.ws.gpa, .{ .entry_point = .{
1817 try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{
18011818 .addr = addr,
18021819 .coverage_id = coverage_id.?,
18031820 } });
......@@ -1900,6 +1917,22 @@ fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index:
19001917 try file.writeAll(full_msg);
19011918}
19021919
1920fn sendRunFuzzTestMessage(
1921 file: std.fs.File,
1922 index: u32,
1923 kind: std.Build.abi.fuzz.LimitKind,
1924 amount_or_instance: u64,
1925) !void {
1926 const header: std.zig.Client.Message.Header = .{
1927 .tag = .start_fuzzing,
1928 .bytes_len = 4 + 1 + 8,
1929 };
1930 const full_msg = std.mem.asBytes(&header) ++ std.mem.asBytes(&index) ++
1931 std.mem.asBytes(&kind) ++ std.mem.asBytes(&amount_or_instance);
1932
1933 try file.writeAll(full_msg);
1934}
1935
19031936fn evalGeneric(run: *Run, child: *std.process.Child) !StdIoResult {
19041937 const b = run.step.owner;
19051938 const arena = b.allocator;
lib/std/Build/WebServer.zig+9-1
......@@ -219,12 +219,20 @@ pub fn finishBuild(ws: *WebServer, opts: struct {
219219 // Affects or affected by issues #5185, #22523, and #22464.
220220 std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
221221 }
222
222223 assert(ws.fuzz == null);
223224
224225 ws.build_status.store(.fuzz_init, .monotonic);
225226 ws.notifyUpdate();
226227
227 ws.fuzz = Fuzz.init(ws) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)});
228 ws.fuzz = Fuzz.init(
229 ws.gpa,
230 ws.thread_pool,
231 ws.all_steps,
232 ws.root_prog_node,
233 ws.ttyconf,
234 .{ .forever = .{ .ws = ws } },
235 ) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)});
228236 ws.fuzz.?.start();
229237 }
230238
lib/std/Build/abi.zig+14-2
......@@ -140,10 +140,10 @@ pub const Rebuild = extern struct {
140140pub const fuzz = struct {
141141 pub const TestOne = *const fn (Slice) callconv(.c) void;
142142 pub extern fn fuzzer_init(cache_dir_path: Slice) void;
143 pub extern fn fuzzer_coverage_id() u64;
143 pub extern fn fuzzer_coverage() Coverage;
144144 pub extern fn fuzzer_init_test(test_one: TestOne, unit_test_name: Slice) void;
145145 pub extern fn fuzzer_new_input(bytes: Slice) void;
146 pub extern fn fuzzer_main() void;
146 pub extern fn fuzzer_main(limit_kind: LimitKind, amount: u64) void;
147147
148148 pub const Slice = extern struct {
149149 ptr: [*]const u8,
......@@ -158,6 +158,8 @@ pub const fuzz = struct {
158158 }
159159 };
160160
161 pub const LimitKind = enum(u8) { forever, iterations };
162
161163 /// libfuzzer uses this and its usize is the one that counts. To match the ABI,
162164 /// make the ints be the size of the target used with libfuzzer.
163165 ///
......@@ -251,6 +253,16 @@ pub const fuzz = struct {
251253 return .{ .locs_len_raw = @bitCast(locs_len) };
252254 }
253255 };
256
257 /// Sent by lib/fuzzer to test_runner to obtain information about the
258 /// active memory mapped input file and cumulative stats about previous
259 /// fuzzing runs.
260 pub const Coverage = extern struct {
261 id: u64,
262 runs: u64,
263 unique: u64,
264 seen: u64,
265 };
254266};
255267
256268/// ABI bits specifically relating to the time report interface.
lib/std/zig/Client.zig+10-2
......@@ -33,10 +33,18 @@ pub const Message = struct {
3333 /// Ask the test runner to run a particular test.
3434 /// The message body is a u32 test index.
3535 run_test,
36 /// Ask the test runner to start fuzzing a particular test.
37 /// The message body is a u32 test index.
36 /// Ask the test runner to start fuzzing a particular test forever or for a given amount of time/iterations.
37 /// The message body is:
38 /// - a u32 test index.
39 /// - a u8 test limit kind (std.Build.api.fuzz.LimitKind)
40 /// - a u64 value whose meaning depends on FuzzLimitKind (either a limit amount or an instance id)
3841 start_fuzzing,
3942
4043 _,
4144 };
45
46 comptime {
47 const std = @import("std");
48 std.debug.assert(@sizeOf(std.Build.abi.fuzz.LimitKind) == 1);
49 }
4250};
lib/std/zig/Server.zig+26-3
......@@ -42,9 +42,13 @@ pub const Message = struct {
4242 /// The remaining bytes is the file path relative to that prefix.
4343 /// The prefixes are hard-coded in Compilation.create (cwd, zig lib dir, local cache dir)
4444 file_system_inputs,
45 /// Body is a u64le that indicates the file path within the cache used
46 /// to store coverage information. The integer is a hash of the PCs
47 /// stored within that file.
45 /// Body is:
46 /// - a u64le that indicates the file path within the cache used
47 /// to store coverage information. The integer is a hash of the PCs
48 /// stored within that file.
49 /// - u64le of total runs accumulated
50 /// - u64le of unique runs accumulated
51 /// - u64le of coverage accumulated
4852 coverage_id,
4953 /// Body is a u64le that indicates the function pointer virtual memory
5054 /// address of the fuzz unit test. This is used to provide a starting
......@@ -141,9 +145,15 @@ pub fn receiveMessage(s: *Server) !InMessage.Header {
141145 return s.in.takeStruct(InMessage.Header, .little);
142146}
143147
148pub fn receiveBody_u8(s: *Server) !u8 {
149 return s.in.takeInt(u8, .little);
150}
144151pub fn receiveBody_u32(s: *Server) !u32 {
145152 return s.in.takeInt(u32, .little);
146153}
154pub fn receiveBody_u64(s: *Server) !u64 {
155 return s.in.takeInt(u64, .little);
156}
147157
148158pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void {
149159 try s.serveMessageHeader(.{
......@@ -160,6 +170,7 @@ pub fn serveMessageHeader(s: *const Server, header: OutMessage.Header) !void {
160170}
161171
162172pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void {
173 assert(tag != .coverage_id);
163174 try serveMessageHeader(s, .{
164175 .tag = tag,
165176 .bytes_len = @sizeOf(u64),
......@@ -168,6 +179,18 @@ pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void {
168179 try s.out.flush();
169180}
170181
182pub fn serveCoverageIdMessage(s: *const Server, id: u64, runs: u64, unique: u64, cov: u64) !void {
183 try serveMessageHeader(s, .{
184 .tag = .coverage_id,
185 .bytes_len = @sizeOf(u64) + @sizeOf(u64) + @sizeOf(u64) + @sizeOf(u64),
186 });
187 try s.out.writeInt(u64, id, .little);
188 try s.out.writeInt(u64, runs, .little);
189 try s.out.writeInt(u64, unique, .little);
190 try s.out.writeInt(u64, cov, .little);
191 try s.out.flush();
192}
193
171194pub fn serveEmitDigest(
172195 s: *Server,
173196 digest: *const [Cache.bin_digest_len]u8,
src/Compilation.zig+1-1
......@@ -8120,7 +8120,7 @@ pub fn addLinkLib(comp: *Compilation, lib_name: []const u8) !void {
81208120/// compiler-rt, libcxx, libc, libunwind, etc.
81218121pub fn compilerRtOptMode(comp: Compilation) std.builtin.OptimizeMode {
81228122 if (comp.debug_compiler_runtime_libs) {
8123 return comp.root_mod.optimize_mode;
8123 return .Debug;
81248124 }
81258125 const target = &comp.root_mod.resolved_target.result;
81268126 switch (comp.root_mod.optimize_mode) {
test/standalone/libfuzzer/main.zig+1-1
......@@ -24,7 +24,7 @@ pub fn main() !void {
2424 abi.fuzzer_new_input(.fromSlice(""));
2525 abi.fuzzer_new_input(.fromSlice("hello"));
2626
27 const pc_digest = abi.fuzzer_coverage_id();
27 const pc_digest = abi.fuzzer_coverage().id;
2828 const coverage_file_path = "v/" ++ std.fmt.hex(pc_digest);
2929 const coverage_file = try cache_dir.openFile(coverage_file_path, .{});
3030 defer coverage_file.close();