authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-09-26 05:28:46-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-09-26 05:28:46-07:00
loge0dc2e4e3ffe72e5e637e30bdf1d2c59b56f3cb6
tree23cafd6ae8e80026c2f257d291b91bae932e5730
parent3b365a1f9b277dd2cf7f7dac51e71647e164ff3c
parent52a13f6a7fb0933c065348128ee3e9aecd64255b
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #25342 from ziglang/fuzz-limit

fuzzing: implement limited fuzzing

11 files changed, 445 insertions(+), 127 deletions(-)

lib/compiler/build_runner.zig+88-6
...@@ -112,7 +112,7 @@ pub fn main() !void {...@@ -112,7 +112,7 @@ pub fn main() !void {
112 var steps_menu = false;112 var steps_menu = false;
113 var output_tmp_nonce: ?[16]u8 = null;113 var output_tmp_nonce: ?[16]u8 = null;
114 var watch = false;114 var watch = false;
115 var fuzz = false;115 var fuzz: ?std.Build.Fuzz.Mode = null;
116 var debounce_interval_ms: u16 = 50;116 var debounce_interval_ms: u16 = 50;
117 var webui_listen: ?std.net.Address = null;117 var webui_listen: ?std.net.Address = null;
118118
...@@ -274,10 +274,44 @@ pub fn main() !void {...@@ -274,10 +274,44 @@ pub fn main() !void {
274 webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;274 webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
275 }275 }
276 } else if (mem.eql(u8, arg, "--fuzz")) {276 } else if (mem.eql(u8, arg, "--fuzz")) {
277 fuzz = true;277 fuzz = .{ .forever = undefined };
278 if (webui_listen == null) {278 if (webui_listen == null) {
279 webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;279 webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
280 }280 }
281 } else if (mem.startsWith(u8, arg, "--fuzz=")) {
282 const value = arg["--fuzz=".len..];
283 if (value.len == 0) fatal("missing argument to --fuzz", .{});
284
285 const unit: u8 = value[value.len - 1];
286 const digits = switch (unit) {
287 '0'...'9' => value,
288 'K', 'M', 'G' => value[0 .. value.len - 1],
289 else => fatal(
290 "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
291 .{},
292 ),
293 };
294
295 const amount = std.fmt.parseInt(u64, digits, 10) catch {
296 fatal(
297 "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
298 .{},
299 );
300 };
301
302 const normalized_amount = std.math.mul(u64, amount, switch (unit) {
303 else => unreachable,
304 '0'...'9' => 1,
305 'K' => 1000,
306 'M' => 1_000_000,
307 'G' => 1_000_000_000,
308 }) catch fatal("fuzzing limit amount overflows u64", .{});
309
310 fuzz = .{
311 .limit = .{
312 .amount = normalized_amount,
313 },
314 };
281 } else if (mem.eql(u8, arg, "-fincremental")) {315 } else if (mem.eql(u8, arg, "-fincremental")) {
282 graph.incremental = true;316 graph.incremental = true;
283 } else if (mem.eql(u8, arg, "-fno-incremental")) {317 } else if (mem.eql(u8, arg, "-fno-incremental")) {
...@@ -476,6 +510,7 @@ pub fn main() !void {...@@ -476,6 +510,7 @@ pub fn main() !void {
476 targets.items,510 targets.items,
477 main_progress_node,511 main_progress_node,
478 &run,512 &run,
513 fuzz,
479 ) catch |err| switch (err) {514 ) catch |err| switch (err) {
480 error.UncleanExit => {515 error.UncleanExit => {
481 assert(!run.watch and run.web_server == null);516 assert(!run.watch and run.web_server == null);
...@@ -485,7 +520,12 @@ pub fn main() !void {...@@ -485,7 +520,12 @@ pub fn main() !void {
485 };520 };
486521
487 if (run.web_server) |*web_server| {522 if (run.web_server) |*web_server| {
488 web_server.finishBuild(.{ .fuzz = fuzz });523 if (fuzz) |mode| if (mode != .forever) fatal(
524 "error: limited fuzzing is not implemented yet for --webui",
525 .{},
526 );
527
528 web_server.finishBuild(.{ .fuzz = fuzz != null });
489 }529 }
490530
491 if (!watch and run.web_server == null) {531 if (!watch and run.web_server == null) {
...@@ -651,6 +691,7 @@ fn runStepNames(...@@ -651,6 +691,7 @@ fn runStepNames(
651 step_names: []const []const u8,691 step_names: []const []const u8,
652 parent_prog_node: std.Progress.Node,692 parent_prog_node: std.Progress.Node,
653 run: *Run,693 run: *Run,
694 fuzz: ?std.Build.Fuzz.Mode,
654) !void {695) !void {
655 const gpa = run.gpa;696 const gpa = run.gpa;
656 const step_stack = &run.step_stack;697 const step_stack = &run.step_stack;
...@@ -676,6 +717,7 @@ fn runStepNames(...@@ -676,6 +717,7 @@ fn runStepNames(
676 });717 });
677 }718 }
678 }719 }
720
679 assert(run.memory_blocked_steps.items.len == 0);721 assert(run.memory_blocked_steps.items.len == 0);
680722
681 var test_skip_count: usize = 0;723 var test_skip_count: usize = 0;
...@@ -724,6 +766,45 @@ fn runStepNames(...@@ -724,6 +766,45 @@ fn runStepNames(
724 }766 }
725 }767 }
726768
769 const ttyconf = run.ttyconf;
770
771 if (fuzz) |mode| blk: {
772 switch (builtin.os.tag) {
773 // Current implementation depends on two things that need to be ported to Windows:
774 // * Memory-mapping to share data between the fuzzer and build runner.
775 // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
776 // many addresses to source locations).
777 .windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}),
778 else => {},
779 }
780 if (@bitSizeOf(usize) != 64) {
781 // Current implementation depends on posix.mmap()'s second parameter, `length: usize`,
782 // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case
783 // on 32-bit platforms.
784 // Affects or affected by issues #5185, #22523, and #22464.
785 fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
786 }
787
788 switch (mode) {
789 .forever => break :blk,
790 .limit => {},
791 }
792
793 assert(mode == .limit);
794 var f = std.Build.Fuzz.init(
795 gpa,
796 thread_pool,
797 step_stack.keys(),
798 parent_prog_node,
799 ttyconf,
800 mode,
801 ) catch |err| fatal("failed to start fuzzer: {s}", .{@errorName(err)});
802 defer f.deinit();
803
804 f.start();
805 f.waitAndPrintReport();
806 }
807
727 // A proper command line application defaults to silently succeeding.808 // A proper command line application defaults to silently succeeding.
728 // The user may request verbose mode if they have a different preference.809 // The user may request verbose mode if they have a different preference.
729 const failures_only = switch (run.summary) {810 const failures_only = switch (run.summary) {
...@@ -737,8 +818,6 @@ fn runStepNames(...@@ -737,8 +818,6 @@ fn runStepNames(
737 std.Progress.setStatus(.failure);818 std.Progress.setStatus(.failure);
738 }819 }
739820
740 const ttyconf = run.ttyconf;
741
742 if (run.summary != .none) {821 if (run.summary != .none) {
743 const w = std.debug.lockStderrWriter(&stdio_buffer_allocation);822 const w = std.debug.lockStderrWriter(&stdio_buffer_allocation);
744 defer std.debug.unlockStderrWriter();823 defer std.debug.unlockStderrWriter();
...@@ -1366,7 +1445,10 @@ fn printUsage(b: *std.Build, w: *Writer) !void {...@@ -1366,7 +1445,10 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
1366 \\ --watch Continuously rebuild when source files are modified1445 \\ --watch Continuously rebuild when source files are modified
1367 \\ --debounce <ms> Delay before rebuilding after changed file detected1446 \\ --debounce <ms> Delay before rebuilding after changed file detected
1368 \\ --webui[=ip] Enable the web interface on the given IP address1447 \\ --webui[=ip] Enable the web interface on the given IP address
1369 \\ --fuzz Continuously search for unit test failures (implies '--webui')1448 \\ --fuzz[=limit] Continuously search for unit test failures with an optional
1449 \\ limit to the max number of iterations. The argument supports
1450 \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies
1451 \\ '--webui' when no limit is specified.
1370 \\ --time-report Force full rebuild and provide detailed information on1452 \\ --time-report Force full rebuild and provide detailed information on
1371 \\ compilation time of Zig source code (implies '--webui')1453 \\ compilation time of Zig source code (implies '--webui')
1372 \\ -fincremental Enable incremental compilation1454 \\ -fincremental Enable incremental compilation
lib/compiler/test_runner.zig+27-9
...@@ -2,6 +2,7 @@...@@ -2,6 +2,7 @@
2const builtin = @import("builtin");2const builtin = @import("builtin");
33
4const std = @import("std");4const std = @import("std");
5const fatal = std.process.fatal;
5const testing = std.testing;6const testing = std.testing;
6const assert = std.debug.assert;7const assert = std.debug.assert;
7const fuzz_abi = std.Build.abi.fuzz;8const fuzz_abi = std.Build.abi.fuzz;
...@@ -55,12 +56,13 @@ pub fn main() void {...@@ -55,12 +56,13 @@ pub fn main() void {
55 }56 }
56 }57 }
5758
58 fba.reset();
59 if (builtin.fuzz) {59 if (builtin.fuzz) {
60 const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument");60 const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument");
61 fuzz_abi.fuzzer_init(.fromSlice(cache_dir));61 fuzz_abi.fuzzer_init(.fromSlice(cache_dir));
62 }62 }
6363
64 fba.reset();
65
64 if (listen) {66 if (listen) {
65 return mainServer() catch @panic("internal test runner failure");67 return mainServer() catch @panic("internal test runner failure");
66 } else {68 } else {
...@@ -79,8 +81,13 @@ fn mainServer() !void {...@@ -79,8 +81,13 @@ fn mainServer() !void {
79 });81 });
8082
81 if (builtin.fuzz) {83 if (builtin.fuzz) {
82 const coverage_id = fuzz_abi.fuzzer_coverage_id();84 const coverage = fuzz_abi.fuzzer_coverage();
83 try server.serveU64Message(.coverage_id, coverage_id);85 try server.serveCoverageIdMessage(
86 coverage.id,
87 coverage.runs,
88 coverage.unique,
89 coverage.seen,
90 );
84 }91 }
8592
86 while (true) {93 while (true) {
...@@ -158,6 +165,9 @@ fn mainServer() !void {...@@ -158,6 +165,9 @@ fn mainServer() !void {
158 if (!builtin.fuzz) unreachable;165 if (!builtin.fuzz) unreachable;
159166
160 const index = try server.receiveBody_u32();167 const index = try server.receiveBody_u32();
168 const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());
169 const amount_or_instance = try server.receiveBody_u64();
170
161 const test_fn = builtin.test_functions[index];171 const test_fn = builtin.test_functions[index];
162 const entry_addr = @intFromPtr(test_fn.func);172 const entry_addr = @intFromPtr(test_fn.func);
163173
...@@ -165,6 +175,8 @@ fn mainServer() !void {...@@ -165,6 +175,8 @@ fn mainServer() !void {
165 defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);175 defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
166 is_fuzz_test = false;176 is_fuzz_test = false;
167 fuzz_test_index = index;177 fuzz_test_index = index;
178 fuzz_mode = mode;
179 fuzz_amount_or_instance = amount_or_instance;
168180
169 test_fn.func() catch |err| switch (err) {181 test_fn.func() catch |err| switch (err) {
170 error.SkipZigTest => return,182 error.SkipZigTest => return,
...@@ -172,12 +184,14 @@ fn mainServer() !void {...@@ -172,12 +184,14 @@ fn mainServer() !void {
172 if (@errorReturnTrace()) |trace| {184 if (@errorReturnTrace()) |trace| {
173 std.debug.dumpStackTrace(trace.*);185 std.debug.dumpStackTrace(trace.*);
174 }186 }
175 std.debug.print("failed with error.{s}\n", .{@errorName(err)});187 std.debug.print("failed with error.{t}\n", .{err});
176 std.process.exit(1);188 std.process.exit(1);
177 },189 },
178 };190 };
179 if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");191 if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");
180 if (log_err_count != 0) @panic("error logs detected");192 if (log_err_count != 0) @panic("error logs detected");
193 assert(mode != .forever);
194 std.process.exit(0);
181 },195 },
182196
183 else => {197 else => {
...@@ -240,11 +254,11 @@ fn mainTerminal() void {...@@ -240,11 +254,11 @@ fn mainTerminal() void {
240 else => {254 else => {
241 fail_count += 1;255 fail_count += 1;
242 if (have_tty) {256 if (have_tty) {
243 std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{257 std.debug.print("{d}/{d} {s}...FAIL ({t})\n", .{
244 i + 1, test_fn_list.len, test_fn.name, @errorName(err),258 i + 1, test_fn_list.len, test_fn.name, err,
245 });259 });
246 } else {260 } else {
247 std.debug.print("FAIL ({s})\n", .{@errorName(err)});261 std.debug.print("FAIL ({t})\n", .{err});
248 }262 }
249 if (@errorReturnTrace()) |trace| {263 if (@errorReturnTrace()) |trace| {
250 std.debug.dumpStackTrace(trace.*);264 std.debug.dumpStackTrace(trace.*);
...@@ -343,6 +357,8 @@ pub fn mainSimple() anyerror!void {...@@ -343,6 +357,8 @@ pub fn mainSimple() anyerror!void {
343357
344var is_fuzz_test: bool = undefined;358var is_fuzz_test: bool = undefined;
345var fuzz_test_index: u32 = undefined;359var fuzz_test_index: u32 = undefined;
360var fuzz_mode: fuzz_abi.LimitKind = undefined;
361var fuzz_amount_or_instance: u64 = undefined;
346362
347pub fn fuzz(363pub fn fuzz(
348 context: anytype,364 context: anytype,
...@@ -383,7 +399,7 @@ pub fn fuzz(...@@ -383,7 +399,7 @@ pub fn fuzz(
383 else => {399 else => {
384 std.debug.lockStdErr();400 std.debug.lockStdErr();
385 if (@errorReturnTrace()) |trace| std.debug.dumpStackTrace(trace.*);401 if (@errorReturnTrace()) |trace| std.debug.dumpStackTrace(trace.*);
386 std.debug.print("failed with error.{s}\n", .{@errorName(err)});402 std.debug.print("failed with error.{t}\n", .{err});
387 std.process.exit(1);403 std.process.exit(1);
388 },404 },
389 };405 };
...@@ -401,9 +417,11 @@ pub fn fuzz(...@@ -401,9 +417,11 @@ pub fn fuzz(
401417
402 global.ctx = context;418 global.ctx = context;
403 fuzz_abi.fuzzer_init_test(&global.test_one, .fromSlice(builtin.test_functions[fuzz_test_index].name));419 fuzz_abi.fuzzer_init_test(&global.test_one, .fromSlice(builtin.test_functions[fuzz_test_index].name));
420
404 for (options.corpus) |elem|421 for (options.corpus) |elem|
405 fuzz_abi.fuzzer_new_input(.fromSlice(elem));422 fuzz_abi.fuzzer_new_input(.fromSlice(elem));
406 fuzz_abi.fuzzer_main();423
424 fuzz_abi.fuzzer_main(fuzz_mode, fuzz_amount_or_instance);
407 return;425 return;
408 }426 }
409427
lib/fuzzer.zig+64-48
...@@ -1,5 +1,6 @@...@@ -1,5 +1,6 @@
1const builtin = @import("builtin");1const builtin = @import("builtin");
2const std = @import("std");2const std = @import("std");
3const fatal = std.process.fatal;
3const mem = std.mem;4const mem = std.mem;
4const math = std.math;5const math = std.math;
5const Allocator = mem.Allocator;6const Allocator = mem.Allocator;
...@@ -105,6 +106,7 @@ const Executable = struct {...@@ -105,6 +106,7 @@ const Executable = struct {
105 const coverage_file_len = @sizeOf(abi.SeenPcsHeader) +106 const coverage_file_len = @sizeOf(abi.SeenPcsHeader) +
106 pc_bitset_usizes * @sizeOf(usize) +107 pc_bitset_usizes * @sizeOf(usize) +
107 pcs.len * @sizeOf(usize);108 pcs.len * @sizeOf(usize);
109
108 if (populate) {110 if (populate) {
109 defer coverage_file.lock(.shared) catch |e| panic(111 defer coverage_file.lock(.shared) catch |e| panic(
110 "failed to demote lock for coverage file '{s}': {t}",112 "failed to demote lock for coverage file '{s}': {t}",
...@@ -510,7 +512,7 @@ const Fuzzer = struct {...@@ -510,7 +512,7 @@ const Fuzzer = struct {
510 self.corpus_pos = 0;512 self.corpus_pos = 0;
511513
512 const rng = self.rng.random();514 const rng = self.rng.random();
513 while (true) {515 const m = while (true) {
514 const m = self.mutations.items[rng.uintLessThanBiased(usize, self.mutations.items.len)];516 const m = self.mutations.items[rng.uintLessThanBiased(usize, self.mutations.items.len)];
515 if (!m.mutate(517 if (!m.mutate(
516 rng,518 rng,
...@@ -522,53 +524,53 @@ const Fuzzer = struct {...@@ -522,53 +524,53 @@ const Fuzzer = struct {
522 inst.const_vals8.items,524 inst.const_vals8.items,
523 inst.const_vals16.items,525 inst.const_vals16.items,
524 )) continue;526 )) continue;
527 break m;
528 };
525529
526 self.run();530 self.run();
527 if (inst.isFresh()) {
528 @branchHint(.unlikely);
529
530 const header = mem.bytesAsValue(
531 abi.SeenPcsHeader,
532 exec.shared_seen_pcs.items[0..@sizeOf(abi.SeenPcsHeader)],
533 );
534 _ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
535531
536 inst.setFresh();532 if (inst.isFresh()) {
537 self.minimizeInput();533 @branchHint(.unlikely);
538 inst.updateSeen();
539
540 // An empty-input has always been tried, so if an empty input is fresh then the
541 // test has to be non-deterministic. This has to be checked as duplicate empty
542 // entries are not allowed.
543 if (self.input.items.len - 8 == 0) {
544 std.log.warn("non-deterministic test (empty input produces different hits)", .{});
545 _ = @atomicRmw(usize, &header.unique_runs, .Sub, 1, .monotonic);
546 return;
547 }
548534
549 const arena = self.arena_ctx.allocator();535 const header = mem.bytesAsValue(
550 const bytes = arena.dupe(u8, @volatileCast(self.input.items[8..])) catch @panic("OOM");536 abi.SeenPcsHeader,
551537 exec.shared_seen_pcs.items[0..@sizeOf(abi.SeenPcsHeader)],
552 self.corpus.append(gpa, bytes) catch @panic("OOM");538 );
553 self.mutations.appendNTimes(gpa, m, 6) catch @panic("OOM");539 _ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
554540
555 // Write new corpus to cache541 inst.setFresh();
556 var name_buf: [@sizeOf(usize) * 2]u8 = undefined;542 self.minimizeInput();
557 self.corpus_dir.writeFile(.{543 inst.updateSeen();
558 .sub_path = std.fmt.bufPrint(544
559 &name_buf,545 // An empty-input has always been tried, so if an empty input is fresh then the
560 "{x}",546 // test has to be non-deterministic. This has to be checked as duplicate empty
561 .{self.corpus_dir_idx},547 // entries are not allowed.
562 ) catch unreachable,548 if (self.input.items.len - 8 == 0) {
563 .data = bytes,549 std.log.warn("non-deterministic test (empty input produces different hits)", .{});
564 }) catch |e| panic(550 _ = @atomicRmw(usize, &header.unique_runs, .Sub, 1, .monotonic);
565 "failed to write corpus file '{x}': {t}",551 return;
566 .{ self.corpus_dir_idx, e },
567 );
568 self.corpus_dir_idx += 1;
569 }552 }
570553
571 break;554 const arena = self.arena_ctx.allocator();
555 const bytes = arena.dupe(u8, @volatileCast(self.input.items[8..])) catch @panic("OOM");
556
557 self.corpus.append(gpa, bytes) catch @panic("OOM");
558 self.mutations.appendNTimes(gpa, m, 6) catch @panic("OOM");
559
560 // Write new corpus to cache
561 var name_buf: [@sizeOf(usize) * 2]u8 = undefined;
562 self.corpus_dir.writeFile(.{
563 .sub_path = std.fmt.bufPrint(
564 &name_buf,
565 "{x}",
566 .{self.corpus_dir_idx},
567 ) catch unreachable,
568 .data = bytes,
569 }) catch |e| panic(
570 "failed to write corpus file '{x}': {t}",
571 .{ self.corpus_dir_idx, e },
572 );
573 self.corpus_dir_idx += 1;
572 }574 }
573 }575 }
574};576};
...@@ -581,8 +583,21 @@ export fn fuzzer_init(cache_dir_path: abi.Slice) void {...@@ -581,8 +583,21 @@ export fn fuzzer_init(cache_dir_path: abi.Slice) void {
581}583}
582584
583/// Invalid until `fuzzer_init` is called.585/// Invalid until `fuzzer_init` is called.
584export fn fuzzer_coverage_id() u64 {586export fn fuzzer_coverage() abi.Coverage {
585 return exec.pc_digest;587 const coverage_id = exec.pc_digest;
588 const header: *const abi.SeenPcsHeader = @ptrCast(@volatileCast(exec.shared_seen_pcs.items.ptr));
589
590 var seen_count: usize = 0;
591 for (header.seenBits()) |chunk| {
592 seen_count += @popCount(chunk);
593 }
594
595 return .{
596 .id = coverage_id,
597 .runs = header.n_runs,
598 .unique = header.unique_runs,
599 .seen = seen_count,
600 };
586}601}
587602
588/// fuzzer_init must be called beforehand603/// fuzzer_init must be called beforehand
...@@ -600,9 +615,10 @@ export fn fuzzer_new_input(bytes: abi.Slice) void {...@@ -600,9 +615,10 @@ export fn fuzzer_new_input(bytes: abi.Slice) void {
600}615}
601616
602/// fuzzer_init_test must be called first617/// fuzzer_init_test must be called first
603export fn fuzzer_main() void {618export fn fuzzer_main(limit_kind: abi.LimitKind, amount: u64) void {
604 while (true) {619 switch (limit_kind) {
605 fuzzer.cycle();620 .forever => while (true) fuzzer.cycle(),
621 .iterations => for (0..amount) |_| fuzzer.cycle(),
606 }622 }
607}623}
608624
lib/std/Build/Fuzz.zig+162-44
...@@ -8,17 +8,22 @@ const Allocator = std.mem.Allocator;...@@ -8,17 +8,22 @@ const Allocator = std.mem.Allocator;
8const log = std.log;8const log = std.log;
9const Coverage = std.debug.Coverage;9const Coverage = std.debug.Coverage;
10const abi = Build.abi.fuzz;10const abi = Build.abi.fuzz;
11const tty = std.Io.tty;
1112
12const Fuzz = @This();13const Fuzz = @This();
13const build_runner = @import("root");14const build_runner = @import("root");
1415
15ws: *Build.WebServer,16gpa: Allocator,
17mode: Mode,
1618
17/// Allocated into `ws.gpa`.19/// Allocated into `gpa`.
18run_steps: []const *Step.Run,20run_steps: []const *Step.Run,
1921
20wait_group: std.Thread.WaitGroup,22wait_group: std.Thread.WaitGroup,
23root_prog_node: std.Progress.Node,
21prog_node: std.Progress.Node,24prog_node: std.Progress.Node,
25thread_pool: *std.Thread.Pool,
26ttyconf: tty.Config,
2227
23/// Protects `coverage_files`.28/// Protects `coverage_files`.
24coverage_mutex: std.Thread.Mutex,29coverage_mutex: std.Thread.Mutex,
...@@ -28,9 +33,23 @@ queue_mutex: std.Thread.Mutex,...@@ -28,9 +33,23 @@ queue_mutex: std.Thread.Mutex,
28queue_cond: std.Thread.Condition,33queue_cond: std.Thread.Condition,
29msg_queue: std.ArrayListUnmanaged(Msg),34msg_queue: std.ArrayListUnmanaged(Msg),
3035
36pub const Mode = union(enum) {
37 forever: struct { ws: *Build.WebServer },
38 limit: Limited,
39
40 pub const Limited = struct {
41 amount: u64,
42 };
43};
44
31const Msg = union(enum) {45const Msg = union(enum) {
32 coverage: struct {46 coverage: struct {
33 id: u64,47 id: u64,
48 cumulative: struct {
49 runs: u64,
50 unique: u64,
51 coverage: u64,
52 },
34 run: *Step.Run,53 run: *Step.Run,
35 },54 },
36 entry_point: struct {55 entry_point: struct {
...@@ -54,23 +73,28 @@ const CoverageMap = struct {...@@ -54,23 +73,28 @@ const CoverageMap = struct {
54 }73 }
55};74};
5675
57pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {76pub fn init(
58 const gpa = ws.gpa;77 gpa: Allocator,
5978 thread_pool: *std.Thread.Pool,
79 all_steps: []const *Build.Step,
80 root_prog_node: std.Progress.Node,
81 ttyconf: tty.Config,
82 mode: Mode,
83) Allocator.Error!Fuzz {
60 const run_steps: []const *Step.Run = steps: {84 const run_steps: []const *Step.Run = steps: {
61 var steps: std.ArrayListUnmanaged(*Step.Run) = .empty;85 var steps: std.ArrayListUnmanaged(*Step.Run) = .empty;
62 defer steps.deinit(gpa);86 defer steps.deinit(gpa);
63 const rebuild_node = ws.root_prog_node.start("Rebuilding Unit Tests", 0);87 const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);
64 defer rebuild_node.end();88 defer rebuild_node.end();
65 var rebuild_wg: std.Thread.WaitGroup = .{};89 var rebuild_wg: std.Thread.WaitGroup = .{};
66 defer rebuild_wg.wait();90 defer rebuild_wg.wait();
6791
68 for (ws.all_steps) |step| {92 for (all_steps) |step| {
69 const run = step.cast(Step.Run) orelse continue;93 const run = step.cast(Step.Run) orelse continue;
70 if (run.producer == null) continue;94 if (run.producer == null) continue;
71 if (run.fuzz_tests.items.len == 0) continue;95 if (run.fuzz_tests.items.len == 0) continue;
72 try steps.append(gpa, run);96 try steps.append(gpa, run);
73 ws.thread_pool.spawnWg(&rebuild_wg, rebuildTestsWorkerRun, .{ run, gpa, ws.ttyconf, rebuild_node });97 thread_pool.spawnWg(&rebuild_wg, rebuildTestsWorkerRun, .{ run, gpa, ttyconf, rebuild_node });
74 }98 }
7599
76 if (steps.items.len == 0) fatal("no fuzz tests found", .{});100 if (steps.items.len == 0) fatal("no fuzz tests found", .{});
...@@ -86,9 +110,13 @@ pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {...@@ -86,9 +110,13 @@ pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {
86 }110 }
87111
88 return .{112 return .{
89 .ws = ws,113 .gpa = gpa,
114 .mode = mode,
90 .run_steps = run_steps,115 .run_steps = run_steps,
91 .wait_group = .{},116 .wait_group = .{},
117 .thread_pool = thread_pool,
118 .ttyconf = ttyconf,
119 .root_prog_node = root_prog_node,
92 .prog_node = .none,120 .prog_node = .none,
93 .coverage_files = .empty,121 .coverage_files = .empty,
94 .coverage_mutex = .{},122 .coverage_mutex = .{},
...@@ -99,32 +127,31 @@ pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {...@@ -99,32 +127,31 @@ pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {
99}127}
100128
101pub fn start(fuzz: *Fuzz) void {129pub fn start(fuzz: *Fuzz) void {
102 const ws = fuzz.ws;130 fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", fuzz.run_steps.len);
103 fuzz.prog_node = ws.root_prog_node.start("Fuzzing", fuzz.run_steps.len);131
104132 if (fuzz.mode == .forever) {
105 // For polling messages and sending updates to subscribers.133 // For polling messages and sending updates to subscribers.
106 fuzz.wait_group.start();134 fuzz.wait_group.start();
107 _ = std.Thread.spawn(.{}, coverageRun, .{fuzz}) catch |err| {135 _ = std.Thread.spawn(.{}, coverageRun, .{fuzz}) catch |err| {
108 fuzz.wait_group.finish();136 fuzz.wait_group.finish();
109 fatal("unable to spawn coverage thread: {s}", .{@errorName(err)});137 fatal("unable to spawn coverage thread: {s}", .{@errorName(err)});
110 };138 };
139 }
111140
112 for (fuzz.run_steps) |run| {141 for (fuzz.run_steps) |run| {
113 for (run.fuzz_tests.items) |unit_test_index| {142 for (run.fuzz_tests.items) |unit_test_index| {
114 assert(run.rebuilt_executable != null);143 assert(run.rebuilt_executable != null);
115 ws.thread_pool.spawnWg(&fuzz.wait_group, fuzzWorkerRun, .{144 fuzz.thread_pool.spawnWg(&fuzz.wait_group, fuzzWorkerRun, .{
116 fuzz, run, unit_test_index,145 fuzz, run, unit_test_index,
117 });146 });
118 }147 }
119 }148 }
120}149}
150
121pub fn deinit(fuzz: *Fuzz) void {151pub fn deinit(fuzz: *Fuzz) void {
122 if (true) @panic("TODO: terminate the fuzzer processes");152 if (!fuzz.wait_group.isDone()) @panic("TODO: terminate the fuzzer processes");
123 fuzz.wait_group.wait();
124 fuzz.prog_node.end();153 fuzz.prog_node.end();
125154 fuzz.gpa.free(fuzz.run_steps);
126 const gpa = fuzz.ws.gpa;
127 gpa.free(fuzz.run_steps);
128}155}
129156
130fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, ttyconf: std.Io.tty.Config, parent_prog_node: std.Progress.Node) void {157fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, ttyconf: std.Io.tty.Config, parent_prog_node: std.Progress.Node) void {
...@@ -177,7 +204,7 @@ fn fuzzWorkerRun(...@@ -177,7 +204,7 @@ fn fuzzWorkerRun(
177 var buf: [256]u8 = undefined;204 var buf: [256]u8 = undefined;
178 const w = std.debug.lockStderrWriter(&buf);205 const w = std.debug.lockStderrWriter(&buf);
179 defer std.debug.unlockStderrWriter();206 defer std.debug.unlockStderrWriter();
180 build_runner.printErrorMessages(gpa, &run.step, .{ .ttyconf = fuzz.ws.ttyconf }, w, false) catch {};207 build_runner.printErrorMessages(gpa, &run.step, .{ .ttyconf = fuzz.ttyconf }, w, false) catch {};
181 return;208 return;
182 },209 },
183 else => {210 else => {
...@@ -190,20 +217,20 @@ fn fuzzWorkerRun(...@@ -190,20 +217,20 @@ fn fuzzWorkerRun(
190}217}
191218
192pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {219pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
193 const gpa = fuzz.ws.gpa;220 assert(fuzz.mode == .forever);
194221
195 var arena_state: std.heap.ArenaAllocator = .init(gpa);222 var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa);
196 defer arena_state.deinit();223 defer arena_state.deinit();
197 const arena = arena_state.allocator();224 const arena = arena_state.allocator();
198225
199 const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);226 const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
200 var dedup_table: DedupTable = .empty;227 var dedup_table: DedupTable = .empty;
201 defer dedup_table.deinit(gpa);228 defer dedup_table.deinit(fuzz.gpa);
202229
203 for (fuzz.run_steps) |run_step| {230 for (fuzz.run_steps) |run_step| {
204 const compile_inputs = run_step.producer.?.step.inputs.table;231 const compile_inputs = run_step.producer.?.step.inputs.table;
205 for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| {232 for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| {
206 try dedup_table.ensureUnusedCapacity(gpa, file_list.items.len);233 try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len);
207 for (file_list.items) |sub_path| {234 for (file_list.items) |sub_path| {
208 if (!std.mem.endsWith(u8, sub_path, ".zig")) continue;235 if (!std.mem.endsWith(u8, sub_path, ".zig")) continue;
209 const joined_path = try dir_path.join(arena, sub_path);236 const joined_path = try dir_path.join(arena, sub_path);
...@@ -224,13 +251,18 @@ pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {...@@ -224,13 +251,18 @@ pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
224 }251 }
225 };252 };
226 std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);253 std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
227 return fuzz.ws.serveTarFile(req, deduped_paths);254 return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths);
228}255}
229256
230pub const Previous = struct {257pub const Previous = struct {
231 unique_runs: usize,258 unique_runs: usize,
232 entry_points: usize,259 entry_points: usize,
233 pub const init: Previous = .{ .unique_runs = 0, .entry_points = 0 };260 sent_source_index: bool,
261 pub const init: Previous = .{
262 .unique_runs = 0,
263 .entry_points = 0,
264 .sent_source_index = false,
265 };
234};266};
235pub fn sendUpdate(267pub fn sendUpdate(
236 fuzz: *Fuzz,268 fuzz: *Fuzz,
...@@ -253,7 +285,8 @@ pub fn sendUpdate(...@@ -253,7 +285,8 @@ pub fn sendUpdate(
253 const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic);285 const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic);
254 const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic);286 const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic);
255 {287 {
256 if (unique_runs != 0 and prev.unique_runs == 0) {288 if (!prev.sent_source_index) {
289 prev.sent_source_index = true;
257 // We need to send initial context.290 // We need to send initial context.
258 const header: abi.SourceIndexHeader = .{291 const header: abi.SourceIndexHeader = .{
259 .directories_len = @intCast(coverage_map.coverage.directories.entries.len),292 .directories_len = @intCast(coverage_map.coverage.directories.entries.len),
...@@ -319,13 +352,13 @@ fn coverageRun(fuzz: *Fuzz) void {...@@ -319,13 +352,13 @@ fn coverageRun(fuzz: *Fuzz) void {
319 }352 }
320}353}
321fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported }!void {354fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported }!void {
322 const ws = fuzz.ws;355 assert(fuzz.mode == .forever);
323 const gpa = ws.gpa;356 const ws = fuzz.mode.forever.ws;
324357
325 fuzz.coverage_mutex.lock();358 fuzz.coverage_mutex.lock();
326 defer fuzz.coverage_mutex.unlock();359 defer fuzz.coverage_mutex.unlock();
327360
328 const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id);361 const gop = try fuzz.coverage_files.getOrPut(fuzz.gpa, coverage_id);
329 if (gop.found_existing) {362 if (gop.found_existing) {
330 // We are fuzzing the same executable with multiple threads.363 // We are fuzzing the same executable with multiple threads.
331 // Perhaps the same unit test; perhaps a different one. In any364 // Perhaps the same unit test; perhaps a different one. In any
...@@ -343,16 +376,16 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO...@@ -343,16 +376,16 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
343 .entry_points = .{},376 .entry_points = .{},
344 .start_timestamp = ws.now(),377 .start_timestamp = ws.now(),
345 };378 };
346 errdefer gop.value_ptr.coverage.deinit(gpa);379 errdefer gop.value_ptr.coverage.deinit(fuzz.gpa);
347380
348 const rebuilt_exe_path = run_step.rebuilt_executable.?;381 const rebuilt_exe_path = run_step.rebuilt_executable.?;
349 var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {382 var debug_info = std.debug.Info.load(fuzz.gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {
350 log.err("step '{s}': failed to load debug information for '{f}': {s}", .{383 log.err("step '{s}': failed to load debug information for '{f}': {s}", .{
351 run_step.step.name, rebuilt_exe_path, @errorName(err),384 run_step.step.name, rebuilt_exe_path, @errorName(err),
352 });385 });
353 return error.AlreadyReported;386 return error.AlreadyReported;
354 };387 };
355 defer debug_info.deinit(gpa);388 defer debug_info.deinit(fuzz.gpa);
356389
357 const coverage_file_path: Build.Cache.Path = .{390 const coverage_file_path: Build.Cache.Path = .{
358 .root_dir = run_step.step.owner.cache_root,391 .root_dir = run_step.step.owner.cache_root,
...@@ -386,14 +419,14 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO...@@ -386,14 +419,14 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
386419
387 const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);420 const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
388 const pcs = header.pcAddrs();421 const pcs = header.pcAddrs();
389 const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len);422 const source_locations = try fuzz.gpa.alloc(Coverage.SourceLocation, pcs.len);
390 errdefer gpa.free(source_locations);423 errdefer fuzz.gpa.free(source_locations);
391424
392 // Unfortunately the PCs array that LLVM gives us from the 8-bit PC425 // Unfortunately the PCs array that LLVM gives us from the 8-bit PC
393 // counters feature is not sorted.426 // counters feature is not sorted.
394 var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .{};427 var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .{};
395 defer sorted_pcs.deinit(gpa);428 defer sorted_pcs.deinit(fuzz.gpa);
396 try sorted_pcs.resize(gpa, pcs.len);429 try sorted_pcs.resize(fuzz.gpa, pcs.len);
397 @memcpy(sorted_pcs.items(.pc), pcs);430 @memcpy(sorted_pcs.items(.pc), pcs);
398 for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i);431 for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i);
399 sorted_pcs.sortUnstable(struct {432 sorted_pcs.sortUnstable(struct {
...@@ -404,7 +437,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO...@@ -404,7 +437,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
404 }437 }
405 }{ .addrs = sorted_pcs.items(.pc) });438 }{ .addrs = sorted_pcs.items(.pc) });
406439
407 debug_info.resolveAddresses(gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {440 debug_info.resolveAddresses(fuzz.gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {
408 log.err("failed to resolve addresses to source locations: {s}", .{@errorName(err)});441 log.err("failed to resolve addresses to source locations: {s}", .{@errorName(err)});
409 return error.AlreadyReported;442 return error.AlreadyReported;
410 };443 };
...@@ -414,6 +447,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO...@@ -414,6 +447,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
414447
415 ws.notifyUpdate();448 ws.notifyUpdate();
416}449}
450
417fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory }!void {451fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory }!void {
418 fuzz.coverage_mutex.lock();452 fuzz.coverage_mutex.lock();
419 defer fuzz.coverage_mutex.unlock();453 defer fuzz.coverage_mutex.unlock();
...@@ -445,5 +479,89 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte...@@ -445,5 +479,89 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte
445 addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1],479 addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1],
446 });480 });
447 }481 }
448 try coverage_map.entry_points.append(fuzz.ws.gpa, @intCast(index));482 try coverage_map.entry_points.append(fuzz.gpa, @intCast(index));
483}
484
485pub fn waitAndPrintReport(fuzz: *Fuzz) void {
486 assert(fuzz.mode == .limit);
487
488 fuzz.wait_group.wait();
489 fuzz.wait_group.reset();
490
491 std.debug.print("======= FUZZING REPORT =======\n", .{});
492 for (fuzz.msg_queue.items) |msg| {
493 if (msg != .coverage) continue;
494
495 const cov = msg.coverage;
496 const coverage_file_path: std.Build.Cache.Path = .{
497 .root_dir = cov.run.step.owner.cache_root,
498 .sub_path = "v/" ++ std.fmt.hex(cov.id),
499 };
500 var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| {
501 fatal("step '{s}': failed to load coverage file '{f}': {s}", .{
502 cov.run.step.name, coverage_file_path, @errorName(err),
503 });
504 };
505 defer coverage_file.close();
506
507 const fuzz_abi = std.Build.abi.fuzz;
508 var rbuf: [0x1000]u8 = undefined;
509 var r = coverage_file.reader(&rbuf);
510
511 var header: fuzz_abi.SeenPcsHeader = undefined;
512 r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| {
513 fatal("step '{s}': failed to read from coverage file '{f}': {s}", .{
514 cov.run.step.name, coverage_file_path, @errorName(err),
515 });
516 };
517
518 if (header.pcs_len == 0) {
519 fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{
520 cov.run.step.name, coverage_file_path,
521 });
522 }
523
524 var seen_count: usize = 0;
525 const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len);
526 for (0..chunk_count) |_| {
527 const seen = r.interface.takeInt(usize, .little) catch |err| {
528 fatal("step '{s}': failed to read from coverage file '{f}': {s}", .{
529 cov.run.step.name, coverage_file_path, @errorName(err),
530 });
531 };
532 seen_count += @popCount(seen);
533 }
534
535 const seen_f: f64 = @floatFromInt(seen_count);
536 const total_f: f64 = @floatFromInt(header.pcs_len);
537 const ratio = seen_f / total_f;
538 std.debug.print(
539 \\Step: {s}
540 \\Fuzz test: "{s}" ({x})
541 \\Runs: {} -> {}
542 \\Unique runs: {} -> {}
543 \\Coverage: {}/{} -> {}/{} ({:.02}%)
544 \\
545 , .{
546 cov.run.step.name,
547 cov.run.cached_test_metadata.?.testName(cov.run.fuzz_tests.items[0]),
548 cov.id,
549 cov.cumulative.runs,
550 header.n_runs,
551 cov.cumulative.unique,
552 header.unique_runs,
553 cov.cumulative.coverage,
554 header.pcs_len,
555 seen_count,
556 header.pcs_len,
557 ratio * 100,
558 });
559
560 std.debug.print("------------------------------\n", .{});
561 }
562 std.debug.print(
563 \\Values are accumulated across multiple runs when preserving the cache.
564 \\==============================
565 \\
566 , .{});
449}567}
lib/std/Build/Step/Run.zig+43-10
...@@ -1662,12 +1662,24 @@ fn evalZigTest(...@@ -1662,12 +1662,24 @@ fn evalZigTest(
1662 // If this is `true`, we avoid ever entering the polling loop below, because the stdin pipe has1662 // If this is `true`, we avoid ever entering the polling loop below, because the stdin pipe has
1663 // somehow already closed; instead, we go straight to capturing stderr in case it has anything1663 // somehow already closed; instead, we go straight to capturing stderr in case it has anything
1664 // useful.1664 // useful.
1665 const first_write_failed = if (fuzz_context) |fuzz| failed: {1665 const first_write_failed = if (fuzz_context) |fctx| failed: {
1666 sendRunTestMessage(child.stdin.?, .start_fuzzing, fuzz.unit_test_index) catch |err| {1666 switch (fctx.fuzz.mode) {
1667 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});1667 .forever => {
1668 break :failed true;1668 const instance_id = 0; // will be used by mutiprocess forever fuzzing
1669 };1669 sendRunFuzzTestMessage(child.stdin.?, fctx.unit_test_index, .forever, instance_id) catch |err| {
1670 break :failed false;1670 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
1671 break :failed true;
1672 };
1673 break :failed false;
1674 },
1675 .limit => |limit| {
1676 sendRunFuzzTestMessage(child.stdin.?, fctx.unit_test_index, .iterations, limit.amount) catch |err| {
1677 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
1678 break :failed true;
1679 };
1680 break :failed false;
1681 },
1682 }
1671 } else failed: {1683 } else failed: {
1672 run.fuzz_tests.clearRetainingCapacity();1684 run.fuzz_tests.clearRetainingCapacity();
1673 sendMessage(child.stdin.?, .query_test_metadata) catch |err| {1685 sendMessage(child.stdin.?, .query_test_metadata) catch |err| {
...@@ -1778,13 +1790,18 @@ fn evalZigTest(...@@ -1778,13 +1790,18 @@ fn evalZigTest(
1778 },1790 },
1779 .coverage_id => {1791 .coverage_id => {
1780 const fuzz = fuzz_context.?.fuzz;1792 const fuzz = fuzz_context.?.fuzz;
1781 const msg_ptr: *align(1) const u64 = @ptrCast(body);1793 const msg_ptr: *align(1) const [4]u64 = @ptrCast(body);
1782 coverage_id = msg_ptr.*;1794 coverage_id = msg_ptr[0];
1783 {1795 {
1784 fuzz.queue_mutex.lock();1796 fuzz.queue_mutex.lock();
1785 defer fuzz.queue_mutex.unlock();1797 defer fuzz.queue_mutex.unlock();
1786 try fuzz.msg_queue.append(fuzz.ws.gpa, .{ .coverage = .{1798 try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{
1787 .id = coverage_id.?,1799 .id = coverage_id.?,
1800 .cumulative = .{
1801 .runs = msg_ptr[1],
1802 .unique = msg_ptr[2],
1803 .coverage = msg_ptr[3],
1804 },
1788 .run = run,1805 .run = run,
1789 } });1806 } });
1790 fuzz.queue_cond.signal();1807 fuzz.queue_cond.signal();
...@@ -1797,7 +1814,7 @@ fn evalZigTest(...@@ -1797,7 +1814,7 @@ fn evalZigTest(
1797 {1814 {
1798 fuzz.queue_mutex.lock();1815 fuzz.queue_mutex.lock();
1799 defer fuzz.queue_mutex.unlock();1816 defer fuzz.queue_mutex.unlock();
1800 try fuzz.msg_queue.append(fuzz.ws.gpa, .{ .entry_point = .{1817 try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{
1801 .addr = addr,1818 .addr = addr,
1802 .coverage_id = coverage_id.?,1819 .coverage_id = coverage_id.?,
1803 } });1820 } });
...@@ -1900,6 +1917,22 @@ fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index:...@@ -1900,6 +1917,22 @@ fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index:
1900 try file.writeAll(full_msg);1917 try file.writeAll(full_msg);
1901}1918}
19021919
1920fn sendRunFuzzTestMessage(
1921 file: std.fs.File,
1922 index: u32,
1923 kind: std.Build.abi.fuzz.LimitKind,
1924 amount_or_instance: u64,
1925) !void {
1926 const header: std.zig.Client.Message.Header = .{
1927 .tag = .start_fuzzing,
1928 .bytes_len = 4 + 1 + 8,
1929 };
1930 const full_msg = std.mem.asBytes(&header) ++ std.mem.asBytes(&index) ++
1931 std.mem.asBytes(&kind) ++ std.mem.asBytes(&amount_or_instance);
1932
1933 try file.writeAll(full_msg);
1934}
1935
1903fn evalGeneric(run: *Run, child: *std.process.Child) !StdIoResult {1936fn evalGeneric(run: *Run, child: *std.process.Child) !StdIoResult {
1904 const b = run.step.owner;1937 const b = run.step.owner;
1905 const arena = b.allocator;1938 const arena = b.allocator;
lib/std/Build/WebServer.zig+9-1
...@@ -219,12 +219,20 @@ pub fn finishBuild(ws: *WebServer, opts: struct {...@@ -219,12 +219,20 @@ pub fn finishBuild(ws: *WebServer, opts: struct {
219 // Affects or affected by issues #5185, #22523, and #22464.219 // Affects or affected by issues #5185, #22523, and #22464.
220 std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});220 std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
221 }221 }
222
222 assert(ws.fuzz == null);223 assert(ws.fuzz == null);
223224
224 ws.build_status.store(.fuzz_init, .monotonic);225 ws.build_status.store(.fuzz_init, .monotonic);
225 ws.notifyUpdate();226 ws.notifyUpdate();
226227
227 ws.fuzz = Fuzz.init(ws) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)});228 ws.fuzz = Fuzz.init(
229 ws.gpa,
230 ws.thread_pool,
231 ws.all_steps,
232 ws.root_prog_node,
233 ws.ttyconf,
234 .{ .forever = .{ .ws = ws } },
235 ) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)});
228 ws.fuzz.?.start();236 ws.fuzz.?.start();
229 }237 }
230238
lib/std/Build/abi.zig+14-2
...@@ -140,10 +140,10 @@ pub const Rebuild = extern struct {...@@ -140,10 +140,10 @@ pub const Rebuild = extern struct {
140pub const fuzz = struct {140pub const fuzz = struct {
141 pub const TestOne = *const fn (Slice) callconv(.c) void;141 pub const TestOne = *const fn (Slice) callconv(.c) void;
142 pub extern fn fuzzer_init(cache_dir_path: Slice) void;142 pub extern fn fuzzer_init(cache_dir_path: Slice) void;
143 pub extern fn fuzzer_coverage_id() u64;143 pub extern fn fuzzer_coverage() Coverage;
144 pub extern fn fuzzer_init_test(test_one: TestOne, unit_test_name: Slice) void;144 pub extern fn fuzzer_init_test(test_one: TestOne, unit_test_name: Slice) void;
145 pub extern fn fuzzer_new_input(bytes: Slice) void;145 pub extern fn fuzzer_new_input(bytes: Slice) void;
146 pub extern fn fuzzer_main() void;146 pub extern fn fuzzer_main(limit_kind: LimitKind, amount: u64) void;
147147
148 pub const Slice = extern struct {148 pub const Slice = extern struct {
149 ptr: [*]const u8,149 ptr: [*]const u8,
...@@ -158,6 +158,8 @@ pub const fuzz = struct {...@@ -158,6 +158,8 @@ pub const fuzz = struct {
158 }158 }
159 };159 };
160160
161 pub const LimitKind = enum(u8) { forever, iterations };
162
161 /// libfuzzer uses this and its usize is the one that counts. To match the ABI,163 /// libfuzzer uses this and its usize is the one that counts. To match the ABI,
162 /// make the ints be the size of the target used with libfuzzer.164 /// make the ints be the size of the target used with libfuzzer.
163 ///165 ///
...@@ -251,6 +253,16 @@ pub const fuzz = struct {...@@ -251,6 +253,16 @@ pub const fuzz = struct {
251 return .{ .locs_len_raw = @bitCast(locs_len) };253 return .{ .locs_len_raw = @bitCast(locs_len) };
252 }254 }
253 };255 };
256
257 /// Sent by lib/fuzzer to test_runner to obtain information about the
258 /// active memory mapped input file and cumulative stats about previous
259 /// fuzzing runs.
260 pub const Coverage = extern struct {
261 id: u64,
262 runs: u64,
263 unique: u64,
264 seen: u64,
265 };
254};266};
255267
256/// ABI bits specifically relating to the time report interface.268/// ABI bits specifically relating to the time report interface.
lib/std/zig/Client.zig+10-2
...@@ -33,10 +33,18 @@ pub const Message = struct {...@@ -33,10 +33,18 @@ pub const Message = struct {
33 /// Ask the test runner to run a particular test.33 /// Ask the test runner to run a particular test.
34 /// The message body is a u32 test index.34 /// The message body is a u32 test index.
35 run_test,35 run_test,
36 /// Ask the test runner to start fuzzing a particular test.36 /// Ask the test runner to start fuzzing a particular test forever or for a given amount of time/iterations.
37 /// The message body is a u32 test index.37 /// The message body is:
38 /// - a u32 test index.
39 /// - a u8 test limit kind (std.Build.api.fuzz.LimitKind)
40 /// - a u64 value whose meaning depends on FuzzLimitKind (either a limit amount or an instance id)
38 start_fuzzing,41 start_fuzzing,
3942
40 _,43 _,
41 };44 };
45
46 comptime {
47 const std = @import("std");
48 std.debug.assert(@sizeOf(std.Build.abi.fuzz.LimitKind) == 1);
49 }
42};50};
lib/std/zig/Server.zig+26-3
...@@ -42,9 +42,13 @@ pub const Message = struct {...@@ -42,9 +42,13 @@ pub const Message = struct {
42 /// The remaining bytes is the file path relative to that prefix.42 /// The remaining bytes is the file path relative to that prefix.
43 /// The prefixes are hard-coded in Compilation.create (cwd, zig lib dir, local cache dir)43 /// The prefixes are hard-coded in Compilation.create (cwd, zig lib dir, local cache dir)
44 file_system_inputs,44 file_system_inputs,
45 /// Body is a u64le that indicates the file path within the cache used45 /// Body is:
46 /// to store coverage information. The integer is a hash of the PCs46 /// - a u64le that indicates the file path within the cache used
47 /// stored within that file.47 /// to store coverage information. The integer is a hash of the PCs
48 /// stored within that file.
49 /// - u64le of total runs accumulated
50 /// - u64le of unique runs accumulated
51 /// - u64le of coverage accumulated
48 coverage_id,52 coverage_id,
49 /// Body is a u64le that indicates the function pointer virtual memory53 /// Body is a u64le that indicates the function pointer virtual memory
50 /// address of the fuzz unit test. This is used to provide a starting54 /// address of the fuzz unit test. This is used to provide a starting
...@@ -141,9 +145,15 @@ pub fn receiveMessage(s: *Server) !InMessage.Header {...@@ -141,9 +145,15 @@ pub fn receiveMessage(s: *Server) !InMessage.Header {
141 return s.in.takeStruct(InMessage.Header, .little);145 return s.in.takeStruct(InMessage.Header, .little);
142}146}
143147
148pub fn receiveBody_u8(s: *Server) !u8 {
149 return s.in.takeInt(u8, .little);
150}
144pub fn receiveBody_u32(s: *Server) !u32 {151pub fn receiveBody_u32(s: *Server) !u32 {
145 return s.in.takeInt(u32, .little);152 return s.in.takeInt(u32, .little);
146}153}
154pub fn receiveBody_u64(s: *Server) !u64 {
155 return s.in.takeInt(u64, .little);
156}
147157
148pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void {158pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void {
149 try s.serveMessageHeader(.{159 try s.serveMessageHeader(.{
...@@ -160,6 +170,7 @@ pub fn serveMessageHeader(s: *const Server, header: OutMessage.Header) !void {...@@ -160,6 +170,7 @@ pub fn serveMessageHeader(s: *const Server, header: OutMessage.Header) !void {
160}170}
161171
162pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void {172pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void {
173 assert(tag != .coverage_id);
163 try serveMessageHeader(s, .{174 try serveMessageHeader(s, .{
164 .tag = tag,175 .tag = tag,
165 .bytes_len = @sizeOf(u64),176 .bytes_len = @sizeOf(u64),
...@@ -168,6 +179,18 @@ pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void {...@@ -168,6 +179,18 @@ pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void {
168 try s.out.flush();179 try s.out.flush();
169}180}
170181
182pub fn serveCoverageIdMessage(s: *const Server, id: u64, runs: u64, unique: u64, cov: u64) !void {
183 try serveMessageHeader(s, .{
184 .tag = .coverage_id,
185 .bytes_len = @sizeOf(u64) + @sizeOf(u64) + @sizeOf(u64) + @sizeOf(u64),
186 });
187 try s.out.writeInt(u64, id, .little);
188 try s.out.writeInt(u64, runs, .little);
189 try s.out.writeInt(u64, unique, .little);
190 try s.out.writeInt(u64, cov, .little);
191 try s.out.flush();
192}
193
171pub fn serveEmitDigest(194pub fn serveEmitDigest(
172 s: *Server,195 s: *Server,
173 digest: *const [Cache.bin_digest_len]u8,196 digest: *const [Cache.bin_digest_len]u8,
src/Compilation.zig+1-1
...@@ -8120,7 +8120,7 @@ pub fn addLinkLib(comp: *Compilation, lib_name: []const u8) !void {...@@ -8120,7 +8120,7 @@ pub fn addLinkLib(comp: *Compilation, lib_name: []const u8) !void {
8120/// compiler-rt, libcxx, libc, libunwind, etc.8120/// compiler-rt, libcxx, libc, libunwind, etc.
8121pub fn compilerRtOptMode(comp: Compilation) std.builtin.OptimizeMode {8121pub fn compilerRtOptMode(comp: Compilation) std.builtin.OptimizeMode {
8122 if (comp.debug_compiler_runtime_libs) {8122 if (comp.debug_compiler_runtime_libs) {
8123 return comp.root_mod.optimize_mode;8123 return .Debug;
8124 }8124 }
8125 const target = &comp.root_mod.resolved_target.result;8125 const target = &comp.root_mod.resolved_target.result;
8126 switch (comp.root_mod.optimize_mode) {8126 switch (comp.root_mod.optimize_mode) {
test/standalone/libfuzzer/main.zig+1-1
...@@ -24,7 +24,7 @@ pub fn main() !void {...@@ -24,7 +24,7 @@ pub fn main() !void {
24 abi.fuzzer_new_input(.fromSlice(""));24 abi.fuzzer_new_input(.fromSlice(""));
25 abi.fuzzer_new_input(.fromSlice("hello"));25 abi.fuzzer_new_input(.fromSlice("hello"));
2626
27 const pc_digest = abi.fuzzer_coverage_id();27 const pc_digest = abi.fuzzer_coverage().id;
28 const coverage_file_path = "v/" ++ std.fmt.hex(pc_digest);28 const coverage_file_path = "v/" ++ std.fmt.hex(pc_digest);
29 const coverage_file = try cache_dir.openFile(coverage_file_path, .{});29 const coverage_file = try cache_dir.openFile(coverage_file_path, .{});
30 defer coverage_file.close();30 defer coverage_file.close();