authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-29 21:44:38-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-08-07 00:48:32-07:00
log97643c1ecc9621b088807f446f06fe51e2db8fb6
tree22b8667d5fb4f762281d6b7ed54bb23774f7bda4
parenta60810b5a31f7c3a9d1b1f5152c889e45d9ee2f7

fuzzer: track code coverage from all runs

When a unique run is encountered, track it in a bit set memory-mapped into the fuzz directory so it can be observed by other processes, even while the fuzzer is running.

2 files changed, 177 insertions(+), 20 deletions(-)

lib/compiler/test_runner.zig+2
...@@ -41,6 +41,7 @@ pub fn main() void {...@@ -41,6 +41,7 @@ pub fn main() void {
41 }41 }
4242
43 fba.reset();43 fba.reset();
44 if (builtin.fuzz) fuzzer_init();
4445
45 if (listen) {46 if (listen) {
46 return mainServer() catch @panic("internal test runner failure");47 return mainServer() catch @panic("internal test runner failure");
...@@ -323,6 +324,7 @@ const FuzzerSlice = extern struct {...@@ -323,6 +324,7 @@ const FuzzerSlice = extern struct {
323var is_fuzz_test: bool = undefined;324var is_fuzz_test: bool = undefined;
324325
325extern fn fuzzer_next() FuzzerSlice;326extern fn fuzzer_next() FuzzerSlice;
327extern fn fuzzer_init() void;
326328
327pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 {329pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 {
328 @disableInstrumentation();330 @disableInstrumentation();
lib/fuzzer.zig+175-20
...@@ -2,6 +2,7 @@ const builtin = @import("builtin");...@@ -2,6 +2,7 @@ const builtin = @import("builtin");
2const std = @import("std");2const std = @import("std");
3const Allocator = std.mem.Allocator;3const Allocator = std.mem.Allocator;
4const assert = std.debug.assert;4const assert = std.debug.assert;
5const fatal = std.process.fatal;
56
6pub const std_options = .{7pub const std_options = .{
7 .logFn = logOverride,8 .logFn = logOverride,
...@@ -17,7 +18,7 @@ fn logOverride(...@@ -17,7 +18,7 @@ fn logOverride(
17) void {18) void {
18 if (builtin.mode != .Debug) return;19 if (builtin.mode != .Debug) return;
19 const f = if (log_file) |f| f else f: {20 const f = if (log_file) |f| f else f: {
20 const f = std.fs.cwd().createFile("libfuzzer.log", .{}) catch @panic("failed to open fuzzer log file");21 const f = fuzzer.dir.createFile("libfuzzer.log", .{}) catch @panic("failed to open fuzzer log file");
21 log_file = f;22 log_file = f;
22 break :f f;23 break :f f;
23 };24 };
...@@ -28,16 +29,17 @@ fn logOverride(...@@ -28,16 +29,17 @@ fn logOverride(
2829
29export threadlocal var __sancov_lowest_stack: usize = 0;30export threadlocal var __sancov_lowest_stack: usize = 0;
3031
31export fn __sanitizer_cov_8bit_counters_init(start: [*]u8, stop: [*]u8) void {32var module_count_8bc: usize = 0;
32 std.log.debug("__sanitizer_cov_8bit_counters_init start={*}, stop={*}", .{ start, stop });33var module_count_pcs: usize = 0;
34
35export fn __sanitizer_cov_8bit_counters_init(start: [*]u8, end: [*]u8) void {
36 assert(@atomicRmw(usize, &module_count_8bc, .Add, 1, .monotonic) == 0);
37 fuzzer.pc_counters = start[0 .. end - start];
33}38}
3439
35export fn __sanitizer_cov_pcs_init(pc_start: [*]const usize, pc_end: [*]const usize) void {40export fn __sanitizer_cov_pcs_init(start: [*]const Fuzzer.FlaggedPc, end: [*]const Fuzzer.FlaggedPc) void {
36 std.log.debug("__sanitizer_cov_pcs_init pc_start={*}, pc_end={*}", .{ pc_start, pc_end });41 assert(@atomicRmw(usize, &module_count_pcs, .Add, 1, .monotonic) == 0);
37 fuzzer.pc_range = .{42 fuzzer.flagged_pcs = start[0 .. end - start];
38 .start = @intFromPtr(pc_start),
39 .end = @intFromPtr(pc_start),
40 };
41}43}
4244
43export fn __sanitizer_cov_trace_const_cmp1(arg1: u8, arg2: u8) void {45export fn __sanitizer_cov_trace_const_cmp1(arg1: u8, arg2: u8) void {
...@@ -102,11 +104,25 @@ const Fuzzer = struct {...@@ -102,11 +104,25 @@ const Fuzzer = struct {
102 gpa: Allocator,104 gpa: Allocator,
103 rng: std.Random.DefaultPrng,105 rng: std.Random.DefaultPrng,
104 input: std.ArrayListUnmanaged(u8),106 input: std.ArrayListUnmanaged(u8),
105 pc_range: PcRange,107 flagged_pcs: []const FlaggedPc,
106 count: usize,108 pc_counters: []u8,
109 n_runs: usize,
107 recent_cases: RunMap,110 recent_cases: RunMap,
108 deduplicated_runs: usize,111 deduplicated_runs: usize,
112 /// Data collected from code coverage instrumentation from one execution of
113 /// the test function.
109 coverage: Coverage,114 coverage: Coverage,
115 /// Tracks which PCs have been seen across all runs that do not crash the fuzzer process.
116 /// Stored in a memory-mapped file so that it can be shared with other
117 /// processes and viewed while the fuzzer is running.
118 seen_pcs: MemoryMappedList,
119 dir: std.fs.Dir,
120
121 const SeenPcsHeader = extern struct {
122 n_runs: usize,
123 pcs_len: usize,
124 lowest_stack: usize,
125 };
110126
111 const RunMap = std.ArrayHashMapUnmanaged(Run, void, Run.HashContext, false);127 const RunMap = std.ArrayHashMapUnmanaged(Run, void, Run.HashContext, false);
112128
...@@ -161,9 +177,12 @@ const Fuzzer = struct {...@@ -161,9 +177,12 @@ const Fuzzer = struct {
161 }177 }
162 };178 };
163179
164 const PcRange = struct {180 const FlaggedPc = extern struct {
165 start: usize,181 addr: usize,
166 end: usize,182 flags: packed struct(usize) {
183 entry: bool,
184 _: @Type(.{ .Int = .{ .signedness = .unsigned, .bits = @bitSizeOf(usize) - 1 } }),
185 },
167 };186 };
168187
169 const Analysis = struct {188 const Analysis = struct {
...@@ -171,6 +190,56 @@ const Fuzzer = struct {...@@ -171,6 +190,56 @@ const Fuzzer = struct {
171 id: Run.Id,190 id: Run.Id,
172 };191 };
173192
193 fn init(f: *Fuzzer, dir: std.fs.Dir) !void {
194 f.dir = dir;
195
196 // Layout of this file:
197 // - Header
198 // - list of PC addresses (usize elements)
199 // - list of hit flag, 1 bit per address (stored in u8 elements)
200 const coverage_file = dir.createFile("coverage", .{
201 .read = true,
202 .truncate = false,
203 }) catch |err| fatal("unable to create coverage file: {s}", .{@errorName(err)});
204 const flagged_pcs = f.flagged_pcs;
205 const n_bitset_elems = (flagged_pcs.len + 7) / 8;
206 const bytes_len = @sizeOf(SeenPcsHeader) + flagged_pcs.len * @sizeOf(usize) + n_bitset_elems;
207 const existing_len = coverage_file.getEndPos() catch |err| {
208 fatal("unable to check len of coverage file: {s}", .{@errorName(err)});
209 };
210 if (existing_len == 0) {
211 coverage_file.setEndPos(bytes_len) catch |err| {
212 fatal("unable to set len of coverage file: {s}", .{@errorName(err)});
213 };
214 } else if (existing_len != bytes_len) {
215 fatal("incompatible existing coverage file (differing lengths)", .{});
216 }
217 f.seen_pcs = MemoryMappedList.init(coverage_file, existing_len, bytes_len) catch |err| {
218 fatal("unable to init coverage memory map: {s}", .{@errorName(err)});
219 };
220 if (existing_len != 0) {
221 const existing_pcs = std.mem.bytesAsSlice(usize, f.seen_pcs.items[@sizeOf(SeenPcsHeader)..][0 .. flagged_pcs.len * @sizeOf(usize)]);
222 for (existing_pcs, flagged_pcs, 0..) |old, new, i| {
223 if (old != new.addr) {
224 fatal("incompatible existing coverage file (differing PC at index {d}: {x} != {x})", .{
225 i, old, new.addr,
226 });
227 }
228 }
229 } else {
230 const header: SeenPcsHeader = .{
231 .n_runs = 0,
232 .pcs_len = flagged_pcs.len,
233 .lowest_stack = std.math.maxInt(usize),
234 };
235 f.seen_pcs.appendSliceAssumeCapacity(std.mem.asBytes(&header));
236 for (flagged_pcs) |flagged_pc| {
237 f.seen_pcs.appendSliceAssumeCapacity(std.mem.asBytes(&flagged_pc.addr));
238 }
239 f.seen_pcs.appendNTimesAssumeCapacity(0, n_bitset_elems);
240 }
241 }
242
174 fn analyzeLastRun(f: *Fuzzer) Analysis {243 fn analyzeLastRun(f: *Fuzzer) Analysis {
175 return .{244 return .{
176 .id = f.coverage.run_id_hasher.final(),245 .id = f.coverage.run_id_hasher.final(),
...@@ -194,7 +263,7 @@ const Fuzzer = struct {...@@ -194,7 +263,7 @@ const Fuzzer = struct {
194 .score = 0,263 .score = 0,
195 }, {});264 }, {});
196 } else {265 } else {
197 if (f.count % 1000 == 0) f.dumpStats();266 if (f.n_runs % 1000 == 0) f.dumpStats();
198267
199 const analysis = f.analyzeLastRun();268 const analysis = f.analyzeLastRun();
200 const gop = f.recent_cases.getOrPutAssumeCapacity(.{269 const gop = f.recent_cases.getOrPutAssumeCapacity(.{
...@@ -217,6 +286,25 @@ const Fuzzer = struct {...@@ -217,6 +286,25 @@ const Fuzzer = struct {
217 .input = try gpa.dupe(u8, f.input.items),286 .input = try gpa.dupe(u8, f.input.items),
218 .score = analysis.score,287 .score = analysis.score,
219 };288 };
289
290 // Track code coverage from all runs.
291 {
292 const seen_pcs = f.seen_pcs.items[@sizeOf(SeenPcsHeader) + f.flagged_pcs.len * @sizeOf(usize) ..];
293 for (seen_pcs, 0..) |*elem, i| {
294 const byte_i = i / 8;
295 const mask: u8 =
296 (@as(u8, @intFromBool(f.pc_counters[byte_i + 0] != 0)) << 0) |
297 (@as(u8, @intFromBool(f.pc_counters[byte_i + 1] != 0)) << 1) |
298 (@as(u8, @intFromBool(f.pc_counters[byte_i + 2] != 0)) << 2) |
299 (@as(u8, @intFromBool(f.pc_counters[byte_i + 3] != 0)) << 3) |
300 (@as(u8, @intFromBool(f.pc_counters[byte_i + 4] != 0)) << 4) |
301 (@as(u8, @intFromBool(f.pc_counters[byte_i + 5] != 0)) << 5) |
302 (@as(u8, @intFromBool(f.pc_counters[byte_i + 6] != 0)) << 6) |
303 (@as(u8, @intFromBool(f.pc_counters[byte_i + 7] != 0)) << 7);
304
305 _ = @atomicRmw(u8, elem, .Or, mask, .monotonic);
306 }
307 }
220 }308 }
221309
222 if (f.recent_cases.entries.len >= 100) {310 if (f.recent_cases.entries.len >= 100) {
...@@ -244,8 +332,12 @@ const Fuzzer = struct {...@@ -244,8 +332,12 @@ const Fuzzer = struct {
244 f.input.appendSliceAssumeCapacity(run.input);332 f.input.appendSliceAssumeCapacity(run.input);
245 try f.mutate();333 try f.mutate();
246334
335 f.n_runs += 1;
336 const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]);
337 _ = @atomicRmw(usize, &header.n_runs, .Add, 1, .monotonic);
338 _ = @atomicRmw(usize, &header.lowest_stack, .Min, __sancov_lowest_stack, .monotonic);
339 @memset(f.pc_counters, 0);
247 f.coverage.reset();340 f.coverage.reset();
248 f.count += 1;
249 return f.input.items;341 return f.input.items;
250 }342 }
251343
...@@ -257,8 +349,7 @@ const Fuzzer = struct {...@@ -257,8 +349,7 @@ const Fuzzer = struct {
257349
258 fn dumpStats(f: *Fuzzer) void {350 fn dumpStats(f: *Fuzzer) void {
259 std.log.info("stats: runs={d} deduplicated={d}", .{351 std.log.info("stats: runs={d} deduplicated={d}", .{
260 f.count,352 f.n_runs, f.deduplicated_runs,
261 f.deduplicated_runs,
262 });353 });
263 for (f.recent_cases.keys()[0..@min(f.recent_cases.entries.len, 5)], 0..) |run, i| {354 for (f.recent_cases.keys()[0..@min(f.recent_cases.entries.len, 5)], 0..) |run, i| {
264 std.log.info("best[{d}] id={x} score={d} input: '{}'", .{355 std.log.info("best[{d}] id={x} score={d} input: '{}'", .{
...@@ -303,11 +394,14 @@ var fuzzer: Fuzzer = .{...@@ -303,11 +394,14 @@ var fuzzer: Fuzzer = .{
303 .gpa = general_purpose_allocator.allocator(),394 .gpa = general_purpose_allocator.allocator(),
304 .rng = std.Random.DefaultPrng.init(0),395 .rng = std.Random.DefaultPrng.init(0),
305 .input = .{},396 .input = .{},
306 .pc_range = .{ .start = 0, .end = 0 },397 .flagged_pcs = undefined,
307 .count = 0,398 .pc_counters = undefined,
399 .n_runs = 0,
308 .deduplicated_runs = 0,400 .deduplicated_runs = 0,
309 .recent_cases = .{},401 .recent_cases = .{},
310 .coverage = undefined,402 .coverage = undefined,
403 .dir = undefined,
404 .seen_pcs = undefined,
311};405};
312406
313export fn fuzzer_next() Fuzzer.Slice {407export fn fuzzer_next() Fuzzer.Slice {
...@@ -315,3 +409,64 @@ export fn fuzzer_next() Fuzzer.Slice {...@@ -315,3 +409,64 @@ export fn fuzzer_next() Fuzzer.Slice {
315 error.OutOfMemory => @panic("out of memory"),409 error.OutOfMemory => @panic("out of memory"),
316 });410 });
317}411}
412
413export fn fuzzer_init() void {
414 if (module_count_8bc == 0) fatal("__sanitizer_cov_8bit_counters_init was never called", .{});
415 if (module_count_pcs == 0) fatal("__sanitizer_cov_pcs_init was never called", .{});
416
417 // TODO: move this to .zig-cache/f
418 const fuzz_dir = std.fs.cwd().makeOpenPath("f", .{ .iterate = true }) catch |err| {
419 fatal("unable to open fuzz directory 'f': {s}", .{@errorName(err)});
420 };
421 fuzzer.init(fuzz_dir) catch |err| fatal("unable to init fuzzer: {s}", .{@errorName(err)});
422}
423
424/// Like `std.ArrayListUnmanaged(u8)` but backed by memory mapping.
425pub const MemoryMappedList = struct {
426 /// Contents of the list.
427 ///
428 /// Pointers to elements in this slice are invalidated by various functions
429 /// of this ArrayList in accordance with the respective documentation. In
430 /// all cases, "invalidated" means that the memory has been passed to this
431 /// allocator's resize or free function.
432 items: []align(std.mem.page_size) volatile u8,
433 /// How many bytes this list can hold without allocating additional memory.
434 capacity: usize,
435
436 pub fn init(file: std.fs.File, length: usize, capacity: usize) !MemoryMappedList {
437 const ptr = try std.posix.mmap(
438 null,
439 capacity,
440 std.posix.PROT.READ | std.posix.PROT.WRITE,
441 .{ .TYPE = .SHARED },
442 file.handle,
443 0,
444 );
445 return .{
446 .items = ptr[0..length],
447 .capacity = capacity,
448 };
449 }
450
451 /// Append the slice of items to the list.
452 /// Asserts that the list can hold the additional items.
453 pub fn appendSliceAssumeCapacity(l: *MemoryMappedList, items: []const u8) void {
454 const old_len = l.items.len;
455 const new_len = old_len + items.len;
456 assert(new_len <= l.capacity);
457 l.items.len = new_len;
458 @memcpy(l.items[old_len..][0..items.len], items);
459 }
460
461 /// Append a value to the list `n` times.
462 /// Never invalidates element pointers.
463 /// The function is inline so that a comptime-known `value` parameter will
464 /// have better memset codegen in case it has a repeated byte pattern.
465 /// Asserts that the list can hold the additional items.
466 pub inline fn appendNTimesAssumeCapacity(l: *MemoryMappedList, value: u8, n: usize) void {
467 const new_len = l.items.len + n;
468 assert(new_len <= l.capacity);
469 @memset(l.items.ptr[l.items.len..new_len], value);
470 l.items.len = new_len;
471 }
472};