authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-09-09 19:36:52-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-09-11 13:41:29-07:00
log892ce7ef527c863d84f3085f79f1a5aec7161c2c
tree7b7d2b934bcf59666e535fdc1df3a54bef8a9e53
parent218cf059dd215282aa96d6b4715e68d533a4238e

rework fuzzing API

The previous API used `std.testing.fuzzInput(.{})` however that has the problem that users call it multiple times incorrectly, and there might be work happening to obtain the corpus which should not be included in coverage analysis, and which must not slow down iteration speed. This commit restructures it so that the main loop lives in libfuzzer and directly calls the "test one" function. In this commit I was a little too aggressive because I made the test runner export `fuzzer_one` for this purpose. This was motivated by performance, but it causes "exported symbol collision: fuzzer_one" to occur when more than one fuzz test is provided. There are three ways to solve this: 1. libfuzzer needs to be passed a function pointer instead. Possible performance downside. 2. build runner needs to build a different process per fuzz test. Potentially wasteful and unclear how to isolate them. 3. test runner needs to perform a relocation at runtime to point the function call to the relevant unit test. Portability issues and dubious performance gains.

4 files changed, 131 insertions(+), 74 deletions(-)

lib/compiler/test_runner.zig+75-31
......@@ -145,31 +145,27 @@ fn mainServer() !void {
145145 .start_fuzzing => {
146146 if (!builtin.fuzz) unreachable;
147147 const index = try server.receiveBody_u32();
148 var first = true;
149148 const test_fn = builtin.test_functions[index];
150 while (true) {
151 testing.allocator_instance = .{};
152 defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
153 log_err_count = 0;
154 is_fuzz_test = false;
155 test_fn.func() catch |err| switch (err) {
156 error.SkipZigTest => continue,
157 else => {
158 if (@errorReturnTrace()) |trace| {
159 std.debug.dumpStackTrace(trace.*);
160 }
161 std.debug.print("failed with error.{s}\n", .{@errorName(err)});
162 std.process.exit(1);
163 },
164 };
165 if (!is_fuzz_test) @panic("missed call to std.testing.fuzzInput");
166 if (log_err_count != 0) @panic("error logs detected");
167 if (first) {
168 first = false;
169 const entry_addr = @intFromPtr(test_fn.func);
170 try server.serveU64Message(.fuzz_start_addr, entry_addr);
171 }
149 const entry_addr = @intFromPtr(test_fn.func);
150 try server.serveU64Message(.fuzz_start_addr, entry_addr);
151 const prev_allocator_state = testing.allocator_instance;
152 defer {
153 testing.allocator_instance = prev_allocator_state;
154 if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
172155 }
156 is_fuzz_test = false;
157 test_fn.func() catch |err| switch (err) {
158 error.SkipZigTest => return,
159 else => {
160 if (@errorReturnTrace()) |trace| {
161 std.debug.dumpStackTrace(trace.*);
162 }
163 std.debug.print("failed with error.{s}\n", .{@errorName(err)});
164 std.process.exit(1);
165 },
166 };
167 if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");
168 if (log_err_count != 0) @panic("error logs detected");
173169 },
174170
175171 else => {
......@@ -349,19 +345,67 @@ const FuzzerSlice = extern struct {
349345
350346var is_fuzz_test: bool = undefined;
351347
352extern fn fuzzer_next() FuzzerSlice;
348extern fn fuzzer_start() void;
353349extern fn fuzzer_init(cache_dir: FuzzerSlice) void;
354350extern fn fuzzer_coverage_id() u64;
355351
356pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 {
352pub fn fuzz(
353 comptime testOne: fn ([]const u8) anyerror!void,
354 options: testing.FuzzInputOptions,
355) anyerror!void {
356 // Prevent this function from confusing the fuzzer by omitting its own code
357 // coverage from being considered.
357358 @disableInstrumentation();
358 if (crippled) return "";
359
360 // Some compiler backends are not capable of handling fuzz testing yet but
361 // we still want CI test coverage enabled.
362 if (crippled) return;
363
364 // Smoke test to ensure the test did not use conditional compilation to
365 // contradict itself by making it not actually be a fuzz test when the test
366 // is built in fuzz mode.
359367 is_fuzz_test = true;
368
369 // Ensure no test failure occurred before starting fuzzing.
370 if (log_err_count != 0) @panic("error logs detected");
371
372 // libfuzzer is in a separate compilation unit so that its own code can be
373 // excluded from code coverage instrumentation. It needs a function pointer
374 // it can call for checking exactly one input. Inside this function we do
375 // our standard unit test checks such as memory leaks, and interaction with
376 // error logs.
377 const global = struct {
378 fn fuzzer_one(input_ptr: [*]const u8, input_len: usize) callconv(.C) void {
379 @disableInstrumentation();
380 testing.allocator_instance = .{};
381 defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
382 log_err_count = 0;
383 testOne(input_ptr[0..input_len]) catch |err| switch (err) {
384 error.SkipZigTest => return,
385 else => {
386 if (@errorReturnTrace()) |trace| {
387 std.debug.dumpStackTrace(trace.*);
388 }
389 std.debug.print("failed with error.{s}\n", .{@errorName(err)});
390 std.process.exit(1);
391 },
392 };
393 if (log_err_count != 0) @panic("error logs detected");
394 }
395 };
360396 if (builtin.fuzz) {
361 return fuzzer_next().toSlice();
397 @export(&global.fuzzer_one, .{ .name = "fuzzer_one" });
398 fuzzer_start();
399 return;
400 }
401
402 // When the unit test executable is not built in fuzz mode, only run the
403 // provided corpus.
404 for (options.corpus) |input| {
405 try testOne(input);
362406 }
363 if (options.corpus.len == 0) return "";
364 var prng = std.Random.DefaultPrng.init(testing.random_seed);
365 const random = prng.random();
366 return options.corpus[random.uintLessThan(usize, options.corpus.len)];
407
408 // In case there is no provided corpus, also use an empty
409 // string as a smoke test.
410 try testOne("");
367411}
lib/fuzzer.zig+45-39
......@@ -235,22 +235,41 @@ const Fuzzer = struct {
235235 };
236236 }
237237
238 fn next(f: *Fuzzer) ![]const u8 {
238 fn start(f: *Fuzzer) !void {
239239 const gpa = f.gpa;
240240 const rng = fuzzer.rng.random();
241241
242 if (f.recent_cases.entries.len == 0) {
243 // Prepare initial input.
244 try f.recent_cases.ensureUnusedCapacity(gpa, 100);
245 const len = rng.uintLessThanBiased(usize, 80);
246 try f.input.resize(gpa, len);
247 rng.bytes(f.input.items);
248 f.recent_cases.putAssumeCapacity(.{
249 .id = 0,
250 .input = try gpa.dupe(u8, f.input.items),
251 .score = 0,
252 }, {});
253 } else {
242 // Prepare initial input.
243 assert(f.recent_cases.entries.len == 0);
244 assert(f.n_runs == 0);
245 try f.recent_cases.ensureUnusedCapacity(gpa, 100);
246 const len = rng.uintLessThanBiased(usize, 80);
247 try f.input.resize(gpa, len);
248 rng.bytes(f.input.items);
249 f.recent_cases.putAssumeCapacity(.{
250 .id = 0,
251 .input = try gpa.dupe(u8, f.input.items),
252 .score = 0,
253 }, {});
254
255 const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]);
256
257 while (true) {
258 const chosen_index = rng.uintLessThanBiased(usize, f.recent_cases.entries.len);
259 const run = &f.recent_cases.keys()[chosen_index];
260 f.input.clearRetainingCapacity();
261 f.input.appendSliceAssumeCapacity(run.input);
262 try f.mutate();
263
264 _ = @atomicRmw(usize, &header.lowest_stack, .Min, __sancov_lowest_stack, .monotonic);
265 @memset(f.pc_counters, 0);
266 f.coverage.reset();
267
268 fuzzer_one(f.input.items.ptr, f.input.items.len);
269
270 f.n_runs += 1;
271 _ = @atomicRmw(usize, &header.n_runs, .Add, 1, .monotonic);
272
254273 if (f.n_runs % 10000 == 0) f.dumpStats();
255274
256275 const analysis = f.analyzeLastRun();
......@@ -301,7 +320,6 @@ const Fuzzer = struct {
301320 }
302321 }
303322
304 const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]);
305323 _ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
306324 }
307325
......@@ -317,26 +335,12 @@ const Fuzzer = struct {
317335 // This has to be done before deinitializing the deleted items.
318336 const doomed_runs = f.recent_cases.keys()[cap..];
319337 f.recent_cases.shrinkRetainingCapacity(cap);
320 for (doomed_runs) |*run| {
321 std.log.info("culling score={d} id={d}", .{ run.score, run.id });
322 run.deinit(gpa);
338 for (doomed_runs) |*doomed_run| {
339 std.log.info("culling score={d} id={d}", .{ doomed_run.score, doomed_run.id });
340 doomed_run.deinit(gpa);
323341 }
324342 }
325343 }
326
327 const chosen_index = rng.uintLessThanBiased(usize, f.recent_cases.entries.len);
328 const run = &f.recent_cases.keys()[chosen_index];
329 f.input.clearRetainingCapacity();
330 f.input.appendSliceAssumeCapacity(run.input);
331 try f.mutate();
332
333 f.n_runs += 1;
334 const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]);
335 _ = @atomicRmw(usize, &header.n_runs, .Add, 1, .monotonic);
336 _ = @atomicRmw(usize, &header.lowest_stack, .Min, __sancov_lowest_stack, .monotonic);
337 @memset(f.pc_counters, 0);
338 f.coverage.reset();
339 return f.input.items;
340344 }
341345
342346 fn visitPc(f: *Fuzzer, pc: usize) void {
......@@ -419,10 +423,12 @@ export fn fuzzer_coverage_id() u64 {
419423 return fuzzer.coverage_id;
420424}
421425
422export fn fuzzer_next() Fuzzer.Slice {
423 return Fuzzer.Slice.fromZig(fuzzer.next() catch |err| switch (err) {
424 error.OutOfMemory => @panic("out of memory"),
425 });
426extern fn fuzzer_one(input_ptr: [*]const u8, input_len: usize) callconv(.C) void;
427
428export fn fuzzer_start() void {
429 fuzzer.start() catch |err| switch (err) {
430 error.OutOfMemory => fatal("out of memory", .{}),
431 };
426432}
427433
428434export fn fuzzer_init(cache_dir_struct: Fuzzer.Slice) void {
......@@ -432,24 +438,24 @@ export fn fuzzer_init(cache_dir_struct: Fuzzer.Slice) void {
432438 const pc_counters_start = @extern([*]u8, .{
433439 .name = "__start___sancov_cntrs",
434440 .linkage = .weak,
435 }) orelse fatal("missing __start___sancov_cntrs symbol");
441 }) orelse fatal("missing __start___sancov_cntrs symbol", .{});
436442
437443 const pc_counters_end = @extern([*]u8, .{
438444 .name = "__stop___sancov_cntrs",
439445 .linkage = .weak,
440 }) orelse fatal("missing __stop___sancov_cntrs symbol");
446 }) orelse fatal("missing __stop___sancov_cntrs symbol", .{});
441447
442448 const pc_counters = pc_counters_start[0 .. pc_counters_end - pc_counters_start];
443449
444450 const pcs_start = @extern([*]usize, .{
445451 .name = "__start___sancov_pcs1",
446452 .linkage = .weak,
447 }) orelse fatal("missing __start___sancov_pcs1 symbol");
453 }) orelse fatal("missing __start___sancov_pcs1 symbol", .{});
448454
449455 const pcs_end = @extern([*]usize, .{
450456 .name = "__stop___sancov_pcs1",
451457 .linkage = .weak,
452 }) orelse fatal("missing __stop___sancov_pcs1 symbol");
458 }) orelse fatal("missing __stop___sancov_pcs1 symbol", .{});
453459
454460 const pcs = pcs_start[0 .. pcs_end - pcs_start];
455461
lib/std/testing.zig+6-2
......@@ -1141,6 +1141,10 @@ pub const FuzzInputOptions = struct {
11411141 corpus: []const []const u8 = &.{},
11421142};
11431143
1144pub inline fn fuzzInput(options: FuzzInputOptions) []const u8 {
1145 return @import("root").fuzzInput(options);
1144/// Inline to avoid coverage instrumentation.
1145pub inline fn fuzz(
1146 comptime testOne: fn (input: []const u8) anyerror!void,
1147 options: FuzzInputOptions,
1148) anyerror!void {
1149 return @import("root").fuzz(testOne, options);
11461150}
lib/std/zig/tokenizer.zig+5-2
......@@ -1708,6 +1708,10 @@ test "invalid tabs and carriage returns" {
17081708 try testTokenize("\rpub\rswitch\r", &.{ .keyword_pub, .keyword_switch });
17091709}
17101710
1711test "fuzzable properties upheld" {
1712 return std.testing.fuzz(testPropertiesUpheld, .{});
1713}
1714
17111715fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
17121716 var tokenizer = Tokenizer.init(source);
17131717 for (expected_token_tags) |expected_token_tag| {
......@@ -1723,8 +1727,7 @@ fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !v
17231727 try std.testing.expectEqual(source.len, last_token.loc.end);
17241728}
17251729
1726test "fuzzable properties upheld" {
1727 const source = std.testing.fuzzInput(.{});
1730fn testPropertiesUpheld(source: []const u8) anyerror!void {
17281731 const source0 = try std.testing.allocator.dupeZ(u8, source);
17291732 defer std.testing.allocator.free(source0);
17301733 var tokenizer = Tokenizer.init(source0);