authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-22 18:06:58-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-25 18:52:20-07:00
log6f3767862d6886d5fde7e3734455a30f168ba80b
treebd2585a84cc1dd1db4725e2f044ba1da30f6e702
parent3256df2ff80be565095993b03f8e1edfd5072367

implement std.testing.fuzzInput

For now this returns a dummy fuzz input.

4 files changed, 125 insertions(+), 53 deletions(-)

lib/compiler/test_runner.zig+69-22
......@@ -1,18 +1,26 @@
11//! Default test runner for unit tests.
2const builtin = @import("builtin");
23const std = @import("std");
34const io = std.io;
4const builtin = @import("builtin");
5const testing = std.testing;
56
67pub const std_options = .{
78 .logFn = log,
89};
910
1011var log_err_count: usize = 0;
11var cmdline_buffer: [4096]u8 = undefined;
12var fba = std.heap.FixedBufferAllocator.init(&cmdline_buffer);
12var fba_buffer: [8192]u8 = undefined;
13var fba = std.heap.FixedBufferAllocator.init(&fba_buffer);
14
15const crippled = switch (builtin.zig_backend) {
16 .stage2_riscv64 => true,
17 else => false,
18};
1319
1420pub fn main() void {
15 if (builtin.zig_backend == .stage2_riscv64) {
21 @disableInstrumentation();
22
23 if (crippled) {
1624 return mainSimple() catch @panic("test failure\n");
1725 }
1826
......@@ -25,13 +33,15 @@ pub fn main() void {
2533 if (std.mem.eql(u8, arg, "--listen=-")) {
2634 listen = true;
2735 } else if (std.mem.startsWith(u8, arg, "--seed=")) {
28 std.testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch
36 testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch
2937 @panic("unable to parse --seed command line argument");
3038 } else {
3139 @panic("unrecognized command line argument");
3240 }
3341 }
3442
43 fba.reset();
44
3545 if (listen) {
3646 return mainServer() catch @panic("internal test runner failure");
3747 } else {
......@@ -40,6 +50,7 @@ pub fn main() void {
4050}
4151
4252fn mainServer() !void {
53 @disableInstrumentation();
4354 var server = try std.zig.Server.init(.{
4455 .gpa = fba.allocator(),
4556 .in = std.io.getStdIn(),
......@@ -55,24 +66,24 @@ fn mainServer() !void {
5566 return std.process.exit(0);
5667 },
5768 .query_test_metadata => {
58 std.testing.allocator_instance = .{};
59 defer if (std.testing.allocator_instance.deinit() == .leak) {
69 testing.allocator_instance = .{};
70 defer if (testing.allocator_instance.deinit() == .leak) {
6071 @panic("internal test runner memory leak");
6172 };
6273
6374 var string_bytes: std.ArrayListUnmanaged(u8) = .{};
64 defer string_bytes.deinit(std.testing.allocator);
65 try string_bytes.append(std.testing.allocator, 0); // Reserve 0 for null.
75 defer string_bytes.deinit(testing.allocator);
76 try string_bytes.append(testing.allocator, 0); // Reserve 0 for null.
6677
6778 const test_fns = builtin.test_functions;
68 const names = try std.testing.allocator.alloc(u32, test_fns.len);
69 defer std.testing.allocator.free(names);
70 const expected_panic_msgs = try std.testing.allocator.alloc(u32, test_fns.len);
71 defer std.testing.allocator.free(expected_panic_msgs);
79 const names = try testing.allocator.alloc(u32, test_fns.len);
80 defer testing.allocator.free(names);
81 const expected_panic_msgs = try testing.allocator.alloc(u32, test_fns.len);
82 defer testing.allocator.free(expected_panic_msgs);
7283
7384 for (test_fns, names, expected_panic_msgs) |test_fn, *name, *expected_panic_msg| {
7485 name.* = @as(u32, @intCast(string_bytes.items.len));
75 try string_bytes.ensureUnusedCapacity(std.testing.allocator, test_fn.name.len + 1);
86 try string_bytes.ensureUnusedCapacity(testing.allocator, test_fn.name.len + 1);
7687 string_bytes.appendSliceAssumeCapacity(test_fn.name);
7788 string_bytes.appendAssumeCapacity(0);
7889 expected_panic_msg.* = 0;
......@@ -86,13 +97,13 @@ fn mainServer() !void {
8697 },
8798
8899 .run_test => {
89 std.testing.allocator_instance = .{};
100 testing.allocator_instance = .{};
90101 log_err_count = 0;
91102 const index = try server.receiveBody_u32();
92103 const test_fn = builtin.test_functions[index];
93104 var fail = false;
94105 var skip = false;
95 var leak = false;
106 is_fuzz_test = false;
96107 test_fn.func() catch |err| switch (err) {
97108 error.SkipZigTest => skip = true,
98109 else => {
......@@ -102,13 +113,14 @@ fn mainServer() !void {
102113 }
103114 },
104115 };
105 leak = std.testing.allocator_instance.deinit() == .leak;
116 const leak = testing.allocator_instance.deinit() == .leak;
106117 try server.serveTestResults(.{
107118 .index = index,
108119 .flags = .{
109120 .fail = fail,
110121 .skip = skip,
111122 .leak = leak,
123 .fuzz = is_fuzz_test,
112124 .log_err_count = std.math.lossyCast(
113125 @TypeOf(@as(std.zig.Server.Message.TestResults.Flags, undefined).log_err_count),
114126 log_err_count,
......@@ -118,7 +130,7 @@ fn mainServer() !void {
118130 },
119131
120132 else => {
121 std.debug.print("unsupported message: {x}", .{@intFromEnum(hdr.tag)});
133 std.debug.print("unsupported message: {x}\n", .{@intFromEnum(hdr.tag)});
122134 std.process.exit(1);
123135 },
124136 }
......@@ -126,6 +138,7 @@ fn mainServer() !void {
126138}
127139
128140fn mainTerminal() void {
141 @disableInstrumentation();
129142 const test_fn_list = builtin.test_functions;
130143 var ok_count: usize = 0;
131144 var skip_count: usize = 0;
......@@ -143,18 +156,19 @@ fn mainTerminal() void {
143156
144157 var leaks: usize = 0;
145158 for (test_fn_list, 0..) |test_fn, i| {
146 std.testing.allocator_instance = .{};
159 testing.allocator_instance = .{};
147160 defer {
148 if (std.testing.allocator_instance.deinit() == .leak) {
161 if (testing.allocator_instance.deinit() == .leak) {
149162 leaks += 1;
150163 }
151164 }
152 std.testing.log_level = .warn;
165 testing.log_level = .warn;
153166
154167 const test_node = root_node.start(test_fn.name, 0);
155168 if (!have_tty) {
156169 std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name });
157170 }
171 // Track in a global variable so that `fuzzInput` can see it.
158172 if (test_fn.func()) |_| {
159173 ok_count += 1;
160174 test_node.end();
......@@ -208,10 +222,11 @@ pub fn log(
208222 comptime format: []const u8,
209223 args: anytype,
210224) void {
225 @disableInstrumentation();
211226 if (@intFromEnum(message_level) <= @intFromEnum(std.log.Level.err)) {
212227 log_err_count +|= 1;
213228 }
214 if (@intFromEnum(message_level) <= @intFromEnum(std.testing.log_level)) {
229 if (@intFromEnum(message_level) <= @intFromEnum(testing.log_level)) {
215230 std.debug.print(
216231 "[" ++ @tagName(scope) ++ "] (" ++ @tagName(message_level) ++ "): " ++ format ++ "\n",
217232 args,
......@@ -222,6 +237,7 @@ pub fn log(
222237/// Simpler main(), exercising fewer language features, so that
223238/// work-in-progress backends can handle it.
224239pub fn mainSimple() anyerror!void {
240 @disableInstrumentation();
225241 // is the backend capable of printing to stderr?
226242 const enable_print = switch (builtin.zig_backend) {
227243 else => false,
......@@ -266,3 +282,34 @@ pub fn mainSimple() anyerror!void {
266282 }
267283 if (failed != 0) std.process.exit(1);
268284}
285
286const FuzzerSlice = extern struct {
287 ptr: [*]const u8,
288 len: usize,
289
290 inline fn toSlice(s: FuzzerSlice) []const u8 {
291 return s.ptr[0..s.len];
292 }
293};
294
295var is_fuzz_test: bool = undefined;
296
297extern fn fuzzer_next() FuzzerSlice;
298
299pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 {
300 @disableInstrumentation();
301 if (crippled) {
302 return "";
303 } else if (builtin.fuzz) {
304 return fuzzer_next().toSlice();
305 } else {
306 is_fuzz_test = true;
307 if (options.corpus.len == 0) {
308 return "";
309 } else {
310 var prng = std.Random.DefaultPrng.init(testing.random_seed);
311 const random = prng.random();
312 return options.corpus[random.uintLessThan(usize, options.corpus.len)];
313 }
314 }
315}
lib/fuzzer.zig+51-5
......@@ -1,13 +1,14 @@
11const std = @import("std");
2const Allocator = std.mem.Allocator;
23
34export threadlocal var __sancov_lowest_stack: usize = 0;
45
56export fn __sanitizer_cov_8bit_counters_init(start: [*]u8, stop: [*]u8) void {
6 std.debug.print("__sanitizer_cov_8bit_counters_init start={*}, stop={*}\n", .{ start, stop });
7 std.log.debug("__sanitizer_cov_8bit_counters_init start={*}, stop={*}", .{ start, stop });
78}
89
910export fn __sanitizer_cov_pcs_init(pcs_beg: [*]const usize, pcs_end: [*]const usize) void {
10 std.debug.print("__sanitizer_cov_pcs_init pcs_beg={*}, pcs_end={*}\n", .{ pcs_beg, pcs_end });
11 std.log.debug("__sanitizer_cov_pcs_init pcs_beg={*}, pcs_end={*}", .{ pcs_beg, pcs_end });
1112}
1213
1314export fn __sanitizer_cov_trace_const_cmp1(arg1: u8, arg2: u8) void {
......@@ -47,16 +48,61 @@ export fn __sanitizer_cov_trace_switch(val: u64, cases_ptr: [*]u64) void {
4748 const len = cases_ptr[0];
4849 const val_size_in_bits = cases_ptr[1];
4950 const cases = cases_ptr[2..][0..len];
50 std.debug.print("0x{x}: switch on value {d} ({d} bits) with {d} cases\n", .{
51 std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{
5152 pc, val, val_size_in_bits, cases.len,
5253 });
5354}
5455
5556export fn __sanitizer_cov_trace_pc_indir(callee: usize) void {
5657 const pc = @returnAddress();
57 std.debug.print("0x{x}: indirect call to 0x{x}\n", .{ pc, callee });
58 std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee });
5859}
5960
6061fn handleCmp(pc: usize, arg1: u64, arg2: u64) void {
61 std.debug.print("0x{x}: comparison of {d} and {d}\n", .{ pc, arg1, arg2 });
62 std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 });
63}
64
65const Fuzzer = struct {
66 gpa: Allocator,
67 rng: std.Random.DefaultPrng,
68 input: std.ArrayListUnmanaged(u8),
69
70 const Slice = extern struct {
71 ptr: [*]const u8,
72 len: usize,
73
74 fn toSlice(s: Slice) []const u8 {
75 return s.ptr[0..s.len];
76 }
77
78 fn fromSlice(s: []const u8) Slice {
79 return .{
80 .ptr = s.ptr,
81 .len = s.len,
82 };
83 }
84 };
85
86 fn next(f: *Fuzzer) ![]const u8 {
87 const gpa = f.gpa;
88 const rng = fuzzer.rng.random();
89 const len = rng.uintLessThan(usize, 64);
90 try f.input.resize(gpa, len);
91 rng.bytes(f.input.items);
92 return f.input.items;
93 }
94};
95
96var general_purpose_allocator: std.heap.GeneralPurposeAllocator(.{}) = .{};
97
98var fuzzer: Fuzzer = .{
99 .gpa = general_purpose_allocator.allocator(),
100 .rng = std.Random.DefaultPrng.init(0),
101 .input = .{},
102};
103
104export fn fuzzer_next() Fuzzer.Slice {
105 return Fuzzer.Slice.fromSlice(fuzzer.next() catch |err| switch (err) {
106 error.OutOfMemory => @panic("out of memory"),
107 });
62108}
lib/std/testing.zig+2-24
......@@ -1137,32 +1137,10 @@ pub fn refAllDeclsRecursive(comptime T: type) void {
11371137 }
11381138}
11391139
1140const FuzzerSlice = extern struct {
1141 ptr: [*]const u8,
1142 len: usize,
1143
1144 fn toSlice(s: FuzzerSlice) []const u8 {
1145 return s.ptr[0..s.len];
1146 }
1147};
1148
1149extern fn fuzzer_next() FuzzerSlice;
1150
11511140pub const FuzzInputOptions = struct {
11521141 corpus: []const []const u8 = &.{},
11531142};
11541143
1155pub fn fuzzInput(options: FuzzInputOptions) []const u8 {
1156 @disableInstrumentation();
1157 if (builtin.fuzz) {
1158 return fuzzer_next().toSlice();
1159 } else {
1160 if (options.corpus.len == 0) {
1161 return "";
1162 } else {
1163 var prng = std.Random.DefaultPrng.init(std.testing.random_seed);
1164 const random = prng.random();
1165 return options.corpus[random.uintLessThan(usize, options.corpus.len)];
1166 }
1167 }
1144pub inline fn fuzzInput(options: FuzzInputOptions) []const u8 {
1145 return @import("root").fuzzInput(options);
11681146}
lib/std/zig/Server.zig+3-2
......@@ -53,7 +53,7 @@ pub const Message = struct {
5353 /// - null-terminated string_bytes index
5454 /// * expected_panic_msg: [tests_len]u32,
5555 /// - null-terminated string_bytes index
56 /// - 0 means does not expect pani
56 /// - 0 means does not expect panic
5757 /// * string_bytes: [string_bytes_len]u8,
5858 pub const TestMetadata = extern struct {
5959 string_bytes_len: u32,
......@@ -68,7 +68,8 @@ pub const Message = struct {
6868 fail: bool,
6969 skip: bool,
7070 leak: bool,
71 log_err_count: u29 = 0,
71 fuzz: bool,
72 log_err_count: u28 = 0,
7273 };
7374 };
7475