authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-22 18:06:58-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-25 18:52:20-07:00
log6f3767862d6886d5fde7e3734455a30f168ba80b
treebd2585a84cc1dd1db4725e2f044ba1da30f6e702
parent3256df2ff80be565095993b03f8e1edfd5072367

implement std.testing.fuzzInput

For now this returns a dummy fuzz input.

4 files changed, 125 insertions(+), 53 deletions(-)

lib/compiler/test_runner.zig+69-22
...@@ -1,18 +1,26 @@...@@ -1,18 +1,26 @@
1//! Default test runner for unit tests.1//! Default test runner for unit tests.
2const builtin = @import("builtin");
2const std = @import("std");3const std = @import("std");
3const io = std.io;4const io = std.io;
4const builtin = @import("builtin");5const testing = std.testing;
56
6pub const std_options = .{7pub const std_options = .{
7 .logFn = log,8 .logFn = log,
8};9};
910
10var log_err_count: usize = 0;11var log_err_count: usize = 0;
11var cmdline_buffer: [4096]u8 = undefined;12var fba_buffer: [8192]u8 = undefined;
12var fba = std.heap.FixedBufferAllocator.init(&cmdline_buffer);13var fba = std.heap.FixedBufferAllocator.init(&fba_buffer);
14
15const crippled = switch (builtin.zig_backend) {
16 .stage2_riscv64 => true,
17 else => false,
18};
1319
14pub fn main() void {20pub fn main() void {
15 if (builtin.zig_backend == .stage2_riscv64) {21 @disableInstrumentation();
22
23 if (crippled) {
16 return mainSimple() catch @panic("test failure\n");24 return mainSimple() catch @panic("test failure\n");
17 }25 }
1826
...@@ -25,13 +33,15 @@ pub fn main() void {...@@ -25,13 +33,15 @@ pub fn main() void {
25 if (std.mem.eql(u8, arg, "--listen=-")) {33 if (std.mem.eql(u8, arg, "--listen=-")) {
26 listen = true;34 listen = true;
27 } else if (std.mem.startsWith(u8, arg, "--seed=")) {35 } else if (std.mem.startsWith(u8, arg, "--seed=")) {
28 std.testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch36 testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch
29 @panic("unable to parse --seed command line argument");37 @panic("unable to parse --seed command line argument");
30 } else {38 } else {
31 @panic("unrecognized command line argument");39 @panic("unrecognized command line argument");
32 }40 }
33 }41 }
3442
43 fba.reset();
44
35 if (listen) {45 if (listen) {
36 return mainServer() catch @panic("internal test runner failure");46 return mainServer() catch @panic("internal test runner failure");
37 } else {47 } else {
...@@ -40,6 +50,7 @@ pub fn main() void {...@@ -40,6 +50,7 @@ pub fn main() void {
40}50}
4151
42fn mainServer() !void {52fn mainServer() !void {
53 @disableInstrumentation();
43 var server = try std.zig.Server.init(.{54 var server = try std.zig.Server.init(.{
44 .gpa = fba.allocator(),55 .gpa = fba.allocator(),
45 .in = std.io.getStdIn(),56 .in = std.io.getStdIn(),
...@@ -55,24 +66,24 @@ fn mainServer() !void {...@@ -55,24 +66,24 @@ fn mainServer() !void {
55 return std.process.exit(0);66 return std.process.exit(0);
56 },67 },
57 .query_test_metadata => {68 .query_test_metadata => {
58 std.testing.allocator_instance = .{};69 testing.allocator_instance = .{};
59 defer if (std.testing.allocator_instance.deinit() == .leak) {70 defer if (testing.allocator_instance.deinit() == .leak) {
60 @panic("internal test runner memory leak");71 @panic("internal test runner memory leak");
61 };72 };
6273
63 var string_bytes: std.ArrayListUnmanaged(u8) = .{};74 var string_bytes: std.ArrayListUnmanaged(u8) = .{};
64 defer string_bytes.deinit(std.testing.allocator);75 defer string_bytes.deinit(testing.allocator);
65 try string_bytes.append(std.testing.allocator, 0); // Reserve 0 for null.76 try string_bytes.append(testing.allocator, 0); // Reserve 0 for null.
6677
67 const test_fns = builtin.test_functions;78 const test_fns = builtin.test_functions;
68 const names = try std.testing.allocator.alloc(u32, test_fns.len);79 const names = try testing.allocator.alloc(u32, test_fns.len);
69 defer std.testing.allocator.free(names);80 defer testing.allocator.free(names);
70 const expected_panic_msgs = try std.testing.allocator.alloc(u32, test_fns.len);81 const expected_panic_msgs = try testing.allocator.alloc(u32, test_fns.len);
71 defer std.testing.allocator.free(expected_panic_msgs);82 defer testing.allocator.free(expected_panic_msgs);
7283
73 for (test_fns, names, expected_panic_msgs) |test_fn, *name, *expected_panic_msg| {84 for (test_fns, names, expected_panic_msgs) |test_fn, *name, *expected_panic_msg| {
74 name.* = @as(u32, @intCast(string_bytes.items.len));85 name.* = @as(u32, @intCast(string_bytes.items.len));
75 try string_bytes.ensureUnusedCapacity(std.testing.allocator, test_fn.name.len + 1);86 try string_bytes.ensureUnusedCapacity(testing.allocator, test_fn.name.len + 1);
76 string_bytes.appendSliceAssumeCapacity(test_fn.name);87 string_bytes.appendSliceAssumeCapacity(test_fn.name);
77 string_bytes.appendAssumeCapacity(0);88 string_bytes.appendAssumeCapacity(0);
78 expected_panic_msg.* = 0;89 expected_panic_msg.* = 0;
...@@ -86,13 +97,13 @@ fn mainServer() !void {...@@ -86,13 +97,13 @@ fn mainServer() !void {
86 },97 },
8798
88 .run_test => {99 .run_test => {
89 std.testing.allocator_instance = .{};100 testing.allocator_instance = .{};
90 log_err_count = 0;101 log_err_count = 0;
91 const index = try server.receiveBody_u32();102 const index = try server.receiveBody_u32();
92 const test_fn = builtin.test_functions[index];103 const test_fn = builtin.test_functions[index];
93 var fail = false;104 var fail = false;
94 var skip = false;105 var skip = false;
95 var leak = false;106 is_fuzz_test = false;
96 test_fn.func() catch |err| switch (err) {107 test_fn.func() catch |err| switch (err) {
97 error.SkipZigTest => skip = true,108 error.SkipZigTest => skip = true,
98 else => {109 else => {
...@@ -102,13 +113,14 @@ fn mainServer() !void {...@@ -102,13 +113,14 @@ fn mainServer() !void {
102 }113 }
103 },114 },
104 };115 };
105 leak = std.testing.allocator_instance.deinit() == .leak;116 const leak = testing.allocator_instance.deinit() == .leak;
106 try server.serveTestResults(.{117 try server.serveTestResults(.{
107 .index = index,118 .index = index,
108 .flags = .{119 .flags = .{
109 .fail = fail,120 .fail = fail,
110 .skip = skip,121 .skip = skip,
111 .leak = leak,122 .leak = leak,
123 .fuzz = is_fuzz_test,
112 .log_err_count = std.math.lossyCast(124 .log_err_count = std.math.lossyCast(
113 @TypeOf(@as(std.zig.Server.Message.TestResults.Flags, undefined).log_err_count),125 @TypeOf(@as(std.zig.Server.Message.TestResults.Flags, undefined).log_err_count),
114 log_err_count,126 log_err_count,
...@@ -118,7 +130,7 @@ fn mainServer() !void {...@@ -118,7 +130,7 @@ fn mainServer() !void {
118 },130 },
119131
120 else => {132 else => {
121 std.debug.print("unsupported message: {x}", .{@intFromEnum(hdr.tag)});133 std.debug.print("unsupported message: {x}\n", .{@intFromEnum(hdr.tag)});
122 std.process.exit(1);134 std.process.exit(1);
123 },135 },
124 }136 }
...@@ -126,6 +138,7 @@ fn mainServer() !void {...@@ -126,6 +138,7 @@ fn mainServer() !void {
126}138}
127139
128fn mainTerminal() void {140fn mainTerminal() void {
141 @disableInstrumentation();
129 const test_fn_list = builtin.test_functions;142 const test_fn_list = builtin.test_functions;
130 var ok_count: usize = 0;143 var ok_count: usize = 0;
131 var skip_count: usize = 0;144 var skip_count: usize = 0;
...@@ -143,18 +156,19 @@ fn mainTerminal() void {...@@ -143,18 +156,19 @@ fn mainTerminal() void {
143156
144 var leaks: usize = 0;157 var leaks: usize = 0;
145 for (test_fn_list, 0..) |test_fn, i| {158 for (test_fn_list, 0..) |test_fn, i| {
146 std.testing.allocator_instance = .{};159 testing.allocator_instance = .{};
147 defer {160 defer {
148 if (std.testing.allocator_instance.deinit() == .leak) {161 if (testing.allocator_instance.deinit() == .leak) {
149 leaks += 1;162 leaks += 1;
150 }163 }
151 }164 }
152 std.testing.log_level = .warn;165 testing.log_level = .warn;
153166
154 const test_node = root_node.start(test_fn.name, 0);167 const test_node = root_node.start(test_fn.name, 0);
155 if (!have_tty) {168 if (!have_tty) {
156 std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name });169 std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name });
157 }170 }
171 // Track in a global variable so that `fuzzInput` can see it.
158 if (test_fn.func()) |_| {172 if (test_fn.func()) |_| {
159 ok_count += 1;173 ok_count += 1;
160 test_node.end();174 test_node.end();
...@@ -208,10 +222,11 @@ pub fn log(...@@ -208,10 +222,11 @@ pub fn log(
208 comptime format: []const u8,222 comptime format: []const u8,
209 args: anytype,223 args: anytype,
210) void {224) void {
225 @disableInstrumentation();
211 if (@intFromEnum(message_level) <= @intFromEnum(std.log.Level.err)) {226 if (@intFromEnum(message_level) <= @intFromEnum(std.log.Level.err)) {
212 log_err_count +|= 1;227 log_err_count +|= 1;
213 }228 }
214 if (@intFromEnum(message_level) <= @intFromEnum(std.testing.log_level)) {229 if (@intFromEnum(message_level) <= @intFromEnum(testing.log_level)) {
215 std.debug.print(230 std.debug.print(
216 "[" ++ @tagName(scope) ++ "] (" ++ @tagName(message_level) ++ "): " ++ format ++ "\n",231 "[" ++ @tagName(scope) ++ "] (" ++ @tagName(message_level) ++ "): " ++ format ++ "\n",
217 args,232 args,
...@@ -222,6 +237,7 @@ pub fn log(...@@ -222,6 +237,7 @@ pub fn log(
222/// Simpler main(), exercising fewer language features, so that237/// Simpler main(), exercising fewer language features, so that
223/// work-in-progress backends can handle it.238/// work-in-progress backends can handle it.
224pub fn mainSimple() anyerror!void {239pub fn mainSimple() anyerror!void {
240 @disableInstrumentation();
225 // is the backend capable of printing to stderr?241 // is the backend capable of printing to stderr?
226 const enable_print = switch (builtin.zig_backend) {242 const enable_print = switch (builtin.zig_backend) {
227 else => false,243 else => false,
...@@ -266,3 +282,34 @@ pub fn mainSimple() anyerror!void {...@@ -266,3 +282,34 @@ pub fn mainSimple() anyerror!void {
266 }282 }
267 if (failed != 0) std.process.exit(1);283 if (failed != 0) std.process.exit(1);
268}284}
285
286const FuzzerSlice = extern struct {
287 ptr: [*]const u8,
288 len: usize,
289
290 inline fn toSlice(s: FuzzerSlice) []const u8 {
291 return s.ptr[0..s.len];
292 }
293};
294
295var is_fuzz_test: bool = undefined;
296
297extern fn fuzzer_next() FuzzerSlice;
298
299pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 {
300 @disableInstrumentation();
301 if (crippled) {
302 return "";
303 } else if (builtin.fuzz) {
304 return fuzzer_next().toSlice();
305 } else {
306 is_fuzz_test = true;
307 if (options.corpus.len == 0) {
308 return "";
309 } else {
310 var prng = std.Random.DefaultPrng.init(testing.random_seed);
311 const random = prng.random();
312 return options.corpus[random.uintLessThan(usize, options.corpus.len)];
313 }
314 }
315}
lib/fuzzer.zig+51-5
...@@ -1,13 +1,14 @@...@@ -1,13 +1,14 @@
1const std = @import("std");1const std = @import("std");
2const Allocator = std.mem.Allocator;
23
3export threadlocal var __sancov_lowest_stack: usize = 0;4export threadlocal var __sancov_lowest_stack: usize = 0;
45
5export fn __sanitizer_cov_8bit_counters_init(start: [*]u8, stop: [*]u8) void {6export fn __sanitizer_cov_8bit_counters_init(start: [*]u8, stop: [*]u8) void {
6 std.debug.print("__sanitizer_cov_8bit_counters_init start={*}, stop={*}\n", .{ start, stop });7 std.log.debug("__sanitizer_cov_8bit_counters_init start={*}, stop={*}", .{ start, stop });
7}8}
89
9export fn __sanitizer_cov_pcs_init(pcs_beg: [*]const usize, pcs_end: [*]const usize) void {10export fn __sanitizer_cov_pcs_init(pcs_beg: [*]const usize, pcs_end: [*]const usize) void {
10 std.debug.print("__sanitizer_cov_pcs_init pcs_beg={*}, pcs_end={*}\n", .{ pcs_beg, pcs_end });11 std.log.debug("__sanitizer_cov_pcs_init pcs_beg={*}, pcs_end={*}", .{ pcs_beg, pcs_end });
11}12}
1213
13export fn __sanitizer_cov_trace_const_cmp1(arg1: u8, arg2: u8) void {14export fn __sanitizer_cov_trace_const_cmp1(arg1: u8, arg2: u8) void {
...@@ -47,16 +48,61 @@ export fn __sanitizer_cov_trace_switch(val: u64, cases_ptr: [*]u64) void {...@@ -47,16 +48,61 @@ export fn __sanitizer_cov_trace_switch(val: u64, cases_ptr: [*]u64) void {
47 const len = cases_ptr[0];48 const len = cases_ptr[0];
48 const val_size_in_bits = cases_ptr[1];49 const val_size_in_bits = cases_ptr[1];
49 const cases = cases_ptr[2..][0..len];50 const cases = cases_ptr[2..][0..len];
50 std.debug.print("0x{x}: switch on value {d} ({d} bits) with {d} cases\n", .{51 std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{
51 pc, val, val_size_in_bits, cases.len,52 pc, val, val_size_in_bits, cases.len,
52 });53 });
53}54}
5455
55export fn __sanitizer_cov_trace_pc_indir(callee: usize) void {56export fn __sanitizer_cov_trace_pc_indir(callee: usize) void {
56 const pc = @returnAddress();57 const pc = @returnAddress();
57 std.debug.print("0x{x}: indirect call to 0x{x}\n", .{ pc, callee });58 std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee });
58}59}
5960
60fn handleCmp(pc: usize, arg1: u64, arg2: u64) void {61fn handleCmp(pc: usize, arg1: u64, arg2: u64) void {
61 std.debug.print("0x{x}: comparison of {d} and {d}\n", .{ pc, arg1, arg2 });62 std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 });
63}
64
65const Fuzzer = struct {
66 gpa: Allocator,
67 rng: std.Random.DefaultPrng,
68 input: std.ArrayListUnmanaged(u8),
69
70 const Slice = extern struct {
71 ptr: [*]const u8,
72 len: usize,
73
74 fn toSlice(s: Slice) []const u8 {
75 return s.ptr[0..s.len];
76 }
77
78 fn fromSlice(s: []const u8) Slice {
79 return .{
80 .ptr = s.ptr,
81 .len = s.len,
82 };
83 }
84 };
85
86 fn next(f: *Fuzzer) ![]const u8 {
87 const gpa = f.gpa;
88 const rng = fuzzer.rng.random();
89 const len = rng.uintLessThan(usize, 64);
90 try f.input.resize(gpa, len);
91 rng.bytes(f.input.items);
92 return f.input.items;
93 }
94};
95
96var general_purpose_allocator: std.heap.GeneralPurposeAllocator(.{}) = .{};
97
98var fuzzer: Fuzzer = .{
99 .gpa = general_purpose_allocator.allocator(),
100 .rng = std.Random.DefaultPrng.init(0),
101 .input = .{},
102};
103
104export fn fuzzer_next() Fuzzer.Slice {
105 return Fuzzer.Slice.fromSlice(fuzzer.next() catch |err| switch (err) {
106 error.OutOfMemory => @panic("out of memory"),
107 });
62}108}
lib/std/testing.zig+2-24
...@@ -1137,32 +1137,10 @@ pub fn refAllDeclsRecursive(comptime T: type) void {...@@ -1137,32 +1137,10 @@ pub fn refAllDeclsRecursive(comptime T: type) void {
1137 }1137 }
1138}1138}
11391139
1140const FuzzerSlice = extern struct {
1141 ptr: [*]const u8,
1142 len: usize,
1143
1144 fn toSlice(s: FuzzerSlice) []const u8 {
1145 return s.ptr[0..s.len];
1146 }
1147};
1148
1149extern fn fuzzer_next() FuzzerSlice;
1150
1151pub const FuzzInputOptions = struct {1140pub const FuzzInputOptions = struct {
1152 corpus: []const []const u8 = &.{},1141 corpus: []const []const u8 = &.{},
1153};1142};
11541143
1155pub fn fuzzInput(options: FuzzInputOptions) []const u8 {1144pub inline fn fuzzInput(options: FuzzInputOptions) []const u8 {
1156 @disableInstrumentation();1145 return @import("root").fuzzInput(options);
1157 if (builtin.fuzz) {
1158 return fuzzer_next().toSlice();
1159 } else {
1160 if (options.corpus.len == 0) {
1161 return "";
1162 } else {
1163 var prng = std.Random.DefaultPrng.init(std.testing.random_seed);
1164 const random = prng.random();
1165 return options.corpus[random.uintLessThan(usize, options.corpus.len)];
1166 }
1167 }
1168}1146}
lib/std/zig/Server.zig+3-2
...@@ -53,7 +53,7 @@ pub const Message = struct {...@@ -53,7 +53,7 @@ pub const Message = struct {
53 /// - null-terminated string_bytes index53 /// - null-terminated string_bytes index
54 /// * expected_panic_msg: [tests_len]u32,54 /// * expected_panic_msg: [tests_len]u32,
55 /// - null-terminated string_bytes index55 /// - null-terminated string_bytes index
56 /// - 0 means does not expect pani56 /// - 0 means does not expect panic
57 /// * string_bytes: [string_bytes_len]u8,57 /// * string_bytes: [string_bytes_len]u8,
58 pub const TestMetadata = extern struct {58 pub const TestMetadata = extern struct {
59 string_bytes_len: u32,59 string_bytes_len: u32,
...@@ -68,7 +68,8 @@ pub const Message = struct {...@@ -68,7 +68,8 @@ pub const Message = struct {
68 fail: bool,68 fail: bool,
69 skip: bool,69 skip: bool,
70 leak: bool,70 leak: bool,
71 log_err_count: u29 = 0,71 fuzz: bool,
72 log_err_count: u28 = 0,
72 };73 };
73 };74 };
7475