authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-23 22:59:25-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-25 18:52:21-07:00
logbce3b1efb0879ba2f0da4d215c3190f3e8a4345b
tree74b75d45e7b284b78614995313c8fc6febf867fb
parent711ed56ce361fd9051fcf6039de48022b8dbc2d1

build runner sends a start_fuzzing message to test runner

4 files changed, 166 insertions(+), 31 deletions(-)

lib/compiler/build_runner.zig+10-6
......@@ -401,7 +401,7 @@ pub fn main() !void {
401401 else => return err,
402402 };
403403 if (fuzz) {
404 Fuzz.start(&run.thread_pool, run.step_stack.keys(), main_progress_node);
404 Fuzz.start(&run.thread_pool, run.step_stack.keys(), run.ttyconf, main_progress_node);
405405 }
406406
407407 if (!watch) return cleanExit();
......@@ -1072,7 +1072,7 @@ fn workerMakeOneStep(
10721072 std.debug.lockStdErr();
10731073 defer std.debug.unlockStdErr();
10741074
1075 printErrorMessages(b, s, run) catch {};
1075 printErrorMessages(b, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {};
10761076 }
10771077
10781078 handle_result: {
......@@ -1125,10 +1125,14 @@ fn workerMakeOneStep(
11251125 }
11261126}
11271127
1128fn printErrorMessages(b: *std.Build, failing_step: *Step, run: *const Run) !void {
1128pub fn printErrorMessages(
1129 b: *std.Build,
1130 failing_step: *Step,
1131 ttyconf: std.io.tty.Config,
1132 stderr: File,
1133 prominent_compile_errors: bool,
1134) !void {
11291135 const gpa = b.allocator;
1130 const stderr = run.stderr;
1131 const ttyconf = run.ttyconf;
11321136
11331137 // Provide context for where these error messages are coming from by
11341138 // printing the corresponding Step subtree.
......@@ -1166,7 +1170,7 @@ fn printErrorMessages(b: *std.Build, failing_step: *Step, run: *const Run) !void
11661170 }
11671171 }
11681172
1169 if (!run.prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0)
1173 if (!prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0)
11701174 try failing_step.result_error_bundle.renderToWriter(renderOptions(ttyconf), stderr.writer());
11711175
11721176 for (failing_step.result_error_msgs.items) |msg| {
lib/std/Build/Fuzz.zig+70-12
......@@ -3,9 +3,15 @@ const Fuzz = @This();
33const Step = std.Build.Step;
44const assert = std.debug.assert;
55const fatal = std.process.fatal;
6const build_runner = @import("root");
67
7pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void {
8 {
8pub fn start(
9 thread_pool: *std.Thread.Pool,
10 all_steps: []const *Step,
11 ttyconf: std.io.tty.Config,
12 prog_node: std.Progress.Node,
13) void {
14 const count = block: {
915 const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0);
1016 defer rebuild_node.end();
1117 var count: usize = 0;
......@@ -14,13 +20,14 @@ pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node:
1420 for (all_steps) |step| {
1521 const run = step.cast(Step.Run) orelse continue;
1622 if (run.fuzz_tests.items.len > 0 and run.producer != null) {
17 thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node });
23 thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, ttyconf, rebuild_node });
1824 count += 1;
1925 }
2026 }
2127 if (count == 0) fatal("no fuzz tests found", .{});
2228 rebuild_node.setEstimatedTotalItems(count);
23 }
29 break :block count;
30 };
2431
2532 // Detect failure.
2633 for (all_steps) |step| {
......@@ -29,18 +36,69 @@ pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node:
2936 fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{});
3037 }
3138
32 @panic("TODO do something with the rebuilt unit tests");
39 {
40 const rebuild_node = prog_node.start("Fuzzing", count);
41 defer rebuild_node.end();
42 var wait_group: std.Thread.WaitGroup = .{};
43 defer wait_group.wait();
44
45 for (all_steps) |step| {
46 const run = step.cast(Step.Run) orelse continue;
47 for (run.fuzz_tests.items) |unit_test_index| {
48 assert(run.rebuilt_executable != null);
49 thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{ run, unit_test_index, ttyconf, prog_node });
50 }
51 }
52 }
53
54 fatal("all fuzz workers crashed", .{});
3355}
3456
35fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void {
57fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
3658 const compile_step = run.producer.?;
3759 const prog_node = parent_prog_node.start(compile_step.step.name, 0);
3860 defer prog_node.end();
39 const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| {
40 std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{
41 compile_step.step.name, @errorName(err),
42 });
43 return;
61 if (compile_step.rebuildInFuzzMode(prog_node)) |rebuilt_bin_path| {
62 run.rebuilt_executable = rebuilt_bin_path;
63 } else |err| switch (err) {
64 error.MakeFailed => {
65 const b = run.step.owner;
66 const stderr = std.io.getStdErr();
67 std.debug.lockStdErr();
68 defer std.debug.unlockStdErr();
69 build_runner.printErrorMessages(b, &compile_step.step, ttyconf, stderr, false) catch {};
70 },
71 else => {
72 std.debug.print("step '{s}': failed to rebuild in fuzz mode: {s}\n", .{
73 compile_step.step.name, @errorName(err),
74 });
75 },
76 }
77}
78
79fn fuzzWorkerRun(
80 run: *Step.Run,
81 unit_test_index: u32,
82 ttyconf: std.io.tty.Config,
83 parent_prog_node: std.Progress.Node,
84) void {
85 const test_name = run.cached_test_metadata.?.testName(unit_test_index);
86
87 const prog_node = parent_prog_node.start(test_name, 0);
88 defer prog_node.end();
89
90 run.rerunInFuzzMode(unit_test_index, prog_node) catch |err| switch (err) {
91 error.MakeFailed => {
92 const b = run.step.owner;
93 const stderr = std.io.getStdErr();
94 std.debug.lockStdErr();
95 defer std.debug.unlockStdErr();
96 build_runner.printErrorMessages(b, &run.step, ttyconf, stderr, false) catch {};
97 },
98 else => {
99 std.debug.print("step '{s}': failed to rebuild '{s}' in fuzz mode: {s}\n", .{
100 run.step.name, test_name, @errorName(err),
101 });
102 },
44103 };
45 run.rebuilt_executable = rebuilt_bin_path;
46104}
lib/std/Build/Step/Run.zig+83-13
......@@ -89,6 +89,8 @@ has_side_effects: bool,
8989/// If this is a Zig unit test binary, this tracks the indexes of the unit
9090/// tests that are also fuzz tests.
9191fuzz_tests: std.ArrayListUnmanaged(u32),
92cached_test_metadata: ?CachedTestMetadata = null,
93
9294/// Populated during the fuzz phase if this run step corresponds to a unit test
9395/// executable that contains fuzz tests.
9496rebuilt_executable: ?[]const u8,
......@@ -754,7 +756,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
754756 b.fmt("{s}{s}", .{ placeholder.output.prefix, output_path });
755757 }
756758
757 try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node);
759 try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node, null);
758760 if (!has_side_effects) try step.writeManifestAndWatch(&man);
759761 return;
760762 };
......@@ -784,7 +786,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
784786 b.fmt("{s}{s}", .{ placeholder.output.prefix, output_path });
785787 }
786788
787 try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node);
789 try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, null);
788790
789791 const dep_file_dir = std.fs.cwd();
790792 const dep_file_basename = dep_output_file.generated_file.getPath();
......@@ -843,6 +845,38 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
843845 );
844846}
845847
848pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress.Node) !void {
849 const step = &run.step;
850 const b = step.owner;
851 const arena = b.allocator;
852 var argv_list: std.ArrayListUnmanaged([]const u8) = .{};
853 for (run.argv.items) |arg| {
854 switch (arg) {
855 .bytes => |bytes| {
856 try argv_list.append(arena, bytes);
857 },
858 .lazy_path => |file| {
859 const file_path = file.lazy_path.getPath2(b, step);
860 try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, file_path }));
861 },
862 .directory_source => |file| {
863 const file_path = file.lazy_path.getPath2(b, step);
864 try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, file_path }));
865 },
866 .artifact => |pa| {
867 const artifact = pa.artifact;
868 const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?;
869 try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path }));
870 },
871 .output_file, .output_directory => unreachable,
872 }
873 }
874 const has_side_effects = false;
875 const rand_int = std.crypto.random.int(u64);
876 const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
877 try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, unit_test_index);
878}
879
846880fn populateGeneratedPaths(
847881 arena: std.mem.Allocator,
848882 output_placeholders: []const IndexedOutput,
......@@ -921,6 +955,7 @@ fn runCommand(
921955 has_side_effects: bool,
922956 output_dir_path: []const u8,
923957 prog_node: std.Progress.Node,
958 fuzz_unit_test_index: ?u32,
924959) !void {
925960 const step = &run.step;
926961 const b = step.owner;
......@@ -939,7 +974,7 @@ fn runCommand(
939974 var interp_argv = std.ArrayList([]const u8).init(b.allocator);
940975 defer interp_argv.deinit();
941976
942 const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node) catch |err| term: {
977 const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node, fuzz_unit_test_index) catch |err| term: {
943978 // InvalidExe: cpu arch mismatch
944979 // FileNotFound: can happen with a wrong dynamic linker path
945980 if (err == error.InvalidExe or err == error.FileNotFound) interpret: {
......@@ -1075,7 +1110,7 @@ fn runCommand(
10751110
10761111 try Step.handleVerbose2(step.owner, cwd, run.env_map, interp_argv.items);
10771112
1078 break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node) catch |e| {
1113 break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node, fuzz_unit_test_index) catch |e| {
10791114 if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped;
10801115
10811116 return step.fail("unable to spawn interpreter {s}: {s}", .{
......@@ -1090,6 +1125,15 @@ fn runCommand(
10901125 step.result_duration_ns = result.elapsed_ns;
10911126 step.result_peak_rss = result.peak_rss;
10921127 step.test_results = result.stdio.test_results;
1128 if (result.stdio.test_metadata) |tm|
1129 run.cached_test_metadata = tm.toCachedTestMetadata();
1130
1131 const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items;
1132
1133 if (fuzz_unit_test_index != null) {
1134 try step.handleChildProcessTerm(result.term, cwd, final_argv);
1135 return;
1136 }
10931137
10941138 // Capture stdout and stderr to GeneratedFile objects.
10951139 const Stream = struct {
......@@ -1126,8 +1170,6 @@ fn runCommand(
11261170 }
11271171 }
11281172
1129 const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items;
1130
11311173 switch (run.stdio) {
11321174 .check => |checks| for (checks.items) |check| switch (check) {
11331175 .expect_stderr_exact => |expected_bytes| {
......@@ -1253,10 +1295,16 @@ fn spawnChildAndCollect(
12531295 argv: []const []const u8,
12541296 has_side_effects: bool,
12551297 prog_node: std.Progress.Node,
1298 fuzz_unit_test_index: ?u32,
12561299) !ChildProcResult {
12571300 const b = run.step.owner;
12581301 const arena = b.allocator;
12591302
1303 if (fuzz_unit_test_index != null) {
1304 assert(!has_side_effects);
1305 assert(run.stdio == .zig_test);
1306 }
1307
12601308 var child = std.process.Child.init(argv, arena);
12611309 if (run.cwd) |lazy_cwd| {
12621310 child.cwd = lazy_cwd.getPath2(b, &run.step);
......@@ -1306,7 +1354,7 @@ fn spawnChildAndCollect(
13061354 var timer = try std.time.Timer.start();
13071355
13081356 const result = if (run.stdio == .zig_test)
1309 evalZigTest(run, &child, prog_node)
1357 evalZigTest(run, &child, prog_node, fuzz_unit_test_index)
13101358 else
13111359 evalGeneric(run, &child);
13121360
......@@ -1332,6 +1380,7 @@ fn evalZigTest(
13321380 run: *Run,
13331381 child: *std.process.Child,
13341382 prog_node: std.Progress.Node,
1383 fuzz_unit_test_index: ?u32,
13351384) !StdIoResult {
13361385 const gpa = run.step.owner.allocator;
13371386 const arena = run.step.owner.allocator;
......@@ -1342,7 +1391,12 @@ fn evalZigTest(
13421391 });
13431392 defer poller.deinit();
13441393
1345 try sendMessage(child.stdin.?, .query_test_metadata);
1394 if (fuzz_unit_test_index) |index| {
1395 try sendRunTestMessage(child.stdin.?, .start_fuzzing, index);
1396 } else {
1397 run.fuzz_tests.clearRetainingCapacity();
1398 try sendMessage(child.stdin.?, .query_test_metadata);
1399 }
13461400
13471401 const Header = std.zig.Server.Message.Header;
13481402
......@@ -1360,8 +1414,6 @@ fn evalZigTest(
13601414 var sub_prog_node: ?std.Progress.Node = null;
13611415 defer if (sub_prog_node) |n| n.end();
13621416
1363 run.fuzz_tests.clearRetainingCapacity();
1364
13651417 poll: while (true) {
13661418 while (stdout.readableLength() < @sizeOf(Header)) {
13671419 if (!(try poller.poll())) break :poll;
......@@ -1382,6 +1434,7 @@ fn evalZigTest(
13821434 }
13831435 },
13841436 .test_metadata => {
1437 assert(fuzz_unit_test_index == null);
13851438 const TmHdr = std.zig.Server.Message.TestMetadata;
13861439 const tm_hdr = @as(*align(1) const TmHdr, @ptrCast(body));
13871440 test_count = tm_hdr.tests_len;
......@@ -1410,6 +1463,7 @@ fn evalZigTest(
14101463 try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node);
14111464 },
14121465 .test_results => {
1466 assert(fuzz_unit_test_index == null);
14131467 const md = metadata.?;
14141468
14151469 const TrHdr = std.zig.Server.Message.TestResults;
......@@ -1479,7 +1533,23 @@ const TestMetadata = struct {
14791533 next_index: u32,
14801534 prog_node: std.Progress.Node,
14811535
1536 fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata {
1537 return .{
1538 .names = tm.names,
1539 .string_bytes = tm.string_bytes,
1540 };
1541 }
1542
14821543 fn testName(tm: TestMetadata, index: u32) []const u8 {
1544 return tm.toCachedTestMetadata().testName(index);
1545 }
1546};
1547
1548pub const CachedTestMetadata = struct {
1549 names: []const u32,
1550 string_bytes: []const u8,
1551
1552 pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 {
14831553 return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0);
14841554 }
14851555};
......@@ -1495,7 +1565,7 @@ fn requestNextTest(in: fs.File, metadata: *TestMetadata, sub_prog_node: *?std.Pr
14951565 if (sub_prog_node.*) |n| n.end();
14961566 sub_prog_node.* = metadata.prog_node.start(name, 0);
14971567
1498 try sendRunTestMessage(in, i);
1568 try sendRunTestMessage(in, .run_test, i);
14991569 return;
15001570 } else {
15011571 try sendMessage(in, .exit);
......@@ -1510,9 +1580,9 @@ fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
15101580 try file.writeAll(std.mem.asBytes(&header));
15111581}
15121582
1513fn sendRunTestMessage(file: std.fs.File, index: u32) !void {
1583fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index: u32) !void {
15141584 const header: std.zig.Client.Message.Header = .{
1515 .tag = .run_test,
1585 .tag = tag,
15161586 .bytes_len = 4,
15171587 };
15181588 const full_msg = std.mem.asBytes(&header) ++ std.mem.asBytes(&index);
lib/std/zig/Client.zig+3
......@@ -33,6 +33,9 @@ pub const Message = struct {
3333 /// Ask the test runner to run a particular test.
3434 /// The message body is a u32 test index.
3535 run_test,
36 /// Ask the test runner to start fuzzing a particular test.
37 /// The message body is a u32 test index.
38 start_fuzzing,
3639
3740 _,
3841 };