authorgravatar for sahnvour@pm.meSahnvour <sahnvour@pm.me> 2023-08-12 13:15:05+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-19 14:24:35-04:00
logb87353a17f4fe297b83d2f75cbf1c737ebb71c74
tree51c461fec1a46d80df5339ad5984cc6ff8bfd748
parent530dc0405c9f48fa3b241f078cf815164e4448ab

std.Build: add --seed argument to randomize step dependencies spawning

help detect possibly hidden dependencies on the running order of steps, especially in -j1 mode

2 files changed, 53 insertions(+), 5 deletions(-)

lib/build_runner.zig+43-5
...@@ -95,6 +95,7 @@ pub fn main() !void {...@@ -95,6 +95,7 @@ pub fn main() !void {
95 var max_rss: usize = 0;95 var max_rss: usize = 0;
96 var skip_oom_steps: bool = false;96 var skip_oom_steps: bool = false;
97 var color: Color = .auto;97 var color: Color = .auto;
98 var seed: u32 = 0;
9899
99 const stderr_stream = io.getStdErr().writer();100 const stderr_stream = io.getStdErr().writer();
100 const stdout_stream = io.getStdOut().writer();101 const stdout_stream = io.getStdOut().writer();
...@@ -196,6 +197,15 @@ pub fn main() !void {...@@ -196,6 +197,15 @@ pub fn main() !void {
196 std.debug.print("Expected argument after {s}\n\n", .{arg});197 std.debug.print("Expected argument after {s}\n\n", .{arg});
197 usageAndErr(builder, false, stderr_stream);198 usageAndErr(builder, false, stderr_stream);
198 } };199 } };
200 } else if (mem.eql(u8, arg, "--seed")) {
201 const next_arg = nextArg(args, &arg_idx) orelse {
202 std.debug.print("Expected u32 after {s}\n\n", .{arg});
203 usageAndErr(builder, false, stderr_stream);
204 };
205 seed = std.fmt.parseUnsigned(u32, next_arg, 10) catch |err| {
206 std.debug.print("unable to parse seed '{s}' as u32: {s}", .{ next_arg, @errorName(err) });
207 process.exit(1);
208 };
199 } else if (mem.eql(u8, arg, "--debug-log")) {209 } else if (mem.eql(u8, arg, "--debug-log")) {
200 const next_arg = nextArg(args, &arg_idx) orelse {210 const next_arg = nextArg(args, &arg_idx) orelse {
201 std.debug.print("Expected argument after {s}\n\n", .{arg});211 std.debug.print("Expected argument after {s}\n\n", .{arg});
...@@ -329,6 +339,7 @@ pub fn main() !void {...@@ -329,6 +339,7 @@ pub fn main() !void {
329 main_progress_node,339 main_progress_node,
330 thread_pool_options,340 thread_pool_options,
331 &run,341 &run,
342 seed,
332 ) catch |err| switch (err) {343 ) catch |err| switch (err) {
333 error.UncleanExit => process.exit(1),344 error.UncleanExit => process.exit(1),
334 else => return err,345 else => return err,
...@@ -355,6 +366,7 @@ fn runStepNames(...@@ -355,6 +366,7 @@ fn runStepNames(
355 parent_prog_node: *std.Progress.Node,366 parent_prog_node: *std.Progress.Node,
356 thread_pool_options: std.Thread.Pool.Options,367 thread_pool_options: std.Thread.Pool.Options,
357 run: *Run,368 run: *Run,
369 seed: u32,
358) !void {370) !void {
359 const gpa = b.allocator;371 const gpa = b.allocator;
360 var step_stack: std.AutoArrayHashMapUnmanaged(*Step, void) = .{};372 var step_stack: std.AutoArrayHashMapUnmanaged(*Step, void) = .{};
...@@ -375,8 +387,13 @@ fn runStepNames(...@@ -375,8 +387,13 @@ fn runStepNames(
375 }387 }
376388
377 const starting_steps = try arena.dupe(*Step, step_stack.keys());389 const starting_steps = try arena.dupe(*Step, step_stack.keys());
390
391 var rng = std.rand.DefaultPrng.init(seed);
392 const rand = rng.random();
393 rand.shuffle(*Step, starting_steps);
394
378 for (starting_steps) |s| {395 for (starting_steps) |s| {
379 checkForDependencyLoop(b, s, &step_stack) catch |err| switch (err) {396 constructGraphAndCheckForDependencyLoop(b, s, &step_stack, rand) catch |err| switch (err) {
380 error.DependencyLoopDetected => return error.UncleanExit,397 error.DependencyLoopDetected => return error.UncleanExit,
381 else => |e| return e,398 else => |e| return e,
382 };399 };
...@@ -509,7 +526,9 @@ fn runStepNames(...@@ -509,7 +526,9 @@ fn runStepNames(
509 stderr.writeAll(" (disable with --summary none)") catch {};526 stderr.writeAll(" (disable with --summary none)") catch {};
510 ttyconf.setColor(stderr, .reset) catch {};527 ttyconf.setColor(stderr, .reset) catch {};
511 }528 }
512 stderr.writeAll("\n") catch {};529 ttyconf.setColor(stderr, .dim) catch {};
530 stderr.writer().print("\nseed is {}\n", .{seed}) catch {};
531 ttyconf.setColor(stderr, .reset) catch {};
513 const failures_only = run.summary != Summary.all;532 const failures_only = run.summary != Summary.all;
514533
515 // Print a fancy tree with build results.534 // Print a fancy tree with build results.
...@@ -748,10 +767,22 @@ fn printTreeStep(...@@ -748,10 +767,22 @@ fn printTreeStep(
748 }767 }
749}768}
750769
751fn checkForDependencyLoop(770/// Traverse the dependency graph depth-first and make it undirected by having
771/// steps know their dependants (they only know dependencies at start).
772/// Along the way, check that there is no dependency loop, and record the steps
773/// in traversal order in `step_stack`.
774/// Each step has its dependencies traversed in random order, this accomplishes
775/// two things:
776/// - `step_stack` will be in randomized-depth-first order, so the build runner
777/// spawns steps in a random (but optimized) order
778/// - each step's `dependants` list is also filled in a random order, so that
779/// when it finishes executing in `workerMakeOneStep`, it spawns next steps
780/// to run in random order
781fn constructGraphAndCheckForDependencyLoop(
752 b: *std.Build,782 b: *std.Build,
753 s: *Step,783 s: *Step,
754 step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void),784 step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void),
785 rand: std.rand.Random,
755) !void {786) !void {
756 switch (s.state) {787 switch (s.state) {
757 .precheck_started => {788 .precheck_started => {
...@@ -762,10 +793,16 @@ fn checkForDependencyLoop(...@@ -762,10 +793,16 @@ fn checkForDependencyLoop(
762 s.state = .precheck_started;793 s.state = .precheck_started;
763794
764 try step_stack.ensureUnusedCapacity(b.allocator, s.dependencies.items.len);795 try step_stack.ensureUnusedCapacity(b.allocator, s.dependencies.items.len);
765 for (s.dependencies.items) |dep| {796
797 // We dupe to avoid shuffling the steps in the summary, it depends
798 // on s.dependencies' order.
799 const deps = b.allocator.dupe(*Step, s.dependencies.items) catch @panic("OOM");
800 rand.shuffle(*Step, deps);
801
802 for (deps) |dep| {
766 try step_stack.put(b.allocator, dep, {});803 try step_stack.put(b.allocator, dep, {});
767 try dep.dependants.append(b.allocator, s);804 try dep.dependants.append(b.allocator, s);
768 checkForDependencyLoop(b, dep, step_stack) catch |err| {805 constructGraphAndCheckForDependencyLoop(b, dep, step_stack, rand) catch |err| {
769 if (err == error.DependencyLoopDetected) {806 if (err == error.DependencyLoopDetected) {
770 std.debug.print(" {s}\n", .{s.name});807 std.debug.print(" {s}\n", .{s.name});
771 }808 }
...@@ -1034,6 +1071,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi...@@ -1034,6 +1071,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi
1034 \\ --global-cache-dir [path] Override path to global Zig cache directory1071 \\ --global-cache-dir [path] Override path to global Zig cache directory
1035 \\ --zig-lib-dir [arg] Override path to Zig lib directory1072 \\ --zig-lib-dir [arg] Override path to Zig lib directory
1036 \\ --build-runner [file] Override path to build runner1073 \\ --build-runner [file] Override path to build runner
1074 \\ --seed [integer] For shuffling dependency traversal order (default: random)
1037 \\ --debug-log [scope] Enable debugging the compiler1075 \\ --debug-log [scope] Enable debugging the compiler
1038 \\ --debug-pkg-config Fail if unknown pkg-config flags encountered1076 \\ --debug-pkg-config Fail if unknown pkg-config flags encountered
1039 \\ --verbose-link Enable compiler debug output for linking1077 \\ --verbose-link Enable compiler debug output for linking
src/main.zig+10
...@@ -4930,6 +4930,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -4930,6 +4930,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4930 var reference_trace: ?u32 = null;4930 var reference_trace: ?u32 = null;
4931 var debug_compile_errors = false;4931 var debug_compile_errors = false;
4932 var fetch_only = false;4932 var fetch_only = false;
4933 const micros: u32 = @truncate(@as(u64, @bitCast(std.time.microTimestamp())));
4934 var seed: []const u8 = try std.fmt.allocPrint(arena, "{}", .{micros});
49334935
4934 const argv_index_exe = child_argv.items.len;4936 const argv_index_exe = child_argv.items.len;
4935 _ = try child_argv.addOne();4937 _ = try child_argv.addOne();
...@@ -4945,6 +4947,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -4945,6 +4947,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4945 const argv_index_global_cache_dir = child_argv.items.len;4947 const argv_index_global_cache_dir = child_argv.items.len;
4946 _ = try child_argv.addOne();4948 _ = try child_argv.addOne();
49474949
4950 try child_argv.appendSlice(&[_][]const u8{ "--seed", seed });
4951 const argv_index_seed = child_argv.items.len - 1;
4952
4948 {4953 {
4949 var i: usize = 0;4954 var i: usize = 0;
4950 while (i < args.len) : (i += 1) {4955 while (i < args.len) : (i += 1) {
...@@ -4993,6 +4998,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -4993,6 +4998,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4993 } else if (mem.eql(u8, arg, "--debug-compile-errors")) {4998 } else if (mem.eql(u8, arg, "--debug-compile-errors")) {
4994 try child_argv.append(arg);4999 try child_argv.append(arg);
4995 debug_compile_errors = true;5000 debug_compile_errors = true;
5001 } else if (mem.eql(u8, arg, "--seed")) {
5002 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5003 i += 1;
5004 child_argv.items[argv_index_seed] = args[i];
5005 continue;
4996 }5006 }
4997 }5007 }
4998 try child_argv.append(arg);5008 try child_argv.append(arg);