authorgravatar for sahnvour@pm.meSahnvour <sahnvour@pm.me> 2023-08-12 13:15:05+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-19 14:24:35-04:00
logb87353a17f4fe297b83d2f75cbf1c737ebb71c74
tree51c461fec1a46d80df5339ad5984cc6ff8bfd748
parent530dc0405c9f48fa3b241f078cf815164e4448ab

std.Build: add --seed argument to randomize step dependencies spawning

help detect possibly hidden dependencies on the running order of steps, especially in -j1 mode

2 files changed, 53 insertions(+), 5 deletions(-)

lib/build_runner.zig+43-5
......@@ -95,6 +95,7 @@ pub fn main() !void {
9595 var max_rss: usize = 0;
9696 var skip_oom_steps: bool = false;
9797 var color: Color = .auto;
98 var seed: u32 = 0;
9899
99100 const stderr_stream = io.getStdErr().writer();
100101 const stdout_stream = io.getStdOut().writer();
......@@ -196,6 +197,15 @@ pub fn main() !void {
196197 std.debug.print("Expected argument after {s}\n\n", .{arg});
197198 usageAndErr(builder, false, stderr_stream);
198199 } };
200 } else if (mem.eql(u8, arg, "--seed")) {
201 const next_arg = nextArg(args, &arg_idx) orelse {
202 std.debug.print("Expected u32 after {s}\n\n", .{arg});
203 usageAndErr(builder, false, stderr_stream);
204 };
205 seed = std.fmt.parseUnsigned(u32, next_arg, 10) catch |err| {
206 std.debug.print("unable to parse seed '{s}' as u32: {s}", .{ next_arg, @errorName(err) });
207 process.exit(1);
208 };
199209 } else if (mem.eql(u8, arg, "--debug-log")) {
200210 const next_arg = nextArg(args, &arg_idx) orelse {
201211 std.debug.print("Expected argument after {s}\n\n", .{arg});
......@@ -329,6 +339,7 @@ pub fn main() !void {
329339 main_progress_node,
330340 thread_pool_options,
331341 &run,
342 seed,
332343 ) catch |err| switch (err) {
333344 error.UncleanExit => process.exit(1),
334345 else => return err,
......@@ -355,6 +366,7 @@ fn runStepNames(
355366 parent_prog_node: *std.Progress.Node,
356367 thread_pool_options: std.Thread.Pool.Options,
357368 run: *Run,
369 seed: u32,
358370) !void {
359371 const gpa = b.allocator;
360372 var step_stack: std.AutoArrayHashMapUnmanaged(*Step, void) = .{};
......@@ -375,8 +387,13 @@ fn runStepNames(
375387 }
376388
377389 const starting_steps = try arena.dupe(*Step, step_stack.keys());
390
391 var rng = std.rand.DefaultPrng.init(seed);
392 const rand = rng.random();
393 rand.shuffle(*Step, starting_steps);
394
378395 for (starting_steps) |s| {
379 checkForDependencyLoop(b, s, &step_stack) catch |err| switch (err) {
396 constructGraphAndCheckForDependencyLoop(b, s, &step_stack, rand) catch |err| switch (err) {
380397 error.DependencyLoopDetected => return error.UncleanExit,
381398 else => |e| return e,
382399 };
......@@ -509,7 +526,9 @@ fn runStepNames(
509526 stderr.writeAll(" (disable with --summary none)") catch {};
510527 ttyconf.setColor(stderr, .reset) catch {};
511528 }
512 stderr.writeAll("\n") catch {};
529 ttyconf.setColor(stderr, .dim) catch {};
530 stderr.writer().print("\nseed is {}\n", .{seed}) catch {};
531 ttyconf.setColor(stderr, .reset) catch {};
513532 const failures_only = run.summary != Summary.all;
514533
515534 // Print a fancy tree with build results.
......@@ -748,10 +767,22 @@ fn printTreeStep(
748767 }
749768}
750769
751fn checkForDependencyLoop(
770/// Traverse the dependency graph depth-first and make it undirected by having
771/// steps know their dependants (they only know dependencies at start).
772/// Along the way, check that there is no dependency loop, and record the steps
773/// in traversal order in `step_stack`.
774/// Each step has its dependencies traversed in random order, this accomplishes
775/// two things:
776/// - `step_stack` will be in randomized-depth-first order, so the build runner
777/// spawns steps in a random (but optimized) order
778/// - each step's `dependants` list is also filled in a random order, so that
779/// when it finishes executing in `workerMakeOneStep`, it spawns next steps
780/// to run in random order
781fn constructGraphAndCheckForDependencyLoop(
752782 b: *std.Build,
753783 s: *Step,
754784 step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void),
785 rand: std.rand.Random,
755786) !void {
756787 switch (s.state) {
757788 .precheck_started => {
......@@ -762,10 +793,16 @@ fn checkForDependencyLoop(
762793 s.state = .precheck_started;
763794
764795 try step_stack.ensureUnusedCapacity(b.allocator, s.dependencies.items.len);
765 for (s.dependencies.items) |dep| {
796
797 // We dupe to avoid shuffling the steps in the summary, it depends
798 // on s.dependencies' order.
799 const deps = b.allocator.dupe(*Step, s.dependencies.items) catch @panic("OOM");
800 rand.shuffle(*Step, deps);
801
802 for (deps) |dep| {
766803 try step_stack.put(b.allocator, dep, {});
767804 try dep.dependants.append(b.allocator, s);
768 checkForDependencyLoop(b, dep, step_stack) catch |err| {
805 constructGraphAndCheckForDependencyLoop(b, dep, step_stack, rand) catch |err| {
769806 if (err == error.DependencyLoopDetected) {
770807 std.debug.print(" {s}\n", .{s.name});
771808 }
......@@ -1034,6 +1071,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi
10341071 \\ --global-cache-dir [path] Override path to global Zig cache directory
10351072 \\ --zig-lib-dir [arg] Override path to Zig lib directory
10361073 \\ --build-runner [file] Override path to build runner
1074 \\ --seed [integer] For shuffling dependency traversal order (default: random)
10371075 \\ --debug-log [scope] Enable debugging the compiler
10381076 \\ --debug-pkg-config Fail if unknown pkg-config flags encountered
10391077 \\ --verbose-link Enable compiler debug output for linking
src/main.zig+10
......@@ -4930,6 +4930,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
49304930 var reference_trace: ?u32 = null;
49314931 var debug_compile_errors = false;
49324932 var fetch_only = false;
4933 const micros: u32 = @truncate(@as(u64, @bitCast(std.time.microTimestamp())));
4934 var seed: []const u8 = try std.fmt.allocPrint(arena, "{}", .{micros});
49334935
49344936 const argv_index_exe = child_argv.items.len;
49354937 _ = try child_argv.addOne();
......@@ -4945,6 +4947,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
49454947 const argv_index_global_cache_dir = child_argv.items.len;
49464948 _ = try child_argv.addOne();
49474949
4950 try child_argv.appendSlice(&[_][]const u8{ "--seed", seed });
4951 const argv_index_seed = child_argv.items.len - 1;
4952
49484953 {
49494954 var i: usize = 0;
49504955 while (i < args.len) : (i += 1) {
......@@ -4993,6 +4998,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
49934998 } else if (mem.eql(u8, arg, "--debug-compile-errors")) {
49944999 try child_argv.append(arg);
49955000 debug_compile_errors = true;
5001 } else if (mem.eql(u8, arg, "--seed")) {
5002 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5003 i += 1;
5004 child_argv.items[argv_index_seed] = args[i];
5005 continue;
49965006 }
49975007 }
49985008 try child_argv.append(arg);