authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-20 15:30:48-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-10-20 15:30:48-04:00
log7de893c085c94934b2923a9659c9cf70b70d71b2
tree7a5942e708267be128b9dd0c84df8f6d6ec4c061
parent45d7dfa8333880ffa0a318723d22a0072bd32b3f
parent10200970bb3700dd36807b4df1b1e327da98bbea
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #16817 from Sahnvour/shuffle-deps

std.Build: add --seed argument to randomize step dependencies spawning

2 files changed, 54 insertions(+), 4 deletions(-)

lib/build_runner.zig+42-4
...@@ -95,6 +95,7 @@ pub fn main() !void {...@@ -95,6 +95,7 @@ pub fn main() !void {
95 var max_rss: usize = 0;95 var max_rss: usize = 0;
96 var skip_oom_steps: bool = false;96 var skip_oom_steps: bool = false;
97 var color: Color = .auto;97 var color: Color = .auto;
98 var seed: u32 = 0;
9899
99 const stderr_stream = io.getStdErr().writer();100 const stderr_stream = io.getStdErr().writer();
100 const stdout_stream = io.getStdOut().writer();101 const stdout_stream = io.getStdOut().writer();
...@@ -196,6 +197,17 @@ pub fn main() !void {...@@ -196,6 +197,17 @@ pub fn main() !void {
196 std.debug.print("Expected argument after {s}\n\n", .{arg});197 std.debug.print("Expected argument after {s}\n\n", .{arg});
197 usageAndErr(builder, false, stderr_stream);198 usageAndErr(builder, false, stderr_stream);
198 } };199 } };
200 } else if (mem.eql(u8, arg, "--seed")) {
201 const next_arg = nextArg(args, &arg_idx) orelse {
202 std.debug.print("Expected u32 after {s}\n\n", .{arg});
203 usageAndErr(builder, false, stderr_stream);
204 };
205 seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| {
206 std.debug.print("unable to parse seed '{s}' as 32-bit integer: {s}", .{
207 next_arg, @errorName(err),
208 });
209 process.exit(1);
210 };
199 } else if (mem.eql(u8, arg, "--debug-log")) {211 } else if (mem.eql(u8, arg, "--debug-log")) {
200 const next_arg = nextArg(args, &arg_idx) orelse {212 const next_arg = nextArg(args, &arg_idx) orelse {
201 std.debug.print("Expected argument after {s}\n\n", .{arg});213 std.debug.print("Expected argument after {s}\n\n", .{arg});
...@@ -329,6 +341,7 @@ pub fn main() !void {...@@ -329,6 +341,7 @@ pub fn main() !void {
329 main_progress_node,341 main_progress_node,
330 thread_pool_options,342 thread_pool_options,
331 &run,343 &run,
344 seed,
332 ) catch |err| switch (err) {345 ) catch |err| switch (err) {
333 error.UncleanExit => process.exit(1),346 error.UncleanExit => process.exit(1),
334 else => return err,347 else => return err,
...@@ -355,6 +368,7 @@ fn runStepNames(...@@ -355,6 +368,7 @@ fn runStepNames(
355 parent_prog_node: *std.Progress.Node,368 parent_prog_node: *std.Progress.Node,
356 thread_pool_options: std.Thread.Pool.Options,369 thread_pool_options: std.Thread.Pool.Options,
357 run: *Run,370 run: *Run,
371 seed: u32,
358) !void {372) !void {
359 const gpa = b.allocator;373 const gpa = b.allocator;
360 var step_stack: std.AutoArrayHashMapUnmanaged(*Step, void) = .{};374 var step_stack: std.AutoArrayHashMapUnmanaged(*Step, void) = .{};
...@@ -375,8 +389,13 @@ fn runStepNames(...@@ -375,8 +389,13 @@ fn runStepNames(
375 }389 }
376390
377 const starting_steps = try arena.dupe(*Step, step_stack.keys());391 const starting_steps = try arena.dupe(*Step, step_stack.keys());
392
393 var rng = std.rand.DefaultPrng.init(seed);
394 const rand = rng.random();
395 rand.shuffle(*Step, starting_steps);
396
378 for (starting_steps) |s| {397 for (starting_steps) |s| {
379 checkForDependencyLoop(b, s, &step_stack) catch |err| switch (err) {398 constructGraphAndCheckForDependencyLoop(b, s, &step_stack, rand) catch |err| switch (err) {
380 error.DependencyLoopDetected => return error.UncleanExit,399 error.DependencyLoopDetected => return error.UncleanExit,
381 else => |e| return e,400 else => |e| return e,
382 };401 };
...@@ -748,10 +767,22 @@ fn printTreeStep(...@@ -748,10 +767,22 @@ fn printTreeStep(
748 }767 }
749}768}
750769
751fn checkForDependencyLoop(770/// Traverse the dependency graph depth-first and make it undirected by having
771/// steps know their dependants (they only know dependencies at start).
772/// Along the way, check that there is no dependency loop, and record the steps
773/// in traversal order in `step_stack`.
774/// Each step has its dependencies traversed in random order, this accomplishes
775/// two things:
776/// - `step_stack` will be in randomized-depth-first order, so the build runner
777/// spawns steps in a random (but optimized) order
778/// - each step's `dependants` list is also filled in a random order, so that
779/// when it finishes executing in `workerMakeOneStep`, it spawns next steps
780/// to run in random order
781fn constructGraphAndCheckForDependencyLoop(
752 b: *std.Build,782 b: *std.Build,
753 s: *Step,783 s: *Step,
754 step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void),784 step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void),
785 rand: std.rand.Random,
755) !void {786) !void {
756 switch (s.state) {787 switch (s.state) {
757 .precheck_started => {788 .precheck_started => {
...@@ -762,10 +793,16 @@ fn checkForDependencyLoop(...@@ -762,10 +793,16 @@ fn checkForDependencyLoop(
762 s.state = .precheck_started;793 s.state = .precheck_started;
763794
764 try step_stack.ensureUnusedCapacity(b.allocator, s.dependencies.items.len);795 try step_stack.ensureUnusedCapacity(b.allocator, s.dependencies.items.len);
765 for (s.dependencies.items) |dep| {796
797 // We dupe to avoid shuffling the steps in the summary, it depends
798 // on s.dependencies' order.
799 const deps = b.allocator.dupe(*Step, s.dependencies.items) catch @panic("OOM");
800 rand.shuffle(*Step, deps);
801
802 for (deps) |dep| {
766 try step_stack.put(b.allocator, dep, {});803 try step_stack.put(b.allocator, dep, {});
767 try dep.dependants.append(b.allocator, s);804 try dep.dependants.append(b.allocator, s);
768 checkForDependencyLoop(b, dep, step_stack) catch |err| {805 constructGraphAndCheckForDependencyLoop(b, dep, step_stack, rand) catch |err| {
769 if (err == error.DependencyLoopDetected) {806 if (err == error.DependencyLoopDetected) {
770 std.debug.print(" {s}\n", .{s.name});807 std.debug.print(" {s}\n", .{s.name});
771 }808 }
...@@ -1034,6 +1071,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi...@@ -1034,6 +1071,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi
1034 \\ --global-cache-dir [path] Override path to global Zig cache directory1071 \\ --global-cache-dir [path] Override path to global Zig cache directory
1035 \\ --zig-lib-dir [arg] Override path to Zig lib directory1072 \\ --zig-lib-dir [arg] Override path to Zig lib directory
1036 \\ --build-runner [file] Override path to build runner1073 \\ --build-runner [file] Override path to build runner
1074 \\ --seed [integer] For shuffling dependency traversal order (default: random)
1037 \\ --debug-log [scope] Enable debugging the compiler1075 \\ --debug-log [scope] Enable debugging the compiler
1038 \\ --debug-pkg-config Fail if unknown pkg-config flags encountered1076 \\ --debug-pkg-config Fail if unknown pkg-config flags encountered
1039 \\ --verbose-link Enable compiler debug output for linking1077 \\ --verbose-link Enable compiler debug output for linking
src/main.zig+12
...@@ -4906,6 +4906,7 @@ pub const usage_build =...@@ -4906,6 +4906,7 @@ pub const usage_build =
4906 \\ --global-cache-dir [path] Override path to global Zig cache directory4906 \\ --global-cache-dir [path] Override path to global Zig cache directory
4907 \\ --zig-lib-dir [arg] Override path to Zig lib directory4907 \\ --zig-lib-dir [arg] Override path to Zig lib directory
4908 \\ --build-runner [file] Override path to build runner4908 \\ --build-runner [file] Override path to build runner
4909 \\ --seed [integer] For shuffling dependency traversal order (default: random)
4909 \\ --fetch Exit after fetching dependency tree4910 \\ --fetch Exit after fetching dependency tree
4910 \\ -h, --help Print this help and exit4911 \\ -h, --help Print this help and exit
4911 \\4912 \\
...@@ -4945,6 +4946,12 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -4945,6 +4946,12 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4945 const argv_index_global_cache_dir = child_argv.items.len;4946 const argv_index_global_cache_dir = child_argv.items.len;
4946 _ = try child_argv.addOne();4947 _ = try child_argv.addOne();
49474948
4949 try child_argv.appendSlice(&.{
4950 "--seed",
4951 try std.fmt.allocPrint(arena, "0x{x}", .{std.crypto.random.int(u32)}),
4952 });
4953 const argv_index_seed = child_argv.items.len - 1;
4954
4948 {4955 {
4949 var i: usize = 0;4956 var i: usize = 0;
4950 while (i < args.len) : (i += 1) {4957 while (i < args.len) : (i += 1) {
...@@ -4993,6 +5000,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -4993,6 +5000,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4993 } else if (mem.eql(u8, arg, "--debug-compile-errors")) {5000 } else if (mem.eql(u8, arg, "--debug-compile-errors")) {
4994 try child_argv.append(arg);5001 try child_argv.append(arg);
4995 debug_compile_errors = true;5002 debug_compile_errors = true;
5003 } else if (mem.eql(u8, arg, "--seed")) {
5004 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5005 i += 1;
5006 child_argv.items[argv_index_seed] = args[i];
5007 continue;
4996 }5008 }
4997 }5009 }
4998 try child_argv.append(arg);5010 try child_argv.append(arg);