authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-03-06 00:20:11-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-03-15 10:48:13-07:00
logf51413d2cf0bd87079dace7f6481d2a361a19ea6
tree4837e28770889be1b56786d163dda254aea61ca5
parent3b29d00c9826d8053b63fe2bcd86c6d1517fcc51

zig build: add an OOM-prevention system

The problem is that one may execute too many subprocesses concurrently that, together, exceed an RSS value that causes the OOM killer to kill something problematic such as the window manager. Or worse, nothing, and the system freezes. This is a real world problem. For example when building LLVM a simple `ninja install` will bring your system to its knees if you don't know that you should add `-DLLVM_PARALLEL_LINK_JOBS=1`. In particular: compiling the zig std lib tests takes about 2G each, which at 16x at once (8 cores + hyperthreading) is using all 32GB of my RAM, causing the OOM killer to kill my window manager The idea here is that you can annotate steps that might use a high amount of system resources with an upper bound. So for example I could mark the std lib tests as having an upper bound peak RSS of 3 GiB. Then the build system will do 2 things: 1. ulimit the child process, so that it will fail if it would exceed that memory limit. 2. Notice how much system RAM is available and avoid running too many concurrent jobs at once that would total more than that. This implements (1) not with an operating system enforced limit, but by checking the maxrss after a child process exits. However it does implement (2) correctly. The available memory used by the build system defaults to the total system memory, regardless of whether it is used by other processes at the time of spawning the build runner. This value can be overridden with the new --maxrss flag to `zig build`. This mechanism will ensure that the sum total of upper bound RSS memory of concurrent tasks will not exceed this value. This system makes it so that project maintainers can annotate problematic subprocesses, avoiding bug reports from users, who can blissfully execute `zig build` without worrying about the project's internals. Nobody's computer crashes, and the build system uses as much parallelism as possible without risking OOM. Users do not need to unnecessarily resort to -j1 when the build system can figure this out for them.

4 files changed, 207 insertions(+), 41 deletions(-)

lib/build_runner.zig+154-35
...@@ -84,20 +84,21 @@ pub fn main() !void {...@@ -84,20 +84,21 @@ pub fn main() !void {
84 );84 );
85 defer builder.destroy();85 defer builder.destroy();
8686
87 const Color = enum { auto, off, on };
88
87 var targets = ArrayList([]const u8).init(arena);89 var targets = ArrayList([]const u8).init(arena);
88 var debug_log_scopes = ArrayList([]const u8).init(arena);90 var debug_log_scopes = ArrayList([]const u8).init(arena);
89 var thread_pool_options: std.Thread.Pool.Options = .{ .allocator = arena };91 var thread_pool_options: std.Thread.Pool.Options = .{ .allocator = arena };
9092
91 const stderr_stream = io.getStdErr().writer();
92 const stdout_stream = io.getStdOut().writer();
93
94 var install_prefix: ?[]const u8 = null;93 var install_prefix: ?[]const u8 = null;
95 var dir_list = std.Build.DirList{};94 var dir_list = std.Build.DirList{};
96 var enable_summary: ?bool = null;95 var enable_summary: ?bool = null;
9796 var max_rss: usize = 0;
98 const Color = enum { auto, off, on };
99 var color: Color = .auto;97 var color: Color = .auto;
10098
99 const stderr_stream = io.getStdErr().writer();
100 const stdout_stream = io.getStdOut().writer();
101
101 while (nextArg(args, &arg_idx)) |arg| {102 while (nextArg(args, &arg_idx)) |arg| {
102 if (mem.startsWith(u8, arg, "-D")) {103 if (mem.startsWith(u8, arg, "-D")) {
103 const option_contents = arg[2..];104 const option_contents = arg[2..];
...@@ -147,6 +148,18 @@ pub fn main() !void {...@@ -147,6 +148,18 @@ pub fn main() !void {
147 usageAndErr(builder, false, stderr_stream);148 usageAndErr(builder, false, stderr_stream);
148 };149 };
149 builder.sysroot = sysroot;150 builder.sysroot = sysroot;
151 } else if (mem.eql(u8, arg, "--maxrss")) {
152 const max_rss_text = nextArg(args, &arg_idx) orelse {
153 std.debug.print("Expected argument after --sysroot\n\n", .{});
154 usageAndErr(builder, false, stderr_stream);
155 };
156 // TODO: support shorthand such as "2GiB", "2GB", or "2G"
157 max_rss = std.fmt.parseInt(usize, max_rss_text, 10) catch |err| {
158 std.debug.print("invalid byte size: '{s}': {s}\n", .{
159 max_rss_text, @errorName(err),
160 });
161 process.exit(1);
162 };
150 } else if (mem.eql(u8, arg, "--search-prefix")) {163 } else if (mem.eql(u8, arg, "--search-prefix")) {
151 const search_prefix = nextArg(args, &arg_idx) orelse {164 const search_prefix = nextArg(args, &arg_idx) orelse {
152 std.debug.print("Expected argument after --search-prefix\n\n", .{});165 std.debug.print("Expected argument after --search-prefix\n\n", .{});
...@@ -280,30 +293,55 @@ pub fn main() !void {...@@ -280,30 +293,55 @@ pub fn main() !void {
280 if (builder.validateUserInputDidItFail())293 if (builder.validateUserInputDidItFail())
281 usageAndErr(builder, true, stderr_stream);294 usageAndErr(builder, true, stderr_stream);
282295
296 var run: Run = .{
297 .max_rss = max_rss,
298 .max_rss_is_default = false,
299 .max_rss_mutex = .{},
300 .memory_blocked_steps = std.ArrayList(*Step).init(arena),
301
302 .claimed_rss = 0,
303 .enable_summary = enable_summary,
304 .ttyconf = ttyconf,
305 .stderr = stderr,
306 };
307
308 if (run.max_rss == 0) {
309 run.max_rss = process.totalSystemMemory() catch std.math.maxInt(usize);
310 run.max_rss_is_default = true;
311 }
312
283 runStepNames(313 runStepNames(
284 arena,314 arena,
285 builder,315 builder,
286 targets.items,316 targets.items,
287 main_progress_node,317 main_progress_node,
288 thread_pool_options,318 thread_pool_options,
289 ttyconf,319 &run,
290 stderr,
291 enable_summary,
292 ) catch |err| switch (err) {320 ) catch |err| switch (err) {
293 error.UncleanExit => process.exit(1),321 error.UncleanExit => process.exit(1),
294 else => return err,322 else => return err,
295 };323 };
296}324}
297325
326const Run = struct {
327 max_rss: usize,
328 max_rss_is_default: bool,
329 max_rss_mutex: std.Thread.Mutex,
330 memory_blocked_steps: std.ArrayList(*Step),
331
332 claimed_rss: usize,
333 enable_summary: ?bool,
334 ttyconf: std.debug.TTY.Config,
335 stderr: std.fs.File,
336};
337
298fn runStepNames(338fn runStepNames(
299 arena: std.mem.Allocator,339 arena: std.mem.Allocator,
300 b: *std.Build,340 b: *std.Build,
301 step_names: []const []const u8,341 step_names: []const []const u8,
302 parent_prog_node: *std.Progress.Node,342 parent_prog_node: *std.Progress.Node,
303 thread_pool_options: std.Thread.Pool.Options,343 thread_pool_options: std.Thread.Pool.Options,
304 ttyconf: std.debug.TTY.Config,344 run: *Run,
305 stderr: std.fs.File,
306 enable_summary: ?bool,
307) !void {345) !void {
308 const gpa = b.allocator;346 const gpa = b.allocator;
309 var step_stack: std.AutoArrayHashMapUnmanaged(*Step, void) = .{};347 var step_stack: std.AutoArrayHashMapUnmanaged(*Step, void) = .{};
...@@ -331,6 +369,26 @@ fn runStepNames(...@@ -331,6 +369,26 @@ fn runStepNames(
331 };369 };
332 }370 }
333371
372 {
373 // Check that we have enough memory to complete the build.
374 var any_problems = false;
375 for (step_stack.keys()) |s| {
376 if (s.max_rss == 0) continue;
377 if (s.max_rss > run.max_rss) {
378 std.debug.print("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory\n", .{
379 s.owner.dep_prefix, s.name, s.max_rss, run.max_rss,
380 });
381 any_problems = true;
382 }
383 }
384 if (any_problems) {
385 if (run.max_rss_is_default) {
386 std.debug.print("note: use --maxrss to override the default", .{});
387 }
388 return error.UncleanExit;
389 }
390 }
391
334 var thread_pool: std.Thread.Pool = undefined;392 var thread_pool: std.Thread.Pool = undefined;
335 try thread_pool.init(thread_pool_options);393 try thread_pool.init(thread_pool_options);
336 defer thread_pool.deinit();394 defer thread_pool.deinit();
...@@ -353,10 +411,11 @@ fn runStepNames(...@@ -353,10 +411,11 @@ fn runStepNames(
353411
354 wait_group.start();412 wait_group.start();
355 thread_pool.spawn(workerMakeOneStep, .{413 thread_pool.spawn(workerMakeOneStep, .{
356 &wait_group, &thread_pool, b, step, &step_prog, ttyconf,414 &wait_group, &thread_pool, b, step, &step_prog, run,
357 }) catch @panic("OOM");415 }) catch @panic("OOM");
358 }416 }
359 }417 }
418 assert(run.memory_blocked_steps.items.len == 0);
360419
361 var success_count: usize = 0;420 var success_count: usize = 0;
362 var skipped_count: usize = 0;421 var skipped_count: usize = 0;
...@@ -396,9 +455,12 @@ fn runStepNames(...@@ -396,9 +455,12 @@ fn runStepNames(
396455
397 // A proper command line application defaults to silently succeeding.456 // A proper command line application defaults to silently succeeding.
398 // The user may request verbose mode if they have a different preference.457 // The user may request verbose mode if they have a different preference.
399 if (failure_count == 0 and enable_summary != true) return cleanExit();458 if (failure_count == 0 and run.enable_summary != true) return cleanExit();
459
460 const ttyconf = run.ttyconf;
461 const stderr = run.stderr;
400462
401 if (enable_summary != false) {463 if (run.enable_summary != false) {
402 const total_count = success_count + failure_count + pending_count + skipped_count;464 const total_count = success_count + failure_count + pending_count + skipped_count;
403 ttyconf.setColor(stderr, .Cyan) catch {};465 ttyconf.setColor(stderr, .Cyan) catch {};
404 stderr.writeAll("Build Summary:") catch {};466 stderr.writeAll("Build Summary:") catch {};
...@@ -407,7 +469,7 @@ fn runStepNames(...@@ -407,7 +469,7 @@ fn runStepNames(
407 if (skipped_count > 0) stderr.writer().print("; {d} skipped", .{skipped_count}) catch {};469 if (skipped_count > 0) stderr.writer().print("; {d} skipped", .{skipped_count}) catch {};
408 if (failure_count > 0) stderr.writer().print("; {d} failed", .{failure_count}) catch {};470 if (failure_count > 0) stderr.writer().print("; {d} failed", .{failure_count}) catch {};
409471
410 if (enable_summary == null) {472 if (run.enable_summary == null) {
411 ttyconf.setColor(stderr, .Dim) catch {};473 ttyconf.setColor(stderr, .Dim) catch {};
412 stderr.writeAll(" (disable with -fno-summary)") catch {};474 stderr.writeAll(" (disable with -fno-summary)") catch {};
413 ttyconf.setColor(stderr, .Reset) catch {};475 ttyconf.setColor(stderr, .Reset) catch {};
...@@ -623,7 +685,7 @@ fn workerMakeOneStep(...@@ -623,7 +685,7 @@ fn workerMakeOneStep(
623 b: *std.Build,685 b: *std.Build,
624 s: *Step,686 s: *Step,
625 prog_node: *std.Progress.Node,687 prog_node: *std.Progress.Node,
626 ttyconf: std.debug.TTY.Config,688 run: *Run,
627) void {689) void {
628 defer wg.finish();690 defer wg.finish();
629691
...@@ -646,10 +708,32 @@ fn workerMakeOneStep(...@@ -646,10 +708,32 @@ fn workerMakeOneStep(
646 }708 }
647 }709 }
648710
649 // Avoid running steps twice.711 if (s.max_rss != 0) {
650 if (@cmpxchgStrong(Step.State, &s.state, .precheck_done, .running, .SeqCst, .SeqCst) != null) {712 run.max_rss_mutex.lock();
651 // Another worker got the job.713 defer run.max_rss_mutex.unlock();
652 return;714
715 // Avoid running steps twice.
716 if (s.state != .precheck_done) {
717 // Another worker got the job.
718 return;
719 }
720
721 const new_claimed_rss = run.claimed_rss + s.max_rss;
722 if (new_claimed_rss > run.max_rss) {
723 // Running this step right now could possibly exceed the allotted RSS.
724 // Add this step to the queue of memory-blocked steps.
725 run.memory_blocked_steps.append(s) catch @panic("OOM");
726 return;
727 }
728
729 run.claimed_rss = new_claimed_rss;
730 s.state = .running;
731 } else {
732 // Avoid running steps twice.
733 if (@cmpxchgStrong(Step.State, &s.state, .precheck_done, .running, .SeqCst, .SeqCst) != null) {
734 // Another worker got the job.
735 return;
736 }
653 }737 }
654738
655 var sub_prog_node = prog_node.start(s.name, 0);739 var sub_prog_node = prog_node.start(s.name, 0);
...@@ -667,7 +751,8 @@ fn workerMakeOneStep(...@@ -667,7 +751,8 @@ fn workerMakeOneStep(
667 sub_prog_node.context.lock_stderr();751 sub_prog_node.context.lock_stderr();
668 defer sub_prog_node.context.unlock_stderr();752 defer sub_prog_node.context.unlock_stderr();
669753
670 const stderr = std.io.getStdErr();754 const stderr = run.stderr;
755 const ttyconf = run.ttyconf;
671756
672 for (s.result_error_msgs.items) |msg| {757 for (s.result_error_msgs.items) |msg| {
673 // Sometimes it feels like you just can't catch a break. Finally,758 // Sometimes it feels like you just can't catch a break. Finally,
...@@ -684,22 +769,55 @@ fn workerMakeOneStep(...@@ -684,22 +769,55 @@ fn workerMakeOneStep(
684 }769 }
685 }770 }
686771
687 if (make_result) |_| {772 handle_result: {
688 @atomicStore(Step.State, &s.state, .success, .SeqCst);773 if (make_result) |_| {
689 } else |err| switch (err) {774 @atomicStore(Step.State, &s.state, .success, .SeqCst);
690 error.MakeFailed => {775 } else |err| switch (err) {
691 @atomicStore(Step.State, &s.state, .failure, .SeqCst);776 error.MakeFailed => {
692 return;777 @atomicStore(Step.State, &s.state, .failure, .SeqCst);
693 },778 break :handle_result;
694 error.MakeSkipped => @atomicStore(Step.State, &s.state, .skipped, .SeqCst),779 },
780 error.MakeSkipped => @atomicStore(Step.State, &s.state, .skipped, .SeqCst),
781 }
782
783 // Successful completion of a step, so we queue up its dependants as well.
784 for (s.dependants.items) |dep| {
785 wg.start();
786 thread_pool.spawn(workerMakeOneStep, .{
787 wg, thread_pool, b, dep, prog_node, run,
788 }) catch @panic("OOM");
789 }
695 }790 }
696791
697 // Successful completion of a step, so we queue up its dependants as well.792 // If this is a step that claims resources, we must now queue up other
698 for (s.dependants.items) |dep| {793 // steps that are waiting for resources.
699 wg.start();794 if (s.max_rss != 0) {
700 thread_pool.spawn(workerMakeOneStep, .{795 run.max_rss_mutex.lock();
701 wg, thread_pool, b, dep, prog_node, ttyconf,796 defer run.max_rss_mutex.unlock();
702 }) catch @panic("OOM");797
798 // Give the memory back to the scheduler.
799 run.claimed_rss -= s.max_rss;
800 // Avoid kicking off too many tasks that we already know will not have
801 // enough resources.
802 var remaining = run.max_rss - run.claimed_rss;
803 var i: usize = 0;
804 var j: usize = 0;
805 while (j < run.memory_blocked_steps.items.len) : (j += 1) {
806 const dep = run.memory_blocked_steps.items[j];
807 assert(dep.max_rss != 0);
808 if (dep.max_rss <= remaining) {
809 remaining -= dep.max_rss;
810
811 wg.start();
812 thread_pool.spawn(workerMakeOneStep, .{
813 wg, thread_pool, b, dep, prog_node, run,
814 }) catch @panic("OOM");
815 } else {
816 run.memory_blocked_steps.items[i] = dep;
817 i += 1;
818 }
819 }
820 run.memory_blocked_steps.shrinkRetainingCapacity(i);
703 }821 }
704}822}
705823
...@@ -770,6 +888,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi...@@ -770,6 +888,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi
770 \\ --color [auto|off|on] Enable or disable colored error messages888 \\ --color [auto|off|on] Enable or disable colored error messages
771 \\ --prominent-compile-errors Output compile errors formatted for a human to read889 \\ --prominent-compile-errors Output compile errors formatted for a human to read
772 \\ -j<N> Limit concurrent jobs (default is to use all CPU cores)890 \\ -j<N> Limit concurrent jobs (default is to use all CPU cores)
891 \\ --maxrss <bytes> Limit memory usage (default is to use available memory)
773 \\892 \\
774 \\Project-Specific Options:893 \\Project-Specific Options:
775 \\894 \\
lib/std/Build.zig+12
...@@ -453,6 +453,7 @@ pub const ExecutableOptions = struct {...@@ -453,6 +453,7 @@ pub const ExecutableOptions = struct {
453 target: CrossTarget = .{},453 target: CrossTarget = .{},
454 optimize: std.builtin.Mode = .Debug,454 optimize: std.builtin.Mode = .Debug,
455 linkage: ?CompileStep.Linkage = null,455 linkage: ?CompileStep.Linkage = null,
456 max_rss: usize = 0,
456};457};
457458
458pub fn addExecutable(b: *Build, options: ExecutableOptions) *CompileStep {459pub fn addExecutable(b: *Build, options: ExecutableOptions) *CompileStep {
...@@ -464,6 +465,7 @@ pub fn addExecutable(b: *Build, options: ExecutableOptions) *CompileStep {...@@ -464,6 +465,7 @@ pub fn addExecutable(b: *Build, options: ExecutableOptions) *CompileStep {
464 .optimize = options.optimize,465 .optimize = options.optimize,
465 .kind = .exe,466 .kind = .exe,
466 .linkage = options.linkage,467 .linkage = options.linkage,
468 .max_rss = options.max_rss,
467 });469 });
468}470}
469471
...@@ -472,6 +474,7 @@ pub const ObjectOptions = struct {...@@ -472,6 +474,7 @@ pub const ObjectOptions = struct {
472 root_source_file: ?FileSource = null,474 root_source_file: ?FileSource = null,
473 target: CrossTarget,475 target: CrossTarget,
474 optimize: std.builtin.Mode,476 optimize: std.builtin.Mode,
477 max_rss: usize = 0,
475};478};
476479
477pub fn addObject(b: *Build, options: ObjectOptions) *CompileStep {480pub fn addObject(b: *Build, options: ObjectOptions) *CompileStep {
...@@ -481,6 +484,7 @@ pub fn addObject(b: *Build, options: ObjectOptions) *CompileStep {...@@ -481,6 +484,7 @@ pub fn addObject(b: *Build, options: ObjectOptions) *CompileStep {
481 .target = options.target,484 .target = options.target,
482 .optimize = options.optimize,485 .optimize = options.optimize,
483 .kind = .obj,486 .kind = .obj,
487 .max_rss = options.max_rss,
484 });488 });
485}489}
486490
...@@ -490,6 +494,7 @@ pub const SharedLibraryOptions = struct {...@@ -490,6 +494,7 @@ pub const SharedLibraryOptions = struct {
490 version: ?std.builtin.Version = null,494 version: ?std.builtin.Version = null,
491 target: CrossTarget,495 target: CrossTarget,
492 optimize: std.builtin.Mode,496 optimize: std.builtin.Mode,
497 max_rss: usize = 0,
493};498};
494499
495pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *CompileStep {500pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *CompileStep {
...@@ -501,6 +506,7 @@ pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *CompileStep {...@@ -501,6 +506,7 @@ pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *CompileStep {
501 .version = options.version,506 .version = options.version,
502 .target = options.target,507 .target = options.target,
503 .optimize = options.optimize,508 .optimize = options.optimize,
509 .max_rss = options.max_rss,
504 });510 });
505}511}
506512
...@@ -510,6 +516,7 @@ pub const StaticLibraryOptions = struct {...@@ -510,6 +516,7 @@ pub const StaticLibraryOptions = struct {
510 target: CrossTarget,516 target: CrossTarget,
511 optimize: std.builtin.Mode,517 optimize: std.builtin.Mode,
512 version: ?std.builtin.Version = null,518 version: ?std.builtin.Version = null,
519 max_rss: usize = 0,
513};520};
514521
515pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *CompileStep {522pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *CompileStep {
...@@ -521,6 +528,7 @@ pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *CompileStep {...@@ -521,6 +528,7 @@ pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *CompileStep {
521 .version = options.version,528 .version = options.version,
522 .target = options.target,529 .target = options.target,
523 .optimize = options.optimize,530 .optimize = options.optimize,
531 .max_rss = options.max_rss,
524 });532 });
525}533}
526534
...@@ -531,6 +539,7 @@ pub const TestOptions = struct {...@@ -531,6 +539,7 @@ pub const TestOptions = struct {
531 target: CrossTarget = .{},539 target: CrossTarget = .{},
532 optimize: std.builtin.Mode = .Debug,540 optimize: std.builtin.Mode = .Debug,
533 version: ?std.builtin.Version = null,541 version: ?std.builtin.Version = null,
542 max_rss: usize = 0,
534};543};
535544
536pub fn addTest(b: *Build, options: TestOptions) *CompileStep {545pub fn addTest(b: *Build, options: TestOptions) *CompileStep {
...@@ -540,6 +549,7 @@ pub fn addTest(b: *Build, options: TestOptions) *CompileStep {...@@ -540,6 +549,7 @@ pub fn addTest(b: *Build, options: TestOptions) *CompileStep {
540 .root_source_file = options.root_source_file,549 .root_source_file = options.root_source_file,
541 .target = options.target,550 .target = options.target,
542 .optimize = options.optimize,551 .optimize = options.optimize,
552 .max_rss = options.max_rss,
543 });553 });
544}554}
545555
...@@ -548,6 +558,7 @@ pub const AssemblyOptions = struct {...@@ -548,6 +558,7 @@ pub const AssemblyOptions = struct {
548 source_file: FileSource,558 source_file: FileSource,
549 target: CrossTarget,559 target: CrossTarget,
550 optimize: std.builtin.Mode,560 optimize: std.builtin.Mode,
561 max_rss: usize = 0,
551};562};
552563
553pub fn addAssembly(b: *Build, options: AssemblyOptions) *CompileStep {564pub fn addAssembly(b: *Build, options: AssemblyOptions) *CompileStep {
...@@ -557,6 +568,7 @@ pub fn addAssembly(b: *Build, options: AssemblyOptions) *CompileStep {...@@ -557,6 +568,7 @@ pub fn addAssembly(b: *Build, options: AssemblyOptions) *CompileStep {
557 .root_source_file = null,568 .root_source_file = null,
558 .target = options.target,569 .target = options.target,
559 .optimize = options.optimize,570 .optimize = options.optimize,
571 .max_rss = options.max_rss,
560 });572 });
561 obj_step.addAssemblyFileSource(options.source_file.dupe(b));573 obj_step.addAssemblyFileSource(options.source_file.dupe(b));
562 return obj_step;574 return obj_step;
lib/std/Build/CompileStep.zig+2
...@@ -274,6 +274,7 @@ pub const Options = struct {...@@ -274,6 +274,7 @@ pub const Options = struct {
274 kind: Kind,274 kind: Kind,
275 linkage: ?Linkage = null,275 linkage: ?Linkage = null,
276 version: ?std.builtin.Version = null,276 version: ?std.builtin.Version = null,
277 max_rss: usize = 0,
277};278};
278279
279pub const Kind = enum {280pub const Kind = enum {
...@@ -333,6 +334,7 @@ pub fn create(owner: *std.Build, options: Options) *CompileStep {...@@ -333,6 +334,7 @@ pub fn create(owner: *std.Build, options: Options) *CompileStep {
333 .name = step_name,334 .name = step_name,
334 .owner = owner,335 .owner = owner,
335 .makeFn = make,336 .makeFn = make,
337 .max_rss = options.max_rss,
336 }),338 }),
337 .version = options.version,339 .version = options.version,
338 .out_filename = undefined,340 .out_filename = undefined,
lib/std/Build/Step.zig+39-6
...@@ -2,14 +2,32 @@ id: Id,...@@ -2,14 +2,32 @@ id: Id,
2name: []const u8,2name: []const u8,
3owner: *Build,3owner: *Build,
4makeFn: MakeFn,4makeFn: MakeFn,
5
5dependencies: std.ArrayList(*Step),6dependencies: std.ArrayList(*Step),
6/// This field is empty during execution of the user's build script, and7/// This field is empty during execution of the user's build script, and
7/// then populated during dependency loop checking in the build runner.8/// then populated during dependency loop checking in the build runner.
8dependants: std.ArrayListUnmanaged(*Step),9dependants: std.ArrayListUnmanaged(*Step),
9state: State,10state: State,
10/// The return addresss associated with creation of this step that can be useful11/// Set this field to declare an upper bound on the amount of bytes of memory it will
11/// to print along with debugging messages.12/// take to run the step. Zero means no limit.
12debug_stack_trace: [n_debug_stack_frames]usize,13///
14/// The idea to annotate steps that might use a high amount of RAM with an
15/// upper bound. For example, perhaps a particular set of unit tests require 4
16/// GiB of RAM, and those tests will be run under 4 different build
17/// configurations at once. This would potentially require 16 GiB of memory on
18/// the system if all 4 steps executed simultaneously, which could easily be
19/// greater than what is actually available, potentially causing the system to
20/// crash when using `zig build` at the default concurrency level.
21///
22/// This field causes the build runner to do two things:
23/// 1. ulimit child processes, so that they will fail if it would exceed this
24/// memory limit. This serves to enforce that this upper bound value is
25/// correct.
26/// 2. Ensure that the set of concurrent steps at any given time have a total
27/// max_rss value that does not exceed the `max_total_rss` value of the build
28/// runner. This value is configurable on the command line, and defaults to the
29/// total system memory available.
30max_rss: usize,
1331
14result_error_msgs: std.ArrayListUnmanaged([]const u8),32result_error_msgs: std.ArrayListUnmanaged([]const u8),
15result_error_bundle: std.zig.ErrorBundle,33result_error_bundle: std.zig.ErrorBundle,
...@@ -18,6 +36,10 @@ result_duration_ns: ?u64,...@@ -18,6 +36,10 @@ result_duration_ns: ?u64,
18/// 0 means unavailable or not reported.36/// 0 means unavailable or not reported.
19result_peak_rss: usize,37result_peak_rss: usize,
2038
39/// The return addresss associated with creation of this step that can be useful
40/// to print along with debugging messages.
41debug_stack_trace: [n_debug_stack_frames]usize,
42
21pub const MakeFn = *const fn (self: *Step, prog_node: *std.Progress.Node) anyerror!void;43pub const MakeFn = *const fn (self: *Step, prog_node: *std.Progress.Node) anyerror!void;
2244
23const n_debug_stack_frames = 4;45const n_debug_stack_frames = 4;
...@@ -83,6 +105,7 @@ pub const Options = struct {...@@ -83,6 +105,7 @@ pub const Options = struct {
83 owner: *Build,105 owner: *Build,
84 makeFn: MakeFn = makeNoOp,106 makeFn: MakeFn = makeNoOp,
85 first_ret_addr: ?usize = null,107 first_ret_addr: ?usize = null,
108 max_rss: usize = 0,
86};109};
87110
88pub fn init(options: Options) Step {111pub fn init(options: Options) Step {
...@@ -104,6 +127,7 @@ pub fn init(options: Options) Step {...@@ -104,6 +127,7 @@ pub fn init(options: Options) Step {
104 .dependencies = std.ArrayList(*Step).init(arena),127 .dependencies = std.ArrayList(*Step).init(arena),
105 .dependants = .{},128 .dependants = .{},
106 .state = .precheck_unstarted,129 .state = .precheck_unstarted,
130 .max_rss = options.max_rss,
107 .debug_stack_trace = addresses,131 .debug_stack_trace = addresses,
108 .result_error_msgs = .{},132 .result_error_msgs = .{},
109 .result_error_bundle = std.zig.ErrorBundle.empty,133 .result_error_bundle = std.zig.ErrorBundle.empty,
...@@ -117,15 +141,24 @@ pub fn init(options: Options) Step {...@@ -117,15 +141,24 @@ pub fn init(options: Options) Step {
117/// have already reported the error. Otherwise, we add a simple error report141/// have already reported the error. Otherwise, we add a simple error report
118/// here.142/// here.
119pub fn make(s: *Step, prog_node: *std.Progress.Node) error{ MakeFailed, MakeSkipped }!void {143pub fn make(s: *Step, prog_node: *std.Progress.Node) error{ MakeFailed, MakeSkipped }!void {
120 return s.makeFn(s, prog_node) catch |err| switch (err) {144 const arena = s.owner.allocator;
145
146 s.makeFn(s, prog_node) catch |err| switch (err) {
121 error.MakeFailed => return error.MakeFailed,147 error.MakeFailed => return error.MakeFailed,
122 error.MakeSkipped => return error.MakeSkipped,148 error.MakeSkipped => return error.MakeSkipped,
123 else => {149 else => {
124 const gpa = s.dependencies.allocator;150 s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM");
125 s.result_error_msgs.append(gpa, @errorName(err)) catch @panic("OOM");
126 return error.MakeFailed;151 return error.MakeFailed;
127 },152 },
128 };153 };
154
155 if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) {
156 const msg = std.fmt.allocPrint(arena, "memory usage peaked at {d} bytes, exceeding the declared upper bound of {d}", .{
157 s.result_peak_rss, s.max_rss,
158 }) catch @panic("OOM");
159 s.result_error_msgs.append(arena, msg) catch @panic("OOM");
160 return error.MakeFailed;
161 }
129}162}
130163
131pub fn dependOn(self: *Step, other: *Step) void {164pub fn dependOn(self: *Step, other: *Step) void {