authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-05-27 00:48:11+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-05-27 00:48:11+02:00
log4d56c6636251a90926467dfe65c99adf5ff7222a
tree5b5265368a236ab18def983268268f3792272d87
parent7451f5d1174c1113da50c883c7b3f7e8415cea0a
parent33680ced58de7efb5bfe18fc95d2b67fc8496aed

Merge pull request 'Maker: memory usage optimizations' (#35471) from build-runner-process into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/35471

10 files changed, 167 insertions(+), 116 deletions(-)

ci/x86_64-linux-debug-llvm.sh-1
...@@ -49,7 +49,6 @@ stage3-debug/bin/zig build \...@@ -49,7 +49,6 @@ stage3-debug/bin/zig build \
49 -Dno-lib49 -Dno-lib
5050
51stage3-debug/bin/zig build test docs \51stage3-debug/bin/zig build test docs \
52 --maker-opt=Debug \
53 --maxrss ${ZSF_MAX_RSS:-0} \52 --maxrss ${ZSF_MAX_RSS:-0} \
54 -Dlldb=$HOME/deps/lldb-zig/Debug-33ec8d3c11/bin/lldb \53 -Dlldb=$HOME/deps/lldb-zig/Debug-33ec8d3c11/bin/lldb \
55 -Dlibc-test-path=$HOME/deps/libc-test-f2bac77 \54 -Dlibc-test-path=$HOME/deps/libc-test-f2bac77 \
ci/x86_64-linux-debug.sh+1
...@@ -48,6 +48,7 @@ stage3-debug/bin/zig build \...@@ -48,6 +48,7 @@ stage3-debug/bin/zig build \
48 -Dno-lib48 -Dno-lib
4949
50stage3-debug/bin/zig build test docs \50stage3-debug/bin/zig build test docs \
51 --maker-opt=Debug \
51 --maxrss ${ZSF_MAX_RSS:-0} \52 --maxrss ${ZSF_MAX_RSS:-0} \
52 -Dlldb=$HOME/deps/lldb-zig/Debug-33ec8d3c11/bin/lldb \53 -Dlldb=$HOME/deps/lldb-zig/Debug-33ec8d3c11/bin/lldb \
53 -fqemu \54 -fqemu \
lib/compiler/Maker.zig+54-44
...@@ -16,6 +16,7 @@ const fmt = std.fmt;...@@ -16,6 +16,7 @@ const fmt = std.fmt;
16const log = std.log;16const log = std.log;
17const mem = std.mem;17const mem = std.mem;
18const process = std.process;18const process = std.process;
19const Color = std.zig.Color;
1920
20const Fuzz = @import("Maker/Fuzz.zig");21const Fuzz = @import("Maker/Fuzz.zig");
21const Graph = @import("Maker/Graph.zig");22const Graph = @import("Maker/Graph.zig");
...@@ -55,12 +56,59 @@ error_style: ErrorStyle,...@@ -55,12 +56,59 @@ error_style: ErrorStyle,
55multiline_errors: MultilineErrors,56multiline_errors: MultilineErrors,
56summary: Summary,57summary: Summary,
5758
59var safe_allocator_instance: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{});
60var stdio_buffer_allocation: [256]u8 = undefined;
61var stdout_writer_allocation: Io.File.Writer = undefined;
62var debug_maker_leaks: bool = false;
63
64const is_debug_mode = builtin.mode == .Debug;
65const use_safe_allocator = switch (builtin.mode) {
66 .Debug, .ReleaseSafe => true,
67 .ReleaseFast, .ReleaseSmall => false,
68};
69
70const InstallPaths = struct {
71 prefix: Path,
72 lib: Path,
73 bin: Path,
74 include: Path,
75};
76
77const PrintNode = struct {
78 parent: ?*PrintNode,
79 last: bool = false,
80};
81
82const ErrorStyle = enum {
83 verbose,
84 minimal,
85 verbose_clear,
86 minimal_clear,
87 fn verboseContext(s: ErrorStyle) bool {
88 return switch (s) {
89 .verbose, .verbose_clear => true,
90 .minimal, .minimal_clear => false,
91 };
92 }
93 fn clearOnUpdate(s: ErrorStyle) bool {
94 return switch (s) {
95 .verbose, .minimal => false,
96 .verbose_clear, .minimal_clear => true,
97 };
98 }
99};
100const MultilineErrors = enum { indent, newline, none };
101const Summary = enum { all, new, failures, line, none };
102
58pub fn main(init: process.Init.Minimal) !void {103pub fn main(init: process.Init.Minimal) !void {
59 // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not104 // The build runner is long-lived in the following use cases:
60 // always the case. So, we do need a true gpa for some things.105 // * `--watch` mode
61 var safe_gpa_state: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{});106 // * `--webui` mode
62 defer _ = safe_gpa_state.deinit();107 // * A project that has a large, complex build graph.
63 const gpa = safe_gpa_state.allocator();108 const gpa = if (use_safe_allocator) safe_allocator_instance.allocator() else std.heap.smp_allocator;
109 defer if (use_safe_allocator) {
110 _ = safe_allocator_instance.deinit();
111 };
64112
65 var threaded: std.Io.Threaded = .init(gpa, .{113 var threaded: std.Io.Threaded = .init(gpa, .{
66 .environ = init.environ,114 .environ = init.environ,
...@@ -689,13 +737,6 @@ fn countSubProcesses(maker: *Maker) usize {...@@ -689,13 +737,6 @@ fn countSubProcesses(maker: *Maker) usize {
689 return count;737 return count;
690}738}
691739
692const InstallPaths = struct {
693 prefix: Path,
694 lib: Path,
695 bin: Path,
696 include: Path,
697};
698
699pub fn stepByIndex(maker: *const Maker, i: Configuration.Step.Index) *Step {740pub fn stepByIndex(maker: *const Maker, i: Configuration.Step.Index) *Step {
700 return &maker.steps[@intFromEnum(i)];741 return &maker.steps[@intFromEnum(i)];
701}742}
...@@ -1018,6 +1059,7 @@ fn makeStepNames(...@@ -1018,6 +1059,7 @@ fn makeStepNames(
1018fn deinit(maker: *Maker) void {1059fn deinit(maker: *Maker) void {
1019 const gpa = maker.gpa;1060 const gpa = maker.gpa;
1020 for (maker.steps) |*step| {1061 for (maker.steps) |*step| {
1062 step.clearResultStderr(gpa);
1021 step.clearFailedCommand(gpa);1063 step.clearFailedCommand(gpa);
1022 step.clearErrorBundle(gpa);1064 step.clearErrorBundle(gpa);
1023 step.inputs.deinit(gpa);1065 step.inputs.deinit(gpa);
...@@ -1424,11 +1466,6 @@ fn printStepFailure(...@@ -1424,11 +1466,6 @@ fn printStepFailure(
1424 }1466 }
1425}1467}
14261468
1427const PrintNode = struct {
1428 parent: ?*PrintNode,
1429 last: bool = false,
1430};
1431
1432fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void {1469fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void {
1433 const parent = node.parent orelse return;1470 const parent = node.parent orelse return;
1434 const writer = stderr.writer;1471 const writer = stderr.writer;
...@@ -1657,28 +1694,6 @@ fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 {...@@ -1657,28 +1694,6 @@ fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 {
1657 return args[idx..];1694 return args[idx..];
1658}1695}
16591696
1660const Color = std.zig.Color;
1661const ErrorStyle = enum {
1662 verbose,
1663 minimal,
1664 verbose_clear,
1665 minimal_clear,
1666 fn verboseContext(s: ErrorStyle) bool {
1667 return switch (s) {
1668 .verbose, .verbose_clear => true,
1669 .minimal, .minimal_clear => false,
1670 };
1671 }
1672 fn clearOnUpdate(s: ErrorStyle) bool {
1673 return switch (s) {
1674 .verbose, .minimal => false,
1675 .verbose_clear, .minimal_clear => true,
1676 };
1677 }
1678};
1679const MultilineErrors = enum { indent, newline, none };
1680const Summary = enum { all, new, failures, line, none };
1681
1682fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn {1697fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn {
1683 log.info("to access the help menu: zig build -h", .{});1698 log.info("to access the help menu: zig build -h", .{});
1684 fatal(f, args);1699 fatal(f, args);
...@@ -1701,9 +1716,6 @@ fn cleanTmpFiles(maker: *Maker, steps: []const Configuration.Step.Index) void {...@@ -1701,9 +1716,6 @@ fn cleanTmpFiles(maker: *Maker, steps: []const Configuration.Step.Index) void {
1701 }1716 }
1702}1717}
17031718
1704var stdio_buffer_allocation: [256]u8 = undefined;
1705var stdout_writer_allocation: Io.File.Writer = undefined;
1706
1707fn initStdoutWriter(io: Io) *Writer {1719fn initStdoutWriter(io: Io) *Writer {
1708 stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation);1720 stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation);
1709 return &stdout_writer_allocation.interface;1721 return &stdout_writer_allocation.interface;
...@@ -2043,8 +2055,6 @@ fn removePoisonedConfiguration(io: Io, scanned_config: *const ScannedConfig) voi...@@ -2043,8 +2055,6 @@ fn removePoisonedConfiguration(io: Io, scanned_config: *const ScannedConfig) voi
2043 }2055 }
2044}2056}
20452057
2046const is_debug_mode = builtin.mode == .Debug;
2047var debug_maker_leaks: bool = false;
2048inline fn debugMakerLeaks() bool {2058inline fn debugMakerLeaks() bool {
2049 if (!is_debug_mode) return false;2059 if (!is_debug_mode) return false;
2050 return debug_maker_leaks;2060 return debug_maker_leaks;
lib/compiler/Maker/PkgConfig.zig+14-7
...@@ -2,6 +2,7 @@ const std = @import("std");...@@ -2,6 +2,7 @@ const std = @import("std");
2const Io = std.Io;2const Io = std.Io;
3const mem = std.mem;3const mem = std.mem;
4const assert = std.debug.assert;4const assert = std.debug.assert;
5const Allocator = std.mem.Allocator;
56
6const Maker = @import("../Maker.zig");7const Maker = @import("../Maker.zig");
7const Step = @import("Step.zig");8const Step = @import("Step.zig");
...@@ -23,6 +24,7 @@ pub const Result = std.zig.PkgConfig.Parsed;...@@ -23,6 +24,7 @@ pub const Result = std.zig.PkgConfig.Parsed;
23pub fn run(24pub fn run(
24 maker: *Maker,25 maker: *Maker,
25 step: *Step,26 step: *Step,
27 arena: Allocator,
26 progress_node: std.Progress.Node,28 progress_node: std.Progress.Node,
27 lib_name: []const u8,29 lib_name: []const u8,
28 /// If true, reports failure error messages on step rather than returning30 /// If true, reports failure error messages on step rather than returning
...@@ -31,7 +33,6 @@ pub fn run(...@@ -31,7 +33,6 @@ pub fn run(
31) RunError!Result {33) RunError!Result {
32 const pc = &maker.pkg_config;34 const pc = &maker.pkg_config;
33 const graph = maker.graph;35 const graph = maker.graph;
34 const arena = graph.arena; // TODO don't leak into process arena
3536
36 const pkg_config_exe = getExe(graph);37 const pkg_config_exe = getExe(graph);
37 const pkgs = try getPkgs(maker, step, progress_node, force);38 const pkgs = try getPkgs(maker, step, progress_node, force);
...@@ -41,7 +42,7 @@ pub fn run(...@@ -41,7 +42,7 @@ pub fn run(
41 };42 };
42 const pkg = pkgs.all[found_index];43 const pkg = pkgs.all[found_index];
4344
44 const stdout = try captureChildProcess(maker, step, .{45 const stdout = try captureChildProcess(maker, step, arena, .{
45 .argv = &.{ pkg_config_exe, pkg.name, "--cflags", "--libs" },46 .argv = &.{ pkg_config_exe, pkg.name, "--cflags", "--libs" },
46 .progress_node = progress_node,47 .progress_node = progress_node,
47 .allow_failure = !force,48 .allow_failure = !force,
...@@ -69,11 +70,16 @@ fn getExe(graph: *const Graph) []const u8 {...@@ -69,11 +70,16 @@ fn getExe(graph: *const Graph) []const u8 {
69 return std.zig.PkgConfig.exe(&graph.environ_map);70 return std.zig.PkgConfig.exe(&graph.environ_map);
70}71}
7172
72fn getPkgs(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force: bool) RunError!std.zig.PkgConfig {73fn getPkgs(
74 maker: *Maker,
75 step: *Step,
76 progress_node: std.Progress.Node,
77 force: bool,
78) RunError!std.zig.PkgConfig {
73 const graph = maker.graph;79 const graph = maker.graph;
74 const arena = graph.arena; // TODO don't leak into process arena
75 const io = graph.io;80 const io = graph.io;
76 const pc = &maker.pkg_config;81 const pc = &maker.pkg_config;
82 const arena = graph.arena;
7783
78 try pc.mutex.lock(io);84 try pc.mutex.lock(io);
79 defer pc.mutex.unlock(io);85 defer pc.mutex.unlock(io);
...@@ -81,7 +87,7 @@ fn getPkgs(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force:...@@ -81,7 +87,7 @@ fn getPkgs(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force:
81 if (pc.pkgs) |pkgs| return pkgs;87 if (pc.pkgs) |pkgs| return pkgs;
8288
83 const pkg_config_exe = getExe(graph);89 const pkg_config_exe = getExe(graph);
84 const stdout = try captureChildProcess(maker, step, .{90 const stdout = try captureChildProcess(maker, step, arena, .{
85 .argv = &.{ pkg_config_exe, "--list-all" },91 .argv = &.{ pkg_config_exe, "--list-all" },
86 .progress_node = progress_node,92 .progress_node = progress_node,
87 .allow_failure = !force,93 .allow_failure = !force,
...@@ -102,11 +108,12 @@ fn getPkgs(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force:...@@ -102,11 +108,12 @@ fn getPkgs(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force:
102 return result;108 return result;
103}109}
104110
105fn captureChildProcess(maker: *Maker, step: *Step, options: Step.CaptureChildProcessOptions) ![]const u8 {111fn captureChildProcess(maker: *Maker, step: *Step, arena: Allocator, options: Step.CaptureChildProcessOptions) ![]const u8 {
106 const captured = step.captureChildProcess(maker, options) catch |err| switch (err) {112 const captured = step.captureChildProcess(maker, arena, options) catch |err| switch (err) {
107 error.FileNotFound => return error.PkgConfigUnavailable,113 error.FileNotFound => return error.PkgConfigUnavailable,
108 else => |e| return e,114 else => |e| return e,
109 };115 };
116 if (captured.stderr.len != 0) try step.setResultStderr(maker.gpa, captured.stderr);
110 assert(step.result_failed_command != null);117 assert(step.result_failed_command != null);
111 if (captured.term.success()) return captured.stdout;118 if (captured.term.success()) return captured.stdout;
112 if (!options.allow_failure) return step.fail(maker, "{s} {f}", .{ options.argv[0], captured.term });119 if (!options.allow_failure) return step.fail(maker, "{s} {f}", .{ options.argv[0], captured.term });
lib/compiler/Maker/Step.zig+47-16
...@@ -54,8 +54,10 @@ dependants: std.ArrayList(Configuration.Step.Index) = .empty,...@@ -54,8 +54,10 @@ dependants: std.ArrayList(Configuration.Step.Index) = .empty,
54inputs: Inputs = .init,54inputs: Inputs = .init,
55pending_deps: u32 = undefined,55pending_deps: u32 = undefined,
5656
57/// Array list and internal memory owned by process arena.
57result_error_msgs: std.ArrayList([]const u8) = .empty,58result_error_msgs: std.ArrayList([]const u8) = .empty,
58result_error_bundle: std.zig.ErrorBundle = .empty,59result_error_bundle: std.zig.ErrorBundle = .empty,
60/// Owned by `Maker.gpa`.
59result_stderr: []const u8 = "",61result_stderr: []const u8 = "",
60result_cached: bool = false,62result_cached: bool = false,
61/// Indicates error information is missing due to allocation failure.63/// Indicates error information is missing due to allocation failure.
...@@ -310,9 +312,8 @@ pub fn reset(step: *Step, maker: *Maker) void {...@@ -310,9 +312,8 @@ pub fn reset(step: *Step, maker: *Maker) void {
310 const gpa = maker.gpa;312 const gpa = maker.gpa;
311313
312 clearFailedCommand(step, gpa);314 clearFailedCommand(step, gpa);
313315 clearResultStderr(step, gpa);
314 step.result_error_msgs.clearRetainingCapacity();316 step.result_error_msgs.clearRetainingCapacity();
315 step.result_stderr = "";
316 step.result_cached = false;317 step.result_cached = false;
317 step.result_duration_ns = null;318 step.result_duration_ns = null;
318 step.result_peak_rss = 0;319 step.result_peak_rss = 0;
...@@ -332,20 +333,23 @@ pub const CaptureChildProcessOptions = struct {...@@ -332,20 +333,23 @@ pub const CaptureChildProcessOptions = struct {
332 allow_failure: bool = false,333 allow_failure: bool = false,
333};334};
334335
335/// Populates `s.result_failed_command`.336/// Populates `s.result_failed_command` unconditionally.
336pub fn captureChildProcess(s: *Step, maker: *Maker, options: CaptureChildProcessOptions) !std.process.RunResult {337pub fn captureChildProcess(
338 s: *Step,
339 maker: *Maker,
340 allocator: Allocator,
341 options: CaptureChildProcessOptions,
342) !std.process.RunResult {
337 const gpa = maker.gpa;343 const gpa = maker.gpa;
338 const graph = maker.graph;344 const graph = maker.graph;
339 const arena = graph.arena; // TODO stop leaking into process arena
340 const io = graph.io;345 const io = graph.io;
341346
342 clearFailedCommand(s, gpa);347 s.setFailedCommand(gpa, options.argv, .{});
343 s.result_failed_command = try std.zig.allocPrintCmd(gpa, options.argv, .{});
344348
345 try handleChildProcUnsupported(s, maker);349 try handleChildProcUnsupported(s, maker);
346 try graph.handleVerbose(null, null, options.argv);350 try graph.handleVerbose(null, null, options.argv);
347351
348 const result = std.process.run(arena, io, .{352 const result = std.process.run(allocator, io, .{
349 .argv = options.argv,353 .argv = options.argv,
350 .environ_map = options.environ_map orelse &graph.environ_map,354 .environ_map = options.environ_map orelse &graph.environ_map,
351 .progress_node = options.progress_node,355 .progress_node = options.progress_node,
...@@ -358,7 +362,7 @@ pub fn captureChildProcess(s: *Step, maker: *Maker, options: CaptureChildProcess...@@ -358,7 +362,7 @@ pub fn captureChildProcess(s: *Step, maker: *Maker, options: CaptureChildProcess
358 return s.fail(maker, "failed to run {s}: {t}", .{ options.argv[0], err });362 return s.fail(maker, "failed to run {s}: {t}", .{ options.argv[0], err });
359 };363 };
360364
361 if (result.stderr.len > 0) try s.result_error_msgs.append(arena, result.stderr);365 if (result.stderr.len > 0) try s.result_error_msgs.append(graph.arena, result.stderr);
362366
363 return result;367 return result;
364}368}
...@@ -375,6 +379,21 @@ pub fn clearFailedCommand(s: *Step, gpa: Allocator) void {...@@ -375,6 +379,21 @@ pub fn clearFailedCommand(s: *Step, gpa: Allocator) void {
375 }379 }
376}380}
377381
382pub fn setFailedCommand(
383 s: *Step,
384 gpa: Allocator,
385 argv: []const []const u8,
386 options: std.zig.AllocPrintCmdOptions,
387) void {
388 s.clearFailedCommand(gpa);
389 s.result_failed_command = std.zig.allocPrintCmd(gpa, argv, options) catch |err| switch (err) {
390 error.OutOfMemory => {
391 s.result_oom = true;
392 return;
393 },
394 };
395}
396
378pub const FailError = error{ OutOfMemory, MakeFailed };397pub const FailError = error{ OutOfMemory, MakeFailed };
379398
380pub fn fail(step: *Step, maker: *const Maker, comptime fmt: []const u8, args: anytype) FailError {399pub fn fail(step: *Step, maker: *const Maker, comptime fmt: []const u8, args: anytype) FailError {
...@@ -410,7 +429,8 @@ pub const ZigProcess = struct {...@@ -410,7 +429,8 @@ pub const ZigProcess = struct {
410429
411/// Assumes that argv contains `--listen=-` and that the process being spawned430/// Assumes that argv contains `--listen=-` and that the process being spawned
412/// is the zig compiler - the same version that compiled the build runner.431/// is the zig compiler - the same version that compiled the build runner.
413/// Populates `s.result_failed_command`.432///
433/// Populates `s.result_failed_command` on failure.
414pub fn evalZigProcess(434pub fn evalZigProcess(
415 step_index: Configuration.Step.Index,435 step_index: Configuration.Step.Index,
416 maker: *Maker,436 maker: *Maker,
...@@ -424,8 +444,7 @@ pub fn evalZigProcess(...@@ -424,8 +444,7 @@ pub fn evalZigProcess(
424 const io = graph.io;444 const io = graph.io;
425445
426 // If an error occurs, it's happened in this command:446 // If an error occurs, it's happened in this command:
427 clearFailedCommand(s, gpa);447 errdefer s.setFailedCommand(gpa, argv, .{});
428 s.result_failed_command = try std.zig.allocPrintCmd(gpa, argv, .{});
429448
430 if (s.getZigProcess()) |zp| update: {449 if (s.getZigProcess()) |zp| update: {
431 assert(watch);450 assert(watch);
...@@ -683,16 +702,12 @@ fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void {...@@ -683,16 +702,12 @@ fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void {
683 };702 };
684}703}
685704
686/// Asserts that the caller has already populated `s.result_failed_command`.
687pub inline fn handleChildProcUnsupported(s: *Step, maker: *Maker) FailError!void {705pub inline fn handleChildProcUnsupported(s: *Step, maker: *Maker) FailError!void {
688 assert(s.result_failed_command != null);
689 if (!std.process.can_spawn)706 if (!std.process.can_spawn)
690 return s.fail(maker, "unable to spawn process: host cannot spawn child processes", .{});707 return s.fail(maker, "unable to spawn process: host cannot spawn child processes", .{});
691}708}
692709
693/// Asserts that the caller has already populated `s.result_failed_command`.
694pub fn handleChildProcessTerm(s: *Step, maker: *Maker, term: std.process.Child.Term) FailError!void {710pub fn handleChildProcessTerm(s: *Step, maker: *Maker, term: std.process.Child.Term) FailError!void {
695 assert(s.result_failed_command != null);
696 if (!term.success()) return s.fail(maker, "process {f}", .{term});711 if (!term.success()) return s.fail(maker, "process {f}", .{term});
697}712}
698713
...@@ -861,3 +876,19 @@ fn oomWrap(s: *Step, result: error{OutOfMemory}!void) void {...@@ -861,3 +876,19 @@ fn oomWrap(s: *Step, result: error{OutOfMemory}!void) void {
861 s.result_oom = true;876 s.result_oom = true;
862 };877 };
863}878}
879
880pub fn clearResultStderr(step: *Step, gpa: Allocator) void {
881 if (step.result_stderr.len != 0) {
882 gpa.free(step.result_stderr);
883 step.result_stderr = "";
884 }
885}
886
887pub fn setResultStderr(step: *Step, gpa: Allocator, bytes: []const u8) Allocator.Error!void {
888 takeResultStderr(step, gpa, try gpa.dupe(u8, bytes));
889}
890
891pub fn takeResultStderr(step: *Step, gpa: Allocator, owned: []const u8) void {
892 clearResultStderr(step, gpa);
893 step.result_stderr = owned;
894}
lib/compiler/Maker/Step/Compile.zig+6-6
...@@ -328,10 +328,10 @@ fn lowerZigArgs(...@@ -328,10 +328,10 @@ fn lowerZigArgs(
328 const pkg_conf_node = progress_node.start("pkg-config", 0);328 const pkg_conf_node = progress_node.start("pkg-config", 0);
329 defer pkg_conf_node.end();329 defer pkg_conf_node.end();
330330
331 if (PkgConfig.run(maker, step, pkg_conf_node, system_lib_name, force)) |result| {331 if (PkgConfig.run(maker, step, arena, pkg_conf_node, system_lib_name, force)) |pc| {
332 try zig_args.appendSlice(gpa, result.cflags);332 try zig_args.appendSlice(gpa, pc.cflags);
333 try zig_args.appendSlice(gpa, result.libs);333 try zig_args.appendSlice(gpa, pc.libs);
334 try seen_system_libs.put(arena, system_lib.name, result.cflags);334 try seen_system_libs.put(arena, system_lib.name, pc.cflags);
335 break :l;335 break :l;
336 } else |err| switch (err) {336 } else |err| switch (err) {
337 error.PkgConfigUnavailable,337 error.PkgConfigUnavailable,
...@@ -960,8 +960,8 @@ pub fn rebuildInFuzzMode(...@@ -960,8 +960,8 @@ pub fn rebuildInFuzzMode(
960 const arena = arena_allocator.allocator();960 const arena = arena_allocator.allocator();
961961
962 step.result_error_msgs.clearRetainingCapacity();962 step.result_error_msgs.clearRetainingCapacity();
963 step.result_stderr = "";963 step.clearResultStderr(gpa);
964964 step.clearErrorBundle(gpa);
965 step.result_error_bundle.deinit(gpa);965 step.result_error_bundle.deinit(gpa);
966 step.result_error_bundle = std.zig.ErrorBundle.empty;966 step.result_error_bundle = std.zig.ErrorBundle.empty;
967967
lib/compiler/Maker/Step/Fmt.zig+1-1
...@@ -43,7 +43,7 @@ pub fn make(...@@ -43,7 +43,7 @@ pub fn make(
43 argv.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, step_index));43 argv.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, step_index));
44 }44 }
4545
46 const run_result = step.captureChildProcess(maker, .{46 const run_result = step.captureChildProcess(maker, arena, .{
47 .progress_node = progress_node,47 .progress_node = progress_node,
48 .argv = argv.items,48 .argv = argv.items,
49 .allow_failure = false,49 .allow_failure = false,
lib/compiler/Maker/Step/Run.zig+42-39
...@@ -713,7 +713,7 @@ const FuzzTestRunner = struct {...@@ -713,7 +713,7 @@ const FuzzTestRunner = struct {
713 }713 }
714 }714 }
715715
716 fn listen(f: *FuzzTestRunner, arena: Allocator) !void {716 fn listen(f: *FuzzTestRunner) !void {
717 const maker = f.ctx.fuzz.maker;717 const maker = f.ctx.fuzz.maker;
718 const graph = maker.graph;718 const graph = maker.graph;
719 const io = graph.io;719 const io = graph.io;
...@@ -737,7 +737,7 @@ const FuzzTestRunner = struct {...@@ -737,7 +737,7 @@ const FuzzTestRunner = struct {
737 else => |read_e| return read_e,737 else => |read_e| return read_e,
738 }),738 }),
739 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) {739 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) {
740 error.EndOfStream => return f.instanceEos(arena, id),740 error.EndOfStream => return f.instanceEos(id),
741 else => |read_e| return read_e,741 else => |read_e| return read_e,
742 }),742 }),
743 else => unreachable,743 else => unreachable,
...@@ -899,8 +899,9 @@ const FuzzTestRunner = struct {...@@ -899,8 +899,9 @@ const FuzzTestRunner = struct {
899 } });899 } });
900 }900 }
901901
902 fn instanceEos(f: *FuzzTestRunner, arena: Allocator, id: u32) !void {902 fn instanceEos(f: *FuzzTestRunner, id: u32) !void {
903 const maker = f.ctx.fuzz.maker;903 const maker = f.ctx.fuzz.maker;
904 const gpa = maker.gpa;
904 const instance = &f.instances[id];905 const instance = &f.instances[id];
905 const run_index = f.run_index;906 const run_index = f.run_index;
906907
...@@ -912,7 +913,7 @@ const FuzzTestRunner = struct {...@@ -912,7 +913,7 @@ const FuzzTestRunner = struct {
912 instance.child.stdin = null;913 instance.child.stdin = null;
913 const term = try instance.child.wait(io);914 const term = try instance.child.wait(io);
914 if (!termMatches(.{ .exited = 0 }, term)) {915 if (!termMatches(.{ .exited = 0 }, term)) {
915 step.result_stderr = try f.mergedStderr(arena);916 step.takeResultStderr(gpa, try f.mergedStderr(gpa));
916 try f.saveCrash(id, term);917 try f.saveCrash(id, term);
917 return step.fail(maker, "test process unexpectedly {f}", .{fmtTerm(term)});918 return step.fail(maker, "test process unexpectedly {f}", .{fmtTerm(term)});
918 }919 }
...@@ -1055,7 +1056,7 @@ const FuzzTestRunner = struct {...@@ -1055,7 +1056,7 @@ const FuzzTestRunner = struct {
1055 }1056 }
1056 }1057 }
10571058
1058 fn mergedStderr(f: *FuzzTestRunner, arena: Allocator) Allocator.Error![]const u8 {1059 fn mergedStderr(f: *FuzzTestRunner, gpa: Allocator) Allocator.Error![]const u8 {
1059 // Collect any available stderr1060 // Collect any available stderr
1060 while (f.batch.next()) |completion| {1061 while (f.batch.next()) |completion| {
1061 if (completion.index % 3 != 2) continue;1062 if (completion.index % 3 != 2) continue;
...@@ -1065,7 +1066,7 @@ const FuzzTestRunner = struct {...@@ -1065,7 +1066,7 @@ const FuzzTestRunner = struct {
10651066
1066 var stderr_len: usize = 0;1067 var stderr_len: usize = 0;
1067 for (f.instances) |*instance| stderr_len += instance.stderr.items.len;1068 for (f.instances) |*instance| stderr_len += instance.stderr.items.len;
1068 const stderr = try arena.alloc(u8, stderr_len);1069 const stderr = try gpa.alloc(u8, stderr_len);
10691070
1070 stderr_len = 0;1071 stderr_len = 0;
1071 for (f.instances) |*instance| {1072 for (f.instances) |*instance| {
...@@ -1086,13 +1087,12 @@ fn evalFuzzTest(...@@ -1086,13 +1087,12 @@ fn evalFuzzTest(
1086 var f: FuzzTestRunner = try .init(run, run_index, fuzz_context, progress_node, spawn_options);1087 var f: FuzzTestRunner = try .init(run, run_index, fuzz_context, progress_node, spawn_options);
1087 defer f.deinit();1088 defer f.deinit();
1088 try f.startInstances();1089 try f.startInstances();
1089 try f.listen(fuzz_context.fuzz.maker.graph.arena);1090 try f.listen();
1090}1091}
10911092
1092const StdioPollEnum = enum { stdout, stderr };1093const StdioPollEnum = enum { stdout, stderr };
10931094
1094fn evalZigTest(1095fn evalZigTest(
1095 arena: Allocator,
1096 run: *Run,1096 run: *Run,
1097 run_index: Configuration.Step.Index,1097 run_index: Configuration.Step.Index,
1098 maker: *Maker,1098 maker: *Maker,
...@@ -1140,7 +1140,7 @@ fn evalZigTest(...@@ -1140,7 +1140,7 @@ fn evalZigTest(
1140 };1140 };
11411141
1142 switch (try waitZigTest(1142 switch (try waitZigTest(
1143 arena,1143 graph.arena,
1144 run,1144 run,
1145 run_index,1145 run_index,
1146 maker,1146 maker,
...@@ -1158,7 +1158,7 @@ fn evalZigTest(...@@ -1158,7 +1158,7 @@ fn evalZigTest(
1158 error.ReadFailed => return stderr_fr.err.?,1158 error.ReadFailed => return stderr_fr.err.?,
1159 error.EndOfStream => {},1159 error.EndOfStream => {},
1160 }1160 }
1161 step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered());1161 step.takeResultStderr(gpa, try multi_reader.toOwnedSlice(1));
11621162
1163 // Clean up everything and wait for the child to exit.1163 // Clean up everything and wait for the child to exit.
1164 child.stdin.?.close(io);1164 child.stdin.?.close(io);
...@@ -1180,8 +1180,9 @@ fn evalZigTest(...@@ -1180,8 +1180,9 @@ fn evalZigTest(
1180 .no_poll => |no_poll| {1180 .no_poll => |no_poll| {
1181 // This might be a success (we requested exit and the child dutifully closed stdout) or1181 // This might be a success (we requested exit and the child dutifully closed stdout) or
1182 // a crash of some kind. Either way, the child will terminate by itself -- wait for it.1182 // a crash of some kind. Either way, the child will terminate by itself -- wait for it.
1183 const stderr_reader = multi_reader.reader(1);1183 const stderr_owned = try multi_reader.toOwnedSlice(1);
1184 const stderr_owned = try arena.dupe(u8, stderr_reader.buffered());1184 var keep_stderr_owned = false;
1185 defer if (!keep_stderr_owned) gpa.free(stderr_owned);
11851186
1186 // Clean up everything and wait for the child to exit.1187 // Clean up everything and wait for the child to exit.
1187 child.stdin.?.close(io);1188 child.stdin.?.close(io);
...@@ -1209,7 +1210,9 @@ fn evalZigTest(...@@ -1209,7 +1210,9 @@ fn evalZigTest(
1209 }1210 }
12101211
1211 // Report an error if the child terminated uncleanly or if we were still trying to run more tests.1212 // Report an error if the child terminated uncleanly or if we were still trying to run more tests.
1212 step.result_stderr = stderr_owned;1213 step.takeResultStderr(gpa, stderr_owned);
1214 keep_stderr_owned = true;
1215
1213 const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32);1216 const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32);
1214 if (!tests_done or !termMatches(.{ .exited = 0 }, term)) {1217 if (!tests_done or !termMatches(.{ .exited = 0 }, term)) {
1215 // The individual unit test results are irrelevant: the test runner itself broke!1218 // The individual unit test results are irrelevant: the test runner itself broke!
...@@ -1234,9 +1237,10 @@ fn evalZigTest(...@@ -1234,9 +1237,10 @@ fn evalZigTest(
1234 return;1237 return;
1235 },1238 },
1236 .timeout => |timeout| {1239 .timeout => |timeout| {
1237 const stderr_reader = multi_reader.reader(1);1240 const stderr_owned = try multi_reader.toOwnedSlice(1);
1238 const stderr = stderr_reader.buffered();1241 var keep_stderr_owned = false;
1239 stderr_reader.tossBuffered();1242 defer if (!keep_stderr_owned) gpa.free(stderr_owned);
1243
1240 if (timeout.active_test_index) |test_index| {1244 if (timeout.active_test_index) |test_index| {
1241 // A test was running. Report the timeout against that test, and continue on to1245 // A test was running. Report the timeout against that test, and continue on to
1242 // the next test.1246 // the next test.
...@@ -1245,16 +1249,20 @@ fn evalZigTest(...@@ -1245,16 +1249,20 @@ fn evalZigTest(
1245 try step.addError(maker, "'{s}' timed out after {f}{s}{s}", .{1249 try step.addError(maker, "'{s}' timed out after {f}{s}{s}", .{
1246 test_metadata.?.testName(test_index),1250 test_metadata.?.testName(test_index),
1247 Io.Duration{ .nanoseconds = timeout.ns_elapsed },1251 Io.Duration{ .nanoseconds = timeout.ns_elapsed },
1248 if (stderr.len != 0) " with stderr:\n" else "",1252 if (stderr_owned.len != 0) " with stderr:\n" else "",
1249 std.mem.trim(u8, stderr, "\n"),1253 std.mem.trim(u8, stderr_owned, "\n"),
1250 });1254 });
1251 continue;1255 continue;
1252 }1256 }
1253 // Just log an error and let the child be killed.1257 // Just log an error and let the child be killed.
1254 step.result_stderr = try arena.dupe(u8, stderr);1258 step.takeResultStderr(gpa, stderr_owned);
1259 keep_stderr_owned = true;
1260
1255 // The individual unit test results in `results` are irrelevant: the test runner1261 // The individual unit test results in `results` are irrelevant: the test runner
1256 // is broken! Fail immediately without populating `s.test_results`.1262 // is broken! Fail immediately without populating `s.test_results`.
1257 return step.fail(maker, "test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }});1263 return step.fail(maker, "test runner failed to respond for {f}", .{
1264 Io.Duration{ .nanoseconds = timeout.ns_elapsed },
1265 });
1258 },1266 },
1259 }1267 }
1260 comptime unreachable;1268 comptime unreachable;
...@@ -1456,8 +1464,8 @@ fn evalGeneric(...@@ -1456,8 +1464,8 @@ fn evalGeneric(
14561464
1457 try multi_reader.checkAnyError();1465 try multi_reader.checkAnyError();
14581466
1459 stdout_bytes = try multi_reader.toOwnedSlice(0);1467 stdout_bytes = multi_reader.reader(0).buffered();
1460 stderr_bytes = try multi_reader.toOwnedSlice(1);1468 stderr_bytes = multi_reader.reader(1).buffered();
1461 } else {1469 } else {
1462 var stdout_reader = stdout.readerStreaming(io, &.{});1470 var stdout_reader = stdout.readerStreaming(io, &.{});
1463 const stdio_limit: Io.Limit = if (conf_run.stdio_limit.value) |x| .limited64(x) else .unlimited;1471 const stdio_limit: Io.Limit = if (conf_run.stdio_limit.value) |x| .limited64(x) else .unlimited;
...@@ -1484,7 +1492,7 @@ fn evalGeneric(...@@ -1484,7 +1492,7 @@ fn evalGeneric(
1484 else => true,1492 else => true,
1485 };1493 };
1486 if (stderr_is_diagnostic) {1494 if (stderr_is_diagnostic) {
1487 step.result_stderr = bytes;1495 try step.setResultStderr(maker.gpa, bytes);
1488 }1496 }
1489 };1497 };
14901498
...@@ -2084,16 +2092,11 @@ fn runCommand(...@@ -2084,16 +2092,11 @@ fn runCommand(
2084 },2092 },
2085 else => {2093 else => {
2086 // On failure, report captured stderr like normal standard error output.2094 // On failure, report captured stderr like normal standard error output.
2087 const bad_exit = switch (generic_result.term) {2095 if (!generic_result.term.success()) {
2088 .exited => |code| code != 0,
2089 .signal, .stopped, .unknown => true,
2090 };
2091 if (bad_exit) {
2092 if (generic_result.stderr) |bytes| {2096 if (generic_result.stderr) |bytes| {
2093 step.result_stderr = bytes;2097 try step.setResultStderr(gpa, bytes);
2094 }2098 }
2095 }2099 }
2096
2097 try step.handleChildProcessTerm(maker, generic_result.term);2100 try step.handleChildProcessTerm(maker, generic_result.term);
2098 },2101 },
2099 }2102 }
...@@ -2135,13 +2138,13 @@ fn spawnChildAndCollect(...@@ -2135,13 +2138,13 @@ fn spawnChildAndCollect(
2135 .inherit;2138 .inherit;
21362139
2137 // If an error occurs, it's caused by this command:2140 // If an error occurs, it's caused by this command:
2138 step.clearFailedCommand(gpa);2141 const cwd_string = switch (child_cwd) {
2139 step.result_failed_command = try std.zig.allocPrintCmd(gpa, argv, .{2142 .path => |p| p,
2140 .cwd = switch (child_cwd) {2143 .dir => unreachable,
2141 .path => |p| p,2144 .inherit => null,
2142 .dir => unreachable,2145 };
2143 .inherit => null,2146 errdefer step.setFailedCommand(gpa, argv, .{
2144 },2147 .cwd = cwd_string,
2145 .child_env = environ_map,2148 .child_env = environ_map,
2146 .parent_env = &graph.environ_map,2149 .parent_env = &graph.environ_map,
2147 });2150 });
...@@ -2178,7 +2181,7 @@ fn spawnChildAndCollect(...@@ -2178,7 +2181,7 @@ fn spawnChildAndCollect(
21782181
2179 if (conf_run.flags.stdio == .zig_test) {2182 if (conf_run.flags.stdio == .zig_test) {
2180 const started: Io.Clock.Timestamp = .now(io, .awake);2183 const started: Io.Clock.Timestamp = .now(io, .awake);
2181 const result = evalZigTest(graph.arena, run, run_index, maker, progress_node, spawn_options, fuzz_context) catch |err| switch (err) {2184 const result = evalZigTest(run, run_index, maker, progress_node, spawn_options, fuzz_context) catch |err| switch (err) {
2182 error.Canceled => |e| return e,2185 error.Canceled => |e| return e,
2183 else => |e| e,2186 else => |e| e,
2184 };2187 };
...@@ -2198,7 +2201,7 @@ fn spawnChildAndCollect(...@@ -2198,7 +2201,7 @@ fn spawnChildAndCollect(
2198 try setColorEnvironmentVariables(&conf_run, environ_map, terminal_mode);2201 try setColorEnvironmentVariables(&conf_run, environ_map, terminal_mode);
21992202
2200 const started: Io.Clock.Timestamp = .now(io, .awake);2203 const started: Io.Clock.Timestamp = .now(io, .awake);
2201 const result = evalGeneric(graph.arena, run_index, maker, spawn_options) catch |err| switch (err) {2204 const result = evalGeneric(arena, run_index, maker, spawn_options) catch |err| switch (err) {
2202 error.Canceled => |e| return e,2205 error.Canceled => |e| return e,
2203 else => |e| e,2206 else => |e| e,
2204 };2207 };
lib/compiler/Maker/Step/TranslateC.zig+1-1
...@@ -112,7 +112,7 @@ pub fn make(...@@ -112,7 +112,7 @@ pub fn make(
112 const pkg_conf_node = progress_node.start("pkg-config", 0);112 const pkg_conf_node = progress_node.start("pkg-config", 0);
113 defer pkg_conf_node.end();113 defer pkg_conf_node.end();
114114
115 if (PkgConfig.run(maker, step, pkg_conf_node, system_lib_name, force)) |result| {115 if (PkgConfig.run(maker, step, arena, pkg_conf_node, system_lib_name, force)) |result| {
116 try argv.appendSlice(arena, result.cflags);116 try argv.appendSlice(arena, result.cflags);
117 try argv.appendSlice(arena, result.libs);117 try argv.appendSlice(arena, result.libs);
118 try seen_system_libs.put(arena, system_lib.name, result.cflags);118 try seen_system_libs.put(arena, system_lib.name, result.cflags);
src/main.zig+1-1
...@@ -5810,7 +5810,7 @@ const MakeRunner = struct {...@@ -5810,7 +5810,7 @@ const MakeRunner = struct {
5810};5810};
58115811
5812fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunner.Options) !MakeRunner {5812fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunner.Options) !MakeRunner {
5813 const compile_prog_node = options.parent_prog_node.start("Compile Maker", 0);5813 const compile_prog_node = options.parent_prog_node.start("Compiling maker (first time setup)", 0);
5814 defer compile_prog_node.end();5814 defer compile_prog_node.end();
58155815
5816 const strip = options.optimize_mode != .Debug;5816 const strip = options.optimize_mode != .Debug;