authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-01-20 21:40:04-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-01-20 21:40:04-05:00
log216e0f37306d5cded92d547f65f9c2566d7b1523
tree47118995bca2d2ea9f9aa052ea9bbd54168433d8
parent8bcb578507908e17e2081bb03f8c51eb508d51db
parent048e85f27e805f51c214e8e87e91a367a2873328
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #22548 from mlugg/fix-broken-pipe

Wait for reported spawn success or failure before trying to write to the stdio pipe in the build runner. Hopefully fixes `error.BrokenPipe` failures

6 files changed, 79 insertions(+), 95 deletions(-)

lib/std/Build.zig+3-4
...@@ -979,7 +979,7 @@ pub const TestOptions = struct {...@@ -979,7 +979,7 @@ pub const TestOptions = struct {
979 /// Deprecated; use `.filters = &.{filter}` instead of `.filter = filter`.979 /// Deprecated; use `.filters = &.{filter}` instead of `.filter = filter`.
980 filter: ?[]const u8 = null,980 filter: ?[]const u8 = null,
981 filters: []const []const u8 = &.{},981 filters: []const []const u8 = &.{},
982 test_runner: ?LazyPath = null,982 test_runner: ?Step.Compile.TestRunner = null,
983 use_llvm: ?bool = null,983 use_llvm: ?bool = null,
984 use_lld: ?bool = null,984 use_lld: ?bool = null,
985 zig_lib_dir: ?LazyPath = null,985 zig_lib_dir: ?LazyPath = null,
...@@ -1136,9 +1136,8 @@ pub fn addRunArtifact(b: *Build, exe: *Step.Compile) *Step.Run {...@@ -1136,9 +1136,8 @@ pub fn addRunArtifact(b: *Build, exe: *Step.Compile) *Step.Run {
1136 run_step.addArtifactArg(exe);1136 run_step.addArtifactArg(exe);
1137 }1137 }
11381138
1139 if (exe.test_server_mode) {1139 const test_server_mode = if (exe.test_runner) |r| r.mode == .server else true;
1140 run_step.enableTestRunnerMode();1140 if (test_server_mode) run_step.enableTestRunnerMode();
1141 }
1142 } else {1141 } else {
1143 run_step.addArtifactArg(exe);1142 run_step.addArtifactArg(exe);
1144 }1143 }
lib/std/Build/Step/Compile.zig+18-9
...@@ -56,8 +56,7 @@ global_base: ?u64 = null,...@@ -56,8 +56,7 @@ global_base: ?u64 = null,
56zig_lib_dir: ?LazyPath,56zig_lib_dir: ?LazyPath,
57exec_cmd_args: ?[]const ?[]const u8,57exec_cmd_args: ?[]const ?[]const u8,
58filters: []const []const u8,58filters: []const []const u8,
59test_runner: ?LazyPath,59test_runner: ?TestRunner,
60test_server_mode: bool,
61wasi_exec_model: ?std.builtin.WasiExecModel = null,60wasi_exec_model: ?std.builtin.WasiExecModel = null,
6261
63installed_headers: ArrayList(HeaderInstallation),62installed_headers: ArrayList(HeaderInstallation),
...@@ -268,7 +267,7 @@ pub const Options = struct {...@@ -268,7 +267,7 @@ pub const Options = struct {
268 version: ?std.SemanticVersion = null,267 version: ?std.SemanticVersion = null,
269 max_rss: usize = 0,268 max_rss: usize = 0,
270 filters: []const []const u8 = &.{},269 filters: []const []const u8 = &.{},
271 test_runner: ?LazyPath = null,270 test_runner: ?TestRunner = null,
272 use_llvm: ?bool = null,271 use_llvm: ?bool = null,
273 use_lld: ?bool = null,272 use_lld: ?bool = null,
274 zig_lib_dir: ?LazyPath = null,273 zig_lib_dir: ?LazyPath = null,
...@@ -347,6 +346,14 @@ pub const HeaderInstallation = union(enum) {...@@ -347,6 +346,14 @@ pub const HeaderInstallation = union(enum) {
347 }346 }
348};347};
349348
349pub const TestRunner = struct {
350 path: LazyPath,
351 /// Test runners can either be "simple", running tests when spawned and terminating when the
352 /// tests are complete, or they can use `std.zig.Server` over stdio to interact more closely
353 /// with the build system.
354 mode: enum { simple, server },
355};
356
350pub fn create(owner: *std.Build, options: Options) *Compile {357pub fn create(owner: *std.Build, options: Options) *Compile {
351 const name = owner.dupe(options.name);358 const name = owner.dupe(options.name);
352 if (mem.indexOf(u8, name, "/") != null or mem.indexOf(u8, name, "\\") != null) {359 if (mem.indexOf(u8, name, "/") != null or mem.indexOf(u8, name, "\\") != null) {
...@@ -411,8 +418,7 @@ pub fn create(owner: *std.Build, options: Options) *Compile {...@@ -411,8 +418,7 @@ pub fn create(owner: *std.Build, options: Options) *Compile {
411 .zig_lib_dir = null,418 .zig_lib_dir = null,
412 .exec_cmd_args = null,419 .exec_cmd_args = null,
413 .filters = options.filters,420 .filters = options.filters,
414 .test_runner = null,421 .test_runner = null, // set below
415 .test_server_mode = options.test_runner == null,
416 .rdynamic = false,422 .rdynamic = false,
417 .installed_path = null,423 .installed_path = null,
418 .force_undefined_symbols = StringHashMap(void).init(owner.allocator),424 .force_undefined_symbols = StringHashMap(void).init(owner.allocator),
...@@ -438,9 +444,12 @@ pub fn create(owner: *std.Build, options: Options) *Compile {...@@ -438,9 +444,12 @@ pub fn create(owner: *std.Build, options: Options) *Compile {
438 lp.addStepDependencies(&compile.step);444 lp.addStepDependencies(&compile.step);
439 }445 }
440446
441 if (options.test_runner) |lp| {447 if (options.test_runner) |runner| {
442 compile.test_runner = lp.dupe(compile.step.owner);448 compile.test_runner = .{
443 lp.addStepDependencies(&compile.step);449 .path = runner.path.dupe(compile.step.owner),
450 .mode = runner.mode,
451 };
452 runner.path.addStepDependencies(&compile.step);
444 }453 }
445454
446 // Only the PE/COFF format has a Resource Table which is where the manifest455 // Only the PE/COFF format has a Resource Table which is where the manifest
...@@ -1399,7 +1408,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {...@@ -1399,7 +1408,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
13991408
1400 if (compile.test_runner) |test_runner| {1409 if (compile.test_runner) |test_runner| {
1401 try zig_args.append("--test-runner");1410 try zig_args.append("--test-runner");
1402 try zig_args.append(test_runner.getPath2(b, step));1411 try zig_args.append(test_runner.path.getPath2(b, step));
1403 }1412 }
14041413
1405 for (b.debug_log_scopes) |log_scope| {1414 for (b.debug_log_scopes) |log_scope| {
lib/std/Build/Step/Run.zig+3
...@@ -1354,6 +1354,9 @@ fn spawnChildAndCollect(...@@ -1354,6 +1354,9 @@ fn spawnChildAndCollect(
1354 _ = child.kill() catch {};1354 _ = child.kill() catch {};
1355 }1355 }
13561356
1357 // We need to report `error.InvalidExe` *now* if applicable.
1358 try child.waitForSpawn();
1359
1357 var timer = try std.time.Timer.start();1360 var timer = try std.time.Timer.start();
13581361
1359 const result = if (run.stdio == .zig_test)1362 const result = if (run.stdio == .zig_test)
lib/std/process/Child.zig+47-80
...@@ -73,7 +73,7 @@ cwd: ?[]const u8,...@@ -73,7 +73,7 @@ cwd: ?[]const u8,
73/// Once that is done, `cwd` will be deprecated in favor of this field.73/// Once that is done, `cwd` will be deprecated in favor of this field.
74cwd_dir: ?fs.Dir = null,74cwd_dir: ?fs.Dir = null,
7575
76err_pipe: ?if (native_os == .windows) void else [2]posix.fd_t,76err_pipe: if (native_os == .windows) void else ?posix.fd_t,
7777
78expand_arg0: Arg0Expand,78expand_arg0: Arg0Expand,
7979
...@@ -211,7 +211,7 @@ pub fn init(argv: []const []const u8, allocator: mem.Allocator) ChildProcess {...@@ -211,7 +211,7 @@ pub fn init(argv: []const []const u8, allocator: mem.Allocator) ChildProcess {
211 .argv = argv,211 .argv = argv,
212 .id = undefined,212 .id = undefined,
213 .thread_handle = undefined,213 .thread_handle = undefined,
214 .err_pipe = null,214 .err_pipe = if (native_os == .windows) {} else null,
215 .term = null,215 .term = null,
216 .env_map = null,216 .env_map = null,
217 .cwd = null,217 .cwd = null,
...@@ -293,17 +293,49 @@ pub fn killPosix(self: *ChildProcess) !Term {...@@ -293,17 +293,49 @@ pub fn killPosix(self: *ChildProcess) !Term {
293 error.ProcessNotFound => return error.AlreadyTerminated,293 error.ProcessNotFound => return error.AlreadyTerminated,
294 else => return err,294 else => return err,
295 };295 };
296 self.waitUnwrapped();296 self.waitUnwrappedPosix();
297 return self.term.?;297 return self.term.?;
298}298}
299299
300pub const WaitError = SpawnError || std.os.windows.GetProcessMemoryInfoError;300pub const WaitError = SpawnError || std.os.windows.GetProcessMemoryInfoError;
301301
302/// On some targets, `spawn` may not report all spawn errors, such as `error.InvalidExe`.
303/// This function will block until any spawn errors can be reported, and return them.
304pub fn waitForSpawn(self: *ChildProcess) SpawnError!void {
305 if (native_os == .windows) return; // `spawn` reports everything
306 if (self.term) |term| {
307 _ = term catch |spawn_err| return spawn_err;
308 return;
309 }
310
311 const err_pipe = self.err_pipe orelse return;
312 self.err_pipe = null;
313
314 // Wait for the child to report any errors in or before `execvpe`.
315 if (readIntFd(err_pipe)) |child_err_int| {
316 posix.close(err_pipe);
317 const child_err: SpawnError = @errorCast(@errorFromInt(child_err_int));
318 self.term = child_err;
319 return child_err;
320 } else |_| {
321 // Write end closed by CLOEXEC at the time of the `execvpe` call, indicating success!
322 posix.close(err_pipe);
323 }
324}
325
302/// Blocks until child process terminates and then cleans up all resources.326/// Blocks until child process terminates and then cleans up all resources.
303pub fn wait(self: *ChildProcess) WaitError!Term {327pub fn wait(self: *ChildProcess) WaitError!Term {
304 const term = if (native_os == .windows) try self.waitWindows() else self.waitPosix();328 try self.waitForSpawn(); // report spawn errors
329 if (self.term) |term| {
330 self.cleanupStreams();
331 return term;
332 }
333 switch (native_os) {
334 .windows => try self.waitUnwrappedWindows(),
335 else => self.waitUnwrappedPosix(),
336 }
305 self.id = undefined;337 self.id = undefined;
306 return term;338 return self.term.?;
307}339}
308340
309pub const RunResult = struct {341pub const RunResult = struct {
...@@ -405,26 +437,6 @@ pub fn run(args: struct {...@@ -405,26 +437,6 @@ pub fn run(args: struct {
405 };437 };
406}438}
407439
408fn waitWindows(self: *ChildProcess) WaitError!Term {
409 if (self.term) |term| {
410 self.cleanupStreams();
411 return term;
412 }
413
414 try self.waitUnwrappedWindows();
415 return self.term.?;
416}
417
418fn waitPosix(self: *ChildProcess) SpawnError!Term {
419 if (self.term) |term| {
420 self.cleanupStreams();
421 return term;
422 }
423
424 self.waitUnwrapped();
425 return self.term.?;
426}
427
428fn waitUnwrappedWindows(self: *ChildProcess) WaitError!void {440fn waitUnwrappedWindows(self: *ChildProcess) WaitError!void {
429 const result = windows.WaitForSingleObjectEx(self.id, windows.INFINITE, false);441 const result = windows.WaitForSingleObjectEx(self.id, windows.INFINITE, false);
430442
...@@ -447,7 +459,7 @@ fn waitUnwrappedWindows(self: *ChildProcess) WaitError!void {...@@ -447,7 +459,7 @@ fn waitUnwrappedWindows(self: *ChildProcess) WaitError!void {
447 return result;459 return result;
448}460}
449461
450fn waitUnwrapped(self: *ChildProcess) void {462fn waitUnwrappedPosix(self: *ChildProcess) void {
451 const res: posix.WaitPidResult = res: {463 const res: posix.WaitPidResult = res: {
452 if (self.request_resource_usage_statistics) {464 if (self.request_resource_usage_statistics) {
453 switch (native_os) {465 switch (native_os) {
...@@ -469,7 +481,7 @@ fn waitUnwrapped(self: *ChildProcess) void {...@@ -469,7 +481,7 @@ fn waitUnwrapped(self: *ChildProcess) void {
469}481}
470482
471fn handleWaitResult(self: *ChildProcess, status: u32) void {483fn handleWaitResult(self: *ChildProcess, status: u32) void {
472 self.term = self.cleanupAfterWait(status);484 self.term = statusToTerm(status);
473}485}
474486
475fn cleanupStreams(self: *ChildProcess) void {487fn cleanupStreams(self: *ChildProcess) void {
...@@ -487,46 +499,6 @@ fn cleanupStreams(self: *ChildProcess) void {...@@ -487,46 +499,6 @@ fn cleanupStreams(self: *ChildProcess) void {
487 }499 }
488}500}
489501
490fn cleanupAfterWait(self: *ChildProcess, status: u32) !Term {
491 if (self.err_pipe) |err_pipe| {
492 defer destroyPipe(err_pipe);
493
494 if (native_os == .linux) {
495 var fd = [1]posix.pollfd{posix.pollfd{
496 .fd = err_pipe[0],
497 .events = posix.POLL.IN,
498 .revents = undefined,
499 }};
500
501 // Check if the eventfd buffer stores a non-zero value by polling
502 // it, that's the error code returned by the child process.
503 _ = posix.poll(&fd, 0) catch unreachable;
504
505 // According to eventfd(2) the descriptor is readable if the counter
506 // has a value greater than 0
507 if ((fd[0].revents & posix.POLL.IN) != 0) {
508 const err_int = try readIntFd(err_pipe[0]);
509 return @as(SpawnError, @errorCast(@errorFromInt(err_int)));
510 }
511 } else {
512 // Write maxInt(ErrInt) to the write end of the err_pipe. This is after
513 // waitpid, so this write is guaranteed to be after the child
514 // pid potentially wrote an error. This way we can do a blocking
515 // read on the error pipe and either get maxInt(ErrInt) (no error) or
516 // an error code.
517 try writeIntFd(err_pipe[1], maxInt(ErrInt));
518 const err_int = try readIntFd(err_pipe[0]);
519 // Here we potentially return the fork child's error from the parent
520 // pid.
521 if (err_int != maxInt(ErrInt)) {
522 return @as(SpawnError, @errorCast(@errorFromInt(err_int)));
523 }
524 }
525 }
526
527 return statusToTerm(status);
528}
529
530fn statusToTerm(status: u32) Term {502fn statusToTerm(status: u32) Term {
531 return if (posix.W.IFEXITED(status))503 return if (posix.W.IFEXITED(status))
532 Term{ .Exited = posix.W.EXITSTATUS(status) }504 Term{ .Exited = posix.W.EXITSTATUS(status) }
...@@ -636,18 +608,9 @@ fn spawnPosix(self: *ChildProcess) SpawnError!void {...@@ -636,18 +608,9 @@ fn spawnPosix(self: *ChildProcess) SpawnError!void {
636 }608 }
637 };609 };
638610
639 // This pipe is used to communicate errors between the time of fork611 // This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
640 // and execve from the child process to the parent process.612 // It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
641 const err_pipe = blk: {613 const err_pipe: [2]posix.fd_t = try posix.pipe2(.{ .CLOEXEC = true });
642 if (native_os == .linux) {
643 const fd = try posix.eventfd(0, linux.EFD.CLOEXEC);
644 // There's no distinction between the readable and the writeable
645 // end with eventfd
646 break :blk [2]posix.fd_t{ fd, fd };
647 } else {
648 break :blk try posix.pipe2(.{ .CLOEXEC = true });
649 }
650 };
651 errdefer destroyPipe(err_pipe);614 errdefer destroyPipe(err_pipe);
652615
653 const pid_result = try posix.fork();616 const pid_result = try posix.fork();
...@@ -687,6 +650,11 @@ fn spawnPosix(self: *ChildProcess) SpawnError!void {...@@ -687,6 +650,11 @@ fn spawnPosix(self: *ChildProcess) SpawnError!void {
687 }650 }
688651
689 // we are the parent652 // we are the parent
653 errdefer comptime unreachable; // The child is forked; we must not error from now on
654
655 posix.close(err_pipe[1]); // make sure only the child holds the write end open
656 self.err_pipe = err_pipe[0];
657
690 const pid: i32 = @intCast(pid_result);658 const pid: i32 = @intCast(pid_result);
691 if (self.stdin_behavior == .Pipe) {659 if (self.stdin_behavior == .Pipe) {
692 self.stdin = .{ .handle = stdin_pipe[1] };660 self.stdin = .{ .handle = stdin_pipe[1] };
...@@ -705,7 +673,6 @@ fn spawnPosix(self: *ChildProcess) SpawnError!void {...@@ -705,7 +673,6 @@ fn spawnPosix(self: *ChildProcess) SpawnError!void {
705 }673 }
706674
707 self.id = pid;675 self.id = pid;
708 self.err_pipe = err_pipe;
709 self.term = null;676 self.term = null;
710677
711 if (self.stdin_behavior == .Pipe) {678 if (self.stdin_behavior == .Pipe) {
test/standalone/test_runner_module_imports/build.zig+4-1
...@@ -13,7 +13,10 @@ pub fn build(b: *std.Build) void {...@@ -13,7 +13,10 @@ pub fn build(b: *std.Build) void {
1313
14 const t = b.addTest(.{14 const t = b.addTest(.{
15 .root_module = test_mod,15 .root_module = test_mod,
16 .test_runner = b.path("test_runner/main.zig"),16 .test_runner = .{
17 .path = b.path("test_runner/main.zig"),
18 .mode = .simple,
19 },
17 });20 });
1821
19 const test_step = b.step("test", "Run unit tests");22 const test_step = b.step("test", "Run unit tests");
test/standalone/test_runner_path/build.zig+4-1
...@@ -8,7 +8,10 @@ pub fn build(b: *std.Build) void {...@@ -8,7 +8,10 @@ pub fn build(b: *std.Build) void {
8 .target = b.graph.host,8 .target = b.graph.host,
9 .root_source_file = b.path("test.zig"),9 .root_source_file = b.path("test.zig"),
10 }) });10 }) });
11 test_exe.test_runner = b.path("test_runner.zig");11 test_exe.test_runner = .{
12 .path = b.path("test_runner.zig"),
13 .mode = .simple,
14 };
1215
13 const test_run = b.addRunArtifact(test_exe);16 const test_run = b.addRunArtifact(test_exe);
14 test_step.dependOn(&test_run.step);17 test_step.dependOn(&test_run.step);