authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-12-29 11:13:00-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-12-29 11:13:00-07:00
log717cf00fe0d68dc1213fb645b184afe1cbf52104
treebcfdeec3c406aeb2a0427d816d0f3d242354fdce
parent892b37cdae89fe23d2a19b0b512fc1f7a73dd6ce

std.ChildProcess: improvements to collectOutputPosix

* read directly into the ArrayList buffers. * respect max_output_bytes * std.ArrayList: - make `allocatedSlice` public. - add `unusedCapacitySlice`. I removed the Windows implementation of this stuff; I am doing a partial merge of LemonBoy's patch with the understanding that a later patch can add the Windows implementation after it is vetted.

5 files changed, 53 insertions(+), 170 deletions(-)

lib/std/array_list.zig+13-3
...@@ -337,11 +337,21 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {...@@ -337,11 +337,21 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
337 return self.pop();337 return self.pop();
338 }338 }
339339
340 // For a nicer API, `items.len` is the length, not the capacity.340 /// Returns a slice of all the items plus the extra capacity, whose memory
341 // This requires "unsafe" slicing.341 /// contents are undefined.
342 fn allocatedSlice(self: Self) Slice {342 pub fn allocatedSlice(self: Self) Slice {
343 // For a nicer API, `items.len` is the length, not the capacity.
344 // This requires "unsafe" slicing.
343 return self.items.ptr[0..self.capacity];345 return self.items.ptr[0..self.capacity];
344 }346 }
347
348 /// Returns a slice of only the extra capacity after items.
349 /// This can be useful for writing directly into an `ArrayList`.
350 /// Note that such an operation must be followed up with a direct
351 /// modification of `self.items.len`.
352 pub fn unusedCapacitySlice(self: Self) Slice {
353 return self.allocatedSlice()[self.items.len..];
354 }
345 };355 };
346}356}
347357
lib/std/child_process.zig+39-119
...@@ -186,14 +186,22 @@ pub const ChildProcess = struct {...@@ -186,14 +186,22 @@ pub const ChildProcess = struct {
186186
187 pub const exec2 = @compileError("deprecated: exec2 is renamed to exec");187 pub const exec2 = @compileError("deprecated: exec2 is renamed to exec");
188188
189 fn collectOutputPosix(child: *const ChildProcess, stdout: *std.ArrayList(u8), stderr: *std.ArrayList(u8), max_output_bytes: usize) !void {189 fn collectOutputPosix(
190 child: *const ChildProcess,
191 stdout: *std.ArrayList(u8),
192 stderr: *std.ArrayList(u8),
193 max_output_bytes: usize,
194 ) !void {
190 var poll_fds = [_]os.pollfd{195 var poll_fds = [_]os.pollfd{
191 .{ .fd = child.stdout.?.handle, .events = os.POLLIN, .revents = undefined },196 .{ .fd = child.stdout.?.handle, .events = os.POLLIN, .revents = undefined },
192 .{ .fd = child.stderr.?.handle, .events = os.POLLIN, .revents = undefined },197 .{ .fd = child.stderr.?.handle, .events = os.POLLIN, .revents = undefined },
193 };198 };
194199
195 var dead_fds: usize = 0;200 var dead_fds: usize = 0;
196 var loop_buf: [4096]u8 = undefined;201 // We ask for ensureCapacity with this much extra space. This has more of an
202 // effect on small reads because once the reads start to get larger the amount
203 // of space an ArrayList will allocate grows exponentially.
204 const bump_amt = 512;
197205
198 while (dead_fds < poll_fds.len) {206 while (dead_fds < poll_fds.len) {
199 const events = try os.poll(&poll_fds, std.math.maxInt(i32));207 const events = try os.poll(&poll_fds, std.math.maxInt(i32));
...@@ -203,13 +211,17 @@ pub const ChildProcess = struct {...@@ -203,13 +211,17 @@ pub const ChildProcess = struct {
203 // conditions.211 // conditions.
204 if (poll_fds[0].revents & os.POLLIN != 0) {212 if (poll_fds[0].revents & os.POLLIN != 0) {
205 // stdout is ready.213 // stdout is ready.
206 const n = try os.read(poll_fds[0].fd, &loop_buf);214 const new_capacity = std.math.min(stdout.items.len + bump_amt, max_output_bytes);
207 try stdout.appendSlice(loop_buf[0..n]);215 if (new_capacity == stdout.capacity) return error.StdoutStreamTooLong;
216 try stdout.ensureCapacity(new_capacity);
217 stdout.items.len += try os.read(poll_fds[0].fd, stdout.unusedCapacitySlice());
208 }218 }
209 if (poll_fds[1].revents & os.POLLIN != 0) {219 if (poll_fds[1].revents & os.POLLIN != 0) {
210 // stderr is ready.220 // stderr is ready.
211 const n = try os.read(poll_fds[1].fd, &loop_buf);221 const new_capacity = std.math.min(stderr.items.len + bump_amt, max_output_bytes);
212 try stderr.appendSlice(loop_buf[0..n]);222 if (new_capacity == stderr.capacity) return error.StderrStreamTooLong;
223 try stderr.ensureCapacity(new_capacity);
224 stderr.items.len += try os.read(poll_fds[1].fd, stderr.unusedCapacitySlice());
213 }225 }
214226
215 // Exclude the fds that signaled an error.227 // Exclude the fds that signaled an error.
...@@ -224,61 +236,6 @@ pub const ChildProcess = struct {...@@ -224,61 +236,6 @@ pub const ChildProcess = struct {
224 }236 }
225 }237 }
226238
227 fn collectOutputWindows(child: *const ChildProcess, stdout: *std.ArrayList(u8), stderr: *std.ArrayList(u8), max_output_bytes: usize) !void {
228 var wait_objects = [_]windows.kernel32.HANDLE{
229 child.stdout.?.handle, child.stderr.?.handle,
230 };
231 var waiting_objects: u32 = wait_objects.len;
232
233 // XXX: Calling zeroes([2]windows.OVERLAPPED) causes the stage1 compiler
234 // to crash and burn.
235 var overlapped = [_]windows.OVERLAPPED{
236 mem.zeroes(windows.OVERLAPPED),
237 mem.zeroes(windows.OVERLAPPED),
238 };
239 var temp_buf: [2][4096]u8 = undefined;
240
241 // Kickstart the loop by issuing two async reads.
242 // ReadFile returns false and GetLastError returns ERROR_IO_PENDING if
243 // everything is ok.
244 _ = windows.kernel32.ReadFile(wait_objects[0], &temp_buf[0], temp_buf[0].len, null, &overlapped[0]);
245 _ = windows.kernel32.ReadFile(wait_objects[1], &temp_buf[1], temp_buf[1].len, null, &overlapped[1]);
246
247 poll: while (waiting_objects > 0) {
248 const status = windows.kernel32.WaitForMultipleObjects(waiting_objects, &wait_objects, 0, windows.INFINITE);
249 switch (status) {
250 windows.WAIT_OBJECT_0 + 0...windows.WAIT_OBJECT_0 + 1 => {
251 // stdout (or stderr) is ready.
252 const object = status - windows.WAIT_OBJECT_0;
253
254 var read_bytes: u32 = undefined;
255 if (windows.kernel32.GetOverlappedResult(wait_objects[object], &overlapped[object], &read_bytes, 0) == 0) {
256 switch (windows.kernel32.GetLastError()) {
257 .BROKEN_PIPE => {
258 // Move it to the end to remove it.
259 if (object != waiting_objects - 1)
260 mem.swap(windows.kernel32.HANDLE, &wait_objects[object], &wait_objects[waiting_objects - 1]);
261 waiting_objects -= 1;
262 continue :poll;
263 },
264 else => |err| return windows.unexpectedError(err),
265 }
266 }
267 try stdout.appendSlice(temp_buf[object][0..read_bytes]);
268 _ = windows.kernel32.ReadFile(wait_objects[object], &temp_buf[object], temp_buf[object].len, null, &overlapped[object]);
269 },
270 windows.WAIT_FAILED => {
271 switch (windows.kernel32.GetLastError()) {
272 else => |err| return windows.unexpectedError(err),
273 }
274 },
275 // We're waiting with an infinite timeout
276 windows.WAIT_TIMEOUT => unreachable,
277 else => unreachable,
278 }
279 }
280 }
281
282 /// Spawns a child process, waits for it, collecting stdout and stderr, and then returns.239 /// Spawns a child process, waits for it, collecting stdout and stderr, and then returns.
283 /// If it succeeds, the caller owns result.stdout and result.stderr memory.240 /// If it succeeds, the caller owns result.stdout and result.stderr memory.
284 pub fn exec(args: struct {241 pub fn exec(args: struct {
...@@ -303,16 +260,28 @@ pub const ChildProcess = struct {...@@ -303,16 +260,28 @@ pub const ChildProcess = struct {
303260
304 try child.spawn();261 try child.spawn();
305262
263 // TODO collect output in a deadlock-avoiding way on Windows.
264 // https://github.com/ziglang/zig/issues/6343
265 if (builtin.os.tag == .windows) {
266 const stdout_in = child.stdout.?.reader();
267 const stderr_in = child.stderr.?.reader();
268
269 const stdout = try stdout_in.readAllAlloc(args.allocator, args.max_output_bytes);
270 errdefer args.allocator.free(stdout);
271 const stderr = try stderr_in.readAllAlloc(args.allocator, args.max_output_bytes);
272 errdefer args.allocator.free(stderr);
273
274 return ExecResult{
275 .term = try child.wait(),
276 .stdout = stdout,
277 .stderr = stderr,
278 };
279 }
280
306 var stdout = std.ArrayList(u8).init(args.allocator);281 var stdout = std.ArrayList(u8).init(args.allocator);
307 var stderr = std.ArrayList(u8).init(args.allocator);282 var stderr = std.ArrayList(u8).init(args.allocator);
308283
309 // XXX: Respect max_output_bytes284 try collectOutputPosix(child, &stdout, &stderr, args.max_output_bytes);
310 // XXX: Smarter reading logic, read directly into the ArrayList
311 if (builtin.os.tag == .windows) {
312 try collectOutputWindows(child, &stdout, &stderr, args.max_output_bytes);
313 } else {
314 try collectOutputPosix(child, &stdout, &stderr, args.max_output_bytes);
315 }
316285
317 return ExecResult{286 return ExecResult{
318 .term = try child.wait(),287 .term = try child.wait(),
...@@ -650,7 +619,7 @@ pub const ChildProcess = struct {...@@ -650,7 +619,7 @@ pub const ChildProcess = struct {
650 var g_hChildStd_OUT_Wr: ?windows.HANDLE = null;619 var g_hChildStd_OUT_Wr: ?windows.HANDLE = null;
651 switch (self.stdout_behavior) {620 switch (self.stdout_behavior) {
652 StdIo.Pipe => {621 StdIo.Pipe => {
653 try windowsMakePipe(&g_hChildStd_OUT_Rd, &g_hChildStd_OUT_Wr, &saAttr);622 try windowsMakePipeOut(&g_hChildStd_OUT_Rd, &g_hChildStd_OUT_Wr, &saAttr);
654 },623 },
655 StdIo.Ignore => {624 StdIo.Ignore => {
656 g_hChildStd_OUT_Wr = nul_handle;625 g_hChildStd_OUT_Wr = nul_handle;
...@@ -670,7 +639,7 @@ pub const ChildProcess = struct {...@@ -670,7 +639,7 @@ pub const ChildProcess = struct {
670 var g_hChildStd_ERR_Wr: ?windows.HANDLE = null;639 var g_hChildStd_ERR_Wr: ?windows.HANDLE = null;
671 switch (self.stderr_behavior) {640 switch (self.stderr_behavior) {
672 StdIo.Pipe => {641 StdIo.Pipe => {
673 try windowsMakePipe(&g_hChildStd_ERR_Rd, &g_hChildStd_ERR_Wr, &saAttr);642 try windowsMakePipeOut(&g_hChildStd_ERR_Rd, &g_hChildStd_ERR_Wr, &saAttr);
674 },643 },
675 StdIo.Ignore => {644 StdIo.Ignore => {
676 g_hChildStd_ERR_Wr = nul_handle;645 g_hChildStd_ERR_Wr = nul_handle;
...@@ -903,55 +872,6 @@ fn windowsDestroyPipe(rd: ?windows.HANDLE, wr: ?windows.HANDLE) void {...@@ -903,55 +872,6 @@ fn windowsDestroyPipe(rd: ?windows.HANDLE, wr: ?windows.HANDLE) void {
903 if (wr) |h| os.close(h);872 if (wr) |h| os.close(h);
904}873}
905874
906var pipe_name_counter = std.atomic.Int(u32).init(1);
907
908fn windowsMakePipe(rd: *?windows.HANDLE, wr: *?windows.HANDLE, sattr: *const windows.SECURITY_ATTRIBUTES) !void {
909 var tmp_buf: [128]u8 = undefined;
910 // Forge a random path for the pipe.
911 const pipe_path = std.fmt.bufPrintZ(
912 &tmp_buf,
913 "\\\\.\\pipe\\zig-childprocess-{d}-{d}",
914 .{ windows.kernel32.GetCurrentProcessId(), pipe_name_counter.fetchAdd(1) },
915 ) catch unreachable;
916
917 // Create the read handle that can be used with overlapped IO ops.
918 const read_handle = windows.kernel32.CreateNamedPipeA(
919 pipe_path,
920 windows.PIPE_ACCESS_INBOUND | windows.FILE_FLAG_OVERLAPPED,
921 windows.PIPE_TYPE_BYTE,
922 1,
923 0x1000,
924 0x1000,
925 0,
926 sattr,
927 );
928 if (read_handle == windows.INVALID_HANDLE_VALUE) {
929 switch (windows.kernel32.GetLastError()) {
930 else => |err| return windows.unexpectedError(err),
931 }
932 }
933
934 const write_handle = windows.kernel32.CreateFileA(
935 pipe_path,
936 windows.GENERIC_WRITE,
937 0,
938 sattr,
939 windows.OPEN_EXISTING,
940 windows.FILE_ATTRIBUTE_NORMAL,
941 null,
942 );
943 if (write_handle == windows.INVALID_HANDLE_VALUE) {
944 switch (windows.kernel32.GetLastError()) {
945 else => |err| return windows.unexpectedError(err),
946 }
947 }
948
949 try windows.SetHandleInformation(read_handle, windows.HANDLE_FLAG_INHERIT, 0);
950
951 rd.* = read_handle;
952 wr.* = write_handle;
953}
954
955fn windowsMakePipeIn(rd: *?windows.HANDLE, wr: *?windows.HANDLE, sattr: *const windows.SECURITY_ATTRIBUTES) !void {875fn windowsMakePipeIn(rd: *?windows.HANDLE, wr: *?windows.HANDLE, sattr: *const windows.SECURITY_ATTRIBUTES) !void {
956 var rd_h: windows.HANDLE = undefined;876 var rd_h: windows.HANDLE = undefined;
957 var wr_h: windows.HANDLE = undefined;877 var wr_h: windows.HANDLE = undefined;
lib/std/os.zig+1-2
...@@ -5269,8 +5269,7 @@ pub const PollError = error{...@@ -5269,8 +5269,7 @@ pub const PollError = error{
52695269
5270pub fn poll(fds: []pollfd, timeout: i32) PollError!usize {5270pub fn poll(fds: []pollfd, timeout: i32) PollError!usize {
5271 while (true) {5271 while (true) {
5272 const fds_count = math.cast(nfds_t, fds.len) catch5272 const fds_count = math.cast(nfds_t, fds.len) catch return error.SystemResources;
5273 return error.SystemResources;
5274 const rc = system.poll(fds.ptr, fds_count, timeout);5273 const rc = system.poll(fds.ptr, fds_count, timeout);
5275 if (builtin.os.tag == .windows) {5274 if (builtin.os.tag == .windows) {
5276 if (rc == windows.ws2_32.SOCKET_ERROR) {5275 if (rc == windows.ws2_32.SOCKET_ERROR) {
lib/std/os/windows/bits.zig-13
...@@ -438,19 +438,6 @@ pub const SECURITY_ATTRIBUTES = extern struct {...@@ -438,19 +438,6 @@ pub const SECURITY_ATTRIBUTES = extern struct {
438pub const PSECURITY_ATTRIBUTES = *SECURITY_ATTRIBUTES;438pub const PSECURITY_ATTRIBUTES = *SECURITY_ATTRIBUTES;
439pub const LPSECURITY_ATTRIBUTES = *SECURITY_ATTRIBUTES;439pub const LPSECURITY_ATTRIBUTES = *SECURITY_ATTRIBUTES;
440440
441pub const PIPE_ACCESS_INBOUND = 0x00000001;
442pub const PIPE_ACCESS_OUTBOUND = 0x00000002;
443pub const PIPE_ACCESS_DUPLEX = 0x00000003;
444
445pub const PIPE_TYPE_BYTE = 0x00000000;
446pub const PIPE_TYPE_MESSAGE = 0x00000004;
447
448pub const PIPE_READMODE_BYTE = 0x00000000;
449pub const PIPE_READMODE_MESSAGE = 0x00000002;
450
451pub const PIPE_WAIT = 0x00000000;
452pub const PIPE_NOWAIT = 0x00000001;
453
454pub const GENERIC_READ = 0x80000000;441pub const GENERIC_READ = 0x80000000;
455pub const GENERIC_WRITE = 0x40000000;442pub const GENERIC_WRITE = 0x40000000;
456pub const GENERIC_EXECUTE = 0x20000000;443pub const GENERIC_EXECUTE = 0x20000000;
lib/std/os/windows/kernel32.zig-33
...@@ -15,29 +15,6 @@ pub extern "kernel32" fn CloseHandle(hObject: HANDLE) callconv(WINAPI) BOOL;...@@ -15,29 +15,6 @@ pub extern "kernel32" fn CloseHandle(hObject: HANDLE) callconv(WINAPI) BOOL;
15pub extern "kernel32" fn CreateDirectoryW(lpPathName: [*:0]const u16, lpSecurityAttributes: ?*SECURITY_ATTRIBUTES) callconv(WINAPI) BOOL;15pub extern "kernel32" fn CreateDirectoryW(lpPathName: [*:0]const u16, lpSecurityAttributes: ?*SECURITY_ATTRIBUTES) callconv(WINAPI) BOOL;
16pub extern "kernel32" fn SetEndOfFile(hFile: HANDLE) callconv(WINAPI) BOOL;16pub extern "kernel32" fn SetEndOfFile(hFile: HANDLE) callconv(WINAPI) BOOL;
1717
18pub extern "kernel32" fn GetCurrentProcessId() callconv(WINAPI) DWORD;
19
20pub extern "kernel32" fn CreateNamedPipeA(
21 lpName: [*:0]const u8,
22 dwOpenMode: DWORD,
23 dwPipeMode: DWORD,
24 nMaxInstances: DWORD,
25 nOutBufferSize: DWORD,
26 nInBufferSize: DWORD,
27 nDefaultTimeOut: DWORD,
28 lpSecurityAttributes: ?*const SECURITY_ATTRIBUTES,
29) callconv(WINAPI) HANDLE;
30pub extern "kernel32" fn CreateNamedPipeW(
31 lpName: LPCWSTR,
32 dwOpenMode: DWORD,
33 dwPipeMode: DWORD,
34 nMaxInstances: DWORD,
35 nOutBufferSize: DWORD,
36 nInBufferSize: DWORD,
37 nDefaultTimeOut: DWORD,
38 lpSecurityAttributes: ?*const SECURITY_ATTRIBUTES,
39) callconv(WINAPI) HANDLE;
40
41pub extern "kernel32" fn CreateEventExW(18pub extern "kernel32" fn CreateEventExW(
42 lpEventAttributes: ?*SECURITY_ATTRIBUTES,19 lpEventAttributes: ?*SECURITY_ATTRIBUTES,
43 lpName: [*:0]const u16,20 lpName: [*:0]const u16,
...@@ -55,16 +32,6 @@ pub extern "kernel32" fn CreateFileW(...@@ -55,16 +32,6 @@ pub extern "kernel32" fn CreateFileW(
55 hTemplateFile: ?HANDLE,32 hTemplateFile: ?HANDLE,
56) callconv(WINAPI) HANDLE;33) callconv(WINAPI) HANDLE;
5734
58pub extern "kernel32" fn CreateFileA(
59 lpFileName: [*:0]const u8,
60 dwDesiredAccess: DWORD,
61 dwShareMode: DWORD,
62 lpSecurityAttributes: ?*const SECURITY_ATTRIBUTES,
63 dwCreationDisposition: DWORD,
64 dwFlagsAndAttributes: DWORD,
65 hTemplateFile: ?HANDLE,
66) callconv(WINAPI) HANDLE;
67
68pub extern "kernel32" fn CreatePipe(35pub extern "kernel32" fn CreatePipe(
69 hReadPipe: *HANDLE,36 hReadPipe: *HANDLE,
70 hWritePipe: *HANDLE,37 hWritePipe: *HANDLE,