authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-01-13 21:23:44-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-01-30 12:10:01-08:00
log5a2cab21e1cf7c0c5dadb5e6c77c359b0b62ff55
treec2874224d4f29a53173277ae4053735e4ec665f6
parent0cca18e43c2685cfb4a72b3b265b68af1e85ca17

std.Io.File.MultiReader: implementation fixes

4 files changed, 87 insertions(+), 80 deletions(-)

lib/std/Build/Step.zig+20-14
......@@ -381,13 +381,15 @@ pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutO
381381
382382pub const ZigProcess = struct {
383383 child: std.process.Child,
384 multi_reader_buffer: Io.File.MultiReader.Buffer(2),
385 multi_reader: Io.File.MultiReader,
384386 progress_ipc_fd: if (std.Progress.have_ipc) ?std.posix.fd_t else void,
385387
386388 pub const StreamEnum = enum { stdout, stderr };
387389
388 pub fn deinit(zp: *ZigProcess, gpa: Allocator, io: Io) void {
389 _ = gpa;
390 pub fn deinit(zp: *ZigProcess, io: Io) void {
390391 zp.child.kill(io);
392 zp.multi_reader.deinit();
391393 zp.* = undefined;
392394 }
393395};
......@@ -460,14 +462,18 @@ pub fn evalZigProcess(
460462 .request_resource_usage_statistics = true,
461463 .progress_node = prog_node,
462464 }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err });
463 defer if (!watch) zp.child.kill(io);
464465
465466 zp.* = .{
466467 .child = zp.child,
468 .multi_reader_buffer = undefined,
469 .multi_reader = undefined,
467470 .progress_ipc_fd = if (std.Progress.have_ipc) prog_node.getIpcFd() else {},
468471 };
472 zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{
473 zp.child.stdout.?, zp.child.stderr.?,
474 });
469475 if (watch) s.setZigProcess(zp);
470 defer if (!watch) zp.deinit(gpa, io);
476 defer if (!watch) zp.deinit(io);
471477
472478 const result = try zigProcessUpdate(s, zp, watch, web_server, gpa);
473479
......@@ -534,18 +540,18 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*Build.
534540
535541 var result: ?Path = null;
536542
537 var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined;
538 var multi_reader: Io.File.MultiReader = undefined;
539 multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ zp.child.stdout.?, zp.child.stderr.? });
540 defer multi_reader.deinit();
541
542 const stdout = multi_reader.reader(0);
543 const stderr = multi_reader.reader(1);
543 const stdout = zp.multi_reader.fileReader(0);
544544
545545 while (true) {
546546 const Header = std.zig.Server.Message.Header;
547 const header = try stdout.takeStruct(Header, .little);
548 const body = try stdout.take(header.bytes_len);
547 const header = stdout.interface.takeStruct(Header, .little) catch |err| switch (err) {
548 error.EndOfStream => break,
549 error.ReadFailed => return stdout.err.?,
550 };
551 const body = stdout.interface.take(header.bytes_len) catch |err| switch (err) {
552 error.EndOfStream => |e| return e,
553 error.ReadFailed => return stdout.err.?,
554 };
549555 switch (header.tag) {
550556 .zig_version => {
551557 if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
......@@ -636,7 +642,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*Build.
636642
637643 s.result_duration_ns = timer.read();
638644
639 const stderr_contents = stderr.buffered();
645 const stderr_contents = zp.multi_reader.reader(1).buffered();
640646 if (stderr_contents.len > 0) {
641647 try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents));
642648 }
lib/std/Build/Step/Run.zig+5-7
......@@ -1695,11 +1695,9 @@ fn evalZigTest(
16951695 // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured
16961696 // all available stderr to make our error output as useful as possible.
16971697 const stderr_fr = multi_reader.fileReader(1);
1698 while (true) {
1699 stderr_fr.interface.fillMore() catch |e| switch (e) {
1700 error.ReadFailed => return stderr_fr.err.?,
1701 error.EndOfStream => break,
1702 };
1698 while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) {
1699 error.ReadFailed => return stderr_fr.err.?,
1700 error.EndOfStream => {},
17031701 }
17041702 run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered());
17051703
......@@ -1905,7 +1903,7 @@ fn waitZigTest(
19051903 .clock = .awake,
19061904 } } else .none;
19071905
1908 multi_reader.fill(timeout) catch |err| switch (err) {
1906 multi_reader.fill(64, timeout) catch |err| switch (err) {
19091907 error.Timeout, error.EndOfStream => return .{ .no_poll = .{
19101908 .active_test_index = active_test_index,
19111909 .ns_elapsed = if (timer) |*t| t.read() else 0,
......@@ -2227,7 +2225,7 @@ fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResul
22272225 const stdout_reader = multi_reader.reader(0);
22282226 const stderr_reader = multi_reader.reader(1);
22292227
2230 while (multi_reader.fill(.none)) |_| {
2228 while (multi_reader.fill(64, .none)) |_| {
22312229 if (run.stdio_limit.toInt()) |limit| {
22322230 if (stdout_reader.buffered().len > limit)
22332231 return error.StdoutStreamTooLong;
lib/std/Io/File/MultiReader.zig+60-58
......@@ -28,18 +28,23 @@ pub const Streams = extern struct {
2828 len: u32,
2929
3030 pub fn contexts(s: *Streams) []Context {
31 _ = s;
32 @panic("TODO");
31 const base: usize = @intFromPtr(s);
32 const ptr: [*]Context = @ptrFromInt(std.mem.alignForward(usize, base + @sizeOf(Streams), @alignOf(Context)));
33 return ptr[0..s.len];
3334 }
3435
3536 pub fn ring(s: *Streams) []u32 {
36 _ = s;
37 @panic("TODO");
37 const prev = contexts(s);
38 const end = prev.ptr + prev.len;
39 const ptr: [*]u32 = @ptrFromInt(std.mem.alignForward(usize, @intFromPtr(end), @alignOf(u32)));
40 return ptr[0..s.len];
3841 }
3942
4043 pub fn operations(s: *Streams) []Io.Operation {
41 _ = s;
42 @panic("TODO");
44 const prev = ring(s);
45 const end = prev.ptr + prev.len;
46 const ptr: [*]Io.Operation = @ptrFromInt(std.mem.alignForward(usize, @intFromPtr(end), @alignOf(Io.Operation)));
47 return ptr[0..s.len];
4348 }
4449};
4550
......@@ -51,6 +56,7 @@ pub fn Buffer(comptime n: usize) type {
5156 operations: [n][@sizeOf(Io.Operation)]u8 align(@alignOf(Io.Operation)),
5257
5358 pub fn toStreams(b: *@This()) *Streams {
59 b.len = n;
5460 return @ptrCast(b);
5561 }
5662 };
......@@ -157,61 +163,43 @@ fn stream(r: *Io.Reader, w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!u
157163 _ = w;
158164 const fr: *File.Reader = @alignCast(@fieldParentPtr("interface", r));
159165 const context: *Context = @fieldParentPtr("fr", fr);
160 const mr = context.mr;
161 return fillUntimed(mr, context);
166 try fillUntimed(context, 1);
167 return 0;
162168}
163169
164170fn discard(r: *Io.Reader, limit: Io.Limit) Io.Reader.Error!usize {
165171 _ = limit;
166172 const fr: *File.Reader = @alignCast(@fieldParentPtr("interface", r));
167173 const context: *Context = @fieldParentPtr("fr", fr);
168 const mr = context.mr;
169 return fillUntimed(mr, context);
174 try fillUntimed(context, 1);
175 return 0;
170176}
171177
172178fn readVec(r: *Io.Reader, data: [][]u8) Io.Reader.Error!usize {
173179 _ = data;
174180 const fr: *File.Reader = @alignCast(@fieldParentPtr("interface", r));
175181 const context: *Context = @fieldParentPtr("fr", fr);
176 const mr = context.mr;
177 return fillUntimed(mr, context);
182 try fillUntimed(context, 1);
183 return 0;
178184}
179185
180186fn rebase(r: *Io.Reader, capacity: usize) Io.Reader.RebaseError!void {
181187 const fr: *File.Reader = @alignCast(@fieldParentPtr("interface", r));
182188 const context: *Context = @fieldParentPtr("fr", fr);
183 const mr = context.mr;
184
185 return rebaseGrowing(mr, context, capacity) catch |err| {
186 context.err = err;
187 return error.ReadFailed;
188 };
189 try fillUntimed(context, capacity);
189190}
190191
191fn rebaseGrowing(mr: *MultiReader, context: *Context, capacity: usize) Allocator.Error!void {
192 const gpa = mr.gpa;
193 const r = &context.fr.interface;
194 if (r.buffer.len >= capacity) {
195 const data = r.buffer[r.seek..r.end];
196 @memmove(r.buffer[0..data.len], data);
197 r.seek = 0;
198 r.end = data.len;
199 } else {
200 const adjusted_capacity = std.ArrayList(u8).growCapacity(capacity);
201
202 if (r.seek == 0) {
203 if (gpa.remap(r.buffer, adjusted_capacity)) |new_memory| {
204 r.buffer = new_memory;
205 return;
206 }
207 }
208
209 const data = r.buffer[r.seek..r.end];
210 const new = try gpa.alloc(u8, adjusted_capacity);
211 @memcpy(new[0..data.len], data);
212 r.seek = 0;
213 r.end = data.len;
214 }
192fn fillUntimed(context: *Context, capacity: usize) Io.Reader.Error!void {
193 fill(context.mr, capacity, .none) catch |err| switch (err) {
194 error.Timeout, error.UnsupportedClock => unreachable,
195 error.Canceled, error.ConcurrencyUnavailable => |e| {
196 context.err = e;
197 return error.ReadFailed;
198 },
199 error.EndOfStream => |e| return e,
200 };
201 if (context.err != null) return error.ReadFailed;
202 if (context.eos) return error.EndOfStream;
215203}
216204
217205pub const FillError = Io.Batch.WaitError || error{
......@@ -221,7 +209,7 @@ pub const FillError = Io.Batch.WaitError || error{
221209};
222210
223211/// Wait until at least one stream receives more data.
224pub fn fill(mr: *MultiReader, timeout: Io.Timeout) FillError!void {
212pub fn fill(mr: *MultiReader, unused_capacity: usize, timeout: Io.Timeout) FillError!void {
225213 const contexts = mr.streams.contexts();
226214 const operations = mr.streams.operations();
227215 const io = contexts[0].fr.io;
......@@ -243,14 +231,14 @@ pub fn fill(mr: *MultiReader, timeout: Io.Timeout) FillError!void {
243231 }
244232 const r = &context.fr.interface;
245233 r.end += n;
246 if (r.buffer.len - r.end == 0) {
247 rebaseGrowing(mr, context, r.bufferedLen() + 1) catch |err| {
234 if (r.buffer.len - r.end < unused_capacity) {
235 rebaseGrowing(mr, context, r.bufferedLen() + unused_capacity) catch |err| {
248236 context.err = err;
249237 continue;
250238 };
251239 assert(r.seek == 0);
252 context.vec[0] = r.buffer;
253240 }
241 context.vec[0] = r.buffer[r.end..];
254242 operation.file_read_streaming.status = .{ .unstarted = {} };
255243 mr.batch.add(i);
256244 }
......@@ -258,16 +246,30 @@ pub fn fill(mr: *MultiReader, timeout: Io.Timeout) FillError!void {
258246 if (!any_completed) return error.EndOfStream;
259247}
260248
261fn fillUntimed(mr: *MultiReader, context: *Context) Io.Reader.Error!usize {
262 fill(mr, .none) catch |err| switch (err) {
263 error.Timeout, error.UnsupportedClock => unreachable,
264 error.Canceled, error.ConcurrencyUnavailable => |e| {
265 context.err = e;
266 return error.ReadFailed;
267 },
268 error.EndOfStream => |e| return e,
269 };
270 if (context.err != null) return error.ReadFailed;
271 if (context.eos) return error.EndOfStream;
272 return 0;
249fn rebaseGrowing(mr: *MultiReader, context: *Context, capacity: usize) Allocator.Error!void {
250 const gpa = mr.gpa;
251 const r = &context.fr.interface;
252 if (r.buffer.len >= capacity) {
253 const data = r.buffer[r.seek..r.end];
254 @memmove(r.buffer[0..data.len], data);
255 r.seek = 0;
256 r.end = data.len;
257 } else {
258 const adjusted_capacity = std.ArrayList(u8).growCapacity(capacity);
259
260 if (r.seek == 0) {
261 if (gpa.remap(r.buffer, adjusted_capacity)) |new_memory| {
262 r.buffer = new_memory;
263 return;
264 }
265 }
266
267 const data = r.buffer[r.seek..r.end];
268 const new = try gpa.alloc(u8, adjusted_capacity);
269 @memcpy(new[0..data.len], data);
270 gpa.free(r.buffer);
271 r.buffer = new;
272 r.seek = 0;
273 r.end = data.len;
274 }
273275}
lib/std/crypto/tls/Client.zig+2-1
......@@ -336,10 +336,11 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client
336336 // Ensure the input buffer pointer is stable in this scope.
337337 input.rebase(tls.max_ciphertext_record_len) catch |err| switch (err) {
338338 error.EndOfStream => {}, // We have assurance the remainder of stream can be buffered.
339 error.ReadFailed => |e| return e,
339340 };
340341 const record_header = input.peek(tls.record_header_len) catch |err| switch (err) {
341342 error.EndOfStream => return error.TlsConnectionTruncated,
342 error.ReadFailed => return error.ReadFailed,
343 error.ReadFailed => |e| return e,
343344 };
344345 const record_ct = input.takeEnumNonexhaustive(tls.ContentType, .big) catch unreachable; // already peeked
345346 input.toss(2); // legacy_version