authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-04-23 13:32:51-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:28-07:00
log1bb75f9d6276ddf6b69717d9b1c2f68140727425
tree62ec5d7fef20f0047c5ff253db2ae80f06451e0b
parenta90f07b5374723ff76ed2ea888e5515326650e64

stabilize readRemainingArrayList and readRemainingAlloc API

8 files changed, 178 insertions(+), 102 deletions(-)

lib/std/Build.zig+4-4
...@@ -179,7 +179,8 @@ const InitializedDepContext = struct {...@@ -179,7 +179,8 @@ const InitializedDepContext = struct {
179};179};
180180
181pub const RunError = error{181pub const RunError = error{
182 ReadFailure,182 ReadFailed,
183 StreamTooLong,
183 ExitCodeFailure,184 ExitCodeFailure,
184 ProcessTerminated,185 ProcessTerminated,
185 ExecNotSupported,186 ExecNotSupported,
...@@ -2059,9 +2060,8 @@ pub fn runAllowFail(...@@ -2059,9 +2060,8 @@ pub fn runAllowFail(
2059 try Step.handleVerbose2(b, null, child.env_map, argv);2060 try Step.handleVerbose2(b, null, child.env_map, argv);
2060 try child.spawn();2061 try child.spawn();
20612062
2062 const stdout = child.stdout.?.readToEndAlloc(b.allocator, .limited(max_output_size)) catch {2063 var file_reader = child.stdout.?.readerStreaming();
2063 return error.ReadFailure;2064 const stdout = try file_reader.interface().readRemainingAlloc(b.allocator, .limited(max_output_size));
2064 };
2065 errdefer b.allocator.free(stdout);2065 errdefer b.allocator.free(stdout);
20662066
2067 const term = try child.wait();2067 const term = try child.wait();
lib/std/Build/Step/Run.zig+12-2
...@@ -1791,10 +1791,20 @@ fn evalGeneric(run: *Run, child: *std.process.Child) !StdIoResult {...@@ -1791,10 +1791,20 @@ fn evalGeneric(run: *Run, child: *std.process.Child) !StdIoResult {
1791 stdout_bytes = poller.reader(.stdout).bufferContents();1791 stdout_bytes = poller.reader(.stdout).bufferContents();
1792 stderr_bytes = poller.reader(.stderr).bufferContents();1792 stderr_bytes = poller.reader(.stderr).bufferContents();
1793 } else {1793 } else {
1794 stdout_bytes = try stdout.readToEndAlloc(arena, run.stdio_limit);1794 var fr = stdout.readerStreaming();
1795 stdout_bytes = fr.interface().readRemainingAlloc(arena, run.stdio_limit) catch |err| switch (err) {
1796 error.OutOfMemory => return error.OutOfMemory,
1797 error.ReadFailed => return fr.err.?,
1798 error.StreamTooLong => return error.StdoutStreamTooLong,
1799 };
1795 }1800 }
1796 } else if (child.stderr) |stderr| {1801 } else if (child.stderr) |stderr| {
1797 stderr_bytes = try stderr.readToEndAlloc(arena, run.stdio_limit);1802 var fr = stderr.readerStreaming();
1803 stderr_bytes = fr.interface().readRemainingAlloc(arena, run.stdio_limit) catch |err| switch (err) {
1804 error.OutOfMemory => return error.OutOfMemory,
1805 error.ReadFailed => return fr.err.?,
1806 error.StreamTooLong => return error.StderrStreamTooLong,
1807 };
1798 }1808 }
17991809
1800 if (stderr_bytes) |bytes| if (bytes.len > 0) {1810 if (stderr_bytes) |bytes| if (bytes.len > 0) {
lib/std/debug/Dwarf.zig+1-1
...@@ -2243,7 +2243,7 @@ pub const ElfModule = struct {...@@ -2243,7 +2243,7 @@ pub const ElfModule = struct {
22432243
2244 var zlib_stream: std.compress.zlib.Decompressor = .init(&section_reader);2244 var zlib_stream: std.compress.zlib.Decompressor = .init(&section_reader);
22452245
2246 const decompressed_section = zlib_stream.reader().readAlloc(gpa, ch_size) catch continue;2246 const decompressed_section = zlib_stream.reader().readRemainingAlloc(gpa, .limited(ch_size)) catch continue;
2247 if (decompressed_section.len != ch_size) {2247 if (decompressed_section.len != ch_size) {
2248 gpa.free(decompressed_section);2248 gpa.free(decompressed_section);
2249 continue;2249 continue;
lib/std/fs/Dir.zig+17-10
...@@ -1963,6 +1963,8 @@ pub fn readFile(self: Dir, file_path: []const u8, buffer: []u8) ![]u8 {...@@ -1963,6 +1963,8 @@ pub fn readFile(self: Dir, file_path: []const u8, buffer: []u8) ![]u8 {
1963 return buffer[0..end_index];1963 return buffer[0..end_index];
1964}1964}
19651965
1966pub const ReadFileAllocError = File.OpenError || File.ReadError || Allocator.Error || error{StreamTooLong};
1967
1966/// Reads all the bytes from the named file. On success, caller owns returned1968/// Reads all the bytes from the named file. On success, caller owns returned
1967/// buffer.1969/// buffer.
1968pub fn readFileAlloc(1970pub fn readFileAlloc(
...@@ -1972,13 +1974,13 @@ pub fn readFileAlloc(...@@ -1972,13 +1974,13 @@ pub fn readFileAlloc(
1972 /// On other platforms, an opaque sequence of bytes with no particular encoding.1974 /// On other platforms, an opaque sequence of bytes with no particular encoding.
1973 file_path: []const u8,1975 file_path: []const u8,
1974 /// Used to allocate the result.1976 /// Used to allocate the result.
1975 gpa: mem.Allocator,1977 gpa: Allocator,
1976 /// If exceeded:1978 /// If exceeded:
1977 /// * The array list's length is increased by exactly one byte past `limit`.1979 /// * The array list's length is increased by exactly one byte past `limit`.
1978 /// * The file seek position is advanced by exactly one byte past `limit`.1980 /// * The file seek position is advanced by exactly one byte past `limit`.
1979 /// * `error.FileTooBig` is returned.1981 /// * `error.StreamTooLong` is returned.
1980 limit: std.io.Reader.Limit,1982 limit: std.io.Reader.Limit,
1981) (File.OpenError || File.ReadAllocError)![]u8 {1983) ReadFileAllocError![]u8 {
1982 return dir.readFileAllocOptions(file_path, gpa, limit, null, .of(u8), null);1984 return dir.readFileAllocOptions(file_path, gpa, limit, null, .of(u8), null);
1983}1985}
19841986
...@@ -1991,18 +1993,18 @@ pub fn readFileAllocOptions(...@@ -1991,18 +1993,18 @@ pub fn readFileAllocOptions(
1991 /// On other platforms, an opaque sequence of bytes with no particular encoding.1993 /// On other platforms, an opaque sequence of bytes with no particular encoding.
1992 file_path: []const u8,1994 file_path: []const u8,
1993 /// Used to allocate the result.1995 /// Used to allocate the result.
1994 gpa: mem.Allocator,1996 gpa: Allocator,
1995 /// If exceeded:1997 /// If exceeded:
1996 /// * The array list's length is increased by exactly one byte past `limit`.1998 /// * The array list's length is increased by exactly one byte past `limit`.
1997 /// * The file seek position is advanced by exactly one byte past `limit`.1999 /// * The file seek position is advanced by exactly one byte past `limit`.
1998 /// * `error.FileTooBig` is returned.2000 /// * `error.StreamTooLong` is returned.
1999 limit: std.io.Reader.Limit,2001 limit: std.io.Reader.Limit,
2000 /// If specified, the initial buffer size is calculated using this value,2002 /// If specified, the initial buffer size is calculated using this value,
2001 /// otherwise the effective file size is used instead.2003 /// otherwise the effective file size is used instead.
2002 size_hint: ?usize,2004 size_hint: ?usize,
2003 comptime alignment: std.mem.Alignment,2005 comptime alignment: std.mem.Alignment,
2004 comptime optional_sentinel: ?u8,2006 comptime optional_sentinel: ?u8,
2005) (File.OpenError || File.ReadAllocError)!(if (optional_sentinel) |s| [:s]align(alignment.toByteUnits()) u8 else []align(alignment.toByteUnits()) u8) {2007) ReadFileAllocError!(if (optional_sentinel) |s| [:s]align(alignment.toByteUnits()) u8 else []align(alignment.toByteUnits()) u8) {
2006 var buffer: std.ArrayListAlignedUnmanaged(u8, alignment) = .empty;2008 var buffer: std.ArrayListAlignedUnmanaged(u8, alignment) = .empty;
2007 defer buffer.deinit(gpa);2009 defer buffer.deinit(gpa);
2008 try readFileIntoArrayList(2010 try readFileIntoArrayList(
...@@ -2026,7 +2028,7 @@ pub fn readFileAllocOptions(...@@ -2026,7 +2028,7 @@ pub fn readFileAllocOptions(
2026/// If `limit` is exceeded:2028/// If `limit` is exceeded:
2027/// * The array list's length is increased by exactly one byte past `limit`.2029/// * The array list's length is increased by exactly one byte past `limit`.
2028/// * The file seek position is advanced by exactly one byte past `limit`.2030/// * The file seek position is advanced by exactly one byte past `limit`.
2029/// * `error.FileTooBig` is returned.2031/// * `error.StreamTooLong` is returned.
2030pub fn readFileIntoArrayList(2032pub fn readFileIntoArrayList(
2031 dir: Dir,2033 dir: Dir,
2032 /// On Windows, should be encoded as [WTF-8](https://simonsapin.github.io/wtf-8/).2034 /// On Windows, should be encoded as [WTF-8](https://simonsapin.github.io/wtf-8/).
...@@ -2040,7 +2042,7 @@ pub fn readFileIntoArrayList(...@@ -2040,7 +2042,7 @@ pub fn readFileIntoArrayList(
2040 size_hint: ?usize,2042 size_hint: ?usize,
2041 comptime alignment: ?std.mem.Alignment,2043 comptime alignment: ?std.mem.Alignment,
2042 list: *std.ArrayListAlignedUnmanaged(u8, alignment),2044 list: *std.ArrayListAlignedUnmanaged(u8, alignment),
2043) (File.OpenError || File.ReadAllocError)!void {2045) ReadFileAllocError!void {
2044 var file = try dir.openFile(file_path, .{});2046 var file = try dir.openFile(file_path, .{});
2045 defer file.close();2047 defer file.close();
20462048
...@@ -2049,14 +2051,19 @@ pub fn readFileIntoArrayList(...@@ -2049,14 +2051,19 @@ pub fn readFileIntoArrayList(
2049 try list.ensureUnusedCapacity(gpa, size);2051 try list.ensureUnusedCapacity(gpa, size);
2050 } else if (file.getEndPos()) |size| {2052 } else if (file.getEndPos()) |size| {
2051 // If the file size doesn't fit a usize it'll be certainly exceed the limit.2053 // If the file size doesn't fit a usize it'll be certainly exceed the limit.
2052 try list.ensureUnusedCapacity(gpa, std.math.cast(usize, size) orelse return error.FileTooBig);2054 try list.ensureUnusedCapacity(gpa, std.math.cast(usize, size) orelse return error.StreamTooLong);
2053 } else |err| switch (err) {2055 } else |err| switch (err) {
2054 // Ignore most errors; size hint is only an optimization.2056 // Ignore most errors; size hint is only an optimization.
2055 error.Unexpected, error.AccessDenied, error.PermissionDenied => {},2057 error.Unexpected, error.AccessDenied, error.PermissionDenied => {},
2056 else => |e| return e,2058 else => |e| return e,
2057 }2059 }
20582060
2059 try file.readIntoArrayList(gpa, limit, alignment, list);2061 var file_reader = file.reader();
2062 file_reader.interface().readRemainingArrayList(gpa, alignment, list, limit) catch |err| switch (err) {
2063 error.OutOfMemory => return error.OutOfMemory,
2064 error.StreamTooLong => return error.StreamTooLong,
2065 error.ReadFailed => return file_reader.err.?,
2066 };
2060}2067}
20612068
2062pub const DeleteTreeError = error{2069pub const DeleteTreeError = error{
lib/std/fs/File.zig-40
...@@ -788,46 +788,6 @@ pub fn updateTimes(...@@ -788,46 +788,6 @@ pub fn updateTimes(
788 try posix.futimens(self.handle, &times);788 try posix.futimens(self.handle, &times);
789}789}
790790
791pub const ReadAllocError = ReadError || Allocator.Error || error{FileTooBig};
792
793/// Reads all the bytes from the current position to the end of the file.
794///
795/// On success, caller owns returned buffer.
796///
797/// If `limit` is exceeded, returns `error.FileTooBig`.
798pub fn readToEndAlloc(file: File, gpa: Allocator, limit: std.io.Reader.Limit) ReadAllocError![]u8 {
799 var buffer: std.ArrayListUnmanaged(u8) = .empty;
800 defer buffer.deinit(gpa);
801 try buffer.ensureUnusedCapacity(gpa, std.heap.page_size_min);
802 try readIntoArrayList(file, gpa, limit, null, &buffer);
803 return buffer.toOwnedSlice(gpa);
804}
805
806/// Reads all the bytes from the current position to the end of the file,
807/// appending them into the provided array list.
808///
809/// If `limit` is exceeded:
810/// * The array list's length is increased by exactly one byte past `limit`.
811/// * The file seek position is advanced by exactly one byte past `limit`.
812/// * `error.FileTooBig` is returned.
813pub fn readIntoArrayList(
814 file: File,
815 gpa: Allocator,
816 limit: std.io.Reader.Limit,
817 comptime alignment: ?std.mem.Alignment,
818 list: *std.ArrayListAlignedUnmanaged(u8, alignment),
819) ReadAllocError!void {
820 var remaining = limit;
821 while (true) {
822 try list.ensureUnusedCapacity(gpa, 1);
823 const buffer = remaining.slice1(list.unusedCapacitySlice());
824 const n = try read(file, buffer);
825 if (n == 0) return;
826 list.items.len += n;
827 remaining = remaining.subtract(n) orelse return error.FileTooBig;
828 }
829}
830
831pub const ReadError = posix.ReadError;791pub const ReadError = posix.ReadError;
832pub const PReadError = posix.PReadError;792pub const PReadError = posix.PReadError;
833793
lib/std/http/test.zig+23-23
...@@ -70,7 +70,7 @@ test "trailers" {...@@ -70,7 +70,7 @@ test "trailers" {
70 try req.send();70 try req.send();
71 try req.wait();71 try req.wait();
7272
73 const body = try req.reader().readAllAlloc(gpa, 8192);73 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
74 defer gpa.free(body);74 defer gpa.free(body);
7575
76 try expectEqualStrings("Hello, World!\n", body);76 try expectEqualStrings("Hello, World!\n", body);
...@@ -153,7 +153,7 @@ test "HTTP server handles a chunked transfer coding request" {...@@ -153,7 +153,7 @@ test "HTTP server handles a chunked transfer coding request" {
153 "content-type: text/plain\r\n" ++153 "content-type: text/plain\r\n" ++
154 "\r\n" ++154 "\r\n" ++
155 "message from server!\n";155 "message from server!\n";
156 const response = try stream.reader().readAllAlloc(gpa, expected_response.len);156 const response = try stream.reader().readRemainingAlloc(gpa, expected_response.len);
157 defer gpa.free(response);157 defer gpa.free(response);
158 try expectEqualStrings(expected_response, response);158 try expectEqualStrings(expected_response, response);
159}159}
...@@ -206,7 +206,7 @@ test "echo content server" {...@@ -206,7 +206,7 @@ test "echo content server" {
206 // request.head.target,206 // request.head.target,
207 //});207 //});
208208
209 const body = try (try request.reader()).readAllAlloc(std.testing.allocator, 8192);209 const body = try (try request.reader()).readRemainingAlloc(std.testing.allocator, 8192);
210 defer std.testing.allocator.free(body);210 defer std.testing.allocator.free(body);
211211
212 try expect(mem.startsWith(u8, request.head.target, "/echo-content"));212 try expect(mem.startsWith(u8, request.head.target, "/echo-content"));
...@@ -291,7 +291,7 @@ test "Server.Request.respondStreaming non-chunked, unknown content-length" {...@@ -291,7 +291,7 @@ test "Server.Request.respondStreaming non-chunked, unknown content-length" {
291 var writer = stream.writer().unbuffered();291 var writer = stream.writer().unbuffered();
292 try writer.writeAll(request_bytes);292 try writer.writeAll(request_bytes);
293293
294 const response = try stream.reader().readAllAlloc(gpa, 8192);294 const response = try stream.reader().readRemainingAlloc(gpa, 8192);
295 defer gpa.free(response);295 defer gpa.free(response);
296296
297 var expected_response = std.ArrayList(u8).init(gpa);297 var expected_response = std.ArrayList(u8).init(gpa);
...@@ -362,7 +362,7 @@ test "receiving arbitrary http headers from the client" {...@@ -362,7 +362,7 @@ test "receiving arbitrary http headers from the client" {
362 var writer = stream_writer.interface().unbuffered();362 var writer = stream_writer.interface().unbuffered();
363 try writer.writeAll(request_bytes);363 try writer.writeAll(request_bytes);
364364
365 const response = try stream.reader().readAllAlloc(gpa, 8192);365 const response = try stream.reader().readRemainingAlloc(gpa, .limited(8192));
366 defer gpa.free(response);366 defer gpa.free(response);
367367
368 var expected_response = std.ArrayList(u8).init(gpa);368 var expected_response = std.ArrayList(u8).init(gpa);
...@@ -419,7 +419,7 @@ test "general client/server API coverage" {...@@ -419,7 +419,7 @@ test "general client/server API coverage" {
419 });419 });
420420
421 const gpa = std.testing.allocator;421 const gpa = std.testing.allocator;
422 const body = try (try request.reader()).readAllAlloc(gpa, 8192);422 const body = try (try request.reader()).readRemainingAlloc(gpa, .limited(8192));
423 defer gpa.free(body);423 defer gpa.free(body);
424424
425 if (mem.startsWith(u8, request.head.target, "/get")) {425 if (mem.startsWith(u8, request.head.target, "/get")) {
...@@ -568,7 +568,7 @@ test "general client/server API coverage" {...@@ -568,7 +568,7 @@ test "general client/server API coverage" {
568 try req.send();568 try req.send();
569 try req.wait();569 try req.wait();
570570
571 const body = try req.reader().readAllAlloc(gpa, 8192);571 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
572 defer gpa.free(body);572 defer gpa.free(body);
573573
574 try expectEqualStrings("Hello, World!\n", body);574 try expectEqualStrings("Hello, World!\n", body);
...@@ -593,7 +593,7 @@ test "general client/server API coverage" {...@@ -593,7 +593,7 @@ test "general client/server API coverage" {
593 try req.send();593 try req.send();
594 try req.wait();594 try req.wait();
595595
596 const body = try req.reader().readAllAlloc(gpa, 8192 * 1024);596 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192 * 1024));
597 defer gpa.free(body);597 defer gpa.free(body);
598598
599 try expectEqual(@as(usize, 14 * 1024 + 14 * 10), body.len);599 try expectEqual(@as(usize, 14 * 1024 + 14 * 10), body.len);
...@@ -617,7 +617,7 @@ test "general client/server API coverage" {...@@ -617,7 +617,7 @@ test "general client/server API coverage" {
617 try req.send();617 try req.send();
618 try req.wait();618 try req.wait();
619619
620 const body = try req.reader().readAllAlloc(gpa, 8192);620 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
621 defer gpa.free(body);621 defer gpa.free(body);
622622
623 try expectEqualStrings("", body);623 try expectEqualStrings("", body);
...@@ -643,7 +643,7 @@ test "general client/server API coverage" {...@@ -643,7 +643,7 @@ test "general client/server API coverage" {
643 try req.send();643 try req.send();
644 try req.wait();644 try req.wait();
645645
646 const body = try req.reader().readAllAlloc(gpa, 8192);646 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
647 defer gpa.free(body);647 defer gpa.free(body);
648648
649 try expectEqualStrings("Hello, World!\n", body);649 try expectEqualStrings("Hello, World!\n", body);
...@@ -668,7 +668,7 @@ test "general client/server API coverage" {...@@ -668,7 +668,7 @@ test "general client/server API coverage" {
668 try req.send();668 try req.send();
669 try req.wait();669 try req.wait();
670670
671 const body = try req.reader().readAllAlloc(gpa, 8192);671 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
672 defer gpa.free(body);672 defer gpa.free(body);
673673
674 try expectEqualStrings("", body);674 try expectEqualStrings("", body);
...@@ -695,7 +695,7 @@ test "general client/server API coverage" {...@@ -695,7 +695,7 @@ test "general client/server API coverage" {
695 try req.send();695 try req.send();
696 try req.wait();696 try req.wait();
697697
698 const body = try req.reader().readAllAlloc(gpa, 8192);698 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
699 defer gpa.free(body);699 defer gpa.free(body);
700700
701 try expectEqualStrings("Hello, World!\n", body);701 try expectEqualStrings("Hello, World!\n", body);
...@@ -725,7 +725,7 @@ test "general client/server API coverage" {...@@ -725,7 +725,7 @@ test "general client/server API coverage" {
725725
726 try std.testing.expectEqual(.ok, req.response.status);726 try std.testing.expectEqual(.ok, req.response.status);
727727
728 const body = try req.reader().readAllAlloc(gpa, 8192);728 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
729 defer gpa.free(body);729 defer gpa.free(body);
730730
731 try expectEqualStrings("", body);731 try expectEqualStrings("", body);
...@@ -764,7 +764,7 @@ test "general client/server API coverage" {...@@ -764,7 +764,7 @@ test "general client/server API coverage" {
764 try req.send();764 try req.send();
765 try req.wait();765 try req.wait();
766766
767 const body = try req.reader().readAllAlloc(gpa, 8192);767 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
768 defer gpa.free(body);768 defer gpa.free(body);
769769
770 try expectEqualStrings("Hello, World!\n", body);770 try expectEqualStrings("Hello, World!\n", body);
...@@ -788,7 +788,7 @@ test "general client/server API coverage" {...@@ -788,7 +788,7 @@ test "general client/server API coverage" {
788 try req.send();788 try req.send();
789 try req.wait();789 try req.wait();
790790
791 const body = try req.reader().readAllAlloc(gpa, 8192);791 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
792 defer gpa.free(body);792 defer gpa.free(body);
793793
794 try expectEqualStrings("Hello, World!\n", body);794 try expectEqualStrings("Hello, World!\n", body);
...@@ -812,7 +812,7 @@ test "general client/server API coverage" {...@@ -812,7 +812,7 @@ test "general client/server API coverage" {
812 try req.send();812 try req.send();
813 try req.wait();813 try req.wait();
814814
815 const body = try req.reader().readAllAlloc(gpa, 8192);815 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
816 defer gpa.free(body);816 defer gpa.free(body);
817817
818 try expectEqualStrings("Hello, World!\n", body);818 try expectEqualStrings("Hello, World!\n", body);
...@@ -855,7 +855,7 @@ test "general client/server API coverage" {...@@ -855,7 +855,7 @@ test "general client/server API coverage" {
855 try req.send();855 try req.send();
856 try req.wait();856 try req.wait();
857857
858 const body = try req.reader().readAllAlloc(gpa, 8192);858 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
859 defer gpa.free(body);859 defer gpa.free(body);
860860
861 try expectEqualStrings("Encoded redirect successful!\n", body);861 try expectEqualStrings("Encoded redirect successful!\n", body);
...@@ -946,7 +946,7 @@ test "Server streams both reading and writing" {...@@ -946,7 +946,7 @@ test "Server streams both reading and writing" {
946 var server = http.Server.init(&connection_br, &connection_bw);946 var server = http.Server.init(&connection_br, &connection_bw);
947 var request = try server.receiveHead();947 var request = try server.receiveHead();
948 var read_buffer: [100]u8 = undefined;948 var read_buffer: [100]u8 = undefined;
949 var br = try request.reader().buffered(&read_buffer);949 var br = (try request.reader()).buffered(&read_buffer);
950 var response = try request.respondStreaming(.{950 var response = try request.respondStreaming(.{
951 .respond_options = .{951 .respond_options = .{
952 .transfer_encoding = .none, // Causes keep_alive=false952 .transfer_encoding = .none, // Causes keep_alive=false
...@@ -993,7 +993,7 @@ test "Server streams both reading and writing" {...@@ -993,7 +993,7 @@ test "Server streams both reading and writing" {
993993
994 try req.finish();994 try req.finish();
995995
996 const body = try req.reader().readAllAlloc(std.testing.allocator, 8192);996 const body = try req.reader().readRemainingAlloc(std.testing.allocator, .limited(8192));
997 defer std.testing.allocator.free(body);997 defer std.testing.allocator.free(body);
998998
999 try expectEqualStrings("ONE FISH", body);999 try expectEqualStrings("ONE FISH", body);
...@@ -1027,7 +1027,7 @@ fn echoTests(client: *http.Client, port: u16) !void {...@@ -1027,7 +1027,7 @@ fn echoTests(client: *http.Client, port: u16) !void {
10271027
1028 try req.wait();1028 try req.wait();
10291029
1030 const body = try req.reader().readAllAlloc(gpa, 8192);1030 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
1031 defer gpa.free(body);1031 defer gpa.free(body);
10321032
1033 try expectEqualStrings("Hello, World!\n", body);1033 try expectEqualStrings("Hello, World!\n", body);
...@@ -1062,7 +1062,7 @@ fn echoTests(client: *http.Client, port: u16) !void {...@@ -1062,7 +1062,7 @@ fn echoTests(client: *http.Client, port: u16) !void {
10621062
1063 try req.wait();1063 try req.wait();
10641064
1065 const body = try req.reader().readAllAlloc(gpa, 8192);1065 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
1066 defer gpa.free(body);1066 defer gpa.free(body);
10671067
1068 try expectEqualStrings("Hello, World!\n", body);1068 try expectEqualStrings("Hello, World!\n", body);
...@@ -1118,7 +1118,7 @@ fn echoTests(client: *http.Client, port: u16) !void {...@@ -1118,7 +1118,7 @@ fn echoTests(client: *http.Client, port: u16) !void {
1118 try req.wait();1118 try req.wait();
1119 try expectEqual(.ok, req.response.status);1119 try expectEqual(.ok, req.response.status);
11201120
1121 const body = try req.reader().readAllAlloc(gpa, 8192);1121 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
1122 defer gpa.free(body);1122 defer gpa.free(body);
11231123
1124 try expectEqualStrings("Hello, World!\n", body);1124 try expectEqualStrings("Hello, World!\n", body);
...@@ -1258,7 +1258,7 @@ test "redirect to different connection" {...@@ -1258,7 +1258,7 @@ test "redirect to different connection" {
1258 try req.send();1258 try req.send();
1259 try req.wait();1259 try req.wait();
12601260
1261 const body = try req.reader().readAllAlloc(gpa, 8192);1261 const body = try req.reader().readRemainingAlloc(gpa, .limited(8192));
1262 defer gpa.free(body);1262 defer gpa.free(body);
12631263
1264 try expectEqualStrings("good job, you pass", body);1264 try expectEqualStrings("good job, you pass", body);
lib/std/io/BufferedReader.zig+72-3
...@@ -7,6 +7,7 @@ const testing = std.testing;...@@ -7,6 +7,7 @@ const testing = std.testing;
7const BufferedWriter = std.io.BufferedWriter;7const BufferedWriter = std.io.BufferedWriter;
8const Reader = std.io.Reader;8const Reader = std.io.Reader;
9const Allocator = std.mem.Allocator;9const Allocator = std.mem.Allocator;
10const ArrayList = std.ArrayListUnmanaged;
1011
11const BufferedReader = @This();12const BufferedReader = @This();
1213
...@@ -373,6 +374,8 @@ pub fn readShort(br: *BufferedReader, buffer: []u8) Reader.ShortError!usize {...@@ -373,6 +374,8 @@ pub fn readShort(br: *BufferedReader, buffer: []u8) Reader.ShortError!usize {
373 @panic("TODO");374 @panic("TODO");
374}375}
375376
377pub const ReadAllocError = Reader.Error || Allocator.Error;
378
376/// The function is inline to avoid the dead code in case `endian` is379/// The function is inline to avoid the dead code in case `endian` is
377/// comptime-known and matches host endianness.380/// comptime-known and matches host endianness.
378pub inline fn readSliceEndianAlloc(381pub inline fn readSliceEndianAlloc(
...@@ -389,8 +392,6 @@ pub inline fn readSliceEndianAlloc(...@@ -389,8 +392,6 @@ pub inline fn readSliceEndianAlloc(
389 return dest;392 return dest;
390}393}
391394
392pub const ReadAllocError = Reader.Error || Allocator.Error;
393
394pub fn readSliceAlloc(br: *BufferedReader, allocator: Allocator, len: usize) ReadAllocError![]u8 {395pub fn readSliceAlloc(br: *BufferedReader, allocator: Allocator, len: usize) ReadAllocError![]u8 {
395 const dest = try allocator.alloc(u8, len);396 const dest = try allocator.alloc(u8, len);
396 errdefer allocator.free(dest);397 errdefer allocator.free(dest);
...@@ -398,6 +399,74 @@ pub fn readSliceAlloc(br: *BufferedReader, allocator: Allocator, len: usize) Rea...@@ -398,6 +399,74 @@ pub fn readSliceAlloc(br: *BufferedReader, allocator: Allocator, len: usize) Rea
398 return dest;399 return dest;
399}400}
400401
402/// Transfers all bytes from the current position to the end of the stream, up
403/// to `limit`, returning them as a caller-owned allocated slice.
404///
405/// If `limit` is exceeded, returns `error.StreamTooLong`. In such case, the
406/// stream is advanced an unspecified amount, and the consumed data is
407/// unrecoverable. The other function listed below does not have this caveat.
408///
409/// Asserts `br` was initialized with at least one byte of storage capacity.
410///
411/// See also:
412/// * `readRemainingArrayList`
413pub fn readRemainingAlloc(r: Reader, gpa: Allocator, limit: Reader.Limit) Reader.LimitedAllocError![]u8 {
414 var buffer: ArrayList(u8) = .empty;
415 defer buffer.deinit(gpa);
416 try readRemainingArrayList(r, gpa, null, &buffer, limit);
417 return buffer.toOwnedSlice(gpa);
418}
419
420/// Transfers all bytes from the current position to the end of the stream, up
421/// to `limit`, appending them to `list`.
422///
423/// If `limit` would be exceeded, `error.StreamTooLong` is returned instead. In
424/// such case, the stream is in a well-defined state. The next byte that would
425/// be read will be the first one to exceed `limit`, and all preceeding bytes
426/// have been appended to `list`.
427///
428/// Asserts `br` was initialized with at least one byte of storage capacity.
429///
430/// See also:
431/// * `readRemainingAlloc`
432pub fn readRemainingArrayList(
433 br: *BufferedReader,
434 gpa: Allocator,
435 comptime alignment: ?std.mem.Alignment,
436 list: *std.ArrayListAlignedUnmanaged(u8, alignment),
437 limit: Reader.Limit,
438) Reader.LimitedAllocError!void {
439 const buffer = br.buffer;
440 const buffered = buffer[br.seek..br.end];
441 const copy_len = limit.minInt(buffered.len);
442 try list.ensureUnusedCapacity(gpa, copy_len);
443 @memcpy(list.unusedCapacitySlice()[0..copy_len], buffer[0..copy_len]);
444 list.items.len += copy_len;
445 br.seek += copy_len;
446 if (copy_len == buffered.len) {
447 br.seek = 0;
448 br.end = 0;
449 }
450 var remaining = limit.subtract(copy_len).?;
451 while (true) {
452 try list.ensureUnusedCapacity(gpa, 1);
453 const dest = remaining.slice(list.unusedCapacitySlice());
454 const additional_buffer = if (@intFromEnum(remaining) == dest.len) buffer else &.{};
455 const n = br.unbuffered_reader.readVec(&.{ dest, additional_buffer }) catch |err| switch (err) {
456 error.EndOfStream => break,
457 error.ReadFailed => return error.ReadFailed,
458 };
459 if (n >= dest.len) {
460 br.end = n - dest.len;
461 list.items.len += dest.len;
462 if (n == dest.len) return;
463 return error.StreamTooLong;
464 }
465 list.items.len += n;
466 remaining = remaining.subtract(n).?;
467 }
468}
469
401pub const DelimiterInclusiveError = error{470pub const DelimiterInclusiveError = error{
402 /// See the `Reader` implementation for detailed diagnostics.471 /// See the `Reader` implementation for detailed diagnostics.
403 ReadFailed,472 ReadFailed,
...@@ -775,7 +844,7 @@ pub fn writableSliceGreedyAlloc(...@@ -775,7 +844,7 @@ pub fn writableSliceGreedyAlloc(
775 br.seek = 0;844 br.seek = 0;
776 }845 }
777 {846 {
778 var list: std.ArrayListUnmanaged(u8) = .{847 var list: ArrayList(u8) = .{
779 .items = br.buffer[0..br.end],848 .items = br.buffer[0..br.end],
780 .capacity = br.buffer.len,849 .capacity = br.buffer.len,
781 };850 };
lib/std/io/Reader.zig+49-19
...@@ -2,6 +2,9 @@ const std = @import("../std.zig");...@@ -2,6 +2,9 @@ const std = @import("../std.zig");
2const Reader = @This();2const Reader = @This();
3const assert = std.debug.assert;3const assert = std.debug.assert;
4const BufferedWriter = std.io.BufferedWriter;4const BufferedWriter = std.io.BufferedWriter;
5const BufferedReader = std.io.BufferedReader;
6const Allocator = std.mem.Allocator;
7const ArrayList = std.ArrayListUnmanaged;
58
6context: ?*anyopaque,9context: ?*anyopaque,
7vtable: *const VTable,10vtable: *const VTable,
...@@ -166,29 +169,56 @@ pub fn discardRemaining(r: Reader) ShortError!usize {...@@ -166,29 +169,56 @@ pub fn discardRemaining(r: Reader) ShortError!usize {
166 }169 }
167}170}
168171
169pub const ReadAllocError = std.mem.Allocator.Error || ShortError;172pub const LimitedAllocError = Allocator.Error || ShortError || error{StreamTooLong};
170173
171/// Allocates enough memory to hold all the contents of the stream. If the allocated174/// Transfers all bytes from the current position to the end of the stream, up
172/// memory would be greater than `max_size`, returns `error.StreamTooLong`.175/// to `limit`, returning them as a caller-owned allocated slice.
173///176///
174/// Caller owns returned memory.177/// If `limit` is exceeded, returns `error.StreamTooLong`. In such case, the
178/// stream is advanced one byte beyond the limit, and the consumed data is
179/// unrecoverable. Other functions listed below do not have this caveat.
175///180///
176/// If this function returns an error, the contents from the stream read so far are lost.181/// See also:
177pub fn readAlloc(r: Reader, gpa: std.mem.Allocator, max_size: usize) ReadAllocError![]u8 {182/// * `readRemainingArrayList`
178 const readFn = r.vtable.read;183/// * `BufferedReader.readRemainingArrayList`
179 var aw: std.io.AllocatingWriter = undefined;184pub fn readRemainingAlloc(r: Reader, gpa: Allocator, limit: Reader.Limit) LimitedAllocError![]u8 {
180 aw.init(gpa);185 var buffer: ArrayList(u8) = .empty;
181 errdefer aw.deinit();186 defer buffer.deinit(gpa);
182 var remaining = max_size;187 try readRemainingArrayList(r, gpa, null, &buffer, limit);
183 while (remaining > 0) {188 return buffer.toOwnedSlice(gpa);
184 const n = readFn(r.context, &aw.buffered_writer, .limited(remaining)) catch |err| switch (err) {189}
185 error.WriteFailed => return error.OutOfMemory,190
186 error.EndOfStream => break,191/// Transfers all bytes from the current position to the end of the stream, up
192/// to `limit`, appending them to `list`.
193///
194/// If `limit` is exceeded:
195/// * The array list's length is increased by exactly one byte past `limit`.
196/// * The stream seek position is advanced by exactly one byte past `limit`.
197/// * `error.StreamTooLong` is returned.
198///
199/// The other function listed below has different semantics for an exceeded
200/// limit.
201///
202/// See also:
203/// * `BufferedReader.readRemainingArrayList`
204pub fn readRemainingArrayList(
205 r: Reader,
206 gpa: Allocator,
207 comptime alignment: ?std.mem.Alignment,
208 list: *std.ArrayListAlignedUnmanaged(u8, alignment),
209 limit: Limit,
210) LimitedAllocError!void {
211 var remaining = limit;
212 while (true) {
213 try list.ensureUnusedCapacity(gpa, 1);
214 const buffer = remaining.slice1(list.unusedCapacitySlice());
215 const n = r.vtable.readVec(r.context, &.{buffer}) catch |err| switch (err) {
216 error.EndOfStream => return,
187 error.ReadFailed => return error.ReadFailed,217 error.ReadFailed => return error.ReadFailed,
188 };218 };
189 remaining -= n;219 list.items.len += n;
220 remaining = remaining.subtract(n) orelse return error.StreamTooLong;
190 }221 }
191 return aw.toOwnedSlice();
192}222}
193223
194pub const failing: Reader = .{224pub const failing: Reader = .{
...@@ -209,11 +239,11 @@ pub const ending: Reader = .{...@@ -209,11 +239,11 @@ pub const ending: Reader = .{
209 },239 },
210};240};
211241
212pub fn unbuffered(r: Reader) std.io.BufferedReader {242pub fn unbuffered(r: Reader) BufferedReader {
213 return buffered(r, &.{});243 return buffered(r, &.{});
214}244}
215245
216pub fn buffered(r: Reader, buffer: []u8) std.io.BufferedReader {246pub fn buffered(r: Reader, buffer: []u8) BufferedReader {
217 return .{247 return .{
218 .unbuffered_reader = r,248 .unbuffered_reader = r,
219 .seek = 0,249 .seek = 0,