authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-07 16:11:59-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-07 19:54:39-07:00
log5ce8e9325b7aa15cbcc77221fc7075b6c46619cc
treea29cd6b99bc833f566d0f086202d99c78d80bfaa
parent5998a8cebe3973d70c258b2a1440c5c3252d3539

std.http.Client: fix fetching by adding a buffer

3 files changed, 27 insertions(+), 30 deletions(-)

lib/std/http.zig+5-4
...@@ -412,7 +412,7 @@ pub const Reader = struct {...@@ -412,7 +412,7 @@ pub const Reader = struct {
412 /// * `interfaceDecompressing`412 /// * `interfaceDecompressing`
413 pub fn bodyReader(413 pub fn bodyReader(
414 reader: *Reader,414 reader: *Reader,
415 buffer: []u8,415 transfer_buffer: []u8,
416 transfer_encoding: TransferEncoding,416 transfer_encoding: TransferEncoding,
417 content_length: ?u64,417 content_length: ?u64,
418 ) *std.Io.Reader {418 ) *std.Io.Reader {
...@@ -421,7 +421,7 @@ pub const Reader = struct {...@@ -421,7 +421,7 @@ pub const Reader = struct {
421 .chunked => {421 .chunked => {
422 reader.state = .{ .body_remaining_chunk_len = .head };422 reader.state = .{ .body_remaining_chunk_len = .head };
423 reader.interface = .{423 reader.interface = .{
424 .buffer = buffer,424 .buffer = transfer_buffer,
425 .seek = 0,425 .seek = 0,
426 .end = 0,426 .end = 0,
427 .vtable = &.{427 .vtable = &.{
...@@ -435,7 +435,7 @@ pub const Reader = struct {...@@ -435,7 +435,7 @@ pub const Reader = struct {
435 if (content_length) |len| {435 if (content_length) |len| {
436 reader.state = .{ .body_remaining_content_length = len };436 reader.state = .{ .body_remaining_content_length = len };
437 reader.interface = .{437 reader.interface = .{
438 .buffer = buffer,438 .buffer = transfer_buffer,
439 .seek = 0,439 .seek = 0,
440 .end = 0,440 .end = 0,
441 .vtable = &.{441 .vtable = &.{
...@@ -460,6 +460,7 @@ pub const Reader = struct {...@@ -460,6 +460,7 @@ pub const Reader = struct {
460 /// * `interface`460 /// * `interface`
461 pub fn bodyReaderDecompressing(461 pub fn bodyReaderDecompressing(
462 reader: *Reader,462 reader: *Reader,
463 transfer_buffer: []u8,
463 transfer_encoding: TransferEncoding,464 transfer_encoding: TransferEncoding,
464 content_length: ?u64,465 content_length: ?u64,
465 content_encoding: ContentEncoding,466 content_encoding: ContentEncoding,
...@@ -488,7 +489,7 @@ pub const Reader = struct {...@@ -488,7 +489,7 @@ pub const Reader = struct {
488 .compress => unreachable,489 .compress => unreachable,
489 }490 }
490 }491 }
491 const transfer_reader = bodyReader(reader, &.{}, transfer_encoding, content_length);492 const transfer_reader = bodyReader(reader, transfer_buffer, transfer_encoding, content_length);
492 return decompressor.init(transfer_reader, decompression_buffer, content_encoding);493 return decompressor.init(transfer_reader, decompression_buffer, content_encoding);
493 }494 }
494495
lib/std/http/Client.zig+17-22
...@@ -13,8 +13,8 @@ const net = std.net;...@@ -13,8 +13,8 @@ const net = std.net;
13const Uri = std.Uri;13const Uri = std.Uri;
14const Allocator = mem.Allocator;14const Allocator = mem.Allocator;
15const assert = std.debug.assert;15const assert = std.debug.assert;
16const Writer = std.io.Writer;16const Writer = std.Io.Writer;
17const Reader = std.io.Reader;17const Reader = std.Io.Reader;
1818
19const Client = @This();19const Client = @This();
2020
...@@ -704,12 +704,12 @@ pub const Response = struct {...@@ -704,12 +704,12 @@ pub const Response = struct {
704 ///704 ///
705 /// See also:705 /// See also:
706 /// * `readerDecompressing`706 /// * `readerDecompressing`
707 pub fn reader(response: *Response, buffer: []u8) *Reader {707 pub fn reader(response: *Response, transfer_buffer: []u8) *Reader {
708 response.head.invalidateStrings();708 response.head.invalidateStrings();
709 const req = response.request;709 const req = response.request;
710 if (!req.method.responseHasBody()) return .ending;710 if (!req.method.responseHasBody()) return .ending;
711 const head = &response.head;711 const head = &response.head;
712 return req.reader.bodyReader(buffer, head.transfer_encoding, head.content_length);712 return req.reader.bodyReader(transfer_buffer, head.transfer_encoding, head.content_length);
713 }713 }
714714
715 /// If compressed body has been negotiated this will return decompressed bytes.715 /// If compressed body has been negotiated this will return decompressed bytes.
...@@ -723,12 +723,14 @@ pub const Response = struct {...@@ -723,12 +723,14 @@ pub const Response = struct {
723 /// * `reader`723 /// * `reader`
724 pub fn readerDecompressing(724 pub fn readerDecompressing(
725 response: *Response,725 response: *Response,
726 transfer_buffer: []u8,
726 decompressor: *http.Decompressor,727 decompressor: *http.Decompressor,
727 decompression_buffer: []u8,728 decompression_buffer: []u8,
728 ) *Reader {729 ) *Reader {
729 response.head.invalidateStrings();730 response.head.invalidateStrings();
730 const head = &response.head;731 const head = &response.head;
731 return response.request.reader.bodyReaderDecompressing(732 return response.request.reader.bodyReaderDecompressing(
733 transfer_buffer,
732 head.transfer_encoding,734 head.transfer_encoding,
733 head.content_length,735 head.content_length,
734 head.content_encoding,736 head.content_encoding,
...@@ -1322,7 +1324,7 @@ pub const basic_authorization = struct {...@@ -1322,7 +1324,7 @@ pub const basic_authorization = struct {
1322 const user: Uri.Component = uri.user orelse .empty;1324 const user: Uri.Component = uri.user orelse .empty;
1323 const password: Uri.Component = uri.password orelse .empty;1325 const password: Uri.Component = uri.password orelse .empty;
13241326
1325 var dw: std.io.Writer.Discarding = .init(&.{});1327 var dw: Writer.Discarding = .init(&.{});
1326 user.formatUser(&dw.writer) catch unreachable; // discarding1328 user.formatUser(&dw.writer) catch unreachable; // discarding
1327 const user_len = dw.count + dw.writer.end;1329 const user_len = dw.count + dw.writer.end;
13281330
...@@ -1696,8 +1698,8 @@ pub const FetchOptions = struct {...@@ -1696,8 +1698,8 @@ pub const FetchOptions = struct {
1696 /// `null` means it will be heap-allocated.1698 /// `null` means it will be heap-allocated.
1697 decompress_buffer: ?[]u8 = null,1699 decompress_buffer: ?[]u8 = null,
1698 redirect_behavior: ?Request.RedirectBehavior = null,1700 redirect_behavior: ?Request.RedirectBehavior = null,
1699 /// If the server sends a body, it will be stored here.1701 /// If the server sends a body, it will be written here.
1700 response_storage: ?ResponseStorage = null,1702 response_writer: ?*Writer = null,
17011703
1702 location: Location,1704 location: Location,
1703 method: ?http.Method = null,1705 method: ?http.Method = null,
...@@ -1725,7 +1727,7 @@ pub const FetchOptions = struct {...@@ -1725,7 +1727,7 @@ pub const FetchOptions = struct {
1725 list: *std.ArrayListUnmanaged(u8),1727 list: *std.ArrayListUnmanaged(u8),
1726 /// If null then only the existing capacity will be used.1728 /// If null then only the existing capacity will be used.
1727 allocator: ?Allocator = null,1729 allocator: ?Allocator = null,
1728 append_limit: std.io.Limit = .unlimited,1730 append_limit: std.Io.Limit = .unlimited,
1729 };1731 };
1730};1732};
17311733
...@@ -1778,7 +1780,7 @@ pub fn fetch(client: *Client, options: FetchOptions) FetchError!FetchResult {...@@ -1778,7 +1780,7 @@ pub fn fetch(client: *Client, options: FetchOptions) FetchError!FetchResult {
17781780
1779 var response = try req.receiveHead(redirect_buffer);1781 var response = try req.receiveHead(redirect_buffer);
17801782
1781 const storage = options.response_storage orelse {1783 const response_writer = options.response_writer orelse {
1782 const reader = response.reader(&.{});1784 const reader = response.reader(&.{});
1783 _ = reader.discardRemaining() catch |err| switch (err) {1785 _ = reader.discardRemaining() catch |err| switch (err) {
1784 error.ReadFailed => return response.bodyErr().?,1786 error.ReadFailed => return response.bodyErr().?,
...@@ -1794,21 +1796,14 @@ pub fn fetch(client: *Client, options: FetchOptions) FetchError!FetchResult {...@@ -1794,21 +1796,14 @@ pub fn fetch(client: *Client, options: FetchOptions) FetchError!FetchResult {
1794 };1796 };
1795 defer if (options.decompress_buffer == null) client.allocator.free(decompress_buffer);1797 defer if (options.decompress_buffer == null) client.allocator.free(decompress_buffer);
17961798
1799 var transfer_buffer: [64]u8 = undefined;
1797 var decompressor: http.Decompressor = undefined;1800 var decompressor: http.Decompressor = undefined;
1798 const reader = response.readerDecompressing(&decompressor, decompress_buffer);1801 const reader = response.readerDecompressing(&transfer_buffer, &decompressor, decompress_buffer);
1799 const list = storage.list;
18001802
1801 if (storage.allocator) |allocator| {1803 _ = reader.streamRemaining(response_writer) catch |err| switch (err) {
1802 reader.appendRemaining(allocator, null, list, storage.append_limit) catch |err| switch (err) {1804 error.ReadFailed => return response.bodyErr().?,
1803 error.ReadFailed => return response.bodyErr().?,1805 else => |e| return e,
1804 else => |e| return e,1806 };
1805 };
1806 } else {
1807 const buf = storage.append_limit.slice(list.unusedCapacitySlice());
1808 list.items.len += reader.readSliceShort(buf) catch |err| switch (err) {
1809 error.ReadFailed => return response.bodyErr().?,
1810 };
1811 }
18121807
1813 return .{ .status = response.head.status };1808 return .{ .status = response.head.status };
1814}1809}
lib/std/http/test.zig+5-4
...@@ -1006,8 +1006,9 @@ fn echoTests(client: *http.Client, port: u16) !void {...@@ -1006,8 +1006,9 @@ fn echoTests(client: *http.Client, port: u16) !void {
1006 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/echo-content#fetch", .{port});1006 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/echo-content#fetch", .{port});
1007 defer gpa.free(location);1007 defer gpa.free(location);
10081008
1009 var body: std.ArrayListUnmanaged(u8) = .empty;1009 var body: std.Io.Writer.Allocating = .init(gpa);
1010 defer body.deinit(gpa);1010 defer body.deinit();
1011 try body.ensureUnusedCapacity(64);
10111012
1012 const res = try client.fetch(.{1013 const res = try client.fetch(.{
1013 .location = .{ .url = location },1014 .location = .{ .url = location },
...@@ -1016,10 +1017,10 @@ fn echoTests(client: *http.Client, port: u16) !void {...@@ -1016,10 +1017,10 @@ fn echoTests(client: *http.Client, port: u16) !void {
1016 .extra_headers = &.{1017 .extra_headers = &.{
1017 .{ .name = "content-type", .value = "text/plain" },1018 .{ .name = "content-type", .value = "text/plain" },
1018 },1019 },
1019 .response_storage = .{ .allocator = gpa, .list = &body },1020 .response_writer = &body.writer,
1020 });1021 });
1021 try expectEqual(.ok, res.status);1022 try expectEqual(.ok, res.status);
1022 try expectEqualStrings("Hello, World!\n", body.items);1023 try expectEqualStrings("Hello, World!\n", body.getWritten());
1023 }1024 }
10241025
1025 { // expect: 100-continue1026 { // expect: 100-continue