authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-16 21:02:05-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
log743a0c966de933e3d0a271f942c2db525df5dbe8
tree4b5e037f933ca37963a68f68b4dd05a517d8dbca
parent0ddcb8341822cf9e3fd555c29a9e011b9a91d9fc

std.http.Client: remove bad decisions from fetch()

* "storage" is a better name than "strategy". * The most flexible memory-based storage API is appending to an ArrayList. * HTTP method should default to POST if there is a payload. * Avoid storing unnecessary data in the FetchResult * Avoid the need for a deinit() method in the FetchResult The decisions that this logic made about how to handle files is beyond repair: - fail to use sendfile() on a plain connection - redundant stat - does not handle arbitrary streams So, file-based response storage is no longer supported. Users should use the lower-level open() API which allows avoiding these pitfalls.

2 files changed, 50 insertions(+), 78 deletions(-)

lib/std/http/Client.zig+42-73
......@@ -700,7 +700,7 @@ pub const Request = struct {
700700 pub const SendError = Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding };
701701
702702 pub const SendOptions = struct {
703 /// Specifies that the uri should be used as is. You guarantee that the uri is already escaped.
703 /// Specifies that the uri is already escaped.
704704 raw_uri: bool = false,
705705 };
706706
......@@ -1562,12 +1562,16 @@ pub fn open(
15621562
15631563pub const FetchOptions = struct {
15641564 server_header_buffer: ?[]u8 = null,
1565 response_strategy: ResponseStrategy = .{ .storage = .{ .dynamic = 16 * 1024 * 1024 } },
15661565 redirect_behavior: ?Request.RedirectBehavior = null,
15671566
1567 /// If the server sends a body, it will be appended to this ArrayList.
1568 /// `max_append_size` provides an upper limit for how much they can grow.
1569 response_storage: ResponseStorage = .ignore,
1570 max_append_size: ?usize = null,
1571
15681572 location: Location,
1569 method: http.Method = .GET,
1570 payload: Payload = .none,
1573 method: ?http.Method = null,
1574 payload: ?[]const u8 = null,
15711575 raw_uri: bool = false,
15721576
15731577 /// Standard headers that have default, but overridable, behavior.
......@@ -1586,111 +1590,76 @@ pub const FetchOptions = struct {
15861590 uri: Uri,
15871591 };
15881592
1589 pub const Payload = union(enum) {
1590 string: []const u8,
1591 file: std.fs.File,
1592 none,
1593 };
1594
1595 pub const ResponseStrategy = union(enum) {
1596 storage: StorageStrategy,
1597 file: std.fs.File,
1598 none,
1599 };
1600
1601 pub const StorageStrategy = union(enum) {
1602 /// In this case, the client's Allocator will be used to store the
1603 /// entire HTTP header. This value is the maximum total size of
1604 /// HTTP headers allowed, otherwise
1605 /// error.HttpHeadersExceededSizeLimit is returned from read().
1606 dynamic: usize,
1607 /// This is used to store the entire HTTP header. If the HTTP
1608 /// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
1609 /// is returned from read(). When this is used, `error.OutOfMemory`
1610 /// cannot be returned from `read()`.
1611 static: []u8,
1593 pub const ResponseStorage = union(enum) {
1594 ignore,
1595 /// Only the existing capacity will be used.
1596 static: *std.ArrayListUnmanaged(u8),
1597 dynamic: *std.ArrayList(u8),
16121598 };
16131599};
16141600
16151601pub const FetchResult = struct {
16161602 status: http.Status,
1617 body: ?[]const u8 = null,
1618
1619 allocator: Allocator,
1620 options: FetchOptions,
1621
1622 pub fn deinit(res: *FetchResult) void {
1623 if (res.options.response_strategy == .storage and
1624 res.options.response_strategy.storage == .dynamic)
1625 {
1626 if (res.body) |body| res.allocator.free(body);
1627 }
1628 }
16291603};
16301604
16311605/// Perform a one-shot HTTP request with the provided options.
16321606///
16331607/// This function is threadsafe.
1634pub fn fetch(client: *Client, allocator: Allocator, options: FetchOptions) !FetchResult {
1608pub fn fetch(client: *Client, options: FetchOptions) !FetchResult {
16351609 const uri = switch (options.location) {
16361610 .url => |u| try Uri.parse(u),
16371611 .uri => |u| u,
16381612 };
16391613 var server_header_buffer: [16 * 1024]u8 = undefined;
16401614
1641 var req = try open(client, options.method, uri, .{
1615 const method: http.Method = options.method orelse
1616 if (options.payload != null) .POST else .GET;
1617
1618 var req = try open(client, method, uri, .{
16421619 .server_header_buffer = options.server_header_buffer orelse &server_header_buffer,
16431620 .redirect_behavior = options.redirect_behavior orelse
1644 if (options.payload == .none) @enumFromInt(3) else .unhandled,
1621 if (options.payload == null) @enumFromInt(3) else .unhandled,
16451622 .headers = options.headers,
16461623 .extra_headers = options.extra_headers,
16471624 .privileged_headers = options.privileged_headers,
16481625 });
16491626 defer req.deinit();
16501627
1651 switch (options.payload) {
1652 .string => |str| req.transfer_encoding = .{ .content_length = str.len },
1653 .file => |file| req.transfer_encoding = .{ .content_length = (try file.stat()).size },
1654 .none => {},
1655 }
1628 if (options.payload) |payload| req.transfer_encoding = .{ .content_length = payload.len };
16561629
16571630 try req.send(.{ .raw_uri = options.raw_uri });
16581631
1659 switch (options.payload) {
1660 .string => |str| try req.writeAll(str),
1661 .file => |file| {
1662 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();
1663 try fifo.pump(file.reader(), req.writer());
1664 },
1665 .none => {},
1666 }
1632 if (options.payload) |payload| try req.writeAll(payload);
16671633
16681634 try req.finish();
16691635 try req.wait();
16701636
1671 var res: FetchResult = .{
1672 .status = req.response.status,
1673 .allocator = allocator,
1674 .options = options,
1675 };
1676
1677 switch (options.response_strategy) {
1678 .storage => |storage| switch (storage) {
1679 .dynamic => |max| res.body = try req.reader().readAllAlloc(allocator, max),
1680 .static => |buf| res.body = buf[0..try req.reader().readAll(buf)],
1637 switch (options.response_storage) {
1638 .ignore => {
1639 // Take advantage of request internals to discard the response body
1640 // and make the connection available for another request.
1641 req.response.skip = true;
1642 assert(try req.transferRead(&.{}) == 0); // No buffer is necessary when skipping.
16811643 },
1682 .file => |file| {
1683 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();
1684 try fifo.pump(req.reader(), file.writer());
1644 .dynamic => |list| {
1645 const max_append_size = options.max_append_size orelse 2 * 1024 * 1024;
1646 try req.reader().readAllArrayList(list, max_append_size);
16851647 },
1686 .none => { // Take advantage of request internals to discard the response body and make the connection available for another request.
1687 req.response.skip = true;
1688
1689 assert(try req.transferRead(&.{}) == 0); // we're skipping, no buffer is necessary
1648 .static => |list| {
1649 const buf = b: {
1650 const buf = list.unusedCapacitySlice();
1651 if (options.max_append_size) |len| {
1652 if (len < buf.len) break :b buf[0..len];
1653 }
1654 break :b buf;
1655 };
1656 list.items.len += try req.reader().readAll(buf);
16901657 },
16911658 }
16921659
1693 return res;
1660 return .{
1661 .status = req.response.status,
1662 };
16941663}
16951664
16961665test {
test/standalone/http.zig+8-5
......@@ -586,17 +586,20 @@ pub fn main() !void {
586586 defer calloc.free(location);
587587
588588 log.info("{s}", .{location});
589 var res = try client.fetch(calloc, .{
589 var body = std.ArrayList(u8).init(calloc);
590 defer body.deinit();
591
592 const res = try client.fetch(.{
590593 .location = .{ .url = location },
591594 .method = .POST,
592 .payload = .{ .string = "Hello, World!\n" },
595 .payload = "Hello, World!\n",
593596 .extra_headers = &.{
594597 .{ .name = "content-type", .value = "text/plain" },
595598 },
599 .response_storage = .{ .dynamic = &body },
596600 });
597 defer res.deinit();
598
599 try testing.expectEqualStrings("Hello, World!\n", res.body.?);
601 try testing.expectEqual(.ok, res.status);
602 try testing.expectEqualStrings("Hello, World!\n", body.items);
600603 }
601604
602605 { // expect: 100-continue