authorgravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-08-11 20:34:59-05:00
committergravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-08-29 21:42:53-05:00
log5d40338f21b468c82d4bc2a1ac0a35c643126e74
tree61611275d65825176934b9da368ee23fbaeaeaef
parent49075d20557994da4eb341e7431de38a6df2088b
signaturelock-open Commit is signed but in an unrecognized format.

std.http: add Client.fetch and improve redirect logic

3 files changed, 217 insertions(+), 16 deletions(-)

lib/std/http/Client.zig+169-14
...@@ -365,8 +365,11 @@ pub const Response = struct {...@@ -365,8 +365,11 @@ pub const Response = struct {
365 if (trailing) continue;365 if (trailing) continue;
366366
367 if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {367 if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
368 if (res.content_length != null) return error.HttpHeadersInvalid;368 const content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength;
369 res.content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength;369
370 if (res.content_length != null and res.content_length != content_length) return error.HttpHeadersInvalid;
371
372 res.content_length = content_length;
370 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {373 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
371 // Transfer-Encoding: second, first374 // Transfer-Encoding: second, first
372 // Transfer-Encoding: deflate, chunked375 // Transfer-Encoding: deflate, chunked
...@@ -536,6 +539,8 @@ pub const Request = struct {...@@ -536,6 +539,8 @@ pub const Request = struct {
536539
537 /// Send the request to the server.540 /// Send the request to the server.
538 pub fn start(req: *Request) StartError!void {541 pub fn start(req: *Request) StartError!void {
542 if (!req.method.requestHasBody() and req.transfer_encoding != .none) return error.UnsupportedTransferEncoding;
543
539 var buffered = std.io.bufferedWriter(req.connection.?.data.writer());544 var buffered = std.io.bufferedWriter(req.connection.?.data.writer());
540 const w = buffered.writer();545 const w = buffered.writer();
541546
...@@ -607,7 +612,14 @@ pub const Request = struct {...@@ -607,7 +612,14 @@ pub const Request = struct {
607 }612 }
608 }613 }
609614
610 try w.print("{}", .{req.headers});615 for (req.headers.list.items) |entry| {
616 if (entry.value.len == 0) continue;
617
618 try w.writeAll(entry.name);
619 try w.writeAll(": ");
620 try w.writeAll(entry.value);
621 try w.writeAll("\r\n");
622 }
611623
612 try w.writeAll("\r\n");624 try w.writeAll("\r\n");
613625
...@@ -635,13 +647,13 @@ pub const Request = struct {...@@ -635,13 +647,13 @@ pub const Request = struct {
635 return index;647 return index;
636 }648 }
637649
638 pub const WaitError = RequestError || StartError || TransferReadError || proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || Uri.ParseError || error{ TooManyHttpRedirects, CannotRedirect, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported };650 pub const WaitError = RequestError || StartError || TransferReadError || proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || Uri.ParseError || error{ TooManyHttpRedirects, RedirectRequiresResend, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported };
639651
640 /// Waits for a response from the server and parses any headers that are sent.652 /// Waits for a response from the server and parses any headers that are sent.
641 /// This function will block until the final response is received.653 /// This function will block until the final response is received.
642 ///654 ///
643 /// If `handle_redirects` is true and the request has no payload, then this function will automatically follow655 /// If `handle_redirects` is true and the request has no payload, then this function will automatically follow
644 /// redirects. If a request payload is present, then this function will error with error.CannotRedirect.656 /// redirects. If a request payload is present, then this function will error with error.RedirectRequiresResend.
645 pub fn wait(req: *Request) WaitError!void {657 pub fn wait(req: *Request) WaitError!void {
646 while (true) { // handle redirects658 while (true) { // handle redirects
647 while (true) { // read headers659 while (true) { // read headers
...@@ -697,9 +709,10 @@ pub const Request = struct {...@@ -697,9 +709,10 @@ pub const Request = struct {
697 req.response.parser.done = true;709 req.response.parser.done = true;
698 }710 }
699711
700 if (req.transfer_encoding == .none and req.response.status.class() == .redirect and req.handle_redirects) {712 if (req.response.status.class() == .redirect and req.handle_redirects) {
701 req.response.skip = true;713 req.response.skip = true;
702714
715 // skip the body of the redirect response, this will at least leave the connection in a known good state.
703 const empty = @as([*]u8, undefined)[0..0];716 const empty = @as([*]u8, undefined)[0..0];
704 assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary717 assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary
705718
...@@ -715,6 +728,30 @@ pub const Request = struct {...@@ -715,6 +728,30 @@ pub const Request = struct {
715 const new_url = Uri.parse(location_duped) catch try Uri.parseWithoutScheme(location_duped);728 const new_url = Uri.parse(location_duped) catch try Uri.parseWithoutScheme(location_duped);
716 const resolved_url = try req.uri.resolve(new_url, false, arena);729 const resolved_url = try req.uri.resolve(new_url, false, arena);
717730
731 // is the redirect location on the same domain, or a subdomain of the original request?
732 const is_same_domain_or_subdomain = std.ascii.endsWithIgnoreCase(resolved_url.host.?, req.uri.host.?) and (resolved_url.host.?.len == req.uri.host.?.len or resolved_url.host.?[resolved_url.host.?.len - req.uri.host.?.len - 1] == '.');
733
734 if (resolved_url.host == null or !is_same_domain_or_subdomain or !std.ascii.eqlIgnoreCase(resolved_url.scheme, req.uri.scheme)) {
735 // we're redirecting to a different domain, strip privileged headers like cookies
736 _ = req.headers.delete("authorization");
737 _ = req.headers.delete("www-authenticate");
738 _ = req.headers.delete("cookie");
739 _ = req.headers.delete("cookie2");
740 }
741
742 if (req.response.status == .see_other or ((req.response.status == .moved_permanently or req.response.status == .found) and req.method == .POST)) {
743 // we're redirecting to a GET, so we need to change the method and remove the body
744 req.method = .GET;
745 req.transfer_encoding = .none;
746 _ = req.headers.delete("transfer-encoding");
747 _ = req.headers.delete("content-length");
748 _ = req.headers.delete("content-type");
749 }
750
751 if (req.transfer_encoding != .none) {
752 return error.RedirectRequiresResend; // The request body has already been sent. The request is still in a valid state, but the redirect must be handled manually.
753 }
754
718 try req.redirect(resolved_url);755 try req.redirect(resolved_url);
719756
720 try req.start();757 try req.start();
...@@ -735,9 +772,6 @@ pub const Request = struct {...@@ -735,9 +772,6 @@ pub const Request = struct {
735 };772 };
736 }773 }
737774
738 if (req.response.status.class() == .redirect and req.handle_redirects and req.transfer_encoding != .none)
739 return error.CannotRedirect; // The request body has already been sent. The request is still in a valid state, but the redirect must be handled manually.
740
741 break;775 break;
742 }776 }
743 }777 }
...@@ -956,17 +990,17 @@ pub const RequestError = ConnectUnproxiedError || ConnectErrorPartial || Request...@@ -956,17 +990,17 @@ pub const RequestError = ConnectUnproxiedError || ConnectErrorPartial || Request
956 UnsupportedTransferEncoding,990 UnsupportedTransferEncoding,
957};991};
958992
959pub const Options = struct {993pub const RequestOptions = struct {
960 version: http.Version = .@"HTTP/1.1",994 version: http.Version = .@"HTTP/1.1",
961995
962 handle_redirects: bool = true,996 handle_redirects: bool = true,
963 max_redirects: u32 = 3,997 max_redirects: u32 = 3,
964 header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },998 header_strategy: StorageStrategy = .{ .dynamic = 16 * 1024 },
965999
966 /// Must be an already acquired connection.1000 /// Must be an already acquired connection.
967 connection: ?*ConnectionPool.Node = null,1001 connection: ?*ConnectionPool.Node = null,
9681002
969 pub const HeaderStrategy = union(enum) {1003 pub const StorageStrategy = union(enum) {
970 /// In this case, the client's Allocator will be used to store the1004 /// In this case, the client's Allocator will be used to store the
971 /// entire HTTP header. This value is the maximum total size of1005 /// entire HTTP header. This value is the maximum total size of
972 /// HTTP headers allowed, otherwise1006 /// HTTP headers allowed, otherwise
...@@ -988,8 +1022,12 @@ pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{...@@ -988,8 +1022,12 @@ pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{
988});1022});
9891023
990/// Form and send a http request to a server.1024/// Form and send a http request to a server.
1025///
1026/// `uri` must remain alive during the entire request.
1027/// `headers` is cloned and may be freed after this function returns.
1028///
991/// This function is threadsafe.1029/// This function is threadsafe.
992pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Headers, options: Options) RequestError!Request {1030pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Headers, options: RequestOptions) RequestError!Request {
993 const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;1031 const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;
9941032
995 const port: u16 = uri.port orelse switch (protocol) {1033 const port: u16 = uri.port orelse switch (protocol) {
...@@ -1015,7 +1053,7 @@ pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Hea...@@ -1015,7 +1053,7 @@ pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Hea
1015 .uri = uri,1053 .uri = uri,
1016 .client = client,1054 .client = client,
1017 .connection = conn,1055 .connection = conn,
1018 .headers = headers,1056 .headers = try headers.clone(client.allocator), // Headers must be cloned to properly handle header transformations in redirects.
1019 .method = method,1057 .method = method,
1020 .version = options.version,1058 .version = options.version,
1021 .redirects_left = options.max_redirects,1059 .redirects_left = options.max_redirects,
...@@ -1039,6 +1077,123 @@ pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Hea...@@ -1039,6 +1077,123 @@ pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Hea
1039 return req;1077 return req;
1040}1078}
10411079
1080pub const FetchOptions = struct {
1081 pub const Location = union(enum) {
1082 url: []const u8,
1083 uri: Uri,
1084 };
1085
1086 pub const Payload = union(enum) {
1087 string: []const u8,
1088 file: std.fs.File,
1089 none,
1090 };
1091
1092 pub const ResponseStrategy = union(enum) {
1093 storage: RequestOptions.StorageStrategy,
1094 file: std.fs.File,
1095 none,
1096 };
1097
1098 header_strategy: RequestOptions.StorageStrategy = .{ .dynamic = 16 * 1024 },
1099 response_strategy: ResponseStrategy = .{ .storage = .{ .dynamic = 16 * 1024 * 1024 } },
1100
1101 location: Location,
1102 method: http.Method = .GET,
1103 headers: http.Headers = http.Headers{ .allocator = std.heap.page_allocator, .owned = false },
1104 payload: Payload = .none,
1105};
1106
1107pub const FetchResult = struct {
1108 status: http.Status,
1109 body: ?[]const u8 = null,
1110 headers: http.Headers,
1111
1112 allocator: Allocator,
1113 options: FetchOptions,
1114
1115 pub fn deinit(res: *FetchResult) void {
1116 if (res.options.response_strategy == .storage and res.options.response_strategy.storage == .dynamic) {
1117 if (res.body) |body| res.allocator.free(body);
1118 }
1119
1120 res.headers.deinit();
1121 }
1122};
1123
1124pub fn fetch(client: *Client, allocator: Allocator, options: FetchOptions) !FetchResult {
1125 const has_transfer_encoding = options.headers.contains("transfer-encoding");
1126 const has_content_length = options.headers.contains("content-length");
1127
1128 if (has_content_length or has_transfer_encoding) return error.UnsupportedHeader;
1129
1130 const uri = switch (options.location) {
1131 .url => |u| try Uri.parse(u),
1132 .uri => |u| u,
1133 };
1134
1135 var req = try request(client, options.method, uri, options.headers, .{
1136 .header_strategy = options.header_strategy,
1137 .handle_redirects = options.payload == .none,
1138 });
1139 defer req.deinit();
1140
1141 { // Block to maintain lock of file to attempt to prevent a race condition where another process modifies the file while we are reading it.
1142 // This relies on other processes actually obeying the advisory lock, which is not guaranteed.
1143 if (options.payload == .file) try options.payload.file.lock(.shared);
1144 defer if (options.payload == .file) options.payload.file.unlock();
1145
1146 switch (options.payload) {
1147 .string => |str| req.transfer_encoding = .{ .content_length = str.len },
1148 .file => |file| req.transfer_encoding = .{ .content_length = (try file.stat()).size },
1149 .none => {},
1150 }
1151
1152 try req.start();
1153
1154 switch (options.payload) {
1155 .string => |str| try req.writeAll(str),
1156 .file => |file| {
1157 try file.seekTo(0);
1158 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();
1159 try fifo.pump(file.reader(), req.writer());
1160 },
1161 .none => {},
1162 }
1163
1164 try req.finish();
1165 }
1166
1167 try req.wait();
1168
1169 var res = FetchResult{
1170 .status = req.response.status,
1171 .headers = try req.response.headers.clone(allocator),
1172
1173 .allocator = allocator,
1174 .options = options,
1175 };
1176
1177 switch (options.response_strategy) {
1178 .storage => |storage| switch (storage) {
1179 .dynamic => |max| res.body = try req.reader().readAllAlloc(allocator, max),
1180 .static => |buf| res.body = buf[0..try req.reader().readAll(buf)],
1181 },
1182 .file => |file| {
1183 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();
1184 try fifo.pump(req.reader(), file.writer());
1185 },
1186 .none => { // Take advantage of request internals to discard the response body and make the connection available for another request.
1187 req.response.skip = true;
1188
1189 const empty = @as([*]u8, undefined)[0..0];
1190 assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary
1191 },
1192 }
1193
1194 return res;
1195}
1196
1042test {1197test {
1043 const builtin = @import("builtin");1198 const builtin = @import("builtin");
1044 const native_endian = comptime builtin.cpu.arch.endian();1199 const native_endian = comptime builtin.cpu.arch.endian();
lib/std/http/Headers.zig+26-1
...@@ -57,6 +57,18 @@ pub const Headers = struct {...@@ -57,6 +57,18 @@ pub const Headers = struct {
57 return .{ .allocator = allocator };57 return .{ .allocator = allocator };
58 }58 }
5959
60 pub fn initList(allocator: Allocator, list: []const Field) Headers {
61 var new = Headers.init(allocator);
62
63 try new.list.ensureTotalCapacity(allocator, list.len);
64 try new.index.ensureTotalCapacity(allocator, list.len);
65 for (list) |field| {
66 try new.append(field.name, field.value);
67 }
68
69 return new;
70 }
71
60 pub fn deinit(headers: *Headers) void {72 pub fn deinit(headers: *Headers) void {
61 headers.deallocateIndexListsAndFields();73 headers.deallocateIndexListsAndFields();
62 headers.index.deinit(headers.allocator);74 headers.index.deinit(headers.allocator);
...@@ -78,7 +90,7 @@ pub const Headers = struct {...@@ -78,7 +90,7 @@ pub const Headers = struct {
78 entry.name = kv.key_ptr.*;90 entry.name = kv.key_ptr.*;
79 try kv.value_ptr.append(headers.allocator, n);91 try kv.value_ptr.append(headers.allocator, n);
80 } else {92 } else {
81 const name_duped = if (headers.owned) try headers.allocator.dupe(u8, name) else name;93 const name_duped = if (headers.owned) try std.ascii.allocLowerString(headers.allocator, name) else name;
82 errdefer if (headers.owned) headers.allocator.free(name_duped);94 errdefer if (headers.owned) headers.allocator.free(name_duped);
8395
84 entry.name = name_duped;96 entry.name = name_duped;
...@@ -97,6 +109,7 @@ pub const Headers = struct {...@@ -97,6 +109,7 @@ pub const Headers = struct {
97 return headers.index.contains(name);109 return headers.index.contains(name);
98 }110 }
99111
112 /// Removes all headers with the given name.
100 pub fn delete(headers: *Headers, name: []const u8) bool {113 pub fn delete(headers: *Headers, name: []const u8) bool {
101 if (headers.index.fetchRemove(name)) |kv| {114 if (headers.index.fetchRemove(name)) |kv| {
102 var index = kv.value;115 var index = kv.value;
...@@ -268,6 +281,18 @@ pub const Headers = struct {...@@ -268,6 +281,18 @@ pub const Headers = struct {
268 headers.index.clearRetainingCapacity();281 headers.index.clearRetainingCapacity();
269 headers.list.clearRetainingCapacity();282 headers.list.clearRetainingCapacity();
270 }283 }
284
285 pub fn clone(headers: Headers, allocator: Allocator) !Headers {
286 var new = Headers.init(allocator);
287
288 try new.list.ensureTotalCapacity(allocator, headers.list.capacity);
289 try new.index.ensureTotalCapacity(allocator, headers.index.capacity());
290 for (headers.list.items) |field| {
291 try new.append(field.name, field.value);
292 }
293
294 return new;
295 }
271};296};
272297
273test "Headers.append" {298test "Headers.append" {
test/standalone/http.zig+22-1
...@@ -571,7 +571,28 @@ pub fn main() !void {...@@ -571,7 +571,28 @@ pub fn main() !void {
571 // connection has been kept alive571 // connection has been kept alive
572 try testing.expect(client.connection_pool.free_len == 1);572 try testing.expect(client.connection_pool.free_len == 1);
573573
574 { // issue 16282574 { // Client.fetch()
575 var h = http.Headers{ .allocator = calloc };
576 defer h.deinit();
577
578 try h.append("content-type", "text/plain");
579
580 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#fetch", .{port});
581 defer calloc.free(location);
582
583 log.info("{s}", .{location});
584 var res = try client.fetch(calloc, .{
585 .location = .{ .url = location },
586 .method = .POST,
587 .headers = h,
588 .payload = .{ .string = "Hello, World!\n" },
589 });
590 defer res.deinit();
591
592 try testing.expectEqualStrings("Hello, World!\n", res.body.?);
593 }
594
595 { // issue 16282 *** This test leaves the client in an invalid state, it must be last ***
575 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});596 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
576 defer calloc.free(location);597 defer calloc.free(location);
577 const uri = try std.Uri.parse(location);598 const uri = try std.Uri.parse(location);