authorgravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-04-24 11:24:51-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-26 00:02:55-07:00
log7285eedcd26b92afb03c313a167133103a78ded5
tree2144b7f0a2faf6632cc33a37e8286b99f3acdd0b
parent13101295b93fe8ec5467bbd6cdf758a2bf823945

std.http: do -> wait, fix redirects

3 files changed, 67 insertions(+), 33 deletions(-)

lib/std/http/Client.zig+65-30
...@@ -365,7 +365,7 @@ pub const Response = struct {...@@ -365,7 +365,7 @@ pub const Response = struct {
365 CompressionNotSupported,365 CompressionNotSupported,
366 };366 };
367367
368 pub fn parse(res: *Response, bytes: []const u8) ParseError!void {368 pub fn parse(res: *Response, bytes: []const u8, trailing: bool) ParseError!void {
369 var it = mem.tokenize(u8, bytes[0 .. bytes.len - 4], "\r\n");369 var it = mem.tokenize(u8, bytes[0 .. bytes.len - 4], "\r\n");
370370
371 const first_line = it.next() orelse return error.HttpHeadersInvalid;371 const first_line = it.next() orelse return error.HttpHeadersInvalid;
...@@ -398,6 +398,8 @@ pub const Response = struct {...@@ -398,6 +398,8 @@ pub const Response = struct {
398398
399 try res.headers.append(header_name, header_value);399 try res.headers.append(header_name, header_value);
400400
401 if (trailing) continue;
402
401 if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {403 if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
402 if (res.content_length != null) return error.HttpHeadersInvalid;404 if (res.content_length != null) return error.HttpHeadersInvalid;
403 res.content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength;405 res.content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength;
...@@ -480,7 +482,7 @@ pub const Response = struct {...@@ -480,7 +482,7 @@ pub const Response = struct {
480482
481/// A HTTP request that has been sent.483/// A HTTP request that has been sent.
482///484///
483/// Order of operations: request[ -> write -> finish] -> do -> read485/// Order of operations: request -> start[ -> write -> finish] -> wait -> read
484pub const Request = struct {486pub const Request = struct {
485 uri: Uri,487 uri: Uri,
486 client: *Client,488 client: *Client,
...@@ -508,8 +510,9 @@ pub const Request = struct {...@@ -508,8 +510,9 @@ pub const Request = struct {
508 .zstd => |*zstd| zstd.deinit(),510 .zstd => |*zstd| zstd.deinit(),
509 }511 }
510512
513 req.response.headers.deinit();
514
511 if (req.response.parser.header_bytes_owned) {515 if (req.response.parser.header_bytes_owned) {
512 req.response.headers.deinit();
513 req.response.parser.header_bytes.deinit(req.client.allocator);516 req.response.parser.header_bytes.deinit(req.client.allocator);
514 }517 }
515518
...@@ -524,6 +527,44 @@ pub const Request = struct {...@@ -524,6 +527,44 @@ pub const Request = struct {
524 req.* = undefined;527 req.* = undefined;
525 }528 }
526529
530 // This function must deallocate all resources associated with the request, or keep those which will be used
531 // This needs to be kept in sync with deinit and request
532 fn redirect(req: *Request, uri: Uri) !void {
533 assert(req.response.parser.done);
534
535 switch (req.response.compression) {
536 .none => {},
537 .deflate => |*deflate| deflate.deinit(),
538 .gzip => |*gzip| gzip.deinit(),
539 .zstd => |*zstd| zstd.deinit(),
540 }
541
542 req.client.connection_pool.release(req.client, req.connection);
543
544 const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;
545
546 const port: u16 = uri.port orelse switch (protocol) {
547 .plain => 80,
548 .tls => 443,
549 };
550
551 const host = uri.host orelse return error.UriMissingHost;
552
553 req.uri = uri;
554 req.connection = try req.client.connect(host, port, protocol);
555 req.redirects_left -= 1;
556 req.response.headers.clearRetainingCapacity();
557 req.response.parser.reset();
558
559 req.response = .{
560 .status = undefined,
561 .reason = undefined,
562 .version = undefined,
563 .headers = req.response.headers,
564 .parser = req.response.parser,
565 };
566 }
567
527 pub const StartError = BufferedConnection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding };568 pub const StartError = BufferedConnection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding };
528569
529 /// Send the request to the server.570 /// Send the request to the server.
...@@ -627,14 +668,14 @@ pub const Request = struct {...@@ -627,14 +668,14 @@ pub const Request = struct {
627 return index;668 return index;
628 }669 }
629670
630 pub const DoError = RequestError || TransferReadError || proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || Uri.ParseError || error{ TooManyHttpRedirects, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported };671 pub const WaitError = RequestError || StartError || TransferReadError || proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || Uri.ParseError || error{ TooManyHttpRedirects, CannotRedirect, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported };
631672
632 /// Waits for a response from the server and parses any headers that are sent.673 /// Waits for a response from the server and parses any headers that are sent.
633 /// This function will block until the final response is received.674 /// This function will block until the final response is received.
634 ///675 ///
635 /// If `handle_redirects` is true, then this function will automatically follow676 /// If `handle_redirects` is true and the request has no payload, then this function will automatically follow
636 /// redirects.677 /// redirects. If a request payload is present, then this function will error with error.CannotRedirect.
637 pub fn do(req: *Request) DoError!void {678 pub fn wait(req: *Request) WaitError!void {
638 while (true) { // handle redirects679 while (true) { // handle redirects
639 while (true) { // read headers680 while (true) { // read headers
640 try req.connection.data.buffered.fill();681 try req.connection.data.buffered.fill();
...@@ -645,7 +686,7 @@ pub const Request = struct {...@@ -645,7 +686,7 @@ pub const Request = struct {
645 if (req.response.parser.state.isContent()) break;686 if (req.response.parser.state.isContent()) break;
646 }687 }
647688
648 try req.response.parse(req.response.parser.header_bytes.items);689 try req.response.parse(req.response.parser.header_bytes.items, false);
649690
650 if (req.response.status == .switching_protocols) {691 if (req.response.status == .switching_protocols) {
651 req.connection.data.closing = false;692 req.connection.data.closing = false;
...@@ -684,7 +725,7 @@ pub const Request = struct {...@@ -684,7 +725,7 @@ pub const Request = struct {
684 req.response.parser.done = true;725 req.response.parser.done = true;
685 }726 }
686727
687 if (req.response.status.class() == .redirect and req.handle_redirects) {728 if (req.transfer_encoding == .none and req.response.status.class() == .redirect and req.handle_redirects) {
688 req.response.skip = true;729 req.response.skip = true;
689730
690 const empty = @as([*]u8, undefined)[0..0];731 const empty = @as([*]u8, undefined)[0..0];
...@@ -694,26 +735,17 @@ pub const Request = struct {...@@ -694,26 +735,17 @@ pub const Request = struct {
694735
695 const location = req.response.headers.getFirstValue("location") orelse736 const location = req.response.headers.getFirstValue("location") orelse
696 return error.HttpRedirectMissingLocation;737 return error.HttpRedirectMissingLocation;
697 const new_url = Uri.parse(location) catch try Uri.parseWithoutScheme(location);738
698739 const arena = req.arena.allocator();
699 var new_arena = std.heap.ArenaAllocator.init(req.client.allocator);740
700 const resolved_url = try req.uri.resolve(new_url, false, new_arena.allocator());741 const location_duped = try arena.dupe(u8, location);
701 errdefer new_arena.deinit();742
702743 const new_url = Uri.parse(location_duped) catch try Uri.parseWithoutScheme(location_duped);
703 req.arena.deinit();744 const resolved_url = try req.uri.resolve(new_url, false, arena);
704 req.arena = new_arena;745
705746 try req.redirect(resolved_url);
706 const new_req = try req.client.request(req.method, resolved_url, req.headers, .{747
707 .version = req.version,748 try req.start();
708 .max_redirects = req.redirects_left - 1,
709 .header_strategy = if (req.response.parser.header_bytes_owned) .{
710 .dynamic = req.response.parser.max_header_bytes,
711 } else .{
712 .static = req.response.parser.header_bytes.items.ptr[0..req.response.parser.max_header_bytes],
713 },
714 });
715 req.deinit();
716 req.* = new_req;
717 } else {749 } else {
718 req.response.skip = false;750 req.response.skip = false;
719 if (!req.response.parser.done) {751 if (!req.response.parser.done) {
...@@ -731,6 +763,9 @@ pub const Request = struct {...@@ -731,6 +763,9 @@ pub const Request = struct {
731 };763 };
732 }764 }
733765
766 if (req.response.status.class() == .redirect and req.handle_redirects and req.transfer_encoding != .none)
767 return error.CannotRedirect; // The request body has already been sent. The request is still in a valid state, but the redirect must be handled manually.
768
734 break;769 break;
735 }770 }
736 }771 }
...@@ -768,7 +803,7 @@ pub const Request = struct {...@@ -768,7 +803,7 @@ pub const Request = struct {
768803
769 // The response headers before the trailers are already guaranteed to be valid, so they will always be parsed again and cannot return an error.804 // The response headers before the trailers are already guaranteed to be valid, so they will always be parsed again and cannot return an error.
770 // This will *only* fail for a malformed trailer.805 // This will *only* fail for a malformed trailer.
771 req.response.parse(req.response.parser.header_bytes.items) catch return error.InvalidTrailers;806 req.response.parse(req.response.parser.header_bytes.items, true) catch return error.InvalidTrailers;
772 }807 }
773 }808 }
774809
lib/std/http/test.zig+1-1
...@@ -62,7 +62,7 @@ test "client requests server" {...@@ -62,7 +62,7 @@ test "client requests server" {
62 try client_req.writeAll("Hello, ");62 try client_req.writeAll("Hello, ");
63 try client_req.writeAll("World!\n");63 try client_req.writeAll("World!\n");
64 try client_req.finish();64 try client_req.finish();
65 try client_req.do(); // this waits for a response65 try client_req.wait(); // this waits for a response
6666
67 const body = try client_req.reader().readAllAlloc(allocator, 8192 * 1024);67 const body = try client_req.reader().readAllAlloc(allocator, 8192 * 1024);
68 defer allocator.free(body);68 defer allocator.free(body);
src/Package.zig+1-2
...@@ -486,8 +486,7 @@ fn fetchAndUnpack(...@@ -486,8 +486,7 @@ fn fetchAndUnpack(
486 defer req.deinit();486 defer req.deinit();
487487
488 try req.start();488 try req.start();
489489 try req.wait();
490 try req.do();
491490
492 if (mem.endsWith(u8, uri.path, ".tar.gz")) {491 if (mem.endsWith(u8, uri.path, ".tar.gz")) {
493 // I observed the gzip stream to read 1 byte at a time, so I am using a492 // I observed the gzip stream to read 1 byte at a time, so I am using a