authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-09-17 00:41:30-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-09-17 00:41:30-07:00
logf40f81cbfb0a867b32d4406e198550730de268cd
tree5e8b04ffd75a73a927f5f32eeb0c79d119c585f9
parent5389af2c1c8c50942da20dc5b0cc29cdca45e0e9
parent4689d93cb204a4143770105200eb65dcdca5d7a0
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #16929 from truemedian/more-http

std.http: handle Expect: 100-continue, improve redirect logic, add Client.fetch for simple requests

6 files changed, 425 insertions(+), 79 deletions(-)

lib/std/http.zig+37-12
...@@ -1,3 +1,5 @@...@@ -1,3 +1,5 @@
1const std = @import("std.zig");
2
1pub const Client = @import("http/Client.zig");3pub const Client = @import("http/Client.zig");
2pub const Server = @import("http/Server.zig");4pub const Server = @import("http/Server.zig");
3pub const protocol = @import("http/protocol.zig");5pub const protocol = @import("http/protocol.zig");
...@@ -14,16 +16,36 @@ pub const Version = enum {...@@ -14,16 +16,36 @@ pub const Version = enum {
14/// https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods16/// https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods
15/// https://datatracker.ietf.org/doc/html/rfc7231#section-4 Initial definition17/// https://datatracker.ietf.org/doc/html/rfc7231#section-4 Initial definition
16/// https://datatracker.ietf.org/doc/html/rfc5789#section-2 PATCH18/// https://datatracker.ietf.org/doc/html/rfc5789#section-2 PATCH
17pub const Method = enum {19pub const Method = enum(u64) { // TODO: should be u192 or u256, but neither is supported by the C backend, and therefore cannot pass CI
18 GET,20 GET = parse("GET"),
19 HEAD,21 HEAD = parse("HEAD"),
20 POST,22 POST = parse("POST"),
21 PUT,23 PUT = parse("PUT"),
22 DELETE,24 DELETE = parse("DELETE"),
23 CONNECT,25 CONNECT = parse("CONNECT"),
24 OPTIONS,26 OPTIONS = parse("OPTIONS"),
25 TRACE,27 TRACE = parse("TRACE"),
26 PATCH,28 PATCH = parse("PATCH"),
29
30 _,
31
32 /// Converts `s` into a type that may be used as a `Method` field.
33 /// Asserts that `s` is 24 or fewer bytes.
34 pub fn parse(s: []const u8) u64 {
35 var x: u64 = 0;
36 @memcpy(std.mem.asBytes(&x)[0..s.len], s);
37 return x;
38 }
39
40 pub fn write(self: Method, w: anytype) !void {
41 const bytes = std.mem.asBytes(&@intFromEnum(self));
42 const str = std.mem.sliceTo(bytes, 0);
43 try w.writeAll(str);
44 }
45
46 pub fn format(value: Method, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) @TypeOf(writer).Error!void {
47 return try value.write(writer);
48 }
2749
28 /// Returns true if a request of this method is allowed to have a body50 /// Returns true if a request of this method is allowed to have a body
29 /// Actual behavior from servers may vary and should still be checked51 /// Actual behavior from servers may vary and should still be checked
...@@ -31,6 +53,7 @@ pub const Method = enum {...@@ -31,6 +53,7 @@ pub const Method = enum {
31 return switch (self) {53 return switch (self) {
32 .POST, .PUT, .PATCH => true,54 .POST, .PUT, .PATCH => true,
33 .GET, .HEAD, .DELETE, .CONNECT, .OPTIONS, .TRACE => false,55 .GET, .HEAD, .DELETE, .CONNECT, .OPTIONS, .TRACE => false,
56 else => true,
34 };57 };
35 }58 }
3659
...@@ -40,6 +63,7 @@ pub const Method = enum {...@@ -40,6 +63,7 @@ pub const Method = enum {
40 return switch (self) {63 return switch (self) {
41 .GET, .POST, .DELETE, .CONNECT, .OPTIONS, .PATCH => true,64 .GET, .POST, .DELETE, .CONNECT, .OPTIONS, .PATCH => true,
42 .HEAD, .PUT, .TRACE => false,65 .HEAD, .PUT, .TRACE => false,
66 else => true,
43 };67 };
44 }68 }
4569
...@@ -50,6 +74,7 @@ pub const Method = enum {...@@ -50,6 +74,7 @@ pub const Method = enum {
50 return switch (self) {74 return switch (self) {
51 .GET, .HEAD, .OPTIONS, .TRACE => true,75 .GET, .HEAD, .OPTIONS, .TRACE => true,
52 .POST, .PUT, .DELETE, .CONNECT, .PATCH => false,76 .POST, .PUT, .DELETE, .CONNECT, .PATCH => false,
77 else => false,
53 };78 };
54 }79 }
5580
...@@ -60,6 +85,7 @@ pub const Method = enum {...@@ -60,6 +85,7 @@ pub const Method = enum {
60 return switch (self) {85 return switch (self) {
61 .GET, .HEAD, .PUT, .DELETE, .OPTIONS, .TRACE => true,86 .GET, .HEAD, .PUT, .DELETE, .OPTIONS, .TRACE => true,
62 .CONNECT, .POST, .PATCH => false,87 .CONNECT, .POST, .PATCH => false,
88 else => false,
63 };89 };
64 }90 }
6591
...@@ -70,6 +96,7 @@ pub const Method = enum {...@@ -70,6 +96,7 @@ pub const Method = enum {
70 return switch (self) {96 return switch (self) {
71 .GET, .HEAD => true,97 .GET, .HEAD => true,
72 .POST, .PUT, .DELETE, .CONNECT, .OPTIONS, .TRACE, .PATCH => false,98 .POST, .PUT, .DELETE, .CONNECT, .OPTIONS, .TRACE, .PATCH => false,
99 else => false,
73 };100 };
74 }101 }
75};102};
...@@ -269,8 +296,6 @@ pub const Connection = enum {...@@ -269,8 +296,6 @@ pub const Connection = enum {
269 close,296 close,
270};297};
271298
272const std = @import("std.zig");
273
274test {299test {
275 _ = Client;300 _ = Client;
276 _ = Method;301 _ = Method;
lib/std/http/Client.zig+216-24
...@@ -365,8 +365,11 @@ pub const Response = struct {...@@ -365,8 +365,11 @@ pub const Response = struct {
365 if (trailing) continue;365 if (trailing) continue;
366366
367 if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {367 if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
368 if (res.content_length != null) return error.HttpHeadersInvalid;368 const content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength;
369 res.content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength;369
370 if (res.content_length != null and res.content_length != content_length) return error.HttpHeadersInvalid;
371
372 res.content_length = content_length;
370 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {373 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
371 // Transfer-Encoding: second, first374 // Transfer-Encoding: second, first
372 // Transfer-Encoding: deflate, chunked375 // Transfer-Encoding: deflate, chunked
...@@ -475,6 +478,7 @@ pub const Request = struct {...@@ -475,6 +478,7 @@ pub const Request = struct {
475 .zstd => |*zstd| zstd.deinit(),478 .zstd => |*zstd| zstd.deinit(),
476 }479 }
477480
481 req.headers.deinit();
478 req.response.headers.deinit();482 req.response.headers.deinit();
479483
480 if (req.response.parser.header_bytes_owned) {484 if (req.response.parser.header_bytes_owned) {
...@@ -536,10 +540,12 @@ pub const Request = struct {...@@ -536,10 +540,12 @@ pub const Request = struct {
536540
537 /// Send the request to the server.541 /// Send the request to the server.
538 pub fn start(req: *Request) StartError!void {542 pub fn start(req: *Request) StartError!void {
543 if (!req.method.requestHasBody() and req.transfer_encoding != .none) return error.UnsupportedTransferEncoding;
544
539 var buffered = std.io.bufferedWriter(req.connection.?.data.writer());545 var buffered = std.io.bufferedWriter(req.connection.?.data.writer());
540 const w = buffered.writer();546 const w = buffered.writer();
541547
542 try w.writeAll(@tagName(req.method));548 try req.method.write(w);
543 try w.writeByte(' ');549 try w.writeByte(' ');
544550
545 if (req.method == .CONNECT) {551 if (req.method == .CONNECT) {
...@@ -607,22 +613,29 @@ pub const Request = struct {...@@ -607,22 +613,29 @@ pub const Request = struct {
607 }613 }
608 }614 }
609615
610 try w.print("{}", .{req.headers});616 for (req.headers.list.items) |entry| {
617 if (entry.value.len == 0) continue;
618
619 try w.writeAll(entry.name);
620 try w.writeAll(": ");
621 try w.writeAll(entry.value);
622 try w.writeAll("\r\n");
623 }
611624
612 try w.writeAll("\r\n");625 try w.writeAll("\r\n");
613626
614 try buffered.flush();627 try buffered.flush();
615 }628 }
616629
617 pub const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError;630 const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError;
618631
619 pub const TransferReader = std.io.Reader(*Request, TransferReadError, transferRead);632 const TransferReader = std.io.Reader(*Request, TransferReadError, transferRead);
620633
621 pub fn transferReader(req: *Request) TransferReader {634 fn transferReader(req: *Request) TransferReader {
622 return .{ .context = req };635 return .{ .context = req };
623 }636 }
624637
625 pub fn transferRead(req: *Request, buf: []u8) TransferReadError!usize {638 fn transferRead(req: *Request, buf: []u8) TransferReadError!usize {
626 if (req.response.parser.done) return 0;639 if (req.response.parser.done) return 0;
627640
628 var index: usize = 0;641 var index: usize = 0;
...@@ -635,13 +648,13 @@ pub const Request = struct {...@@ -635,13 +648,13 @@ pub const Request = struct {
635 return index;648 return index;
636 }649 }
637650
638 pub const WaitError = RequestError || StartError || TransferReadError || proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || Uri.ParseError || error{ TooManyHttpRedirects, CannotRedirect, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported };651 pub const WaitError = RequestError || StartError || TransferReadError || proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || Uri.ParseError || error{ TooManyHttpRedirects, RedirectRequiresResend, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported };
639652
640 /// Waits for a response from the server and parses any headers that are sent.653 /// Waits for a response from the server and parses any headers that are sent.
641 /// This function will block until the final response is received.654 /// This function will block until the final response is received.
642 ///655 ///
643 /// If `handle_redirects` is true and the request has no payload, then this function will automatically follow656 /// If `handle_redirects` is true and the request has no payload, then this function will automatically follow
644 /// redirects. If a request payload is present, then this function will error with error.CannotRedirect.657 /// redirects. If a request payload is present, then this function will error with error.RedirectRequiresResend.
645 pub fn wait(req: *Request) WaitError!void {658 pub fn wait(req: *Request) WaitError!void {
646 while (true) { // handle redirects659 while (true) { // handle redirects
647 while (true) { // read headers660 while (true) { // read headers
...@@ -655,17 +668,19 @@ pub const Request = struct {...@@ -655,17 +668,19 @@ pub const Request = struct {
655668
656 try req.response.parse(req.response.parser.header_bytes.items, false);669 try req.response.parse(req.response.parser.header_bytes.items, false);
657670
658 if (req.response.status == .switching_protocols) {671 if (req.response.status == .@"continue") {
659 req.connection.?.data.closing = false;672 req.response.parser.done = true; // we're done parsing the continue response, reset to prepare for the real response
660 req.response.parser.done = true;673 req.response.parser.reset();
674 break;
661 }675 }
662676
663 if (req.method == .CONNECT and req.response.status == .ok) {677 // we're switching protocols, so this connection is no longer doing http
678 if (req.response.status == .switching_protocols or (req.method == .CONNECT and req.response.status == .ok)) {
664 req.connection.?.data.closing = false;679 req.connection.?.data.closing = false;
665 req.response.parser.done = true;680 req.response.parser.done = true;
666 }681 }
667682
668 // we default to using keep-alive if not provided683 // we default to using keep-alive if not provided in the client if the server asks for it
669 const req_connection = req.headers.getFirstValue("connection");684 const req_connection = req.headers.getFirstValue("connection");
670 const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);685 const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
671686
...@@ -697,9 +712,10 @@ pub const Request = struct {...@@ -697,9 +712,10 @@ pub const Request = struct {
697 req.response.parser.done = true;712 req.response.parser.done = true;
698 }713 }
699714
700 if (req.transfer_encoding == .none and req.response.status.class() == .redirect and req.handle_redirects) {715 if (req.response.status.class() == .redirect and req.handle_redirects) {
701 req.response.skip = true;716 req.response.skip = true;
702717
718 // skip the body of the redirect response, this will at least leave the connection in a known good state.
703 const empty = @as([*]u8, undefined)[0..0];719 const empty = @as([*]u8, undefined)[0..0];
704 assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary720 assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary
705721
...@@ -715,6 +731,30 @@ pub const Request = struct {...@@ -715,6 +731,30 @@ pub const Request = struct {
715 const new_url = Uri.parse(location_duped) catch try Uri.parseWithoutScheme(location_duped);731 const new_url = Uri.parse(location_duped) catch try Uri.parseWithoutScheme(location_duped);
716 const resolved_url = try req.uri.resolve(new_url, false, arena);732 const resolved_url = try req.uri.resolve(new_url, false, arena);
717733
734 // is the redirect location on the same domain, or a subdomain of the original request?
735 const is_same_domain_or_subdomain = std.ascii.endsWithIgnoreCase(resolved_url.host.?, req.uri.host.?) and (resolved_url.host.?.len == req.uri.host.?.len or resolved_url.host.?[resolved_url.host.?.len - req.uri.host.?.len - 1] == '.');
736
737 if (resolved_url.host == null or !is_same_domain_or_subdomain or !std.ascii.eqlIgnoreCase(resolved_url.scheme, req.uri.scheme)) {
738 // we're redirecting to a different domain, strip privileged headers like cookies
739 _ = req.headers.delete("authorization");
740 _ = req.headers.delete("www-authenticate");
741 _ = req.headers.delete("cookie");
742 _ = req.headers.delete("cookie2");
743 }
744
745 if (req.response.status == .see_other or ((req.response.status == .moved_permanently or req.response.status == .found) and req.method == .POST)) {
746 // we're redirecting to a GET, so we need to change the method and remove the body
747 req.method = .GET;
748 req.transfer_encoding = .none;
749 _ = req.headers.delete("transfer-encoding");
750 _ = req.headers.delete("content-length");
751 _ = req.headers.delete("content-type");
752 }
753
754 if (req.transfer_encoding != .none) {
755 return error.RedirectRequiresResend; // The request body has already been sent. The request is still in a valid state, but the redirect must be handled manually.
756 }
757
718 try req.redirect(resolved_url);758 try req.redirect(resolved_url);
719759
720 try req.start();760 try req.start();
...@@ -735,9 +775,6 @@ pub const Request = struct {...@@ -735,9 +775,6 @@ pub const Request = struct {
735 };775 };
736 }776 }
737777
738 if (req.response.status.class() == .redirect and req.handle_redirects and req.transfer_encoding != .none)
739 return error.CannotRedirect; // The request body has already been sent. The request is still in a valid state, but the redirect must be handled manually.
740
741 break;778 break;
742 }779 }
743 }780 }
...@@ -921,6 +958,40 @@ pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol:...@@ -921,6 +958,40 @@ pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol:
921 return conn;958 return conn;
922}959}
923960
961pub const ConnectUnixError = Allocator.Error || std.os.SocketError || error{ NameTooLong, Unsupported } || std.os.ConnectError;
962
963pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*ConnectionPool.Node {
964 if (!net.has_unix_sockets) return error.Unsupported;
965
966 if (client.connection_pool.findConnection(.{
967 .host = path,
968 .port = 0,
969 .is_tls = false,
970 })) |node|
971 return node;
972
973 const conn = try client.allocator.create(ConnectionPool.Node);
974 errdefer client.allocator.destroy(conn);
975 conn.* = .{ .data = undefined };
976
977 const stream = try std.net.connectUnixSocket(path);
978 errdefer stream.close();
979
980 conn.data = .{
981 .stream = stream,
982 .tls_client = undefined,
983 .protocol = .plain,
984
985 .host = try client.allocator.dupe(u8, path),
986 .port = 0,
987 };
988 errdefer client.allocator.free(conn.data.host);
989
990 client.connection_pool.addUsed(conn);
991
992 return conn;
993}
994
924// Prevents a dependency loop in request()995// Prevents a dependency loop in request()
925const ConnectErrorPartial = ConnectUnproxiedError || error{ UnsupportedUrlScheme, ConnectionRefused };996const ConnectErrorPartial = ConnectUnproxiedError || error{ UnsupportedUrlScheme, ConnectionRefused };
926pub const ConnectError = ConnectErrorPartial || RequestError;997pub const ConnectError = ConnectErrorPartial || RequestError;
...@@ -956,17 +1027,17 @@ pub const RequestError = ConnectUnproxiedError || ConnectErrorPartial || Request...@@ -956,17 +1027,17 @@ pub const RequestError = ConnectUnproxiedError || ConnectErrorPartial || Request
956 UnsupportedTransferEncoding,1027 UnsupportedTransferEncoding,
957};1028};
9581029
959pub const Options = struct {1030pub const RequestOptions = struct {
960 version: http.Version = .@"HTTP/1.1",1031 version: http.Version = .@"HTTP/1.1",
9611032
962 handle_redirects: bool = true,1033 handle_redirects: bool = true,
963 max_redirects: u32 = 3,1034 max_redirects: u32 = 3,
964 header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },1035 header_strategy: StorageStrategy = .{ .dynamic = 16 * 1024 },
9651036
966 /// Must be an already acquired connection.1037 /// Must be an already acquired connection.
967 connection: ?*ConnectionPool.Node = null,1038 connection: ?*ConnectionPool.Node = null,
9681039
969 pub const HeaderStrategy = union(enum) {1040 pub const StorageStrategy = union(enum) {
970 /// In this case, the client's Allocator will be used to store the1041 /// In this case, the client's Allocator will be used to store the
971 /// entire HTTP header. This value is the maximum total size of1042 /// entire HTTP header. This value is the maximum total size of
972 /// HTTP headers allowed, otherwise1043 /// HTTP headers allowed, otherwise
...@@ -988,8 +1059,12 @@ pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{...@@ -988,8 +1059,12 @@ pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{
988});1059});
9891060
990/// Form and send a http request to a server.1061/// Form and send a http request to a server.
1062///
1063/// `uri` must remain alive during the entire request.
1064/// `headers` is cloned and may be freed after this function returns.
1065///
991/// This function is threadsafe.1066/// This function is threadsafe.
992pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Headers, options: Options) RequestError!Request {1067pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Headers, options: RequestOptions) RequestError!Request {
993 const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;1068 const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;
9941069
995 const port: u16 = uri.port orelse switch (protocol) {1070 const port: u16 = uri.port orelse switch (protocol) {
...@@ -1015,7 +1090,7 @@ pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Hea...@@ -1015,7 +1090,7 @@ pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Hea
1015 .uri = uri,1090 .uri = uri,
1016 .client = client,1091 .client = client,
1017 .connection = conn,1092 .connection = conn,
1018 .headers = headers,1093 .headers = try headers.clone(client.allocator), // Headers must be cloned to properly handle header transformations in redirects.
1019 .method = method,1094 .method = method,
1020 .version = options.version,1095 .version = options.version,
1021 .redirects_left = options.max_redirects,1096 .redirects_left = options.max_redirects,
...@@ -1039,6 +1114,123 @@ pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Hea...@@ -1039,6 +1114,123 @@ pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Hea
1039 return req;1114 return req;
1040}1115}
10411116
1117pub const FetchOptions = struct {
1118 pub const Location = union(enum) {
1119 url: []const u8,
1120 uri: Uri,
1121 };
1122
1123 pub const Payload = union(enum) {
1124 string: []const u8,
1125 file: std.fs.File,
1126 none,
1127 };
1128
1129 pub const ResponseStrategy = union(enum) {
1130 storage: RequestOptions.StorageStrategy,
1131 file: std.fs.File,
1132 none,
1133 };
1134
1135 header_strategy: RequestOptions.StorageStrategy = .{ .dynamic = 16 * 1024 },
1136 response_strategy: ResponseStrategy = .{ .storage = .{ .dynamic = 16 * 1024 * 1024 } },
1137
1138 location: Location,
1139 method: http.Method = .GET,
1140 headers: http.Headers = http.Headers{ .allocator = std.heap.page_allocator, .owned = false },
1141 payload: Payload = .none,
1142};
1143
1144pub const FetchResult = struct {
1145 status: http.Status,
1146 body: ?[]const u8 = null,
1147 headers: http.Headers,
1148
1149 allocator: Allocator,
1150 options: FetchOptions,
1151
1152 pub fn deinit(res: *FetchResult) void {
1153 if (res.options.response_strategy == .storage and res.options.response_strategy.storage == .dynamic) {
1154 if (res.body) |body| res.allocator.free(body);
1155 }
1156
1157 res.headers.deinit();
1158 }
1159};
1160
1161pub fn fetch(client: *Client, allocator: Allocator, options: FetchOptions) !FetchResult {
1162 const has_transfer_encoding = options.headers.contains("transfer-encoding");
1163 const has_content_length = options.headers.contains("content-length");
1164
1165 if (has_content_length or has_transfer_encoding) return error.UnsupportedHeader;
1166
1167 const uri = switch (options.location) {
1168 .url => |u| try Uri.parse(u),
1169 .uri => |u| u,
1170 };
1171
1172 var req = try request(client, options.method, uri, options.headers, .{
1173 .header_strategy = options.header_strategy,
1174 .handle_redirects = options.payload == .none,
1175 });
1176 defer req.deinit();
1177
1178 { // Block to maintain lock of file to attempt to prevent a race condition where another process modifies the file while we are reading it.
1179 // This relies on other processes actually obeying the advisory lock, which is not guaranteed.
1180 if (options.payload == .file) try options.payload.file.lock(.shared);
1181 defer if (options.payload == .file) options.payload.file.unlock();
1182
1183 switch (options.payload) {
1184 .string => |str| req.transfer_encoding = .{ .content_length = str.len },
1185 .file => |file| req.transfer_encoding = .{ .content_length = (try file.stat()).size },
1186 .none => {},
1187 }
1188
1189 try req.start();
1190
1191 switch (options.payload) {
1192 .string => |str| try req.writeAll(str),
1193 .file => |file| {
1194 try file.seekTo(0);
1195 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();
1196 try fifo.pump(file.reader(), req.writer());
1197 },
1198 .none => {},
1199 }
1200
1201 try req.finish();
1202 }
1203
1204 try req.wait();
1205
1206 var res = FetchResult{
1207 .status = req.response.status,
1208 .headers = try req.response.headers.clone(allocator),
1209
1210 .allocator = allocator,
1211 .options = options,
1212 };
1213
1214 switch (options.response_strategy) {
1215 .storage => |storage| switch (storage) {
1216 .dynamic => |max| res.body = try req.reader().readAllAlloc(allocator, max),
1217 .static => |buf| res.body = buf[0..try req.reader().readAll(buf)],
1218 },
1219 .file => |file| {
1220 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();
1221 try fifo.pump(req.reader(), file.writer());
1222 },
1223 .none => { // Take advantage of request internals to discard the response body and make the connection available for another request.
1224 req.response.skip = true;
1225
1226 const empty = @as([*]u8, undefined)[0..0];
1227 assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary
1228 },
1229 }
1230
1231 return res;
1232}
1233
1042test {1234test {
1043 const builtin = @import("builtin");1235 const builtin = @import("builtin");
1044 const native_endian = comptime builtin.cpu.arch.endian();1236 const native_endian = comptime builtin.cpu.arch.endian();
lib/std/http/Headers.zig+26-1
...@@ -57,6 +57,18 @@ pub const Headers = struct {...@@ -57,6 +57,18 @@ pub const Headers = struct {
57 return .{ .allocator = allocator };57 return .{ .allocator = allocator };
58 }58 }
5959
60 pub fn initList(allocator: Allocator, list: []const Field) Headers {
61 var new = Headers.init(allocator);
62
63 try new.list.ensureTotalCapacity(allocator, list.len);
64 try new.index.ensureTotalCapacity(allocator, list.len);
65 for (list) |field| {
66 try new.append(field.name, field.value);
67 }
68
69 return new;
70 }
71
60 pub fn deinit(headers: *Headers) void {72 pub fn deinit(headers: *Headers) void {
61 headers.deallocateIndexListsAndFields();73 headers.deallocateIndexListsAndFields();
62 headers.index.deinit(headers.allocator);74 headers.index.deinit(headers.allocator);
...@@ -78,7 +90,7 @@ pub const Headers = struct {...@@ -78,7 +90,7 @@ pub const Headers = struct {
78 entry.name = kv.key_ptr.*;90 entry.name = kv.key_ptr.*;
79 try kv.value_ptr.append(headers.allocator, n);91 try kv.value_ptr.append(headers.allocator, n);
80 } else {92 } else {
81 const name_duped = if (headers.owned) try headers.allocator.dupe(u8, name) else name;93 const name_duped = if (headers.owned) try std.ascii.allocLowerString(headers.allocator, name) else name;
82 errdefer if (headers.owned) headers.allocator.free(name_duped);94 errdefer if (headers.owned) headers.allocator.free(name_duped);
8395
84 entry.name = name_duped;96 entry.name = name_duped;
...@@ -97,6 +109,7 @@ pub const Headers = struct {...@@ -97,6 +109,7 @@ pub const Headers = struct {
97 return headers.index.contains(name);109 return headers.index.contains(name);
98 }110 }
99111
112 /// Removes all headers with the given name.
100 pub fn delete(headers: *Headers, name: []const u8) bool {113 pub fn delete(headers: *Headers, name: []const u8) bool {
101 if (headers.index.fetchRemove(name)) |kv| {114 if (headers.index.fetchRemove(name)) |kv| {
102 var index = kv.value;115 var index = kv.value;
...@@ -268,6 +281,18 @@ pub const Headers = struct {...@@ -268,6 +281,18 @@ pub const Headers = struct {
268 headers.index.clearRetainingCapacity();281 headers.index.clearRetainingCapacity();
269 headers.list.clearRetainingCapacity();282 headers.list.clearRetainingCapacity();
270 }283 }
284
285 pub fn clone(headers: Headers, allocator: Allocator) !Headers {
286 var new = Headers.init(allocator);
287
288 try new.list.ensureTotalCapacity(allocator, headers.list.capacity);
289 try new.index.ensureTotalCapacity(allocator, headers.index.capacity());
290 for (headers.list.items) |field| {
291 try new.append(field.name, field.value);
292 }
293
294 return new;
295 }
271};296};
272297
273test "Headers.append" {298test "Headers.append" {
lib/std/http/Server.zig+46-36
...@@ -185,8 +185,10 @@ pub const Request = struct {...@@ -185,8 +185,10 @@ pub const Request = struct {
185 return error.HttpHeadersInvalid;185 return error.HttpHeadersInvalid;
186186
187 const method_end = mem.indexOfScalar(u8, first_line, ' ') orelse return error.HttpHeadersInvalid;187 const method_end = mem.indexOfScalar(u8, first_line, ' ') orelse return error.HttpHeadersInvalid;
188 if (method_end > 24) return error.HttpHeadersInvalid;
189
188 const method_str = first_line[0..method_end];190 const method_str = first_line[0..method_end];
189 const method = std.meta.stringToEnum(http.Method, method_str) orelse return error.UnknownHttpMethod;191 const method: http.Method = @enumFromInt(http.Method.parse(method_str));
190192
191 const version_start = mem.lastIndexOfScalar(u8, first_line, ' ') orelse return error.HttpHeadersInvalid;193 const version_start = mem.lastIndexOfScalar(u8, first_line, ' ') orelse return error.HttpHeadersInvalid;
192 if (version_start == method_end) return error.HttpHeadersInvalid;194 if (version_start == method_end) return error.HttpHeadersInvalid;
...@@ -411,59 +413,67 @@ pub const Response = struct {...@@ -411,59 +413,67 @@ pub const Response = struct {
411 }413 }
412 try w.writeAll("\r\n");414 try w.writeAll("\r\n");
413415
414 if (!res.headers.contains("server")) {416 if (res.status == .@"continue") {
415 try w.writeAll("Server: zig (std.http)\r\n");417 res.state = .waited; // we still need to send another request after this
416 }418 } else {
419 if (!res.headers.contains("server")) {
420 try w.writeAll("Server: zig (std.http)\r\n");
421 }
417422
418 if (!res.headers.contains("connection")) {423 if (!res.headers.contains("connection")) {
419 const req_connection = res.request.headers.getFirstValue("connection");424 const req_connection = res.request.headers.getFirstValue("connection");
420 const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);425 const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
421426
422 if (req_keepalive) {427 if (req_keepalive) {
423 try w.writeAll("Connection: keep-alive\r\n");428 try w.writeAll("Connection: keep-alive\r\n");
424 } else {429 } else {
425 try w.writeAll("Connection: close\r\n");430 try w.writeAll("Connection: close\r\n");
431 }
426 }432 }
427 }
428433
429 const has_transfer_encoding = res.headers.contains("transfer-encoding");434 const has_transfer_encoding = res.headers.contains("transfer-encoding");
430 const has_content_length = res.headers.contains("content-length");435 const has_content_length = res.headers.contains("content-length");
431436
432 if (!has_transfer_encoding and !has_content_length) {437 if (!has_transfer_encoding and !has_content_length) {
433 switch (res.transfer_encoding) {438 switch (res.transfer_encoding) {
434 .chunked => try w.writeAll("Transfer-Encoding: chunked\r\n"),439 .chunked => try w.writeAll("Transfer-Encoding: chunked\r\n"),
435 .content_length => |content_length| try w.print("Content-Length: {d}\r\n", .{content_length}),440 .content_length => |content_length| try w.print("Content-Length: {d}\r\n", .{content_length}),
436 .none => {},441 .none => {},
437 }
438 } else {
439 if (has_content_length) {
440 const content_length = std.fmt.parseInt(u64, res.headers.getFirstValue("content-length").?, 10) catch return error.InvalidContentLength;
441
442 res.transfer_encoding = .{ .content_length = content_length };
443 } else if (has_transfer_encoding) {
444 const transfer_encoding = res.headers.getFirstValue("transfer-encoding").?;
445 if (std.mem.eql(u8, transfer_encoding, "chunked")) {
446 res.transfer_encoding = .chunked;
447 } else {
448 return error.UnsupportedTransferEncoding;
449 }442 }
450 } else {443 } else {
451 res.transfer_encoding = .none;444 if (has_content_length) {
445 const content_length = std.fmt.parseInt(u64, res.headers.getFirstValue("content-length").?, 10) catch return error.InvalidContentLength;
446
447 res.transfer_encoding = .{ .content_length = content_length };
448 } else if (has_transfer_encoding) {
449 const transfer_encoding = res.headers.getFirstValue("transfer-encoding").?;
450 if (std.mem.eql(u8, transfer_encoding, "chunked")) {
451 res.transfer_encoding = .chunked;
452 } else {
453 return error.UnsupportedTransferEncoding;
454 }
455 } else {
456 res.transfer_encoding = .none;
457 }
452 }458 }
459
460 try w.print("{}", .{res.headers});
453 }461 }
454462
455 try w.print("{}", .{res.headers});463 if (res.request.method == .HEAD) {
464 res.transfer_encoding = .none;
465 }
456466
457 try w.writeAll("\r\n");467 try w.writeAll("\r\n");
458468
459 try buffered.flush();469 try buffered.flush();
460 }470 }
461471
462 pub const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError;472 const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError;
463473
464 pub const TransferReader = std.io.Reader(*Response, TransferReadError, transferRead);474 const TransferReader = std.io.Reader(*Response, TransferReadError, transferRead);
465475
466 pub fn transferReader(res: *Response) TransferReader {476 fn transferReader(res: *Response) TransferReader {
467 return .{ .context = res };477 return .{ .context = res };
468 }478 }
469479
lib/std/http/protocol.zig+6-3
...@@ -534,9 +534,9 @@ pub const HeadersParser = struct {...@@ -534,9 +534,9 @@ pub const HeadersParser = struct {
534534
535 if (r.next_chunk_length == 0) r.done = true;535 if (r.next_chunk_length == 0) r.done = true;
536536
537 return 0;537 return out_index;
538 } else {538 } else if (out_index < buffer.len) {
539 const out_avail = buffer.len;539 const out_avail = buffer.len - out_index;
540540
541 const can_read = @as(usize, @intCast(@min(data_avail, out_avail)));541 const can_read = @as(usize, @intCast(@min(data_avail, out_avail)));
542 const nread = try conn.read(buffer[0..can_read]);542 const nread = try conn.read(buffer[0..can_read]);
...@@ -545,6 +545,8 @@ pub const HeadersParser = struct {...@@ -545,6 +545,8 @@ pub const HeadersParser = struct {
545 if (r.next_chunk_length == 0) r.done = true;545 if (r.next_chunk_length == 0) r.done = true;
546546
547 return nread;547 return nread;
548 } else {
549 return out_index;
548 }550 }
549 },551 },
550 .chunk_data_suffix, .chunk_data_suffix_r, .chunk_head_size, .chunk_head_ext, .chunk_head_r => {552 .chunk_data_suffix, .chunk_data_suffix_r, .chunk_head_size, .chunk_head_ext, .chunk_head_r => {
...@@ -558,6 +560,7 @@ pub const HeadersParser = struct {...@@ -558,6 +560,7 @@ pub const HeadersParser = struct {
558 .chunk_data => if (r.next_chunk_length == 0) {560 .chunk_data => if (r.next_chunk_length == 0) {
559 if (std.mem.eql(u8, conn.peek(), "\r\n")) {561 if (std.mem.eql(u8, conn.peek(), "\r\n")) {
560 r.state = .finished;562 r.state = .finished;
563 r.done = true;
561 } else {564 } else {
562 // The trailer section is formatted identically to the header section.565 // The trailer section is formatted identically to the header section.
563 r.state = .seen_rn;566 r.state = .seen_rn;
test/standalone/http.zig+94-3
...@@ -20,7 +20,19 @@ var server: Server = undefined;...@@ -20,7 +20,19 @@ var server: Server = undefined;
20fn handleRequest(res: *Server.Response) !void {20fn handleRequest(res: *Server.Response) !void {
21 const log = std.log.scoped(.server);21 const log = std.log.scoped(.server);
2222
23 log.info("{s} {s} {s}", .{ @tagName(res.request.method), @tagName(res.request.version), res.request.target });23 log.info("{} {s} {s}", .{ res.request.method, @tagName(res.request.version), res.request.target });
24
25 if (res.request.headers.contains("expect")) {
26 if (mem.eql(u8, res.request.headers.getFirstValue("expect").?, "100-continue")) {
27 res.status = .@"continue";
28 try res.do();
29 res.status = .ok;
30 } else {
31 res.status = .expectation_failed;
32 try res.do();
33 return;
34 }
35 }
2436
25 const body = try res.reader().readAllAlloc(salloc, 8192);37 const body = try res.reader().readAllAlloc(salloc, 8192);
26 defer salloc.free(body);38 defer salloc.free(body);
...@@ -43,6 +55,8 @@ fn handleRequest(res: *Server.Response) !void {...@@ -43,6 +55,8 @@ fn handleRequest(res: *Server.Response) !void {
43 try res.writeAll("Hello, ");55 try res.writeAll("Hello, ");
44 try res.writeAll("World!\n");56 try res.writeAll("World!\n");
45 try res.finish();57 try res.finish();
58 } else {
59 try testing.expectEqual(res.writeAll("errors"), error.NotWriteable);
46 }60 }
47 } else if (mem.startsWith(u8, res.request.target, "/large")) {61 } else if (mem.startsWith(u8, res.request.target, "/large")) {
48 res.transfer_encoding = .{ .content_length = 14 * 1024 + 14 * 10 };62 res.transfer_encoding = .{ .content_length = 14 * 1024 + 14 * 10 };
...@@ -62,7 +76,7 @@ fn handleRequest(res: *Server.Response) !void {...@@ -62,7 +76,7 @@ fn handleRequest(res: *Server.Response) !void {
62 }76 }
6377
64 try res.finish();78 try res.finish();
65 } else if (mem.eql(u8, res.request.target, "/echo-content")) {79 } else if (mem.startsWith(u8, res.request.target, "/echo-content")) {
66 try testing.expectEqualStrings("Hello, World!\n", body);80 try testing.expectEqualStrings("Hello, World!\n", body);
67 try testing.expectEqualStrings("text/plain", res.request.headers.getFirstValue("content-type").?);81 try testing.expectEqualStrings("text/plain", res.request.headers.getFirstValue("content-type").?);
6882
...@@ -571,7 +585,84 @@ pub fn main() !void {...@@ -571,7 +585,84 @@ pub fn main() !void {
571 // connection has been kept alive585 // connection has been kept alive
572 try testing.expect(client.connection_pool.free_len == 1);586 try testing.expect(client.connection_pool.free_len == 1);
573587
574 { // issue 16282588 { // Client.fetch()
589 var h = http.Headers{ .allocator = calloc };
590 defer h.deinit();
591
592 try h.append("content-type", "text/plain");
593
594 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#fetch", .{port});
595 defer calloc.free(location);
596
597 log.info("{s}", .{location});
598 var res = try client.fetch(calloc, .{
599 .location = .{ .url = location },
600 .method = .POST,
601 .headers = h,
602 .payload = .{ .string = "Hello, World!\n" },
603 });
604 defer res.deinit();
605
606 try testing.expectEqualStrings("Hello, World!\n", res.body.?);
607 }
608
609 { // expect: 100-continue
610 var h = http.Headers{ .allocator = calloc };
611 defer h.deinit();
612
613 try h.append("expect", "100-continue");
614 try h.append("content-type", "text/plain");
615
616 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#expect-100", .{port});
617 defer calloc.free(location);
618 const uri = try std.Uri.parse(location);
619
620 log.info("{s}", .{location});
621 var req = try client.request(.POST, uri, h, .{});
622 defer req.deinit();
623
624 req.transfer_encoding = .chunked;
625
626 try req.start();
627 try req.wait();
628 try testing.expectEqual(http.Status.@"continue", req.response.status);
629
630 try req.writeAll("Hello, ");
631 try req.writeAll("World!\n");
632 try req.finish();
633
634 try req.wait();
635 try testing.expectEqual(http.Status.ok, req.response.status);
636
637 const body = try req.reader().readAllAlloc(calloc, 8192);
638 defer calloc.free(body);
639
640 try testing.expectEqualStrings("Hello, World!\n", body);
641 }
642
643 { // expect: garbage
644 var h = http.Headers{ .allocator = calloc };
645 defer h.deinit();
646
647 try h.append("content-type", "text/plain");
648 try h.append("expect", "garbage");
649
650 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#expect-garbage", .{port});
651 defer calloc.free(location);
652 const uri = try std.Uri.parse(location);
653
654 log.info("{s}", .{location});
655 var req = try client.request(.POST, uri, h, .{});
656 defer req.deinit();
657
658 req.transfer_encoding = .chunked;
659
660 try req.start();
661 try req.wait();
662 try testing.expectEqual(http.Status.expectation_failed, req.response.status);
663 }
664
665 { // issue 16282 *** This test leaves the client in an invalid state, it must be last ***
575 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});666 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
576 defer calloc.free(location);667 defer calloc.free(location);
577 const uri = try std.Uri.parse(location);668 const uri = try std.Uri.parse(location);