authorgravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-03-08 08:20:53-06:00
committergravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-03-09 14:55:20-06:00
log634e7155048aeaf15553d866783930f3d22b375c
treee9a610d3cb8cfc6868535f3760b098a4b32937d6
parent0a4130f364c2714b206257d0cf589103da823407
signaturelock-open Commit is signed but in an unrecognized format.

std.http: split Client's parts into their own files

4 files changed, 1010 insertions(+), 977 deletions(-)

lib/std/http.zig+5
...@@ -248,9 +248,14 @@ pub const Status = enum(u10) {...@@ -248,9 +248,14 @@ pub const Status = enum(u10) {
248248
249pub const TransferEncoding = enum {249pub const TransferEncoding = enum {
250 chunked,250 chunked,
251 // compression is intentionally omitted here, as std.http.Client stores it as content-encoding
252};
253
254pub const ContentEncoding = enum {
251 compress,255 compress,
252 deflate,256 deflate,
253 gzip,257 gzip,
258 zstd,
254};259};
255260
256pub const Connection = enum {261pub const Connection = enum {
lib/std/http/Client.zig+11-977
...@@ -13,6 +13,9 @@ const Uri = std.Uri;...@@ -13,6 +13,9 @@ const Uri = std.Uri;
13const Allocator = std.mem.Allocator;13const Allocator = std.mem.Allocator;
14const testing = std.testing;14const testing = std.testing;
1515
16pub const Request = @import("Client/Request.zig");
17pub const Response = @import("Client/Response.zig");
18
16/// Used for tcpConnectToHost and storing HTTP headers when an externally19/// Used for tcpConnectToHost and storing HTTP headers when an externally
17/// managed buffer is not provided.20/// managed buffer is not provided.
18allocator: Allocator,21allocator: Allocator,
...@@ -25,8 +28,8 @@ connection_mutex: std.Thread.Mutex = .{},...@@ -25,8 +28,8 @@ connection_mutex: std.Thread.Mutex = .{},
25connection_pool: ConnectionPool = .{},28connection_pool: ConnectionPool = .{},
26connection_used: ConnectionPool = .{},29connection_used: ConnectionPool = .{},
2730
28const ConnectionPool = std.TailQueue(Connection);31pub const ConnectionPool = std.TailQueue(Connection);
29const ConnectionNode = ConnectionPool.Node;32pub const ConnectionNode = ConnectionPool.Node;
3033
31/// Acquires an existing connection from the connection pool. This function is threadsafe.34/// Acquires an existing connection from the connection pool. This function is threadsafe.
32/// If the caller already holds the connection mutex, it should pass `true` for `held`.35/// If the caller already holds the connection mutex, it should pass `true` for `held`.
...@@ -55,8 +58,9 @@ pub fn release(client: *Client, node: *ConnectionNode) void {...@@ -55,8 +58,9 @@ pub fn release(client: *Client, node: *ConnectionNode) void {
55 client.connection_pool.append(node);58 client.connection_pool.append(node);
56}59}
5760
58const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.ReaderRaw);61pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.ReaderRaw);
59const GzipDecompressor = std.compress.gzip.Decompress(Request.ReaderRaw);62pub const GzipDecompressor = std.compress.gzip.Decompress(Request.ReaderRaw);
63pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.ReaderRaw, .{});
6064
61pub const Connection = struct {65pub const Connection = struct {
62 stream: net.Stream,66 stream: net.Stream,
...@@ -137,976 +141,6 @@ pub const Connection = struct {...@@ -137,976 +141,6 @@ pub const Connection = struct {
137 }141 }
138};142};
139143
140pub const Request = struct {
141 const read_buffer_size = 8192;
142 const ReadBufferIndex = std.math.IntFittingRange(0, read_buffer_size);
143
144 uri: Uri,
145 client: *Client,
146 connection: *ConnectionNode,
147 response: Response,
148 /// These are stored in Request so that they are available when following
149 /// redirects.
150 headers: Headers,
151
152 redirects_left: u32,
153 handle_redirects: bool,
154 compression_init: bool,
155
156 /// Used as a allocator for resolving redirects locations.
157 arena: std.heap.ArenaAllocator,
158
159 /// Read buffer for the connection. This is used to pull in large amounts of data from the connection even if the user asks for a small amount. This can probably be removed with careful planning.
160 read_buffer: [read_buffer_size]u8 = undefined,
161 read_buffer_start: ReadBufferIndex = 0,
162 read_buffer_len: ReadBufferIndex = 0,
163
164 pub const Response = struct {
165 headers: Response.Headers,
166 state: State,
167 header_bytes_owned: bool,
168 /// This could either be a fixed buffer provided by the API user or it
169 /// could be our own array list.
170 header_bytes: std.ArrayListUnmanaged(u8),
171 max_header_bytes: usize,
172 next_chunk_length: u64,
173 done: bool = false,
174
175 compression: union(enum) {
176 deflate: DeflateDecompressor,
177 gzip: GzipDecompressor,
178 none: void,
179 } = .none,
180
181 pub const Headers = struct {
182 status: http.Status,
183 version: http.Version,
184 location: ?[]const u8 = null,
185 content_length: ?u64 = null,
186 transfer_encoding: ?http.TransferEncoding = null, // This should only ever be chunked, compression is handled separately.
187 transfer_compression: ?http.TransferEncoding = null,
188 connection: http.Connection = .close,
189
190 number_of_headers: usize = 0,
191
192 pub fn parse(bytes: []const u8) !Response.Headers {
193 var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
194
195 const first_line = it.first();
196 if (first_line.len < 12)
197 return error.ShortHttpStatusLine;
198
199 const version: http.Version = switch (int64(first_line[0..8])) {
200 int64("HTTP/1.0") => .@"HTTP/1.0",
201 int64("HTTP/1.1") => .@"HTTP/1.1",
202 else => return error.BadHttpVersion,
203 };
204 if (first_line[8] != ' ') return error.HttpHeadersInvalid;
205 const status = @intToEnum(http.Status, parseInt3(first_line[9..12].*));
206
207 var headers: Response.Headers = .{
208 .version = version,
209 .status = status,
210 };
211
212 while (it.next()) |line| {
213 headers.number_of_headers += 1;
214
215 if (line.len == 0) return error.HttpHeadersInvalid;
216 switch (line[0]) {
217 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
218 else => {},
219 }
220 var line_it = mem.split(u8, line, ": ");
221 const header_name = line_it.first();
222 const header_value = line_it.rest();
223 if (std.ascii.eqlIgnoreCase(header_name, "location")) {
224 if (headers.location != null) return error.HttpHeadersInvalid;
225 headers.location = header_value;
226 } else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
227 if (headers.content_length != null) return error.HttpHeadersInvalid;
228 headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
229 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
230 if (headers.transfer_encoding != null or headers.transfer_compression != null) return error.HttpHeadersInvalid;
231
232 // Transfer-Encoding: second, first
233 // Transfer-Encoding: deflate, chunked
234 var iter = std.mem.splitBackwards(u8, header_value, ",");
235
236 if (iter.next()) |first| {
237 const kind = std.meta.stringToEnum(
238 http.TransferEncoding,
239 std.mem.trim(u8, first, " "),
240 ) orelse
241 return error.HttpTransferEncodingUnsupported;
242
243 switch (kind) {
244 .chunked => headers.transfer_encoding = .chunked,
245 .compress => headers.transfer_compression = .compress,
246 .deflate => headers.transfer_compression = .deflate,
247 .gzip => headers.transfer_compression = .gzip,
248 }
249 }
250
251 if (iter.next()) |second| {
252 if (headers.transfer_compression != null) return error.HttpTransferEncodingUnsupported;
253
254 const kind = std.meta.stringToEnum(
255 http.TransferEncoding,
256 std.mem.trim(u8, second, " "),
257 ) orelse
258 return error.HttpTransferEncodingUnsupported;
259
260 switch (kind) {
261 .chunked => return error.HttpHeadersInvalid, // chunked must come last
262 .compress => return error.HttpTransferEncodingUnsupported, // compress not supported
263 .deflate => headers.transfer_compression = .deflate,
264 .gzip => headers.transfer_compression = .gzip,
265 }
266 }
267
268 if (iter.next()) |_| return error.HttpTransferEncodingUnsupported;
269 } else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
270 if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
271
272 const kind = std.meta.stringToEnum(
273 http.TransferEncoding,
274 std.mem.trim(u8, header_value, " "),
275 ) orelse
276 return error.HttpTransferEncodingUnsupported;
277
278 switch (kind) {
279 .chunked => return error.HttpHeadersInvalid, // not transfer encoding
280 .compress => return error.HttpTransferEncodingUnsupported, // compress not supported
281 .deflate => headers.transfer_compression = .deflate,
282 .gzip => headers.transfer_compression = .gzip,
283 }
284 } else if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
285 if (std.ascii.eqlIgnoreCase(header_value, "keep-alive")) {
286 headers.connection = .keep_alive;
287 } else if (std.ascii.eqlIgnoreCase(header_value, "close")) {
288 headers.connection = .close;
289 } else {
290 return error.HttpConnectionHeaderUnsupported;
291 }
292 }
293 }
294
295 return headers;
296 }
297
298 test "parse headers" {
299 const example =
300 "HTTP/1.1 301 Moved Permanently\r\n" ++
301 "Location: https://www.example.com/\r\n" ++
302 "Content-Type: text/html; charset=UTF-8\r\n" ++
303 "Content-Length: 220\r\n\r\n";
304 const parsed = try Response.Headers.parse(example);
305 try testing.expectEqual(http.Version.@"HTTP/1.1", parsed.version);
306 try testing.expectEqual(http.Status.moved_permanently, parsed.status);
307 try testing.expectEqualStrings("https://www.example.com/", parsed.location orelse
308 return error.TestFailed);
309 try testing.expectEqual(@as(?u64, 220), parsed.content_length);
310 }
311
312 test "header continuation" {
313 const example =
314 "HTTP/1.0 200 OK\r\n" ++
315 "Content-Type: text/html;\r\n charset=UTF-8\r\n" ++
316 "Content-Length: 220\r\n\r\n";
317 try testing.expectError(
318 error.HttpHeaderContinuationsUnsupported,
319 Response.Headers.parse(example),
320 );
321 }
322
323 test "extra content length" {
324 const example =
325 "HTTP/1.0 200 OK\r\n" ++
326 "Content-Length: 220\r\n" ++
327 "Content-Type: text/html; charset=UTF-8\r\n" ++
328 "content-length: 220\r\n\r\n";
329 try testing.expectError(
330 error.HttpHeadersInvalid,
331 Response.Headers.parse(example),
332 );
333 }
334 };
335
336 pub const State = enum {
337 /// Begin header parsing states.
338 invalid,
339 start,
340 seen_r,
341 seen_rn,
342 seen_rnr,
343 finished,
344 /// Begin transfer-encoding: chunked parsing states.
345 chunk_size_prefix_r,
346 chunk_size_prefix_n,
347 chunk_size,
348 chunk_r,
349 chunk_data,
350
351 pub fn isContent(self: State) bool {
352 return switch (self) {
353 .invalid, .start, .seen_r, .seen_rn, .seen_rnr => false,
354 .finished, .chunk_size_prefix_r, .chunk_size_prefix_n, .chunk_size, .chunk_r, .chunk_data => true,
355 };
356 }
357 };
358
359 pub fn initDynamic(max: usize) Response {
360 return .{
361 .state = .start,
362 .headers = undefined,
363 .header_bytes = .{},
364 .max_header_bytes = max,
365 .header_bytes_owned = true,
366 .next_chunk_length = undefined,
367 };
368 }
369
370 pub fn initStatic(buf: []u8) Response {
371 return .{
372 .state = .start,
373 .headers = undefined,
374 .header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
375 .max_header_bytes = buf.len,
376 .header_bytes_owned = false,
377 .next_chunk_length = undefined,
378 };
379 }
380
381 /// Returns how many bytes are part of HTTP headers. Always less than or
382 /// equal to bytes.len. If the amount returned is less than bytes.len, it
383 /// means the headers ended and the first byte after the double \r\n\r\n is
384 /// located at `bytes[result]`.
385 pub fn findHeadersEnd(r: *Response, bytes: []const u8) usize {
386 var index: usize = 0;
387
388 // TODO: https://github.com/ziglang/zig/issues/8220
389 state: while (true) {
390 switch (r.state) {
391 .invalid => unreachable,
392 .finished => unreachable,
393 .start => while (true) {
394 switch (bytes.len - index) {
395 0 => return index,
396 1 => {
397 if (bytes[index] == '\r')
398 r.state = .seen_r;
399 return index + 1;
400 },
401 2 => {
402 if (int16(bytes[index..][0..2]) == int16("\r\n")) {
403 r.state = .seen_rn;
404 } else if (bytes[index + 1] == '\r') {
405 r.state = .seen_r;
406 }
407 return index + 2;
408 },
409 3 => {
410 if (int16(bytes[index..][0..2]) == int16("\r\n") and
411 bytes[index + 2] == '\r')
412 {
413 r.state = .seen_rnr;
414 } else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n")) {
415 r.state = .seen_rn;
416 } else if (bytes[index + 2] == '\r') {
417 r.state = .seen_r;
418 }
419 return index + 3;
420 },
421 4...15 => {
422 if (int32(bytes[index..][0..4]) == int32("\r\n\r\n")) {
423 r.state = .finished;
424 return index + 4;
425 } else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n") and
426 bytes[index + 3] == '\r')
427 {
428 r.state = .seen_rnr;
429 index += 4;
430 continue :state;
431 } else if (int16(bytes[index + 2 ..][0..2]) == int16("\r\n")) {
432 r.state = .seen_rn;
433 index += 4;
434 continue :state;
435 } else if (bytes[index + 3] == '\r') {
436 r.state = .seen_r;
437 index += 4;
438 continue :state;
439 }
440 index += 4;
441 continue;
442 },
443 else => {
444 const chunk = bytes[index..][0..16];
445 const v: @Vector(16, u8) = chunk.*;
446 const matches_r = v == @splat(16, @as(u8, '\r'));
447 const iota = std.simd.iota(u8, 16);
448 const default = @splat(16, @as(u8, 16));
449 const sub_index = @reduce(.Min, @select(u8, matches_r, iota, default));
450 switch (sub_index) {
451 0...12 => {
452 index += sub_index + 4;
453 if (int32(chunk[sub_index..][0..4]) == int32("\r\n\r\n")) {
454 r.state = .finished;
455 return index;
456 }
457 continue;
458 },
459 13 => {
460 index += 16;
461 if (int16(chunk[14..][0..2]) == int16("\n\r")) {
462 r.state = .seen_rnr;
463 continue :state;
464 }
465 continue;
466 },
467 14 => {
468 index += 16;
469 if (chunk[15] == '\n') {
470 r.state = .seen_rn;
471 continue :state;
472 }
473 continue;
474 },
475 15 => {
476 r.state = .seen_r;
477 index += 16;
478 continue :state;
479 },
480 16 => {
481 index += 16;
482 continue;
483 },
484 else => unreachable,
485 }
486 },
487 }
488 },
489
490 .seen_r => switch (bytes.len - index) {
491 0 => return index,
492 1 => {
493 switch (bytes[index]) {
494 '\n' => r.state = .seen_rn,
495 '\r' => r.state = .seen_r,
496 else => r.state = .start,
497 }
498 return index + 1;
499 },
500 2 => {
501 if (int16(bytes[index..][0..2]) == int16("\n\r")) {
502 r.state = .seen_rnr;
503 return index + 2;
504 }
505 r.state = .start;
506 return index + 2;
507 },
508 else => {
509 if (int16(bytes[index..][0..2]) == int16("\n\r") and
510 bytes[index + 2] == '\n')
511 {
512 r.state = .finished;
513 return index + 3;
514 }
515 index += 3;
516 r.state = .start;
517 continue :state;
518 },
519 },
520 .seen_rn => switch (bytes.len - index) {
521 0 => return index,
522 1 => {
523 switch (bytes[index]) {
524 '\r' => r.state = .seen_rnr,
525 else => r.state = .start,
526 }
527 return index + 1;
528 },
529 else => {
530 if (int16(bytes[index..][0..2]) == int16("\r\n")) {
531 r.state = .finished;
532 return index + 2;
533 }
534 index += 2;
535 r.state = .start;
536 continue :state;
537 },
538 },
539 .seen_rnr => switch (bytes.len - index) {
540 0 => return index,
541 else => {
542 if (bytes[index] == '\n') {
543 r.state = .finished;
544 return index + 1;
545 }
546 index += 1;
547 r.state = .start;
548 continue :state;
549 },
550 },
551 .chunk_size_prefix_r => unreachable,
552 .chunk_size_prefix_n => unreachable,
553 .chunk_size => unreachable,
554 .chunk_r => unreachable,
555 .chunk_data => unreachable,
556 }
557
558 return index;
559 }
560 }
561
562 pub fn findChunkedLen(r: *Response, bytes: []const u8) usize {
563 var i: usize = 0;
564 if (r.state == .chunk_size) {
565 while (i < bytes.len) : (i += 1) {
566 const digit = switch (bytes[i]) {
567 '0'...'9' => |b| b - '0',
568 'A'...'Z' => |b| b - 'A' + 10,
569 'a'...'z' => |b| b - 'a' + 10,
570 '\r' => {
571 r.state = .chunk_r;
572 i += 1;
573 break;
574 },
575 else => {
576 r.state = .invalid;
577 return i;
578 },
579 };
580 const mul = @mulWithOverflow(r.next_chunk_length, 16);
581 if (mul[1] != 0) {
582 r.state = .invalid;
583 return i;
584 }
585 const add = @addWithOverflow(mul[0], digit);
586 if (add[1] != 0) {
587 r.state = .invalid;
588 return i;
589 }
590 r.next_chunk_length = add[0];
591 } else {
592 return i;
593 }
594 }
595 assert(r.state == .chunk_r);
596 if (i == bytes.len) return i;
597
598 if (bytes[i] == '\n') {
599 r.state = .chunk_data;
600 return i + 1;
601 } else {
602 r.state = .invalid;
603 return i;
604 }
605 }
606
607 fn parseInt3(nnn: @Vector(3, u8)) u10 {
608 const zero: @Vector(3, u8) = .{ '0', '0', '0' };
609 const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
610 return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
611 }
612
613 test parseInt3 {
614 const expectEqual = std.testing.expectEqual;
615 try expectEqual(@as(u10, 0), parseInt3("000".*));
616 try expectEqual(@as(u10, 418), parseInt3("418".*));
617 try expectEqual(@as(u10, 999), parseInt3("999".*));
618 }
619
620 test "find headers end basic" {
621 var buffer: [1]u8 = undefined;
622 var r = Response.initStatic(&buffer);
623 try testing.expectEqual(@as(usize, 10), r.findHeadersEnd("HTTP/1.1 4"));
624 try testing.expectEqual(@as(usize, 2), r.findHeadersEnd("18"));
625 try testing.expectEqual(@as(usize, 8), r.findHeadersEnd(" lol\r\n\r\nblah blah"));
626 }
627
628 test "find headers end vectorized" {
629 var buffer: [1]u8 = undefined;
630 var r = Response.initStatic(&buffer);
631 const example =
632 "HTTP/1.1 301 Moved Permanently\r\n" ++
633 "Location: https://www.example.com/\r\n" ++
634 "Content-Type: text/html; charset=UTF-8\r\n" ++
635 "Content-Length: 220\r\n" ++
636 "\r\ncontent";
637 try testing.expectEqual(@as(usize, 131), r.findHeadersEnd(example));
638 }
639
640 test "find headers end bug" {
641 var buffer: [1]u8 = undefined;
642 var r = Response.initStatic(&buffer);
643 const trail = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
644 const example =
645 "HTTP/1.1 200 OK\r\n" ++
646 "Access-Control-Allow-Origin: https://render.githubusercontent.com\r\n" ++
647 "content-disposition: attachment; filename=zig-0.10.0.tar.gz\r\n" ++
648 "Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'; sandbox\r\n" ++
649 "Content-Type: application/x-gzip\r\n" ++
650 "ETag: \"bfae0af6b01c7c0d89eb667cb5f0e65265968aeebda2689177e6b26acd3155ca\"\r\n" ++
651 "Strict-Transport-Security: max-age=31536000\r\n" ++
652 "Vary: Authorization,Accept-Encoding,Origin\r\n" ++
653 "X-Content-Type-Options: nosniff\r\n" ++
654 "X-Frame-Options: deny\r\n" ++
655 "X-XSS-Protection: 1; mode=block\r\n" ++
656 "Date: Fri, 06 Jan 2023 22:26:22 GMT\r\n" ++
657 "Transfer-Encoding: chunked\r\n" ++
658 "X-GitHub-Request-Id: 89C6:17E9:A7C9E:124B51:63B8A00E\r\n" ++
659 "connection: close\r\n\r\n" ++ trail;
660 try testing.expectEqual(@as(usize, example.len - trail.len), r.findHeadersEnd(example));
661 }
662 };
663
664 pub const RequestTransfer = union(enum) {
665 content_length: u64,
666 chunked: void,
667 none: void,
668 };
669
670 pub const Headers = struct {
671 version: http.Version = .@"HTTP/1.1",
672 method: http.Method = .GET,
673 user_agent: []const u8 = "Zig (std.http)",
674 connection: http.Connection = .keep_alive,
675 transfer_encoding: RequestTransfer = .none,
676
677 custom: []const http.CustomHeader = &[_]http.CustomHeader{},
678 };
679
680 pub const Options = struct {
681 handle_redirects: bool = true,
682 max_redirects: u32 = 3,
683 header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
684
685 pub const HeaderStrategy = union(enum) {
686 /// In this case, the client's Allocator will be used to store the
687 /// entire HTTP header. This value is the maximum total size of
688 /// HTTP headers allowed, otherwise
689 /// error.HttpHeadersExceededSizeLimit is returned from read().
690 dynamic: usize,
691 /// This is used to store the entire HTTP header. If the HTTP
692 /// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
693 /// is returned from read(). When this is used, `error.OutOfMemory`
694 /// cannot be returned from `read()`.
695 static: []u8,
696 };
697 };
698
699 /// Frees all resources associated with the request.
700 pub fn deinit(req: *Request) void {
701 switch (req.response.compression) {
702 .none => {},
703 .deflate => |*deflate| deflate.deinit(),
704 .gzip => |*gzip| gzip.deinit(),
705 }
706
707 if (req.response.header_bytes_owned) {
708 req.response.header_bytes.deinit(req.client.allocator);
709 }
710
711 if (!req.response.done) {
712 // If the response wasn't fully read, then we need to close the connection.
713 req.connection.data.closing = true;
714 req.client.release(req.connection);
715 }
716
717 req.arena.deinit();
718 req.* = undefined;
719 }
720
721 const ReadRawError = Connection.ReadError || Uri.ParseError || RequestError || error{
722 UnexpectedEndOfStream,
723 TooManyHttpRedirects,
724 HttpRedirectMissingLocation,
725 HttpHeadersInvalid,
726 };
727
728 const ReaderRaw = std.io.Reader(*Request, ReadRawError, readRaw);
729
730 /// Read from the underlying stream, without decompressing or parsing the headers. Must be called
731 /// after waitForCompleteHead() has returned successfully.
732 pub fn readRaw(req: *Request, buffer: []u8) ReadRawError!usize {
733 assert(req.response.state.isContent());
734
735 var index: usize = 0;
736 while (index == 0) {
737 const amt = try req.readRawAdvanced(buffer[index..]);
738 if (amt == 0 and req.response.done) break;
739 index += amt;
740 }
741
742 return index;
743 }
744
745 fn checkForCompleteHead(req: *Request, buffer: []u8) !usize {
746 switch (req.response.state) {
747 .invalid => unreachable,
748 .start, .seen_r, .seen_rn, .seen_rnr => {},
749 else => return 0, // No more headers to read.
750 }
751
752 const i = req.response.findHeadersEnd(buffer[0..]);
753 if (req.response.state == .invalid) return error.HttpHeadersInvalid;
754
755 const headers_data = buffer[0..i];
756 if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
757 return error.HttpHeadersExceededSizeLimit;
758 }
759 try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
760
761 if (req.response.state == .finished) {
762 req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
763
764 if (req.response.headers.connection == .keep_alive) {
765 req.connection.data.closing = false;
766 } else {
767 req.connection.data.closing = true;
768 }
769
770 if (req.response.headers.transfer_encoding) |transfer_encoding| {
771 switch (transfer_encoding) {
772 .chunked => {
773 req.response.next_chunk_length = 0;
774 req.response.state = .chunk_size;
775 },
776 .compress => unreachable,
777 .deflate => unreachable,
778 .gzip => unreachable,
779 }
780 } else if (req.response.headers.content_length) |content_length| {
781 req.response.next_chunk_length = content_length;
782
783 if (content_length == 0) req.response.done = true;
784 } else {
785 req.response.done = true;
786 }
787
788 return i;
789 }
790
791 return 0;
792 }
793
794 pub const WaitForCompleteHeadError = ReadRawError || error{
795 UnexpectedEndOfStream,
796
797 HttpHeadersExceededSizeLimit,
798 ShortHttpStatusLine,
799 BadHttpVersion,
800 HttpHeaderContinuationsUnsupported,
801 HttpTransferEncodingUnsupported,
802 HttpConnectionHeaderUnsupported,
803 };
804
805 /// Reads a complete response head. Any leftover data is stored in the request. This function is idempotent.
806 pub fn waitForCompleteHead(req: *Request) WaitForCompleteHeadError!void {
807 if (req.response.state.isContent()) return;
808
809 while (true) {
810 const nread = try req.connection.data.read(req.read_buffer[0..]);
811 const amt = try checkForCompleteHead(req, req.read_buffer[0..nread]);
812
813 if (amt != 0) {
814 req.read_buffer_start = @intCast(ReadBufferIndex, amt);
815 req.read_buffer_len = @intCast(ReadBufferIndex, nread);
816 return;
817 } else if (nread == 0) {
818 return error.UnexpectedEndOfStream;
819 }
820 }
821 }
822
823 /// This one can return 0 without meaning EOF.
824 fn readRawAdvanced(req: *Request, buffer: []u8) !usize {
825 assert(req.response.state.isContent());
826 if (req.response.done) return 0;
827
828 // var in: []const u8 = undefined;
829 if (req.read_buffer_start == req.read_buffer_len) {
830 const nread = try req.connection.data.read(req.read_buffer[0..]);
831 if (nread == 0) return error.UnexpectedEndOfStream;
832
833 req.read_buffer_start = 0;
834 req.read_buffer_len = @intCast(ReadBufferIndex, nread);
835 }
836
837 var out_index: usize = 0;
838 while (true) {
839 switch (req.response.state) {
840 .invalid, .start, .seen_r, .seen_rn, .seen_rnr => unreachable,
841 .finished => {
842 // TODO https://github.com/ziglang/zig/issues/14039
843 const buf_avail = req.read_buffer_len - req.read_buffer_start;
844 const data_avail = req.response.next_chunk_length;
845 const out_avail = buffer.len;
846
847 if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
848 const can_read = @intCast(usize, @min(buf_avail, data_avail));
849 req.response.next_chunk_length -= can_read;
850
851 if (req.response.next_chunk_length == 0) {
852 req.client.release(req.connection);
853 req.connection = undefined;
854 req.response.done = true;
855 }
856
857 return 0; // skip over as much data as possible
858 }
859
860 const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
861 req.response.next_chunk_length -= can_read;
862
863 mem.copy(u8, buffer[0..], req.read_buffer[req.read_buffer_start..][0..can_read]);
864 req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
865
866 if (req.response.next_chunk_length == 0) {
867 req.client.release(req.connection);
868 req.connection = undefined;
869 req.response.done = true;
870 }
871
872 return can_read;
873 },
874 .chunk_size_prefix_r => switch (req.read_buffer_len - req.read_buffer_start) {
875 0 => return out_index,
876 1 => switch (req.read_buffer[req.read_buffer_start]) {
877 '\r' => {
878 req.response.state = .chunk_size_prefix_n;
879 return out_index;
880 },
881 else => {
882 req.response.state = .invalid;
883 return error.HttpHeadersInvalid;
884 },
885 },
886 else => switch (int16(req.read_buffer[req.read_buffer_start..][0..2])) {
887 int16("\r\n") => {
888 req.read_buffer_start += 2;
889 req.response.state = .chunk_size;
890 continue;
891 },
892 else => {
893 req.response.state = .invalid;
894 return error.HttpHeadersInvalid;
895 },
896 },
897 },
898 .chunk_size_prefix_n => switch (req.read_buffer_len - req.read_buffer_start) {
899 0 => return out_index,
900 else => switch (req.read_buffer[req.read_buffer_start]) {
901 '\n' => {
902 req.read_buffer_start += 1;
903 req.response.state = .chunk_size;
904 continue;
905 },
906 else => {
907 req.response.state = .invalid;
908 return error.HttpHeadersInvalid;
909 },
910 },
911 },
912 .chunk_size, .chunk_r => {
913 const i = req.response.findChunkedLen(req.read_buffer[req.read_buffer_start..req.read_buffer_len]);
914 switch (req.response.state) {
915 .invalid => return error.HttpHeadersInvalid,
916 .chunk_data => {
917 if (req.response.next_chunk_length == 0) {
918 req.response.done = true;
919 req.client.release(req.connection);
920 req.connection = undefined;
921
922 return out_index;
923 }
924
925 req.read_buffer_start += @intCast(ReadBufferIndex, i);
926 continue;
927 },
928 .chunk_size => return out_index,
929 else => unreachable,
930 }
931 },
932 .chunk_data => {
933 // TODO https://github.com/ziglang/zig/issues/14039
934 const buf_avail = req.read_buffer_len - req.read_buffer_start;
935 const data_avail = req.response.next_chunk_length;
936 const out_avail = buffer.len - out_index;
937
938 if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
939 const can_read = @intCast(usize, @min(buf_avail, data_avail));
940 req.response.next_chunk_length -= can_read;
941
942 if (req.response.next_chunk_length == 0) {
943 req.client.release(req.connection);
944 req.connection = undefined;
945 req.response.done = true;
946 continue;
947 }
948
949 return 0; // skip over as much data as possible
950 }
951
952 const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
953 req.response.next_chunk_length -= can_read;
954
955 mem.copy(u8, buffer[out_index..], req.read_buffer[req.read_buffer_start..][0..can_read]);
956 req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
957 out_index += can_read;
958
959 if (req.response.next_chunk_length == 0) {
960 req.response.state = .chunk_size_prefix_r;
961
962 continue;
963 }
964
965 return out_index;
966 },
967 }
968 }
969 }
970
971 pub const ReadError = DeflateDecompressor.Error || GzipDecompressor.Error || WaitForCompleteHeadError || error{
972 BadHeader,
973 InvalidCompression,
974 StreamTooLong,
975 InvalidWindowSize,
976 };
977
978 pub const Reader = std.io.Reader(*Request, ReadError, read);
979
980 pub fn reader(req: *Request) Reader {
981 return .{ .context = req };
982 }
983
984 pub fn read(req: *Request, buffer: []u8) ReadError!usize {
985 while (true) {
986 if (!req.response.state.isContent()) try req.waitForCompleteHead();
987
988 if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
989 assert(try req.readRaw(buffer) == 0);
990
991 if (req.redirects_left == 0) return error.TooManyHttpRedirects;
992
993 const location = req.response.headers.location orelse
994 return error.HttpRedirectMissingLocation;
995 const new_url = Uri.parse(location) catch try Uri.parseWithoutScheme(location);
996
997 var new_arena = std.heap.ArenaAllocator.init(req.client.allocator);
998 const resolved_url = try req.uri.resolve(new_url, false, new_arena.allocator());
999 errdefer new_arena.deinit();
1000
1001 req.arena.deinit();
1002 req.arena = new_arena;
1003
1004 const new_req = try req.client.request(resolved_url, req.headers, .{
1005 .max_redirects = req.redirects_left - 1,
1006 .header_strategy = if (req.response.header_bytes_owned) .{
1007 .dynamic = req.response.max_header_bytes,
1008 } else .{
1009 .static = req.response.header_bytes.unusedCapacitySlice(),
1010 },
1011 });
1012 req.deinit();
1013 req.* = new_req;
1014 } else {
1015 break;
1016 }
1017 }
1018
1019 if (req.response.compression == .none) {
1020 if (req.response.headers.transfer_compression) |compression| {
1021 switch (compression) {
1022 .compress => unreachable,
1023 .deflate => req.response.compression = .{
1024 .deflate = try std.compress.zlib.zlibStream(req.client.allocator, ReaderRaw{ .context = req }),
1025 },
1026 .gzip => req.response.compression = .{
1027 .gzip = try std.compress.gzip.decompress(req.client.allocator, ReaderRaw{ .context = req }),
1028 },
1029 .chunked => unreachable,
1030 }
1031 }
1032 }
1033
1034 return switch (req.response.compression) {
1035 .deflate => |*deflate| try deflate.read(buffer),
1036 .gzip => |*gzip| try gzip.read(buffer),
1037 else => try req.readRaw(buffer),
1038 };
1039 }
1040
1041 pub fn readAll(req: *Request, buffer: []u8) !usize {
1042 var index: usize = 0;
1043 while (index < buffer.len) {
1044 const amt = try read(req, buffer[index..]);
1045 if (amt == 0) break;
1046 index += amt;
1047 }
1048 return index;
1049 }
1050
1051 pub const WriteError = Connection.WriteError || error{MessageTooLong};
1052
1053 pub const Writer = std.io.Writer(*Request, WriteError, write);
1054
1055 pub fn writer(req: *Request) Writer {
1056 return .{ .context = req };
1057 }
1058
1059 /// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.
1060 pub fn write(req: *Request, bytes: []const u8) !usize {
1061 switch (req.headers.transfer_encoding) {
1062 .chunked => {
1063 try req.connection.data.writer().print("{x}\r\n", .{bytes.len});
1064 try req.connection.data.writeAll(bytes);
1065 try req.connection.data.writeAll("\r\n");
1066
1067 return bytes.len;
1068 },
1069 .content_length => |*len| {
1070 if (len.* < bytes.len) return error.MessageTooLong;
1071
1072 const amt = try req.connection.data.write(bytes);
1073 len.* -= amt;
1074 return amt;
1075 },
1076 .none => return error.NotWriteable,
1077 }
1078 }
1079
1080 /// Finish the body of a request. This notifies the server that you have no more data to send.
1081 pub fn finish(req: *Request) !void {
1082 switch (req.headers.transfer_encoding) {
1083 .chunked => try req.connection.data.writeAll("0\r\n"),
1084 .content_length => |len| if (len != 0) return error.MessageNotCompleted,
1085 .none => {},
1086 }
1087 }
1088
1089 inline fn int16(array: *const [2]u8) u16 {
1090 return @bitCast(u16, array.*);
1091 }
1092
1093 inline fn int32(array: *const [4]u8) u32 {
1094 return @bitCast(u32, array.*);
1095 }
1096
1097 inline fn int64(array: *const [8]u8) u64 {
1098 return @bitCast(u64, array.*);
1099 }
1100
1101 test {
1102 const builtin = @import("builtin");
1103
1104 if (builtin.os.tag == .wasi) return error.SkipZigTest;
1105
1106 _ = Response;
1107 }
1108};
1109
1110pub fn deinit(client: *Client) void {144pub fn deinit(client: *Client) void {
1111 client.connection_mutex.lock();145 client.connection_mutex.lock();
1112146
...@@ -1231,8 +265,8 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req...@@ -1231,8 +265,8 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req
1231 .handle_redirects = options.handle_redirects,265 .handle_redirects = options.handle_redirects,
1232 .compression_init = false,266 .compression_init = false,
1233 .response = switch (options.header_strategy) {267 .response = switch (options.header_strategy) {
1234 .dynamic => |max| Request.Response.initDynamic(max),268 .dynamic => |max| Response.initDynamic(max),
1235 .static => |buf| Request.Response.initStatic(buf),269 .static => |buf| Response.initStatic(buf),
1236 },270 },
1237 .arena = undefined,271 .arena = undefined,
1238 };272 };
...@@ -1274,7 +308,7 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req...@@ -1274,7 +308,7 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req
1274 } else {308 } else {
1275 try writer.writeAll("\r\nConnection: keep-alive");309 try writer.writeAll("\r\nConnection: keep-alive");
1276 }310 }
1277 try writer.writeAll("\r\nAccept-Encoding: gzip, deflate");311 try writer.writeAll("\r\nAccept-Encoding: gzip, deflate, zstd");
1278312
1279 switch (headers.transfer_encoding) {313 switch (headers.transfer_encoding) {
1280 .chunked => try writer.writeAll("\r\nTransfer-Encoding: chunked"),314 .chunked => try writer.writeAll("\r\nTransfer-Encoding: chunked"),
lib/std/http/Client/Request.zig created+488
...@@ -0,0 +1,488 @@
1const std = @import("std");
2const http = std.http;
3const Uri = std.Uri;
4const mem = std.mem;
5const assert = std.debug.assert;
6
7const Client = @import("../Client.zig");
8const Connection = Client.Connection;
9const ConnectionNode = Client.ConnectionNode;
10const Response = @import("Response.zig");
11
12const Request = @This();
13
14const read_buffer_size = 8192;
15const ReadBufferIndex = std.math.IntFittingRange(0, read_buffer_size);
16
17uri: Uri,
18client: *Client,
19connection: *ConnectionNode,
20response: Response,
21/// These are stored in Request so that they are available when following
22/// redirects.
23headers: Headers,
24
25redirects_left: u32,
26handle_redirects: bool,
27compression_init: bool,
28
29/// Used as a allocator for resolving redirects locations.
30arena: std.heap.ArenaAllocator,
31
32/// Read buffer for the connection. This is used to pull in large amounts of data from the connection even if the user asks for a small amount. This can probably be removed with careful planning.
33read_buffer: [read_buffer_size]u8 = undefined,
34read_buffer_start: ReadBufferIndex = 0,
35read_buffer_len: ReadBufferIndex = 0,
36
37pub const RequestTransfer = union(enum) {
38 content_length: u64,
39 chunked: void,
40 none: void,
41};
42
43pub const Headers = struct {
44 version: http.Version = .@"HTTP/1.1",
45 method: http.Method = .GET,
46 user_agent: []const u8 = "zig (std.http)",
47 connection: http.Connection = .keep_alive,
48 transfer_encoding: RequestTransfer = .none,
49
50 custom: []const http.CustomHeader = &[_]http.CustomHeader{},
51};
52
53pub const Options = struct {
54 handle_redirects: bool = true,
55 max_redirects: u32 = 3,
56 header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
57
58 pub const HeaderStrategy = union(enum) {
59 /// In this case, the client's Allocator will be used to store the
60 /// entire HTTP header. This value is the maximum total size of
61 /// HTTP headers allowed, otherwise
62 /// error.HttpHeadersExceededSizeLimit is returned from read().
63 dynamic: usize,
64 /// This is used to store the entire HTTP header. If the HTTP
65 /// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
66 /// is returned from read(). When this is used, `error.OutOfMemory`
67 /// cannot be returned from `read()`.
68 static: []u8,
69 };
70};
71
72/// Frees all resources associated with the request.
73pub fn deinit(req: *Request) void {
74 switch (req.response.compression) {
75 .none => {},
76 .deflate => |*deflate| deflate.deinit(),
77 .gzip => |*gzip| gzip.deinit(),
78 .zstd => |*zstd| zstd.deinit(),
79 }
80
81 if (req.response.header_bytes_owned) {
82 req.response.header_bytes.deinit(req.client.allocator);
83 }
84
85 if (!req.response.done) {
86 // If the response wasn't fully read, then we need to close the connection.
87 req.connection.data.closing = true;
88 req.client.release(req.connection);
89 }
90
91 req.arena.deinit();
92 req.* = undefined;
93}
94
95pub const ReadRawError = Connection.ReadError || Uri.ParseError || Client.RequestError || error{
96 UnexpectedEndOfStream,
97 TooManyHttpRedirects,
98 HttpRedirectMissingLocation,
99 HttpHeadersInvalid,
100};
101
102pub const ReaderRaw = std.io.Reader(*Request, ReadRawError, readRaw);
103
104/// Read from the underlying stream, without decompressing or parsing the headers. Must be called
105/// after waitForCompleteHead() has returned successfully.
106pub fn readRaw(req: *Request, buffer: []u8) ReadRawError!usize {
107 assert(req.response.state.isContent());
108
109 var index: usize = 0;
110 while (index == 0) {
111 const amt = try req.readRawAdvanced(buffer[index..]);
112 if (amt == 0 and req.response.done) break;
113 index += amt;
114 }
115
116 return index;
117}
118
119fn checkForCompleteHead(req: *Request, buffer: []u8) !usize {
120 switch (req.response.state) {
121 .invalid => unreachable,
122 .start, .seen_r, .seen_rn, .seen_rnr => {},
123 else => return 0, // No more headers to read.
124 }
125
126 const i = req.response.findHeadersEnd(buffer[0..]);
127 if (req.response.state == .invalid) return error.HttpHeadersInvalid;
128
129 const headers_data = buffer[0..i];
130 if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
131 return error.HttpHeadersExceededSizeLimit;
132 }
133 try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
134
135 if (req.response.state == .finished) {
136 req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
137
138 if (req.response.upgrade) |_| {
139 req.connection.data.closing = false;
140 req.response.done = true;
141 return i;
142 }
143
144 if (req.response.headers.connection == .keep_alive) {
145 req.connection.data.closing = false;
146 } else {
147 req.connection.data.closing = true;
148 }
149
150 if (req.response.headers.transfer_encoding) |transfer_encoding| {
151 switch (transfer_encoding) {
152 .chunked => {
153 req.response.next_chunk_length = 0;
154 req.response.state = .chunk_size;
155 },
156 }
157 } else if (req.response.headers.content_length) |content_length| {
158 req.response.next_chunk_length = content_length;
159
160 if (content_length == 0) req.response.done = true;
161 } else {
162 req.response.done = true;
163 }
164
165 return i;
166 }
167
168 return 0;
169}
170
171pub const WaitForCompleteHeadError = ReadRawError || error{
172 UnexpectedEndOfStream,
173
174 HttpHeadersExceededSizeLimit,
175 ShortHttpStatusLine,
176 BadHttpVersion,
177 HttpHeaderContinuationsUnsupported,
178 HttpTransferEncodingUnsupported,
179 HttpConnectionHeaderUnsupported,
180};
181
182/// Reads a complete response head. Any leftover data is stored in the request. This function is idempotent.
183pub fn waitForCompleteHead(req: *Request) WaitForCompleteHeadError!void {
184 if (req.response.state.isContent()) return;
185
186 while (true) {
187 const nread = try req.connection.data.read(req.read_buffer[0..]);
188 const amt = try checkForCompleteHead(req, req.read_buffer[0..nread]);
189
190 if (amt != 0) {
191 req.read_buffer_start = @intCast(ReadBufferIndex, amt);
192 req.read_buffer_len = @intCast(ReadBufferIndex, nread);
193 return;
194 } else if (nread == 0) {
195 return error.UnexpectedEndOfStream;
196 }
197 }
198}
199
200/// This one can return 0 without meaning EOF.
201fn readRawAdvanced(req: *Request, buffer: []u8) !usize {
202 assert(req.response.state.isContent());
203 if (req.response.done) return 0;
204
205 // var in: []const u8 = undefined;
206 if (req.read_buffer_start == req.read_buffer_len) {
207 const nread = try req.connection.data.read(req.read_buffer[0..]);
208 if (nread == 0) return error.UnexpectedEndOfStream;
209
210 req.read_buffer_start = 0;
211 req.read_buffer_len = @intCast(ReadBufferIndex, nread);
212 }
213
214 var out_index: usize = 0;
215 while (true) {
216 switch (req.response.state) {
217 .invalid, .start, .seen_r, .seen_rn, .seen_rnr => unreachable,
218 .finished => {
219 // TODO https://github.com/ziglang/zig/issues/14039
220 const buf_avail = req.read_buffer_len - req.read_buffer_start;
221 const data_avail = req.response.next_chunk_length;
222 const out_avail = buffer.len;
223
224 if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
225 const can_read = @intCast(usize, @min(buf_avail, data_avail));
226 req.response.next_chunk_length -= can_read;
227
228 if (req.response.next_chunk_length == 0) {
229 req.client.release(req.connection);
230 req.connection = undefined;
231 req.response.done = true;
232 }
233
234 return 0; // skip over as much data as possible
235 }
236
237 const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
238 req.response.next_chunk_length -= can_read;
239
240 mem.copy(u8, buffer[0..], req.read_buffer[req.read_buffer_start..][0..can_read]);
241 req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
242
243 if (req.response.next_chunk_length == 0) {
244 req.client.release(req.connection);
245 req.connection = undefined;
246 req.response.done = true;
247 }
248
249 return can_read;
250 },
251 .chunk_size_prefix_r => switch (req.read_buffer_len - req.read_buffer_start) {
252 0 => return out_index,
253 1 => switch (req.read_buffer[req.read_buffer_start]) {
254 '\r' => {
255 req.response.state = .chunk_size_prefix_n;
256 return out_index;
257 },
258 else => {
259 req.response.state = .invalid;
260 return error.HttpHeadersInvalid;
261 },
262 },
263 else => switch (int16(req.read_buffer[req.read_buffer_start..][0..2])) {
264 int16("\r\n") => {
265 req.read_buffer_start += 2;
266 req.response.state = .chunk_size;
267 continue;
268 },
269 else => {
270 req.response.state = .invalid;
271 return error.HttpHeadersInvalid;
272 },
273 },
274 },
275 .chunk_size_prefix_n => switch (req.read_buffer_len - req.read_buffer_start) {
276 0 => return out_index,
277 else => switch (req.read_buffer[req.read_buffer_start]) {
278 '\n' => {
279 req.read_buffer_start += 1;
280 req.response.state = .chunk_size;
281 continue;
282 },
283 else => {
284 req.response.state = .invalid;
285 return error.HttpHeadersInvalid;
286 },
287 },
288 },
289 .chunk_size, .chunk_r => {
290 const i = req.response.findChunkedLen(req.read_buffer[req.read_buffer_start..req.read_buffer_len]);
291 switch (req.response.state) {
292 .invalid => return error.HttpHeadersInvalid,
293 .chunk_data => {
294 if (req.response.next_chunk_length == 0) {
295 req.response.done = true;
296 req.client.release(req.connection);
297 req.connection = undefined;
298
299 return out_index;
300 }
301
302 req.read_buffer_start += @intCast(ReadBufferIndex, i);
303 continue;
304 },
305 .chunk_size => return out_index,
306 else => unreachable,
307 }
308 },
309 .chunk_data => {
310 // TODO https://github.com/ziglang/zig/issues/14039
311 const buf_avail = req.read_buffer_len - req.read_buffer_start;
312 const data_avail = req.response.next_chunk_length;
313 const out_avail = buffer.len - out_index;
314
315 if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
316 const can_read = @intCast(usize, @min(buf_avail, data_avail));
317 req.response.next_chunk_length -= can_read;
318
319 if (req.response.next_chunk_length == 0) {
320 req.client.release(req.connection);
321 req.connection = undefined;
322 req.response.done = true;
323 continue;
324 }
325
326 return 0; // skip over as much data as possible
327 }
328
329 const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
330 req.response.next_chunk_length -= can_read;
331
332 mem.copy(u8, buffer[out_index..], req.read_buffer[req.read_buffer_start..][0..can_read]);
333 req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
334 out_index += can_read;
335
336 if (req.response.next_chunk_length == 0) {
337 req.response.state = .chunk_size_prefix_r;
338
339 continue;
340 }
341
342 return out_index;
343 },
344 }
345 }
346}
347
348pub const ReadError = Client.DeflateDecompressor.Error || Client.GzipDecompressor.Error || Client.ZstdDecompressor.Error || WaitForCompleteHeadError || error{
349 BadHeader,
350 InvalidCompression,
351 StreamTooLong,
352 InvalidWindowSize,
353 CompressionNotSupported
354};
355
356pub const Reader = std.io.Reader(*Request, ReadError, read);
357
358pub fn reader(req: *Request) Reader {
359 return .{ .context = req };
360}
361
362pub fn read(req: *Request, buffer: []u8) ReadError!usize {
363 while (true) {
364 if (!req.response.state.isContent()) try req.waitForCompleteHead();
365
366 if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
367 assert(try req.readRaw(buffer) == 0);
368
369 if (req.redirects_left == 0) return error.TooManyHttpRedirects;
370
371 const location = req.response.headers.location orelse
372 return error.HttpRedirectMissingLocation;
373 const new_url = Uri.parse(location) catch try Uri.parseWithoutScheme(location);
374
375 var new_arena = std.heap.ArenaAllocator.init(req.client.allocator);
376 const resolved_url = try req.uri.resolve(new_url, false, new_arena.allocator());
377 errdefer new_arena.deinit();
378
379 req.arena.deinit();
380 req.arena = new_arena;
381
382 const new_req = try req.client.request(resolved_url, req.headers, .{
383 .max_redirects = req.redirects_left - 1,
384 .header_strategy = if (req.response.header_bytes_owned) .{
385 .dynamic = req.response.max_header_bytes,
386 } else .{
387 .static = req.response.header_bytes.unusedCapacitySlice(),
388 },
389 });
390 req.deinit();
391 req.* = new_req;
392 } else {
393 break;
394 }
395 }
396
397 if (req.response.compression == .none) {
398 if (req.response.headers.transfer_compression) |compression| {
399 switch (compression) {
400 .compress => return error.CompressionNotSupported,
401 .deflate => req.response.compression = .{
402 .deflate = try std.compress.zlib.zlibStream(req.client.allocator, ReaderRaw{ .context = req }),
403 },
404 .gzip => req.response.compression = .{
405 .gzip = try std.compress.gzip.decompress(req.client.allocator, ReaderRaw{ .context = req }),
406 },
407 .zstd => req.response.compression = .{
408 .zstd = std.compress.zstd.decompressStream(req.client.allocator, ReaderRaw{ .context = req }),
409 },
410 }
411 }
412 }
413
414 return switch (req.response.compression) {
415 .deflate => |*deflate| try deflate.read(buffer),
416 .gzip => |*gzip| try gzip.read(buffer),
417 .zstd => |*zstd| try zstd.read(buffer),
418 else => try req.readRaw(buffer),
419 };
420}
421
422pub fn readAll(req: *Request, buffer: []u8) !usize {
423 var index: usize = 0;
424 while (index < buffer.len) {
425 const amt = try read(req, buffer[index..]);
426 if (amt == 0) break;
427 index += amt;
428 }
429 return index;
430}
431
432pub const WriteError = Connection.WriteError || error{MessageTooLong};
433
434pub const Writer = std.io.Writer(*Request, WriteError, write);
435
436pub fn writer(req: *Request) Writer {
437 return .{ .context = req };
438}
439
440/// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.
441pub fn write(req: *Request, bytes: []const u8) !usize {
442 switch (req.headers.transfer_encoding) {
443 .chunked => {
444 try req.connection.data.writer().print("{x}\r\n", .{bytes.len});
445 try req.connection.data.writeAll(bytes);
446 try req.connection.data.writeAll("\r\n");
447
448 return bytes.len;
449 },
450 .content_length => |*len| {
451 if (len.* < bytes.len) return error.MessageTooLong;
452
453 const amt = try req.connection.data.write(bytes);
454 len.* -= amt;
455 return amt;
456 },
457 .none => return error.NotWriteable,
458 }
459}
460
461/// Finish the body of a request. This notifies the server that you have no more data to send.
462pub fn finish(req: *Request) !void {
463 switch (req.headers.transfer_encoding) {
464 .chunked => try req.connection.data.writeAll("0\r\n"),
465 .content_length => |len| if (len != 0) return error.MessageNotCompleted,
466 .none => {},
467 }
468}
469
470inline fn int16(array: *const [2]u8) u16 {
471 return @bitCast(u16, array.*);
472}
473
474inline fn int32(array: *const [4]u8) u32 {
475 return @bitCast(u32, array.*);
476}
477
478inline fn int64(array: *const [8]u8) u64 {
479 return @bitCast(u64, array.*);
480}
481
482test {
483 const builtin = @import("builtin");
484
485 if (builtin.os.tag == .wasi) return error.SkipZigTest;
486
487 _ = Response;
488}
lib/std/http/Client/Response.zig created+506
...@@ -0,0 +1,506 @@
1const std = @import("std");
2const http = std.http;
3const mem = std.mem;
4const testing = std.testing;
5const assert = std.debug.assert;
6
7const Client = @import("../Client.zig");
8const Response = @This();
9
10headers: Headers,
11state: State,
12header_bytes_owned: bool,
13/// This could either be a fixed buffer provided by the API user or it
14/// could be our own array list.
15header_bytes: std.ArrayListUnmanaged(u8),
16max_header_bytes: usize,
17next_chunk_length: u64,
18done: bool = false,
19
20compression: union(enum) {
21 deflate: Client.DeflateDecompressor,
22 gzip: Client.GzipDecompressor,
23 zstd: Client.ZstdDecompressor,
24 none: void,
25} = .none,
26
27pub const Headers = struct {
28 status: http.Status,
29 version: http.Version,
30 location: ?[]const u8 = null,
31 content_length: ?u64 = null,
32 transfer_encoding: ?http.TransferEncoding = null,
33 transfer_compression: ?http.ContentEncoding = null,
34 connection: http.Connection = .close,
35
36 number_of_headers: usize = 0,
37
38 pub fn parse(bytes: []const u8) !Headers {
39 var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
40
41 const first_line = it.first();
42 if (first_line.len < 12)
43 return error.ShortHttpStatusLine;
44
45 const version: http.Version = switch (int64(first_line[0..8])) {
46 int64("HTTP/1.0") => .@"HTTP/1.0",
47 int64("HTTP/1.1") => .@"HTTP/1.1",
48 else => return error.BadHttpVersion,
49 };
50 if (first_line[8] != ' ') return error.HttpHeadersInvalid;
51 const status = @intToEnum(http.Status, parseInt3(first_line[9..12].*));
52
53 var headers: Headers = .{
54 .version = version,
55 .status = status,
56 };
57
58 while (it.next()) |line| {
59 headers.number_of_headers += 1;
60
61 if (line.len == 0) return error.HttpHeadersInvalid;
62 switch (line[0]) {
63 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
64 else => {},
65 }
66 var line_it = mem.split(u8, line, ": ");
67 const header_name = line_it.first();
68 const header_value = line_it.rest();
69 if (std.ascii.eqlIgnoreCase(header_name, "location")) {
70 if (headers.location != null) return error.HttpHeadersInvalid;
71 headers.location = header_value;
72 } else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
73 if (headers.content_length != null) return error.HttpHeadersInvalid;
74 headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
75 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
76 if (headers.transfer_encoding != null or headers.transfer_compression != null) return error.HttpHeadersInvalid;
77
78 // Transfer-Encoding: second, first
79 // Transfer-Encoding: deflate, chunked
80 var iter = std.mem.splitBackwards(u8, header_value, ",");
81
82 if (iter.next()) |first| {
83 const trimmed = std.mem.trim(u8, first, " ");
84
85 if (std.meta.stringToEnum(http.TransferEncoding, trimmed)) |te| {
86 headers.transfer_encoding = te;
87 } else if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) |ce| {
88 headers.transfer_compression = ce;
89 } else {
90 return error.HttpTransferEncodingUnsupported;
91 }
92 }
93
94 if (iter.next()) |second| {
95 if (headers.transfer_compression != null) return error.HttpTransferEncodingUnsupported;
96
97 const trimmed = std.mem.trim(u8, second, " ");
98
99 if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) |ce| {
100 headers.transfer_compression = ce;
101 } else {
102 return error.HttpTransferEncodingUnsupported;
103 }
104 }
105
106 if (iter.next()) |_| return error.HttpTransferEncodingUnsupported;
107 } else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
108 if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
109
110 const trimmed = std.mem.trim(u8, header_value, " ");
111
112 if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) |ce| {
113 headers.transfer_compression = ce;
114 } else {
115 return error.HttpTransferEncodingUnsupported;
116 }
117 } else if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
118 if (std.ascii.eqlIgnoreCase(header_value, "keep-alive")) {
119 headers.connection = .keep_alive;
120 } else if (std.ascii.eqlIgnoreCase(header_value, "close")) {
121 headers.connection = .close;
122 } else {
123 return error.HttpConnectionHeaderUnsupported;
124 }
125 }
126 }
127
128 return headers;
129 }
130
131 test "parse headers" {
132 const example =
133 "HTTP/1.1 301 Moved Permanently\r\n" ++
134 "Location: https://www.example.com/\r\n" ++
135 "Content-Type: text/html; charset=UTF-8\r\n" ++
136 "Content-Length: 220\r\n\r\n";
137 const parsed = try Headers.parse(example);
138 try testing.expectEqual(http.Version.@"HTTP/1.1", parsed.version);
139 try testing.expectEqual(http.Status.moved_permanently, parsed.status);
140 try testing.expectEqualStrings("https://www.example.com/", parsed.location orelse
141 return error.TestFailed);
142 try testing.expectEqual(@as(?u64, 220), parsed.content_length);
143 }
144
145 test "header continuation" {
146 const example =
147 "HTTP/1.0 200 OK\r\n" ++
148 "Content-Type: text/html;\r\n charset=UTF-8\r\n" ++
149 "Content-Length: 220\r\n\r\n";
150 try testing.expectError(
151 error.HttpHeaderContinuationsUnsupported,
152 Headers.parse(example),
153 );
154 }
155
156 test "extra content length" {
157 const example =
158 "HTTP/1.0 200 OK\r\n" ++
159 "Content-Length: 220\r\n" ++
160 "Content-Type: text/html; charset=UTF-8\r\n" ++
161 "content-length: 220\r\n\r\n";
162 try testing.expectError(
163 error.HttpHeadersInvalid,
164 Headers.parse(example),
165 );
166 }
167};
168
169inline fn int16(array: *const [2]u8) u16 {
170 return @bitCast(u16, array.*);
171}
172
173inline fn int32(array: *const [4]u8) u32 {
174 return @bitCast(u32, array.*);
175}
176
177inline fn int64(array: *const [8]u8) u64 {
178 return @bitCast(u64, array.*);
179}
180
181pub const State = enum {
182 /// Begin header parsing states.
183 invalid,
184 start,
185 seen_r,
186 seen_rn,
187 seen_rnr,
188 finished,
189 /// Begin transfer-encoding: chunked parsing states.
190 chunk_size_prefix_r,
191 chunk_size_prefix_n,
192 chunk_size,
193 chunk_r,
194 chunk_data,
195
196 pub fn isContent(self: State) bool {
197 return switch (self) {
198 .invalid, .start, .seen_r, .seen_rn, .seen_rnr => false,
199 .finished, .chunk_size_prefix_r, .chunk_size_prefix_n, .chunk_size, .chunk_r, .chunk_data => true,
200 };
201 }
202};
203
204pub fn initDynamic(max: usize) Response {
205 return .{
206 .state = .start,
207 .headers = undefined,
208 .header_bytes = .{},
209 .max_header_bytes = max,
210 .header_bytes_owned = true,
211 .next_chunk_length = undefined,
212 };
213}
214
215pub fn initStatic(buf: []u8) Response {
216 return .{
217 .state = .start,
218 .headers = undefined,
219 .header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
220 .max_header_bytes = buf.len,
221 .header_bytes_owned = false,
222 .next_chunk_length = undefined,
223 };
224}
225
226/// Returns how many bytes are part of HTTP headers. Always less than or
227/// equal to bytes.len. If the amount returned is less than bytes.len, it
228/// means the headers ended and the first byte after the double \r\n\r\n is
229/// located at `bytes[result]`.
230pub fn findHeadersEnd(r: *Response, bytes: []const u8) usize {
231 var index: usize = 0;
232
233 // TODO: https://github.com/ziglang/zig/issues/8220
234 state: while (true) {
235 switch (r.state) {
236 .invalid => unreachable,
237 .finished => unreachable,
238 .start => while (true) {
239 switch (bytes.len - index) {
240 0 => return index,
241 1 => {
242 if (bytes[index] == '\r')
243 r.state = .seen_r;
244 return index + 1;
245 },
246 2 => {
247 if (int16(bytes[index..][0..2]) == int16("\r\n")) {
248 r.state = .seen_rn;
249 } else if (bytes[index + 1] == '\r') {
250 r.state = .seen_r;
251 }
252 return index + 2;
253 },
254 3 => {
255 if (int16(bytes[index..][0..2]) == int16("\r\n") and
256 bytes[index + 2] == '\r')
257 {
258 r.state = .seen_rnr;
259 } else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n")) {
260 r.state = .seen_rn;
261 } else if (bytes[index + 2] == '\r') {
262 r.state = .seen_r;
263 }
264 return index + 3;
265 },
266 4...15 => {
267 if (int32(bytes[index..][0..4]) == int32("\r\n\r\n")) {
268 r.state = .finished;
269 return index + 4;
270 } else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n") and
271 bytes[index + 3] == '\r')
272 {
273 r.state = .seen_rnr;
274 index += 4;
275 continue :state;
276 } else if (int16(bytes[index + 2 ..][0..2]) == int16("\r\n")) {
277 r.state = .seen_rn;
278 index += 4;
279 continue :state;
280 } else if (bytes[index + 3] == '\r') {
281 r.state = .seen_r;
282 index += 4;
283 continue :state;
284 }
285 index += 4;
286 continue;
287 },
288 else => {
289 const chunk = bytes[index..][0..16];
290 const v: @Vector(16, u8) = chunk.*;
291 const matches_r = v == @splat(16, @as(u8, '\r'));
292 const iota = std.simd.iota(u8, 16);
293 const default = @splat(16, @as(u8, 16));
294 const sub_index = @reduce(.Min, @select(u8, matches_r, iota, default));
295 switch (sub_index) {
296 0...12 => {
297 index += sub_index + 4;
298 if (int32(chunk[sub_index..][0..4]) == int32("\r\n\r\n")) {
299 r.state = .finished;
300 return index;
301 }
302 continue;
303 },
304 13 => {
305 index += 16;
306 if (int16(chunk[14..][0..2]) == int16("\n\r")) {
307 r.state = .seen_rnr;
308 continue :state;
309 }
310 continue;
311 },
312 14 => {
313 index += 16;
314 if (chunk[15] == '\n') {
315 r.state = .seen_rn;
316 continue :state;
317 }
318 continue;
319 },
320 15 => {
321 r.state = .seen_r;
322 index += 16;
323 continue :state;
324 },
325 16 => {
326 index += 16;
327 continue;
328 },
329 else => unreachable,
330 }
331 },
332 }
333 },
334
335 .seen_r => switch (bytes.len - index) {
336 0 => return index,
337 1 => {
338 switch (bytes[index]) {
339 '\n' => r.state = .seen_rn,
340 '\r' => r.state = .seen_r,
341 else => r.state = .start,
342 }
343 return index + 1;
344 },
345 2 => {
346 if (int16(bytes[index..][0..2]) == int16("\n\r")) {
347 r.state = .seen_rnr;
348 return index + 2;
349 }
350 r.state = .start;
351 return index + 2;
352 },
353 else => {
354 if (int16(bytes[index..][0..2]) == int16("\n\r") and
355 bytes[index + 2] == '\n')
356 {
357 r.state = .finished;
358 return index + 3;
359 }
360 index += 3;
361 r.state = .start;
362 continue :state;
363 },
364 },
365 .seen_rn => switch (bytes.len - index) {
366 0 => return index,
367 1 => {
368 switch (bytes[index]) {
369 '\r' => r.state = .seen_rnr,
370 else => r.state = .start,
371 }
372 return index + 1;
373 },
374 else => {
375 if (int16(bytes[index..][0..2]) == int16("\r\n")) {
376 r.state = .finished;
377 return index + 2;
378 }
379 index += 2;
380 r.state = .start;
381 continue :state;
382 },
383 },
384 .seen_rnr => switch (bytes.len - index) {
385 0 => return index,
386 else => {
387 if (bytes[index] == '\n') {
388 r.state = .finished;
389 return index + 1;
390 }
391 index += 1;
392 r.state = .start;
393 continue :state;
394 },
395 },
396 .chunk_size_prefix_r => unreachable,
397 .chunk_size_prefix_n => unreachable,
398 .chunk_size => unreachable,
399 .chunk_r => unreachable,
400 .chunk_data => unreachable,
401 }
402
403 return index;
404 }
405}
406
407pub fn findChunkedLen(r: *Response, bytes: []const u8) usize {
408 var i: usize = 0;
409 if (r.state == .chunk_size) {
410 while (i < bytes.len) : (i += 1) {
411 const digit = switch (bytes[i]) {
412 '0'...'9' => |b| b - '0',
413 'A'...'Z' => |b| b - 'A' + 10,
414 'a'...'z' => |b| b - 'a' + 10,
415 '\r' => {
416 r.state = .chunk_r;
417 i += 1;
418 break;
419 },
420 else => {
421 r.state = .invalid;
422 return i;
423 },
424 };
425 const mul = @mulWithOverflow(r.next_chunk_length, 16);
426 if (mul[1] != 0) {
427 r.state = .invalid;
428 return i;
429 }
430 const add = @addWithOverflow(mul[0], digit);
431 if (add[1] != 0) {
432 r.state = .invalid;
433 return i;
434 }
435 r.next_chunk_length = add[0];
436 } else {
437 return i;
438 }
439 }
440 assert(r.state == .chunk_r);
441 if (i == bytes.len) return i;
442
443 if (bytes[i] == '\n') {
444 r.state = .chunk_data;
445 return i + 1;
446 } else {
447 r.state = .invalid;
448 return i;
449 }
450}
451
452fn parseInt3(nnn: @Vector(3, u8)) u10 {
453 const zero: @Vector(3, u8) = .{ '0', '0', '0' };
454 const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
455 return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
456}
457
458test parseInt3 {
459 const expectEqual = std.testing.expectEqual;
460 try expectEqual(@as(u10, 0), parseInt3("000".*));
461 try expectEqual(@as(u10, 418), parseInt3("418".*));
462 try expectEqual(@as(u10, 999), parseInt3("999".*));
463}
464
465test "find headers end basic" {
466 var buffer: [1]u8 = undefined;
467 var r = Response.initStatic(&buffer);
468 try testing.expectEqual(@as(usize, 10), r.findHeadersEnd("HTTP/1.1 4"));
469 try testing.expectEqual(@as(usize, 2), r.findHeadersEnd("18"));
470 try testing.expectEqual(@as(usize, 8), r.findHeadersEnd(" lol\r\n\r\nblah blah"));
471}
472
473test "find headers end vectorized" {
474 var buffer: [1]u8 = undefined;
475 var r = Response.initStatic(&buffer);
476 const example =
477 "HTTP/1.1 301 Moved Permanently\r\n" ++
478 "Location: https://www.example.com/\r\n" ++
479 "Content-Type: text/html; charset=UTF-8\r\n" ++
480 "Content-Length: 220\r\n" ++
481 "\r\ncontent";
482 try testing.expectEqual(@as(usize, 131), r.findHeadersEnd(example));
483}
484
485test "find headers end bug" {
486 var buffer: [1]u8 = undefined;
487 var r = Response.initStatic(&buffer);
488 const trail = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
489 const example =
490 "HTTP/1.1 200 OK\r\n" ++
491 "Access-Control-Allow-Origin: https://render.githubusercontent.com\r\n" ++
492 "content-disposition: attachment; filename=zig-0.10.0.tar.gz\r\n" ++
493 "Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'; sandbox\r\n" ++
494 "Content-Type: application/x-gzip\r\n" ++
495 "ETag: \"bfae0af6b01c7c0d89eb667cb5f0e65265968aeebda2689177e6b26acd3155ca\"\r\n" ++
496 "Strict-Transport-Security: max-age=31536000\r\n" ++
497 "Vary: Authorization,Accept-Encoding,Origin\r\n" ++
498 "X-Content-Type-Options: nosniff\r\n" ++
499 "X-Frame-Options: deny\r\n" ++
500 "X-XSS-Protection: 1; mode=block\r\n" ++
501 "Date: Fri, 06 Jan 2023 22:26:22 GMT\r\n" ++
502 "Transfer-Encoding: chunked\r\n" ++
503 "X-GitHub-Request-Id: 89C6:17E9:A7C9E:124B51:63B8A00E\r\n" ++
504 "connection: close\r\n\r\n" ++ trail;
505 try testing.expectEqual(@as(usize, example.len - trail.len), r.findHeadersEnd(example));
506}