authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-06 05:29:14-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-01-06 05:29:14-05:00
log7a2d7ff6280c4e75618b5d4de24c3ff4aedd8a18
treefdaca886a1914bad627c35b14a42dd99389d962e
parentdb7b36f3fcf7c14fc2031928d533b7a5b9d8fa0c
parenta7a933d7eef05a947d77de5c1ee42b45addf6ec5
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14224 from ziglang/std.http

std.http.Client: support transfer-encoding: chunked

3 files changed, 173 insertions(+), 40 deletions(-)

lib/std/crypto/tls/Client.zig+1
......@@ -1207,6 +1207,7 @@ const VecPut = struct {
12071207 /// Returns the amount actually put which is always equal to bytes.len
12081208 /// unless the vectors ran out of space.
12091209 fn put(vp: *VecPut, bytes: []const u8) usize {
1210 if (vp.idx >= vp.iovecs.len) return 0;
12101211 var bytes_i: usize = 0;
12111212 while (true) {
12121213 const v = vp.iovecs[vp.idx];
lib/std/http.zig+7
......@@ -246,6 +246,13 @@ pub const Status = enum(u10) {
246246 }
247247};
248248
249pub const TransferEncoding = enum {
250 chunked,
251 compress,
252 deflate,
253 gzip,
254};
255
249256const std = @import("std.zig");
250257
251258test {
lib/std/http/Client.zig+165-40
......@@ -77,12 +77,14 @@ pub const Request = struct {
7777 /// could be our own array list.
7878 header_bytes: std.ArrayListUnmanaged(u8),
7979 max_header_bytes: usize,
80 next_chunk_length: u64,
8081
8182 pub const Headers = struct {
82 location: ?[]const u8 = null,
8383 status: http.Status,
8484 version: http.Version,
85 location: ?[]const u8 = null,
8586 content_length: ?u64 = null,
87 transfer_encoding: ?http.TransferEncoding = null,
8688
8789 pub fn parse(bytes: []const u8) !Response.Headers {
8890 var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
......@@ -119,6 +121,10 @@ pub const Request = struct {
119121 } else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
120122 if (headers.content_length != null) return error.HttpHeadersInvalid;
121123 headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
124 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
125 if (headers.transfer_encoding != null) return error.HttpHeadersInvalid;
126 headers.transfer_encoding = std.meta.stringToEnum(http.TransferEncoding, header_value) orelse
127 return error.HttpTransferEncodingUnsupported;
122128 }
123129 }
124130
......@@ -164,12 +170,24 @@ pub const Request = struct {
164170 };
165171
166172 pub const State = enum {
173 /// Begin header parsing states.
167174 invalid,
168 finished,
169175 start,
170176 seen_r,
171177 seen_rn,
172178 seen_rnr,
179 finished,
180 /// Begin transfer-encoding: chunked parsing states.
181 chunk_size,
182 chunk_r,
183 chunk_data,
184
185 pub fn zeroMeansEnd(state: State) bool {
186 return switch (state) {
187 .finished, .chunk_data => true,
188 else => false,
189 };
190 }
173191 };
174192
175193 pub fn initDynamic(max: usize) Response {
......@@ -179,6 +197,7 @@ pub const Request = struct {
179197 .header_bytes = .{},
180198 .max_header_bytes = max,
181199 .header_bytes_owned = true,
200 .next_chunk_length = undefined,
182201 };
183202 }
184203
......@@ -189,6 +208,7 @@ pub const Request = struct {
189208 .header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
190209 .max_header_bytes = buf.len,
191210 .header_bytes_owned = false,
211 .next_chunk_length = undefined,
192212 };
193213 }
194214
......@@ -362,12 +382,60 @@ pub const Request = struct {
362382 continue :state;
363383 },
364384 },
385 .chunk_size => unreachable,
386 .chunk_r => unreachable,
387 .chunk_data => unreachable,
365388 }
366389
367390 return index;
368391 }
369392 }
370393
394 pub fn findChunkedLen(r: *Response, bytes: []const u8) usize {
395 var i: usize = 0;
396 if (r.state == .chunk_size) {
397 while (i < bytes.len) : (i += 1) {
398 const digit = switch (bytes[i]) {
399 '0'...'9' => |b| b - '0',
400 'A'...'Z' => |b| b - 'A' + 10,
401 'a'...'z' => |b| b - 'a' + 10,
402 '\r' => {
403 r.state = .chunk_r;
404 i += 1;
405 break;
406 },
407 else => {
408 r.state = .invalid;
409 return i;
410 },
411 };
412 const mul = @mulWithOverflow(r.next_chunk_length, 16);
413 if (mul[1] != 0) {
414 r.state = .invalid;
415 return i;
416 }
417 const add = @addWithOverflow(mul[0], digit);
418 if (add[1] != 0) {
419 r.state = .invalid;
420 return i;
421 }
422 r.next_chunk_length = add[0];
423 } else {
424 return i;
425 }
426 }
427 assert(r.state == .chunk_r);
428 if (i == bytes.len) return i;
429
430 if (bytes[i] == '\n') {
431 r.state = .chunk_data;
432 return i + 1;
433 } else {
434 r.state = .invalid;
435 return i;
436 }
437 }
438
371439 fn parseInt3(nnn: @Vector(3, u8)) u10 {
372440 const zero: @Vector(3, u8) = .{ '0', '0', '0' };
373441 const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
......@@ -415,6 +483,7 @@ pub const Request = struct {
415483 };
416484
417485 pub const Headers = struct {
486 version: http.Version = .@"HTTP/1.1",
418487 method: http.Method = .GET,
419488 };
420489
......@@ -456,9 +525,9 @@ pub const Request = struct {
456525 assert(len <= buffer.len);
457526 var index: usize = 0;
458527 while (index < len) {
459 const headers_finished = req.response.state == .finished;
528 const zero_means_end = req.response.state.zeroMeansEnd();
460529 const amt = try readAdvanced(req, buffer[index..]);
461 if (amt == 0 and headers_finished) break;
530 if (amt == 0 and zero_means_end) break;
462531 index += amt;
463532 }
464533 return index;
......@@ -467,47 +536,101 @@ pub const Request = struct {
467536 /// This one can return 0 without meaning EOF.
468537 /// TODO change to readvAdvanced
469538 pub fn readAdvanced(req: *Request, buffer: []u8) !usize {
470 if (req.response.state == .finished) return req.connection.read(buffer);
471
472539 const amt = try req.connection.read(buffer);
473 const data = buffer[0..amt];
474 const i = req.response.findHeadersEnd(data);
475 if (req.response.state == .invalid) return error.HttpHeadersInvalid;
540 var in = buffer[0..amt];
541 var out_index: usize = 0;
542 while (true) {
543 switch (req.response.state) {
544 .invalid => unreachable,
545 .start, .seen_r, .seen_rn, .seen_rnr => {
546 const i = req.response.findHeadersEnd(in);
547 if (req.response.state == .invalid) return error.HttpHeadersInvalid;
548
549 const headers_data = in[0..i];
550 if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
551 return error.HttpHeadersExceededSizeLimit;
552 }
553 try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
554
555 if (req.response.state == .finished) {
556 req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
557
558 if (req.response.headers.status.class() == .redirect) {
559 if (req.redirects_left == 0) return error.TooManyHttpRedirects;
560 const location = req.response.headers.location orelse
561 return error.HttpRedirectMissingLocation;
562 const new_url = try std.Url.parse(location);
563 const new_req = try req.client.request(new_url, req.headers, .{
564 .max_redirects = req.redirects_left - 1,
565 .header_strategy = if (req.response.header_bytes_owned) .{
566 .dynamic = req.response.max_header_bytes,
567 } else .{
568 .static = req.response.header_bytes.unusedCapacitySlice(),
569 },
570 });
571 req.deinit();
572 req.* = new_req;
573 assert(out_index == 0);
574 return readAdvanced(req, buffer);
575 }
476576
477 const headers_data = data[0..i];
478 if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
479 return error.HttpHeadersExceededSizeLimit;
480 }
481 try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
577 if (req.response.headers.transfer_encoding) |transfer_encoding| {
578 switch (transfer_encoding) {
579 .chunked => {
580 req.response.next_chunk_length = 0;
581 req.response.state = .chunk_size;
582 },
583 .compress => return error.HttpTransferEncodingUnsupported,
584 .deflate => return error.HttpTransferEncodingUnsupported,
585 .gzip => return error.HttpTransferEncodingUnsupported,
586 }
587 } else if (req.response.headers.content_length) |content_length| {
588 req.response.next_chunk_length = content_length;
589 } else {
590 return error.HttpContentLengthUnknown;
591 }
482592
483 if (req.response.state == .finished) {
484 req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
485 }
593 in = in[i..];
594 continue;
595 }
486596
487 if (req.response.headers.status.class() == .redirect) {
488 if (req.redirects_left == 0) return error.TooManyHttpRedirects;
489 const location = req.response.headers.location orelse
490 return error.HttpRedirectMissingLocation;
491 const new_url = try std.Url.parse(location);
492 const new_req = try req.client.request(new_url, req.headers, .{
493 .max_redirects = req.redirects_left - 1,
494 .header_strategy = if (req.response.header_bytes_owned) .{
495 .dynamic = req.response.max_header_bytes,
496 } else .{
497 .static = req.response.header_bytes.unusedCapacitySlice(),
597 assert(out_index == 0);
598 return 0;
498599 },
499 });
500 req.deinit();
501 req.* = new_req;
502 return readAdvanced(req, buffer);
503 }
504
505 const body_data = data[i..];
506 if (body_data.len > 0) {
507 mem.copy(u8, buffer, body_data);
508 return body_data.len;
600 .finished => {
601 mem.copy(u8, buffer[out_index..], in);
602 return out_index + in.len;
603 },
604 .chunk_size, .chunk_r => {
605 const i = req.response.findChunkedLen(in);
606 switch (req.response.state) {
607 .invalid => return error.HttpHeadersInvalid,
608 .chunk_data => {
609 if (req.response.next_chunk_length == 0) {
610 req.response.state = .start;
611 return out_index;
612 }
613 in = in[i..];
614 continue;
615 },
616 .chunk_size => return out_index,
617 else => unreachable,
618 }
619 },
620 .chunk_data => {
621 const sub_amt = @min(req.response.next_chunk_length, in.len);
622 mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
623 out_index += sub_amt;
624 req.response.next_chunk_length -= sub_amt;
625 if (req.response.next_chunk_length == 0) {
626 req.response.state = .chunk_size;
627 in = in[sub_amt..];
628 continue;
629 }
630 return out_index;
631 },
632 }
509633 }
510 return 0;
511634 }
512635
513636 test {
......@@ -569,7 +692,9 @@ pub fn request(client: *Client, url: Url, headers: Request.Headers, options: Req
569692 try h.appendSlice(@tagName(headers.method));
570693 try h.appendSlice(" ");
571694 try h.appendSlice(url.path);
572 try h.appendSlice(" HTTP/1.1\r\nHost: ");
695 try h.appendSlice(" ");
696 try h.appendSlice(@tagName(headers.version));
697 try h.appendSlice("\r\nHost: ");
573698 try h.appendSlice(url.host);
574699 try h.appendSlice("\r\nConnection: close\r\n\r\n");
575700