authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-06 05:29:14-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-01-06 05:29:14-05:00
log7a2d7ff6280c4e75618b5d4de24c3ff4aedd8a18
treefdaca886a1914bad627c35b14a42dd99389d962e
parentdb7b36f3fcf7c14fc2031928d533b7a5b9d8fa0c
parenta7a933d7eef05a947d77de5c1ee42b45addf6ec5
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14224 from ziglang/std.http

std.http.Client: support transfer-encoding: chunked

3 files changed, 173 insertions(+), 40 deletions(-)

lib/std/crypto/tls/Client.zig+1
...@@ -1207,6 +1207,7 @@ const VecPut = struct {...@@ -1207,6 +1207,7 @@ const VecPut = struct {
1207 /// Returns the amount actually put which is always equal to bytes.len1207 /// Returns the amount actually put which is always equal to bytes.len
1208 /// unless the vectors ran out of space.1208 /// unless the vectors ran out of space.
1209 fn put(vp: *VecPut, bytes: []const u8) usize {1209 fn put(vp: *VecPut, bytes: []const u8) usize {
1210 if (vp.idx >= vp.iovecs.len) return 0;
1210 var bytes_i: usize = 0;1211 var bytes_i: usize = 0;
1211 while (true) {1212 while (true) {
1212 const v = vp.iovecs[vp.idx];1213 const v = vp.iovecs[vp.idx];
lib/std/http.zig+7
...@@ -246,6 +246,13 @@ pub const Status = enum(u10) {...@@ -246,6 +246,13 @@ pub const Status = enum(u10) {
246 }246 }
247};247};
248248
249pub const TransferEncoding = enum {
250 chunked,
251 compress,
252 deflate,
253 gzip,
254};
255
249const std = @import("std.zig");256const std = @import("std.zig");
250257
251test {258test {
lib/std/http/Client.zig+165-40
...@@ -77,12 +77,14 @@ pub const Request = struct {...@@ -77,12 +77,14 @@ pub const Request = struct {
77 /// could be our own array list.77 /// could be our own array list.
78 header_bytes: std.ArrayListUnmanaged(u8),78 header_bytes: std.ArrayListUnmanaged(u8),
79 max_header_bytes: usize,79 max_header_bytes: usize,
80 next_chunk_length: u64,
8081
81 pub const Headers = struct {82 pub const Headers = struct {
82 location: ?[]const u8 = null,
83 status: http.Status,83 status: http.Status,
84 version: http.Version,84 version: http.Version,
85 location: ?[]const u8 = null,
85 content_length: ?u64 = null,86 content_length: ?u64 = null,
87 transfer_encoding: ?http.TransferEncoding = null,
8688
87 pub fn parse(bytes: []const u8) !Response.Headers {89 pub fn parse(bytes: []const u8) !Response.Headers {
88 var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");90 var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
...@@ -119,6 +121,10 @@ pub const Request = struct {...@@ -119,6 +121,10 @@ pub const Request = struct {
119 } else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {121 } else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
120 if (headers.content_length != null) return error.HttpHeadersInvalid;122 if (headers.content_length != null) return error.HttpHeadersInvalid;
121 headers.content_length = try std.fmt.parseInt(u64, header_value, 10);123 headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
124 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
125 if (headers.transfer_encoding != null) return error.HttpHeadersInvalid;
126 headers.transfer_encoding = std.meta.stringToEnum(http.TransferEncoding, header_value) orelse
127 return error.HttpTransferEncodingUnsupported;
122 }128 }
123 }129 }
124130
...@@ -164,12 +170,24 @@ pub const Request = struct {...@@ -164,12 +170,24 @@ pub const Request = struct {
164 };170 };
165171
166 pub const State = enum {172 pub const State = enum {
173 /// Begin header parsing states.
167 invalid,174 invalid,
168 finished,
169 start,175 start,
170 seen_r,176 seen_r,
171 seen_rn,177 seen_rn,
172 seen_rnr,178 seen_rnr,
179 finished,
180 /// Begin transfer-encoding: chunked parsing states.
181 chunk_size,
182 chunk_r,
183 chunk_data,
184
185 pub fn zeroMeansEnd(state: State) bool {
186 return switch (state) {
187 .finished, .chunk_data => true,
188 else => false,
189 };
190 }
173 };191 };
174192
175 pub fn initDynamic(max: usize) Response {193 pub fn initDynamic(max: usize) Response {
...@@ -179,6 +197,7 @@ pub const Request = struct {...@@ -179,6 +197,7 @@ pub const Request = struct {
179 .header_bytes = .{},197 .header_bytes = .{},
180 .max_header_bytes = max,198 .max_header_bytes = max,
181 .header_bytes_owned = true,199 .header_bytes_owned = true,
200 .next_chunk_length = undefined,
182 };201 };
183 }202 }
184203
...@@ -189,6 +208,7 @@ pub const Request = struct {...@@ -189,6 +208,7 @@ pub const Request = struct {
189 .header_bytes = .{ .items = buf[0..0], .capacity = buf.len },208 .header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
190 .max_header_bytes = buf.len,209 .max_header_bytes = buf.len,
191 .header_bytes_owned = false,210 .header_bytes_owned = false,
211 .next_chunk_length = undefined,
192 };212 };
193 }213 }
194214
...@@ -362,12 +382,60 @@ pub const Request = struct {...@@ -362,12 +382,60 @@ pub const Request = struct {
362 continue :state;382 continue :state;
363 },383 },
364 },384 },
385 .chunk_size => unreachable,
386 .chunk_r => unreachable,
387 .chunk_data => unreachable,
365 }388 }
366389
367 return index;390 return index;
368 }391 }
369 }392 }
370393
394 pub fn findChunkedLen(r: *Response, bytes: []const u8) usize {
395 var i: usize = 0;
396 if (r.state == .chunk_size) {
397 while (i < bytes.len) : (i += 1) {
398 const digit = switch (bytes[i]) {
399 '0'...'9' => |b| b - '0',
400 'A'...'Z' => |b| b - 'A' + 10,
401 'a'...'z' => |b| b - 'a' + 10,
402 '\r' => {
403 r.state = .chunk_r;
404 i += 1;
405 break;
406 },
407 else => {
408 r.state = .invalid;
409 return i;
410 },
411 };
412 const mul = @mulWithOverflow(r.next_chunk_length, 16);
413 if (mul[1] != 0) {
414 r.state = .invalid;
415 return i;
416 }
417 const add = @addWithOverflow(mul[0], digit);
418 if (add[1] != 0) {
419 r.state = .invalid;
420 return i;
421 }
422 r.next_chunk_length = add[0];
423 } else {
424 return i;
425 }
426 }
427 assert(r.state == .chunk_r);
428 if (i == bytes.len) return i;
429
430 if (bytes[i] == '\n') {
431 r.state = .chunk_data;
432 return i + 1;
433 } else {
434 r.state = .invalid;
435 return i;
436 }
437 }
438
371 fn parseInt3(nnn: @Vector(3, u8)) u10 {439 fn parseInt3(nnn: @Vector(3, u8)) u10 {
372 const zero: @Vector(3, u8) = .{ '0', '0', '0' };440 const zero: @Vector(3, u8) = .{ '0', '0', '0' };
373 const mmm: @Vector(3, u10) = .{ 100, 10, 1 };441 const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
...@@ -415,6 +483,7 @@ pub const Request = struct {...@@ -415,6 +483,7 @@ pub const Request = struct {
415 };483 };
416484
417 pub const Headers = struct {485 pub const Headers = struct {
486 version: http.Version = .@"HTTP/1.1",
418 method: http.Method = .GET,487 method: http.Method = .GET,
419 };488 };
420489
...@@ -456,9 +525,9 @@ pub const Request = struct {...@@ -456,9 +525,9 @@ pub const Request = struct {
456 assert(len <= buffer.len);525 assert(len <= buffer.len);
457 var index: usize = 0;526 var index: usize = 0;
458 while (index < len) {527 while (index < len) {
459 const headers_finished = req.response.state == .finished;528 const zero_means_end = req.response.state.zeroMeansEnd();
460 const amt = try readAdvanced(req, buffer[index..]);529 const amt = try readAdvanced(req, buffer[index..]);
461 if (amt == 0 and headers_finished) break;530 if (amt == 0 and zero_means_end) break;
462 index += amt;531 index += amt;
463 }532 }
464 return index;533 return index;
...@@ -467,47 +536,101 @@ pub const Request = struct {...@@ -467,47 +536,101 @@ pub const Request = struct {
467 /// This one can return 0 without meaning EOF.536 /// This one can return 0 without meaning EOF.
468 /// TODO change to readvAdvanced537 /// TODO change to readvAdvanced
469 pub fn readAdvanced(req: *Request, buffer: []u8) !usize {538 pub fn readAdvanced(req: *Request, buffer: []u8) !usize {
470 if (req.response.state == .finished) return req.connection.read(buffer);
471
472 const amt = try req.connection.read(buffer);539 const amt = try req.connection.read(buffer);
473 const data = buffer[0..amt];540 var in = buffer[0..amt];
474 const i = req.response.findHeadersEnd(data);541 var out_index: usize = 0;
475 if (req.response.state == .invalid) return error.HttpHeadersInvalid;542 while (true) {
543 switch (req.response.state) {
544 .invalid => unreachable,
545 .start, .seen_r, .seen_rn, .seen_rnr => {
546 const i = req.response.findHeadersEnd(in);
547 if (req.response.state == .invalid) return error.HttpHeadersInvalid;
548
549 const headers_data = in[0..i];
550 if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
551 return error.HttpHeadersExceededSizeLimit;
552 }
553 try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
554
555 if (req.response.state == .finished) {
556 req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
557
558 if (req.response.headers.status.class() == .redirect) {
559 if (req.redirects_left == 0) return error.TooManyHttpRedirects;
560 const location = req.response.headers.location orelse
561 return error.HttpRedirectMissingLocation;
562 const new_url = try std.Url.parse(location);
563 const new_req = try req.client.request(new_url, req.headers, .{
564 .max_redirects = req.redirects_left - 1,
565 .header_strategy = if (req.response.header_bytes_owned) .{
566 .dynamic = req.response.max_header_bytes,
567 } else .{
568 .static = req.response.header_bytes.unusedCapacitySlice(),
569 },
570 });
571 req.deinit();
572 req.* = new_req;
573 assert(out_index == 0);
574 return readAdvanced(req, buffer);
575 }
476576
477 const headers_data = data[0..i];577 if (req.response.headers.transfer_encoding) |transfer_encoding| {
478 if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {578 switch (transfer_encoding) {
479 return error.HttpHeadersExceededSizeLimit;579 .chunked => {
480 }580 req.response.next_chunk_length = 0;
481 try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);581 req.response.state = .chunk_size;
582 },
583 .compress => return error.HttpTransferEncodingUnsupported,
584 .deflate => return error.HttpTransferEncodingUnsupported,
585 .gzip => return error.HttpTransferEncodingUnsupported,
586 }
587 } else if (req.response.headers.content_length) |content_length| {
588 req.response.next_chunk_length = content_length;
589 } else {
590 return error.HttpContentLengthUnknown;
591 }
482592
483 if (req.response.state == .finished) {593 in = in[i..];
484 req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);594 continue;
485 }595 }
486596
487 if (req.response.headers.status.class() == .redirect) {597 assert(out_index == 0);
488 if (req.redirects_left == 0) return error.TooManyHttpRedirects;598 return 0;
489 const location = req.response.headers.location orelse
490 return error.HttpRedirectMissingLocation;
491 const new_url = try std.Url.parse(location);
492 const new_req = try req.client.request(new_url, req.headers, .{
493 .max_redirects = req.redirects_left - 1,
494 .header_strategy = if (req.response.header_bytes_owned) .{
495 .dynamic = req.response.max_header_bytes,
496 } else .{
497 .static = req.response.header_bytes.unusedCapacitySlice(),
498 },599 },
499 });600 .finished => {
500 req.deinit();601 mem.copy(u8, buffer[out_index..], in);
501 req.* = new_req;602 return out_index + in.len;
502 return readAdvanced(req, buffer);603 },
503 }604 .chunk_size, .chunk_r => {
504605 const i = req.response.findChunkedLen(in);
505 const body_data = data[i..];606 switch (req.response.state) {
506 if (body_data.len > 0) {607 .invalid => return error.HttpHeadersInvalid,
507 mem.copy(u8, buffer, body_data);608 .chunk_data => {
508 return body_data.len;609 if (req.response.next_chunk_length == 0) {
610 req.response.state = .start;
611 return out_index;
612 }
613 in = in[i..];
614 continue;
615 },
616 .chunk_size => return out_index,
617 else => unreachable,
618 }
619 },
620 .chunk_data => {
621 const sub_amt = @min(req.response.next_chunk_length, in.len);
622 mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
623 out_index += sub_amt;
624 req.response.next_chunk_length -= sub_amt;
625 if (req.response.next_chunk_length == 0) {
626 req.response.state = .chunk_size;
627 in = in[sub_amt..];
628 continue;
629 }
630 return out_index;
631 },
632 }
509 }633 }
510 return 0;
511 }634 }
512635
513 test {636 test {
...@@ -569,7 +692,9 @@ pub fn request(client: *Client, url: Url, headers: Request.Headers, options: Req...@@ -569,7 +692,9 @@ pub fn request(client: *Client, url: Url, headers: Request.Headers, options: Req
569 try h.appendSlice(@tagName(headers.method));692 try h.appendSlice(@tagName(headers.method));
570 try h.appendSlice(" ");693 try h.appendSlice(" ");
571 try h.appendSlice(url.path);694 try h.appendSlice(url.path);
572 try h.appendSlice(" HTTP/1.1\r\nHost: ");695 try h.appendSlice(" ");
696 try h.appendSlice(@tagName(headers.version));
697 try h.appendSlice("\r\nHost: ");
573 try h.appendSlice(url.host);698 try h.appendSlice(url.host);
574 try h.appendSlice("\r\nConnection: close\r\n\r\n");699 try h.appendSlice("\r\nConnection: close\r\n\r\n");
575700