authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-21 00:16:03-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
logb4b9f6aa4a5bfd6a54b59444f3e1a3706358eb76
treed20273d53d628d62c2284895f10c187a4d062208
parenta8958c99a9ecfd0a95dc8194b5a4fd172739b30e

std.http.Server: reimplement chunked uploading

* Uncouple std.http.ChunkParser from protocol.zig * Fix receiveHead not passing leftover buffer through the header parser. * Fix content-length read streaming This implementation handles the final chunk length correctly rather than "hoping" that the buffer already contains \r\n.

6 files changed, 299 insertions(+), 180 deletions(-)

lib/std/http.zig+2
......@@ -4,6 +4,7 @@ pub const Client = @import("http/Client.zig");
44pub const Server = @import("http/Server.zig");
55pub const protocol = @import("http/protocol.zig");
66pub const HeadParser = @import("http/HeadParser.zig");
7pub const ChunkParser = @import("http/ChunkParser.zig");
78
89pub const Version = enum {
910 @"HTTP/1.0",
......@@ -313,5 +314,6 @@ test {
313314 _ = Server;
314315 _ = Status;
315316 _ = HeadParser;
317 _ = ChunkParser;
316318 _ = @import("http/test.zig");
317319}
lib/std/http/ChunkParser.zig created+131
......@@ -0,0 +1,131 @@
1//! Parser for transfer-encoding: chunked.
2
3state: State,
4chunk_len: u64,
5
6pub const init: ChunkParser = .{
7 .state = .head_size,
8 .chunk_len = 0,
9};
10
11pub const State = enum {
12 head_size,
13 head_ext,
14 head_r,
15 data,
16 data_suffix,
17 data_suffix_r,
18 invalid,
19};
20
21/// Returns the number of bytes consumed by the chunk size. This is always
22/// less than or equal to `bytes.len`.
23///
24/// After this function returns, `chunk_len` will contain the parsed chunk size
25/// in bytes when `state` is `data`. Alternately, `state` may become `invalid`,
26/// indicating a syntax error in the input stream.
27///
28/// If the amount returned is less than `bytes.len`, the parser is in the
29/// `chunk_data` state and the first byte of the chunk is at `bytes[result]`.
30///
31/// Asserts `state` is neither `data` nor `invalid`.
32pub fn feed(p: *ChunkParser, bytes: []const u8) usize {
33 for (bytes, 0..) |c, i| switch (p.state) {
34 .data_suffix => switch (c) {
35 '\r' => p.state = .data_suffix_r,
36 '\n' => p.state = .head_size,
37 else => {
38 p.state = .invalid;
39 return i;
40 },
41 },
42 .data_suffix_r => switch (c) {
43 '\n' => p.state = .head_size,
44 else => {
45 p.state = .invalid;
46 return i;
47 },
48 },
49 .head_size => {
50 const digit = switch (c) {
51 '0'...'9' => |b| b - '0',
52 'A'...'Z' => |b| b - 'A' + 10,
53 'a'...'z' => |b| b - 'a' + 10,
54 '\r' => {
55 p.state = .head_r;
56 continue;
57 },
58 '\n' => {
59 p.state = .data;
60 return i + 1;
61 },
62 else => {
63 p.state = .head_ext;
64 continue;
65 },
66 };
67
68 const new_len = p.chunk_len *% 16 +% digit;
69 if (new_len <= p.chunk_len and p.chunk_len != 0) {
70 p.state = .invalid;
71 return i;
72 }
73
74 p.chunk_len = new_len;
75 },
76 .head_ext => switch (c) {
77 '\r' => p.state = .head_r,
78 '\n' => {
79 p.state = .data;
80 return i + 1;
81 },
82 else => continue,
83 },
84 .head_r => switch (c) {
85 '\n' => {
86 p.state = .data;
87 return i + 1;
88 },
89 else => {
90 p.state = .invalid;
91 return i;
92 },
93 },
94 .data => unreachable,
95 .invalid => unreachable,
96 };
97 return bytes.len;
98}
99
100const ChunkParser = @This();
101const std = @import("std");
102
103test feed {
104 const testing = std.testing;
105
106 const data = "Ff\r\nf0f000 ; ext\n0\r\nffffffffffffffffffffffffffffffffffffffff\r\n";
107
108 var p = init;
109 const first = p.feed(data[0..]);
110 try testing.expectEqual(@as(u32, 4), first);
111 try testing.expectEqual(@as(u64, 0xff), p.chunk_len);
112 try testing.expectEqual(.data, p.state);
113
114 p = init;
115 const second = p.feed(data[first..]);
116 try testing.expectEqual(@as(u32, 13), second);
117 try testing.expectEqual(@as(u64, 0xf0f000), p.chunk_len);
118 try testing.expectEqual(.data, p.state);
119
120 p = init;
121 const third = p.feed(data[first + second ..]);
122 try testing.expectEqual(@as(u32, 3), third);
123 try testing.expectEqual(@as(u64, 0), p.chunk_len);
124 try testing.expectEqual(.data, p.state);
125
126 p = init;
127 const fourth = p.feed(data[first + second + third ..]);
128 try testing.expectEqual(@as(u32, 16), fourth);
129 try testing.expectEqual(@as(u64, 0xffffffffffffffff), p.chunk_len);
130 try testing.expectEqual(.invalid, p.state);
131}
lib/std/http/HeadParser.zig+8-7
......@@ -1,3 +1,5 @@
1//! Finds the end of an HTTP head in a stream.
2
13state: State = .start,
24
35pub const State = enum {
......@@ -17,13 +19,12 @@ pub const State = enum {
1719/// `bytes[result]`.
1820pub fn feed(p: *HeadParser, bytes: []const u8) usize {
1921 const vector_len: comptime_int = @max(std.simd.suggestVectorLength(u8) orelse 1, 8);
20 const len: u32 = @intCast(bytes.len);
21 var index: u32 = 0;
22 var index: usize = 0;
2223
2324 while (true) {
2425 switch (p.state) {
2526 .finished => return index,
26 .start => switch (len - index) {
27 .start => switch (bytes.len - index) {
2728 0 => return index,
2829 1 => {
2930 switch (bytes[index]) {
......@@ -218,7 +219,7 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {
218219 continue;
219220 },
220221 },
221 .seen_n => switch (len - index) {
222 .seen_n => switch (bytes.len - index) {
222223 0 => return index,
223224 else => {
224225 switch (bytes[index]) {
......@@ -230,7 +231,7 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {
230231 continue;
231232 },
232233 },
233 .seen_r => switch (len - index) {
234 .seen_r => switch (bytes.len - index) {
234235 0 => return index,
235236 1 => {
236237 switch (bytes[index]) {
......@@ -286,7 +287,7 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {
286287 continue;
287288 },
288289 },
289 .seen_rn => switch (len - index) {
290 .seen_rn => switch (bytes.len - index) {
290291 0 => return index,
291292 1 => {
292293 switch (bytes[index]) {
......@@ -317,7 +318,7 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {
317318 continue;
318319 },
319320 },
320 .seen_rnr => switch (len - index) {
321 .seen_rnr => switch (bytes.len - index) {
321322 0 => return index,
322323 else => {
323324 switch (bytes[index]) {
lib/std/http/Server.zig+133-60
......@@ -1,4 +1,5 @@
11//! Blocking HTTP server implementation.
2//! Handles a single connection's lifecycle.
23
34connection: net.Server.Connection,
45/// Keeps track of whether the Server is ready to accept a new request on the
......@@ -62,20 +63,19 @@ pub fn receiveHead(s: *Server) ReceiveHeadError!Request {
6263 // In case of a reused connection, move the next request's bytes to the
6364 // beginning of the buffer.
6465 if (s.next_request_start > 0) {
65 if (s.read_buffer_len > s.next_request_start) {
66 const leftover = s.read_buffer[s.next_request_start..s.read_buffer_len];
67 const dest = s.read_buffer[0..leftover.len];
68 if (leftover.len <= s.next_request_start) {
69 @memcpy(dest, leftover);
70 } else {
71 mem.copyBackwards(u8, dest, leftover);
72 }
73 s.read_buffer_len = leftover.len;
74 }
66 if (s.read_buffer_len > s.next_request_start) rebase(s, 0);
7567 s.next_request_start = 0;
7668 }
7769
7870 var hp: http.HeadParser = .{};
71
72 if (s.read_buffer_len > 0) {
73 const bytes = s.read_buffer[0..s.read_buffer_len];
74 const end = hp.feed(bytes);
75 if (hp.state == .finished)
76 return finishReceivingHead(s, end);
77 }
78
7979 while (true) {
8080 const buf = s.read_buffer[s.read_buffer_len..];
8181 if (buf.len == 0)
......@@ -85,16 +85,21 @@ pub fn receiveHead(s: *Server) ReceiveHeadError!Request {
8585 s.read_buffer_len += read_n;
8686 const bytes = buf[0..read_n];
8787 const end = hp.feed(bytes);
88 if (hp.state == .finished) return .{
89 .server = s,
90 .head_end = end,
91 .head = Request.Head.parse(s.read_buffer[0..end]) catch
92 return error.HttpHeadersInvalid,
93 .reader_state = undefined,
94 };
88 if (hp.state == .finished)
89 return finishReceivingHead(s, s.read_buffer_len - bytes.len + end);
9590 }
9691}
9792
93fn finishReceivingHead(s: *Server, head_end: usize) ReceiveHeadError!Request {
94 return .{
95 .server = s,
96 .head_end = head_end,
97 .head = Request.Head.parse(s.read_buffer[0..head_end]) catch
98 return error.HttpHeadersInvalid,
99 .reader_state = undefined,
100 };
101}
102
98103pub const Request = struct {
99104 server: *Server,
100105 /// Index into Server's read_buffer.
......@@ -102,6 +107,7 @@ pub const Request = struct {
102107 head: Head,
103108 reader_state: union {
104109 remaining_content_length: u64,
110 chunk_parser: http.ChunkParser,
105111 },
106112
107113 pub const Compression = union(enum) {
......@@ -416,51 +422,130 @@ pub const Request = struct {
416422 };
417423 }
418424
419 pub const ReadError = net.Stream.ReadError;
425 pub const ReadError = net.Stream.ReadError || error{ HttpChunkInvalid, HttpHeadersOversize };
420426
421427 fn read_cl(context: *const anyopaque, buffer: []u8) ReadError!usize {
422428 const request: *Request = @constCast(@alignCast(@ptrCast(context)));
423429 const s = request.server;
424430 assert(s.state == .receiving_body);
425
426431 const remaining_content_length = &request.reader_state.remaining_content_length;
427
428432 if (remaining_content_length.* == 0) {
429433 s.state = .ready;
430434 return 0;
431435 }
432
433 const available_bytes = s.read_buffer_len - request.head_end;
434 if (available_bytes == 0)
435 s.read_buffer_len += try s.connection.stream.read(s.read_buffer[request.head_end..]);
436
437 const available_buf = s.read_buffer[request.head_end..s.read_buffer_len];
438 const len = @min(remaining_content_length.*, available_buf.len, buffer.len);
439 @memcpy(buffer[0..len], available_buf[0..len]);
436 const available = try fill(s, request.head_end);
437 const len = @min(remaining_content_length.*, available.len, buffer.len);
438 @memcpy(buffer[0..len], available[0..len]);
440439 remaining_content_length.* -= len;
440 s.next_request_start += len;
441441 if (remaining_content_length.* == 0)
442442 s.state = .ready;
443443 return len;
444444 }
445445
446 fn fill(s: *Server, head_end: usize) ReadError![]u8 {
447 const available = s.read_buffer[s.next_request_start..s.read_buffer_len];
448 if (available.len > 0) return available;
449 s.next_request_start = head_end;
450 s.read_buffer_len = head_end + try s.connection.stream.read(s.read_buffer[head_end..]);
451 return s.read_buffer[head_end..s.read_buffer_len];
452 }
453
446454 fn read_chunked(context: *const anyopaque, buffer: []u8) ReadError!usize {
447455 const request: *Request = @constCast(@alignCast(@ptrCast(context)));
448456 const s = request.server;
449457 assert(s.state == .receiving_body);
450 _ = buffer;
451 @panic("TODO");
452 }
453458
454 pub const ReadAllError = ReadError || error{HttpBodyOversize};
459 const cp = &request.reader_state.chunk_parser;
460 const head_end = request.head_end;
461
462 // Protect against returning 0 before the end of stream.
463 var out_end: usize = 0;
464 while (out_end == 0) {
465 switch (cp.state) {
466 .invalid => return 0,
467 .data => {
468 const available = try fill(s, head_end);
469 const len = @min(cp.chunk_len, available.len, buffer.len);
470 @memcpy(buffer[0..len], available[0..len]);
471 cp.chunk_len -= len;
472 if (cp.chunk_len == 0)
473 cp.state = .data_suffix;
474 out_end += len;
475 s.next_request_start += len;
476 continue;
477 },
478 else => {
479 const available = try fill(s, head_end);
480 const n = cp.feed(available);
481 switch (cp.state) {
482 .invalid => return error.HttpChunkInvalid,
483 .data => {
484 if (cp.chunk_len == 0) {
485 // The next bytes in the stream are trailers,
486 // or \r\n to indicate end of chunked body.
487 //
488 // This function must append the trailers at
489 // head_end so that headers and trailers are
490 // together.
491 //
492 // Since returning 0 would indicate end of
493 // stream, this function must read all the
494 // trailers before returning.
495 if (s.next_request_start > head_end) rebase(s, head_end);
496 var hp: http.HeadParser = .{};
497 {
498 const bytes = s.read_buffer[head_end..s.read_buffer_len];
499 const end = hp.feed(bytes);
500 if (hp.state == .finished) {
501 s.next_request_start = s.read_buffer_len - bytes.len + end;
502 return out_end;
503 }
504 }
505 while (true) {
506 const buf = s.read_buffer[s.read_buffer_len..];
507 if (buf.len == 0)
508 return error.HttpHeadersOversize;
509 const read_n = try s.connection.stream.read(buf);
510 s.read_buffer_len += read_n;
511 const bytes = buf[0..read_n];
512 const end = hp.feed(bytes);
513 if (hp.state == .finished) {
514 s.next_request_start = s.read_buffer_len - bytes.len + end;
515 return out_end;
516 }
517 }
518 }
519 const data = available[n..];
520 const len = @min(cp.chunk_len, data.len, buffer.len);
521 @memcpy(buffer[0..len], data[0..len]);
522 cp.chunk_len -= len;
523 if (cp.chunk_len == 0)
524 cp.state = .data_suffix;
525 out_end += len;
526 s.next_request_start += n + len;
527 continue;
528 },
529 else => continue,
530 }
531 },
532 }
533 }
534 return out_end;
535 }
455536
456537 pub fn reader(request: *Request) std.io.AnyReader {
457538 const s = request.server;
458539 assert(s.state == .received_head);
459540 s.state = .receiving_body;
541 s.next_request_start = request.head_end;
460542 switch (request.head.transfer_encoding) {
461 .chunked => return .{
462 .readFn = read_chunked,
463 .context = request,
543 .chunked => {
544 request.reader_state = .{ .chunk_parser = http.ChunkParser.init };
545 return .{
546 .readFn = read_chunked,
547 .context = request,
548 };
464549 },
465550 .none => {
466551 request.reader_state = .{
......@@ -489,31 +574,8 @@ pub const Request = struct {
489574 const s = request.server;
490575 if (keep_alive and request.head.keep_alive) switch (s.state) {
491576 .received_head => {
492 s.state = .receiving_body;
493 switch (request.head.transfer_encoding) {
494 .none => t: {
495 const len = request.head.content_length orelse break :t;
496 const head_end = request.head_end;
497 var total_body_discarded: usize = 0;
498 while (true) {
499 const available_bytes = s.read_buffer_len - head_end;
500 const remaining_len = len - total_body_discarded;
501 if (available_bytes >= remaining_len) {
502 s.next_request_start = head_end + remaining_len;
503 break :t;
504 }
505 total_body_discarded += available_bytes;
506 // Preserve request header memory until receiveHead is called.
507 const buf = s.read_buffer[head_end..];
508 const read_n = s.connection.stream.read(buf) catch return false;
509 s.read_buffer_len = head_end + read_n;
510 }
511 },
512 .chunked => {
513 @panic("TODO");
514 },
515 }
516 s.state = .ready;
577 _ = request.reader().discard() catch return false;
578 assert(s.state == .ready);
517579 return true;
518580 },
519581 .receiving_body, .ready => return true,
......@@ -799,6 +861,17 @@ pub const Response = struct {
799861 }
800862};
801863
864fn rebase(s: *Server, index: usize) void {
865 const leftover = s.read_buffer[s.next_request_start..s.read_buffer_len];
866 const dest = s.read_buffer[index..][0..leftover.len];
867 if (leftover.len <= s.next_request_start - index) {
868 @memcpy(dest, leftover);
869 } else {
870 mem.copyBackwards(u8, dest, leftover);
871 }
872 s.read_buffer_len = index + leftover.len;
873}
874
802875const std = @import("../std.zig");
803876const http = std.http;
804877const mem = std.mem;
lib/std/http/protocol.zig+24-112
......@@ -97,85 +97,32 @@ pub const HeadersParser = struct {
9797 return @intCast(result);
9898 }
9999
100 /// Returns the number of bytes consumed by the chunk size. This is always
101 /// less than or equal to `bytes.len`.
102 /// You should check `r.state == .chunk_data` after this to check if the
103 /// chunk size has been fully parsed.
104 ///
105 /// If the amount returned is less than `bytes.len`, you may assume that
106 /// the parser is in the `chunk_data` state and that the first byte of the
107 /// chunk is at `bytes[result]`.
108100 pub fn findChunkedLen(r: *HeadersParser, bytes: []const u8) u32 {
109 const len = @as(u32, @intCast(bytes.len));
110
111 for (bytes[0..], 0..) |c, i| {
112 const index = @as(u32, @intCast(i));
113 switch (r.state) {
114 .chunk_data_suffix => switch (c) {
115 '\r' => r.state = .chunk_data_suffix_r,
116 '\n' => r.state = .chunk_head_size,
117 else => {
118 r.state = .invalid;
119 return index;
120 },
121 },
122 .chunk_data_suffix_r => switch (c) {
123 '\n' => r.state = .chunk_head_size,
124 else => {
125 r.state = .invalid;
126 return index;
127 },
128 },
129 .chunk_head_size => {
130 const digit = switch (c) {
131 '0'...'9' => |b| b - '0',
132 'A'...'Z' => |b| b - 'A' + 10,
133 'a'...'z' => |b| b - 'a' + 10,
134 '\r' => {
135 r.state = .chunk_head_r;
136 continue;
137 },
138 '\n' => {
139 r.state = .chunk_data;
140 return index + 1;
141 },
142 else => {
143 r.state = .chunk_head_ext;
144 continue;
145 },
146 };
147
148 const new_len = r.next_chunk_length *% 16 +% digit;
149 if (new_len <= r.next_chunk_length and r.next_chunk_length != 0) {
150 r.state = .invalid;
151 return index;
152 }
153
154 r.next_chunk_length = new_len;
155 },
156 .chunk_head_ext => switch (c) {
157 '\r' => r.state = .chunk_head_r,
158 '\n' => {
159 r.state = .chunk_data;
160 return index + 1;
161 },
162 else => continue,
163 },
164 .chunk_head_r => switch (c) {
165 '\n' => {
166 r.state = .chunk_data;
167 return index + 1;
168 },
169 else => {
170 r.state = .invalid;
171 return index;
172 },
173 },
101 var cp: std.http.ChunkParser = .{
102 .state = switch (r.state) {
103 .chunk_head_size => .head_size,
104 .chunk_head_ext => .head_ext,
105 .chunk_head_r => .head_r,
106 .chunk_data => .data,
107 .chunk_data_suffix => .data_suffix,
108 .chunk_data_suffix_r => .data_suffix_r,
109 .invalid => .invalid,
174110 else => unreachable,
175 }
176 }
177
178 return len;
111 },
112 .chunk_len = r.next_chunk_length,
113 };
114 const result = cp.feed(bytes);
115 r.state = switch (cp.state) {
116 .head_size => .chunk_head_size,
117 .head_ext => .chunk_head_ext,
118 .head_r => .chunk_head_r,
119 .data => .chunk_data,
120 .data_suffix => .chunk_data_suffix,
121 .data_suffix_r => .chunk_data_suffix_r,
122 .invalid => .invalid,
123 };
124 r.next_chunk_length = cp.chunk_len;
125 return @intCast(result);
179126 }
180127
181128 /// Returns whether or not the parser has finished parsing a complete
......@@ -464,41 +411,6 @@ const MockBufferedConnection = struct {
464411 }
465412};
466413
467test "HeadersParser.findChunkedLen" {
468 var r: HeadersParser = undefined;
469 const data = "Ff\r\nf0f000 ; ext\n0\r\nffffffffffffffffffffffffffffffffffffffff\r\n";
470
471 r = HeadersParser.init(&.{});
472 r.state = .chunk_head_size;
473 r.next_chunk_length = 0;
474
475 const first = r.findChunkedLen(data[0..]);
476 try testing.expectEqual(@as(u32, 4), first);
477 try testing.expectEqual(@as(u64, 0xff), r.next_chunk_length);
478 try testing.expectEqual(State.chunk_data, r.state);
479 r.state = .chunk_head_size;
480 r.next_chunk_length = 0;
481
482 const second = r.findChunkedLen(data[first..]);
483 try testing.expectEqual(@as(u32, 13), second);
484 try testing.expectEqual(@as(u64, 0xf0f000), r.next_chunk_length);
485 try testing.expectEqual(State.chunk_data, r.state);
486 r.state = .chunk_head_size;
487 r.next_chunk_length = 0;
488
489 const third = r.findChunkedLen(data[first + second ..]);
490 try testing.expectEqual(@as(u32, 3), third);
491 try testing.expectEqual(@as(u64, 0), r.next_chunk_length);
492 try testing.expectEqual(State.chunk_data, r.state);
493 r.state = .chunk_head_size;
494 r.next_chunk_length = 0;
495
496 const fourth = r.findChunkedLen(data[first + second + third ..]);
497 try testing.expectEqual(@as(u32, 16), fourth);
498 try testing.expectEqual(@as(u64, 0xffffffffffffffff), r.next_chunk_length);
499 try testing.expectEqual(State.invalid, r.state);
500}
501
502414test "HeadersParser.read length" {
503415 // mock BufferedConnection for read
504416 var headers_buf: [256]u8 = undefined;
lib/std/http/test.zig+1-1
......@@ -164,7 +164,7 @@ test "HTTP server handles a chunked transfer coding request" {
164164
165165 const stream = try std.net.tcpConnectToHost(allocator, "127.0.0.1", server_port);
166166 defer stream.close();
167 _ = try stream.writeAll(request_bytes[0..]);
167 try stream.writeAll(request_bytes);
168168
169169 server_thread.join();
170170}