authorgravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-03-30 22:53:59-05:00
committergravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-04-08 09:59:36-05:00
logaecbfa3a1e9aa379368e6a9a999ca42fc4803f18
treec23bdaf7457839f684d53a027ed789412fb4105e
parent08bdaf3bd650ec7682f1424c3644fc3b762ccf27
signaturelock-open Commit is signed but in an unrecognized format.

add buffering to connection instead of the http protocol, to allow passing through upgrades

4 files changed, 282 insertions(+), 388 deletions(-)

lib/std/http/Client.zig+131-31
......@@ -32,7 +32,20 @@ pub const ConnectionPool = struct {
3232 is_tls: bool,
3333 };
3434
35 const Queue = std.TailQueue(Connection);
35 pub const StoredConnection = struct {
36 buffered: BufferedConnection,
37 host: []u8,
38 port: u16,
39
40 closing: bool = false,
41
42 pub fn deinit(self: *StoredConnection, client: *Client) void {
43 self.buffered.close(client);
44 client.allocator.free(self.host);
45 }
46 };
47
48 const Queue = std.TailQueue(StoredConnection);
3649 pub const Node = Queue.Node;
3750
3851 mutex: std.Thread.Mutex = .{},
......@@ -49,7 +62,7 @@ pub const ConnectionPool = struct {
4962
5063 var next = pool.free.last;
5164 while (next) |node| : (next = node.prev) {
52 if ((node.data.protocol == .tls) != criteria.is_tls) continue;
65 if ((node.data.buffered.conn.protocol == .tls) != criteria.is_tls) continue;
5366 if (node.data.port != criteria.port) continue;
5467 if (mem.eql(u8, node.data.host, criteria.host)) continue;
5568
......@@ -85,7 +98,7 @@ pub const ConnectionPool = struct {
8598 pool.used.remove(node);
8699
87100 if (node.data.closing) {
88 node.data.close(client);
101 node.data.deinit(client);
89102
90103 return client.allocator.destroy(node);
91104 }
......@@ -93,7 +106,7 @@ pub const ConnectionPool = struct {
93106 if (pool.free_len + 1 >= pool.free_size) {
94107 const popped = pool.free.popFirst() orelse unreachable;
95108
96 popped.data.close(client);
109 popped.data.deinit(client);
97110
98111 return client.allocator.destroy(popped);
99112 }
......@@ -118,7 +131,7 @@ pub const ConnectionPool = struct {
118131 defer client.allocator.destroy(node);
119132 next = node.next;
120133
121 node.data.close(client);
134 node.data.deinit(client);
122135 }
123136
124137 next = pool.used.first;
......@@ -126,7 +139,7 @@ pub const ConnectionPool = struct {
126139 defer client.allocator.destroy(node);
127140 next = node.next;
128141
129 node.data.close(client);
142 node.data.deinit(client);
130143 }
131144
132145 pool.* = undefined;
......@@ -140,13 +153,8 @@ pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.Transfer
140153pub const Connection = struct {
141154 stream: net.Stream,
142155 /// undefined unless protocol is tls.
143 tls_client: *std.crypto.tls.Client, // TODO: allocate this, it's currently 16 KB.
156 tls_client: *std.crypto.tls.Client,
144157 protocol: Protocol,
145 host: []u8,
146 port: u16,
147
148 // This connection has been part of a non keepalive request and cannot be added to the pool.
149 closing: bool = false,
150158
151159 pub const Protocol = enum { plain, tls };
152160
......@@ -211,8 +219,89 @@ pub const Connection = struct {
211219 }
212220
213221 conn.stream.close();
222 }
223};
224
225pub const BufferedConnection = struct {
226 pub const buffer_size = 0x2000;
227
228 conn: Connection,
229 buf: [buffer_size]u8 = undefined,
230 start: u16 = 0,
231 end: u16 = 0,
232
233 pub fn fill(bconn: *BufferedConnection) ReadError!void {
234 if (bconn.end != bconn.start) return;
235
236 const nread = try bconn.conn.read(bconn.buf[0..]);
237 if (nread == 0) return error.EndOfStream;
238 bconn.start = 0;
239 bconn.end = @truncate(u16, nread);
240 }
241
242 pub fn peek(bconn: *BufferedConnection) []const u8 {
243 return bconn.buf[bconn.start..bconn.end];
244 }
245
246 pub fn clear(bconn: *BufferedConnection, num: u16) void {
247 bconn.start += num;
248 }
249
250 pub fn readAtLeast(bconn: *BufferedConnection, buffer: []u8, len: usize) ReadError!usize {
251 var out_index: u16 = 0;
252 while (out_index < len) {
253 const available = bconn.end - bconn.start;
254 const left = buffer.len - out_index;
255
256 if (available > 0) {
257 const can_read = @truncate(u16, @min(available, left));
258
259 std.mem.copy(u8, buffer[out_index..], bconn.buf[bconn.start..][0..can_read]);
260 out_index += can_read;
261 bconn.start += can_read;
262
263 continue;
264 }
214265
215 client.allocator.free(conn.host);
266 if (left > bconn.buf.len) {
267 // skip the buffer if the output is large enough
268 return bconn.conn.read(buffer[out_index..]);
269 }
270
271 try bconn.fill();
272 }
273
274 return out_index;
275 }
276
277 pub fn read(bconn: *BufferedConnection, buffer: []u8) ReadError!usize {
278 return bconn.readAtLeast(buffer, 1);
279 }
280
281 pub const ReadError = Connection.ReadError || error{EndOfStream};
282 pub const Reader = std.io.Reader(*BufferedConnection, ReadError, read);
283
284 pub fn reader(bconn: *BufferedConnection) Reader {
285 return Reader{ .context = bconn };
286 }
287
288 pub fn writeAll(bconn: *BufferedConnection, buffer: []const u8) WriteError!void {
289 return bconn.conn.writeAll(buffer);
290 }
291
292 pub fn write(bconn: *BufferedConnection, buffer: []const u8) WriteError!usize {
293 return bconn.conn.write(buffer);
294 }
295
296 pub const WriteError = Connection.WriteError;
297 pub const Writer = std.io.Writer(*BufferedConnection, WriteError, write);
298
299 pub fn writer(bconn: *BufferedConnection) Writer {
300 return Writer{ .context = bconn };
301 }
302
303 pub fn close(bconn: *BufferedConnection, client: *const Client) void {
304 bconn.conn.close(client);
216305 }
217306};
218307
......@@ -417,7 +506,7 @@ pub const Request = struct {
417506 req.* = undefined;
418507 }
419508
420 pub const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError;
509 pub const TransferReadError = BufferedConnection.ReadError || proto.HeadersParser.ReadError;
421510
422511 pub const TransferReader = std.io.Reader(*Request, TransferReadError, transferRead);
423512
......@@ -430,7 +519,7 @@ pub const Request = struct {
430519
431520 var index: usize = 0;
432521 while (index == 0) {
433 const amt = try req.response.parser.read(req.connection.data.reader(), buf[index..], req.response.skip);
522 const amt = try req.response.parser.read(&req.connection.data.buffered, buf[index..], req.response.skip);
434523 if (amt == 0 and req.response.parser.isComplete()) break;
435524 index += amt;
436525 }
......@@ -438,10 +527,17 @@ pub const Request = struct {
438527 return index;
439528 }
440529
441 pub const WaitForCompleteHeadError = Connection.ReadError || proto.HeadersParser.WaitForCompleteHeadError || Response.Headers.ParseError || error{ BadHeader, InvalidCompression, StreamTooLong, InvalidWindowSize } || error{CompressionNotSupported};
530 pub const WaitForCompleteHeadError = BufferedConnection.ReadError || proto.HeadersParser.CheckCompleteHeadError || Response.Headers.ParseError || error{ BadHeader, InvalidCompression, StreamTooLong, InvalidWindowSize } || error{CompressionNotSupported};
442531
443532 pub fn waitForCompleteHead(req: *Request) !void {
444 try req.response.parser.waitForCompleteHead(req.connection.data.reader(), req.client.allocator);
533 while (true) {
534 try req.connection.data.buffered.fill();
535
536 const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.data.buffered.peek());
537 req.connection.data.buffered.clear(@intCast(u16, nchecked));
538
539 if (req.response.parser.state.isContent()) break;
540 }
445541
446542 req.response.headers = try Response.Headers.parse(req.response.parser.header_bytes.items);
447543
......@@ -550,7 +646,7 @@ pub const Request = struct {
550646 return index;
551647 }
552648
553 pub const WriteError = Connection.WriteError || error{ NotWriteable, MessageTooLong };
649 pub const WriteError = BufferedConnection.WriteError || error{ NotWriteable, MessageTooLong };
554650
555651 pub const Writer = std.io.Writer(*Request, WriteError, write);
556652
......@@ -562,16 +658,16 @@ pub const Request = struct {
562658 pub fn write(req: *Request, bytes: []const u8) WriteError!usize {
563659 switch (req.headers.transfer_encoding) {
564660 .chunked => {
565 try req.connection.data.writer().print("{x}\r\n", .{bytes.len});
566 try req.connection.data.writeAll(bytes);
567 try req.connection.data.writeAll("\r\n");
661 try req.connection.data.conn.writer().print("{x}\r\n", .{bytes.len});
662 try req.connection.data.conn.writeAll(bytes);
663 try req.connection.data.conn.writeAll("\r\n");
568664
569665 return bytes.len;
570666 },
571667 .content_length => |*len| {
572668 if (len.* < bytes.len) return error.MessageTooLong;
573669
574 const amt = try req.connection.data.write(bytes);
670 const amt = try req.connection.data.conn.write(bytes);
575671 len.* -= amt;
576672 return amt;
577673 },
......@@ -582,7 +678,7 @@ pub const Request = struct {
582678 /// Finish the body of a request. This notifies the server that you have no more data to send.
583679 pub fn finish(req: *Request) !void {
584680 switch (req.headers.transfer_encoding) {
585 .chunked => try req.connection.data.writeAll("0\r\n"),
681 .chunked => try req.connection.data.conn.writeAll("0\r\n"),
586682 .content_length => |len| if (len != 0) return error.MessageNotCompleted,
587683 .none => {},
588684 }
......@@ -610,10 +706,14 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
610706 errdefer client.allocator.destroy(conn);
611707 conn.* = .{ .data = undefined };
612708
709 const stream = try net.tcpConnectToHost(client.allocator, host, port);
710
613711 conn.data = .{
614 .stream = try net.tcpConnectToHost(client.allocator, host, port),
615 .tls_client = undefined,
616 .protocol = protocol,
712 .buffered = .{ .conn = .{
713 .stream = stream,
714 .tls_client = undefined,
715 .protocol = protocol,
716 } },
617717 .host = try client.allocator.dupe(u8, host),
618718 .port = port,
619719 };
......@@ -621,11 +721,11 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
621721 switch (protocol) {
622722 .plain => {},
623723 .tls => {
624 conn.data.tls_client = try client.allocator.create(std.crypto.tls.Client);
625 conn.data.tls_client.* = try std.crypto.tls.Client.init(conn.data.stream, client.ca_bundle, host);
724 conn.data.buffered.conn.tls_client = try client.allocator.create(std.crypto.tls.Client);
725 conn.data.buffered.conn.tls_client.* = try std.crypto.tls.Client.init(stream, client.ca_bundle, host);
626726 // This is appropriate for HTTPS because the HTTP headers contain
627727 // the content length which is used to detect truncation attacks.
628 conn.data.tls_client.allow_truncation_attacks = true;
728 conn.data.buffered.conn.tls_client.allow_truncation_attacks = true;
629729 },
630730 }
631731
......@@ -634,7 +734,7 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
634734 return conn;
635735}
636736
637pub const RequestError = ConnectError || Connection.WriteError || error{
737pub const RequestError = ConnectError || BufferedConnection.WriteError || error{
638738 UnsupportedUrlScheme,
639739 UriMissingHost,
640740
......@@ -708,7 +808,7 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Opt
708808 req.arena = std.heap.ArenaAllocator.init(client.allocator);
709809
710810 {
711 var buffered = std.io.bufferedWriter(req.connection.data.writer());
811 var buffered = std.io.bufferedWriter(req.connection.data.buffered.writer());
712812 const writer = buffered.writer();
713813
714814 const escaped_path = try Uri.escapePath(client.allocator, uri.path);
lib/std/http/Client/Response.zig deleted-276
......@@ -1,276 +0,0 @@
1const std = @import("std");
2const http = std.http;
3const mem = std.mem;
4const testing = std.testing;
5const assert = std.debug.assert;
6
7const protocol = @import("../protocol.zig");
8const Client = @import("../Client.zig");
9const Response = @This();
10
11headers: Headers,
12state: State,
13header_bytes_owned: bool,
14/// This could either be a fixed buffer provided by the API user or it
15/// could be our own array list.
16header_bytes: std.ArrayListUnmanaged(u8),
17max_header_bytes: usize,
18next_chunk_length: u64,
19done: bool = false,
20
21compression: union(enum) {
22 deflate: Client.DeflateDecompressor,
23 gzip: Client.GzipDecompressor,
24 zstd: Client.ZstdDecompressor,
25 none: void,
26} = .none,
27
28pub const Headers = struct {
29 status: http.Status,
30 version: http.Version,
31 location: ?[]const u8 = null,
32 content_length: ?u64 = null,
33 transfer_encoding: ?http.TransferEncoding = null,
34 transfer_compression: ?http.ContentEncoding = null,
35 connection: http.Connection = .close,
36 upgrade: ?[]const u8 = null,
37
38 number_of_headers: usize = 0,
39
40 pub fn parse(bytes: []const u8) !Headers {
41 var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
42
43 const first_line = it.first();
44 if (first_line.len < 12)
45 return error.ShortHttpStatusLine;
46
47 const version: http.Version = switch (int64(first_line[0..8])) {
48 int64("HTTP/1.0") => .@"HTTP/1.0",
49 int64("HTTP/1.1") => .@"HTTP/1.1",
50 else => return error.BadHttpVersion,
51 };
52 if (first_line[8] != ' ') return error.HttpHeadersInvalid;
53 const status = @intToEnum(http.Status, parseInt3(first_line[9..12].*));
54
55 var headers: Headers = .{
56 .version = version,
57 .status = status,
58 };
59
60 while (it.next()) |line| {
61 headers.number_of_headers += 1;
62
63 if (line.len == 0) return error.HttpHeadersInvalid;
64 switch (line[0]) {
65 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
66 else => {},
67 }
68 var line_it = mem.split(u8, line, ": ");
69 const header_name = line_it.first();
70 const header_value = line_it.rest();
71 if (std.ascii.eqlIgnoreCase(header_name, "location")) {
72 if (headers.location != null) return error.HttpHeadersInvalid;
73 headers.location = header_value;
74 } else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
75 if (headers.content_length != null) return error.HttpHeadersInvalid;
76 headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
77 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
78 // Transfer-Encoding: second, first
79 // Transfer-Encoding: deflate, chunked
80 var iter = std.mem.splitBackwards(u8, header_value, ",");
81
82 if (iter.next()) |first| {
83 const trimmed = std.mem.trim(u8, first, " ");
84
85 if (std.meta.stringToEnum(http.TransferEncoding, trimmed)) |te| {
86 if (headers.transfer_encoding != null) return error.HttpHeadersInvalid;
87 headers.transfer_encoding = te;
88 } else if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) |ce| {
89 if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
90 headers.transfer_compression = ce;
91 } else {
92 return error.HttpTransferEncodingUnsupported;
93 }
94 }
95
96 if (iter.next()) |second| {
97 if (headers.transfer_compression != null) return error.HttpTransferEncodingUnsupported;
98
99 const trimmed = std.mem.trim(u8, second, " ");
100
101 if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) |ce| {
102 headers.transfer_compression = ce;
103 } else {
104 return error.HttpTransferEncodingUnsupported;
105 }
106 }
107
108 if (iter.next()) |_| return error.HttpTransferEncodingUnsupported;
109 } else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
110 if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
111
112 const trimmed = std.mem.trim(u8, header_value, " ");
113
114 if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) |ce| {
115 headers.transfer_compression = ce;
116 } else {
117 return error.HttpTransferEncodingUnsupported;
118 }
119 } else if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
120 if (std.ascii.eqlIgnoreCase(header_value, "keep-alive")) {
121 headers.connection = .keep_alive;
122 } else if (std.ascii.eqlIgnoreCase(header_value, "close")) {
123 headers.connection = .close;
124 } else {
125 return error.HttpConnectionHeaderUnsupported;
126 }
127 } else if (std.ascii.eqlIgnoreCase(header_name, "upgrade")) {
128 headers.upgrade = header_value;
129 }
130 }
131
132 return headers;
133 }
134
135 test "parse headers" {
136 const example =
137 "HTTP/1.1 301 Moved Permanently\r\n" ++
138 "Location: https://www.example.com/\r\n" ++
139 "Content-Type: text/html; charset=UTF-8\r\n" ++
140 "Content-Length: 220\r\n\r\n";
141 const parsed = try Headers.parse(example);
142 try testing.expectEqual(http.Version.@"HTTP/1.1", parsed.version);
143 try testing.expectEqual(http.Status.moved_permanently, parsed.status);
144 try testing.expectEqualStrings("https://www.example.com/", parsed.location orelse
145 return error.TestFailed);
146 try testing.expectEqual(@as(?u64, 220), parsed.content_length);
147 }
148
149 test "header continuation" {
150 const example =
151 "HTTP/1.0 200 OK\r\n" ++
152 "Content-Type: text/html;\r\n charset=UTF-8\r\n" ++
153 "Content-Length: 220\r\n\r\n";
154 try testing.expectError(
155 error.HttpHeaderContinuationsUnsupported,
156 Headers.parse(example),
157 );
158 }
159
160 test "extra content length" {
161 const example =
162 "HTTP/1.0 200 OK\r\n" ++
163 "Content-Length: 220\r\n" ++
164 "Content-Type: text/html; charset=UTF-8\r\n" ++
165 "content-length: 220\r\n\r\n";
166 try testing.expectError(
167 error.HttpHeadersInvalid,
168 Headers.parse(example),
169 );
170 }
171};
172
173inline fn int64(array: *const [8]u8) u64 {
174 return @bitCast(u64, array.*);
175}
176
177pub const State = enum {
178 /// Begin header parsing states.
179 invalid,
180 start,
181 seen_r,
182 seen_rn,
183 seen_rnr,
184 finished,
185 /// Begin transfer-encoding: chunked parsing states.
186 chunk_size_prefix_r,
187 chunk_size_prefix_n,
188 chunk_size,
189 chunk_r,
190 chunk_data,
191
192 pub fn isContent(self: State) bool {
193 return switch (self) {
194 .invalid, .start, .seen_r, .seen_rn, .seen_rnr => false,
195 .finished, .chunk_size_prefix_r, .chunk_size_prefix_n, .chunk_size, .chunk_r, .chunk_data => true,
196 };
197 }
198};
199
200pub fn initDynamic(max: usize) Response {
201 return .{
202 .state = .start,
203 .headers = undefined,
204 .header_bytes = .{},
205 .max_header_bytes = max,
206 .header_bytes_owned = true,
207 .next_chunk_length = undefined,
208 };
209}
210
211pub fn initStatic(buf: []u8) Response {
212 return .{
213 .state = .start,
214 .headers = undefined,
215 .header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
216 .max_header_bytes = buf.len,
217 .header_bytes_owned = false,
218 .next_chunk_length = undefined,
219 };
220}
221
222fn parseInt3(nnn: @Vector(3, u8)) u10 {
223 const zero: @Vector(3, u8) = .{ '0', '0', '0' };
224 const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
225 return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
226}
227
228test parseInt3 {
229 const expectEqual = std.testing.expectEqual;
230 try expectEqual(@as(u10, 0), parseInt3("000".*));
231 try expectEqual(@as(u10, 418), parseInt3("418".*));
232 try expectEqual(@as(u10, 999), parseInt3("999".*));
233}
234
235test "find headers end basic" {
236 var buffer: [1]u8 = undefined;
237 var r = Response.initStatic(&buffer);
238 try testing.expectEqual(@as(usize, 10), r.findHeadersEnd("HTTP/1.1 4"));
239 try testing.expectEqual(@as(usize, 2), r.findHeadersEnd("18"));
240 try testing.expectEqual(@as(usize, 8), r.findHeadersEnd(" lol\r\n\r\nblah blah"));
241}
242
243test "find headers end vectorized" {
244 var buffer: [1]u8 = undefined;
245 var r = Response.initStatic(&buffer);
246 const example =
247 "HTTP/1.1 301 Moved Permanently\r\n" ++
248 "Location: https://www.example.com/\r\n" ++
249 "Content-Type: text/html; charset=UTF-8\r\n" ++
250 "Content-Length: 220\r\n" ++
251 "\r\ncontent";
252 try testing.expectEqual(@as(usize, 131), r.findHeadersEnd(example));
253}
254
255test "find headers end bug" {
256 var buffer: [1]u8 = undefined;
257 var r = Response.initStatic(&buffer);
258 const trail = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
259 const example =
260 "HTTP/1.1 200 OK\r\n" ++
261 "Access-Control-Allow-Origin: https://render.githubusercontent.com\r\n" ++
262 "content-disposition: attachment; filename=zig-0.10.0.tar.gz\r\n" ++
263 "Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'; sandbox\r\n" ++
264 "Content-Type: application/x-gzip\r\n" ++
265 "ETag: \"bfae0af6b01c7c0d89eb667cb5f0e65265968aeebda2689177e6b26acd3155ca\"\r\n" ++
266 "Strict-Transport-Security: max-age=31536000\r\n" ++
267 "Vary: Authorization,Accept-Encoding,Origin\r\n" ++
268 "X-Content-Type-Options: nosniff\r\n" ++
269 "X-Frame-Options: deny\r\n" ++
270 "X-XSS-Protection: 1; mode=block\r\n" ++
271 "Date: Fri, 06 Jan 2023 22:26:22 GMT\r\n" ++
272 "Transfer-Encoding: chunked\r\n" ++
273 "X-GitHub-Request-Id: 89C6:17E9:A7C9E:124B51:63B8A00E\r\n" ++
274 "connection: close\r\n\r\n" ++ trail;
275 try testing.expectEqual(@as(usize, example.len - trail.len), r.findHeadersEnd(example));
276}
lib/std/http/Server.zig+102-12
......@@ -74,6 +74,89 @@ pub const Connection = struct {
7474 }
7575};
7676
77pub const BufferedConnection = struct {
78 pub const buffer_size = 0x2000;
79
80 conn: Connection,
81 buf: [buffer_size]u8 = undefined,
82 start: u16 = 0,
83 end: u16 = 0,
84
85 pub fn fill(bconn: *BufferedConnection) ReadError!void {
86 if (bconn.end != bconn.start) return;
87
88 const nread = try bconn.conn.read(bconn.buf[0..]);
89 if (nread == 0) return error.EndOfStream;
90 bconn.start = 0;
91 bconn.end = @truncate(u16, nread);
92 }
93
94 pub fn peek(bconn: *BufferedConnection) []const u8 {
95 return bconn.buf[bconn.start..bconn.end];
96 }
97
98 pub fn clear(bconn: *BufferedConnection, num: u16) void {
99 bconn.start += num;
100 }
101
102 pub fn readAtLeast(bconn: *BufferedConnection, buffer: []u8, len: usize) ReadError!usize {
103 var out_index: u16 = 0;
104 while (out_index < len) {
105 const available = bconn.end - bconn.start;
106 const left = buffer.len - out_index;
107
108 if (available > 0) {
109 const can_read = @truncate(u16, @min(available, left));
110
111 std.mem.copy(u8, buffer[out_index..], bconn.buf[bconn.start..][0..can_read]);
112 out_index += can_read;
113 bconn.start += can_read;
114
115 continue;
116 }
117
118 if (left > bconn.buf.len) {
119 // skip the buffer if the output is large enough
120 return bconn.conn.read(buffer[out_index..]);
121 }
122
123 try bconn.fill();
124 }
125
126 return out_index;
127 }
128
129 pub fn read(bconn: *BufferedConnection, buffer: []u8) ReadError!usize {
130 return bconn.readAtLeast(buffer, 1);
131 }
132
133 pub const ReadError = Connection.ReadError || error{EndOfStream};
134 pub const Reader = std.io.Reader(*BufferedConnection, ReadError, read);
135
136 pub fn reader(bconn: *BufferedConnection) Reader {
137 return Reader{ .context = bconn };
138 }
139
140 pub fn writeAll(bconn: *BufferedConnection, buffer: []const u8) WriteError!void {
141 return bconn.conn.writeAll(buffer);
142 }
143
144 pub fn write(bconn: *BufferedConnection, buffer: []const u8) WriteError!usize {
145 return bconn.conn.write(buffer);
146 }
147
148 pub const WriteError = Connection.WriteError;
149 pub const Writer = std.io.Writer(*BufferedConnection, WriteError, write);
150
151 pub fn writer(bconn: *BufferedConnection) Writer {
152 return Writer{ .context = bconn };
153 }
154
155 pub fn close(bconn: *BufferedConnection) void {
156 bconn.conn.close();
157 }
158};
159
77160pub const Request = struct {
78161 pub const Headers = struct {
79162 method: http.Method,
......@@ -222,7 +305,7 @@ pub const Response = struct {
222305
223306 server: *Server,
224307 address: net.Address,
225 connection: Connection,
308 connection: BufferedConnection,
226309
227310 headers: Headers = .{},
228311 request: Request,
......@@ -237,10 +320,10 @@ pub const Response = struct {
237320
238321 if (!res.request.parser.done) {
239322 // If the response wasn't fully read, then we need to close the connection.
240 res.connection.closing = true;
323 res.connection.conn.closing = true;
241324 }
242325
243 if (res.connection.closing) {
326 if (res.connection.conn.closing) {
244327 res.connection.close();
245328
246329 if (res.request.parser.header_bytes_owned) {
......@@ -296,7 +379,7 @@ pub const Response = struct {
296379 try buffered.flush();
297380 }
298381
299 pub const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError;
382 pub const TransferReadError = BufferedConnection.ReadError || proto.HeadersParser.ReadError;
300383
301384 pub const TransferReader = std.io.Reader(*Response, TransferReadError, transferRead);
302385
......@@ -309,7 +392,7 @@ pub const Response = struct {
309392
310393 var index: usize = 0;
311394 while (index == 0) {
312 const amt = try res.request.parser.read(res.connection.reader(), buf[index..], false);
395 const amt = try res.request.parser.read(&res.connection, buf[index..], false);
313396 if (amt == 0 and res.request.parser.isComplete()) break;
314397 index += amt;
315398 }
......@@ -317,17 +400,24 @@ pub const Response = struct {
317400 return index;
318401 }
319402
320 pub const WaitForCompleteHeadError = Connection.ReadError || proto.HeadersParser.WaitForCompleteHeadError || Request.Headers.ParseError || error{ BadHeader, InvalidCompression, StreamTooLong, InvalidWindowSize } || error{CompressionNotSupported};
403 pub const WaitForCompleteHeadError = BufferedConnection.ReadError || proto.HeadersParser.WaitForCompleteHeadError || Request.Headers.ParseError || error{ BadHeader, InvalidCompression, StreamTooLong, InvalidWindowSize } || error{CompressionNotSupported};
321404
322405 pub fn waitForCompleteHead(res: *Response) !void {
323 try res.request.parser.waitForCompleteHead(res.connection.reader(), res.server.allocator);
406 while (true) {
407 try res.connection.fill();
408
409 const nchecked = try res.request.parser.checkCompleteHead(res.server.allocator, res.connection.peek());
410 res.connection.clear(@intCast(u16, nchecked));
411
412 if (res.request.parser.state.isContent()) break;
413 }
324414
325415 res.request.headers = try Request.Headers.parse(res.request.parser.header_bytes.items);
326416
327417 if (res.headers.connection == .keep_alive and res.request.headers.connection == .keep_alive) {
328 res.connection.closing = false;
418 res.connection.conn.closing = false;
329419 } else {
330 res.connection.closing = true;
420 res.connection.conn.closing = true;
331421 }
332422
333423 if (res.request.headers.transfer_encoding) |te| {
......@@ -388,7 +478,7 @@ pub const Response = struct {
388478 return index;
389479 }
390480
391 pub const WriteError = Connection.WriteError || error{ NotWriteable, MessageTooLong };
481 pub const WriteError = BufferedConnection.WriteError || error{ NotWriteable, MessageTooLong };
392482
393483 pub const Writer = std.io.Writer(*Response, WriteError, write);
394484
......@@ -479,10 +569,10 @@ pub fn accept(server: *Server, options: HeaderStrategy) AcceptError!*Response {
479569 res.* = .{
480570 .server = server,
481571 .address = in.address,
482 .connection = .{
572 .connection = .{ .conn = .{
483573 .stream = in.stream,
484574 .protocol = .plain,
485 },
575 } },
486576 .request = .{
487577 .parser = switch (options) {
488578 .dynamic => |max| proto.HeadersParser.initDynamic(max),
lib/std/http/protocol.zig+49-69
......@@ -29,9 +29,6 @@ pub const State = enum {
2929 }
3030};
3131
32const read_buffer_size = 0x4000;
33const ReadBufferIndex = std.math.IntFittingRange(0, read_buffer_size);
34
3532pub const HeadersParser = struct {
3633 state: State = .start,
3734 /// Wether or not `header_bytes` is allocated or was provided as a fixed buffer.
......@@ -46,10 +43,6 @@ pub const HeadersParser = struct {
4643 /// A message is only done when the entire payload has been read
4744 done: bool = false,
4845
49 read_buffer: [read_buffer_size]u8 = undefined,
50 read_buffer_start: ReadBufferIndex = 0,
51 read_buffer_len: ReadBufferIndex = 0,
52
5346 pub fn initDynamic(max: usize) HeadersParser {
5447 return .{
5548 .header_bytes = .{},
......@@ -232,7 +225,7 @@ pub const HeadersParser = struct {
232225 }
233226 },
234227 4...vector_len - 1 => {
235 for (0..vector_len - 4) |i_usize| {
228 inline for (0..vector_len - 3) |i_usize| {
236229 const i = @truncate(u32, i_usize);
237230
238231 const b32 = int32(chunk[i..][0..4]);
......@@ -246,6 +239,27 @@ pub const HeadersParser = struct {
246239 return index + i + 2;
247240 }
248241 }
242
243 const b24 = int24(chunk[vector_len - 3 ..][0..3]);
244 const b16 = intShift(u16, b24);
245 const b8 = intShift(u8, b24);
246
247 switch (b8) {
248 '\r' => r.state = .seen_r,
249 '\n' => r.state = .seen_n,
250 else => {},
251 }
252
253 switch (b16) {
254 int16("\r\n") => r.state = .seen_rn,
255 int16("\n\n") => r.state = .finished,
256 else => {},
257 }
258
259 switch (b24) {
260 int24("\r\n\r") => r.state = .seen_rnr,
261 else => {},
262 }
249263 },
250264 else => unreachable,
251265 }
......@@ -475,30 +489,6 @@ pub const HeadersParser = struct {
475489 return i;
476490 }
477491
478 /// Set of errors that `waitForCompleteHead` can throw except any errors inherited by `reader`
479 pub const WaitForCompleteHeadError = CheckCompleteHeadError || error{UnexpectedEndOfStream};
480
481 /// Waits for the complete head to be available. This function will continue trying to read until the head is complete
482 /// or an error occurs.
483 pub fn waitForCompleteHead(r: *HeadersParser, reader: anytype, allocator: std.mem.Allocator) !void {
484 if (r.state.isContent()) return;
485
486 while (true) {
487 if (r.read_buffer_start == r.read_buffer_len) {
488 const nread = try reader.read(r.read_buffer[0..]);
489 if (nread == 0) return error.UnexpectedEndOfStream;
490
491 r.read_buffer_start = 0;
492 r.read_buffer_len = @intCast(ReadBufferIndex, nread);
493 }
494
495 const amt = try r.checkCompleteHead(allocator, r.read_buffer[r.read_buffer_start..r.read_buffer_len]);
496 r.read_buffer_start += @intCast(ReadBufferIndex, amt);
497
498 if (amt != 0) return;
499 }
500 }
501
502492 pub const ReadError = error{
503493 UnexpectedEndOfStream,
504494 HttpHeadersExceededSizeLimit,
......@@ -507,48 +497,40 @@ pub const HeadersParser = struct {
507497
508498 /// Reads the body of the message into `buffer`. If `skip` is true, the buffer will be unused and the body will be
509499 /// skipped. Returns the number of bytes placed in the buffer.
510 pub fn read(r: *HeadersParser, reader: anytype, buffer: []u8, skip: bool) !usize {
500 pub fn read(r: *HeadersParser, bconn: anytype, buffer: []u8, skip: bool) !usize {
511501 assert(r.state.isContent());
512502 if (r.done) return 0;
513503
514 if (r.read_buffer_start == r.read_buffer_len) {
515 const nread = try reader.read(r.read_buffer[0..]);
516 if (nread == 0) return error.UnexpectedEndOfStream;
517
518 r.read_buffer_start = 0;
519 r.read_buffer_len = @intCast(ReadBufferIndex, nread);
520 }
521
522504 var out_index: usize = 0;
523505 while (true) {
524506 switch (r.state) {
525507 .invalid, .start, .seen_n, .seen_r, .seen_rn, .seen_rnr => unreachable,
526508 .finished => {
527 const buf_avail = r.read_buffer_len - r.read_buffer_start;
528509 const data_avail = r.next_chunk_length;
529 const out_avail = buffer.len;
530510
531 // TODO https://github.com/ziglang/zig/issues/14039
532 const read_available = @intCast(usize, @min(buf_avail, data_avail));
533511 if (skip) {
534 r.next_chunk_length -= read_available;
535 r.read_buffer_start += @intCast(ReadBufferIndex, read_available);
536 } else {
537 const can_read = @min(read_available, out_avail);
538 r.next_chunk_length -= can_read;
512 try bconn.fill();
539513
540 mem.copy(u8, buffer[out_index..], r.read_buffer[r.read_buffer_start..][0..can_read]);
541 r.read_buffer_start += @intCast(ReadBufferIndex, can_read);
542 out_index += can_read;
514 const nread = @min(bconn.peek().len, data_avail);
515 bconn.clear(@intCast(u16, nread));
516 r.next_chunk_length -= nread;
517
518 return 0;
543519 }
544520
545 if (r.next_chunk_length == 0) r.done = true;
521 const out_avail = buffer.len;
546522
547 return out_index;
523 const can_read = @min(data_avail, out_avail);
524 const nread = try bconn.read(buffer[0..can_read]);
525 r.next_chunk_length -= nread;
526
527 return nread;
548528 },
549529 .chunk_data_suffix, .chunk_data_suffix_r, .chunk_head_size, .chunk_head_ext, .chunk_head_r => {
550 const i = r.findChunkedLen(r.read_buffer[r.read_buffer_start..r.read_buffer_len]);
551 r.read_buffer_start += @intCast(ReadBufferIndex, i);
530 try bconn.fill();
531
532 const i = r.findChunkedLen(bconn.peek());
533 bconn.clear(@intCast(u16, i));
552534
553535 switch (r.state) {
554536 .invalid => return error.HttpChunkInvalid,
......@@ -565,22 +547,20 @@ pub const HeadersParser = struct {
565547 continue;
566548 },
567549 .chunk_data => {
568 const buf_avail = r.read_buffer_len - r.read_buffer_start;
569550 const data_avail = r.next_chunk_length;
570 const out_avail = buffer.len;
551 const out_avail = buffer.len - out_index;
571552
572 // TODO https://github.com/ziglang/zig/issues/14039
573 const read_available = @intCast(usize, @min(buf_avail, data_avail));
574553 if (skip) {
575 r.next_chunk_length -= read_available;
576 r.read_buffer_start += @intCast(ReadBufferIndex, read_available);
577 } else {
578 const can_read = @min(read_available, out_avail);
579 r.next_chunk_length -= can_read;
554 try bconn.fill();
580555
581 mem.copy(u8, buffer[out_index..], r.read_buffer[r.read_buffer_start..][0..can_read]);
582 r.read_buffer_start += @intCast(ReadBufferIndex, can_read);
583 out_index += can_read;
556 const nread = @min(bconn.peek().len, data_avail);
557 bconn.clear(@intCast(u16, nread));
558 r.next_chunk_length -= nread;
559 } else {
560 const can_read = @min(data_avail, out_avail);
561 const nread = try bconn.read(buffer[out_index..][0..can_read]);
562 r.next_chunk_length -= nread;
563 out_index += nread;
584564 }
585565
586566 if (r.next_chunk_length == 0) {