authorgravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-05-27 07:40:56-05:00
committergravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-06-01 13:43:23-05:00
log8136123aa7cdd6d53c682572405eb6c1d5e0f0a0
tree1588f253155fbbf11719d67dfd229be7ecd70f89
parent6c2f3745564aefa669b336e249888bb7390b3a3f
signaturelock-open Commit is signed but in an unrecognized format.

std.http.Client: collapse BufferedConnection into Connection

2 files changed, 111 insertions(+), 174 deletions(-)

lib/std/http/Client.zig+101-164
...@@ -36,21 +36,7 @@ pub const ConnectionPool = struct {...@@ -36,21 +36,7 @@ pub const ConnectionPool = struct {
36 is_tls: bool,36 is_tls: bool,
37 };37 };
3838
39 pub const StoredConnection = struct {39 const Queue = std.TailQueue(Connection);
40 buffered: BufferedConnection,
41 host: []u8,
42 port: u16,
43
44 proxied: bool = false,
45 closing: bool = false,
46
47 pub fn deinit(self: *StoredConnection, client: *Client) void {
48 self.buffered.close(client);
49 client.allocator.free(self.host);
50 }
51 };
52
53 const Queue = std.TailQueue(StoredConnection);
54 pub const Node = Queue.Node;40 pub const Node = Queue.Node;
5541
56 mutex: std.Thread.Mutex = .{},42 mutex: std.Thread.Mutex = .{},
...@@ -69,7 +55,7 @@ pub const ConnectionPool = struct {...@@ -69,7 +55,7 @@ pub const ConnectionPool = struct {
6955
70 var next = pool.free.last;56 var next = pool.free.last;
71 while (next) |node| : (next = node.prev) {57 while (next) |node| : (next = node.prev) {
72 if ((node.data.buffered.conn.protocol == .tls) != criteria.is_tls) continue;58 if ((node.data.protocol == .tls) != criteria.is_tls) continue;
73 if (node.data.port != criteria.port) continue;59 if (node.data.port != criteria.port) continue;
74 if (!mem.eql(u8, node.data.host, criteria.host)) continue;60 if (!mem.eql(u8, node.data.host, criteria.host)) continue;
7561
...@@ -160,27 +146,25 @@ pub const ConnectionPool = struct {...@@ -160,27 +146,25 @@ pub const ConnectionPool = struct {
160146
161/// An interface to either a plain or TLS connection.147/// An interface to either a plain or TLS connection.
162pub const Connection = struct {148pub const Connection = struct {
149 pub const buffer_size = std.crypto.tls.max_ciphertext_record_len;
150 pub const Protocol = enum { plain, tls };
151
163 stream: net.Stream,152 stream: net.Stream,
164 /// undefined unless protocol is tls.153 /// undefined unless protocol is tls.
165 tls_client: *std.crypto.tls.Client,154 tls_client: *std.crypto.tls.Client,
155
166 protocol: Protocol,156 protocol: Protocol,
157 host: []u8,
158 port: u16,
167159
168 pub const Protocol = enum { plain, tls };160 proxied: bool = false,
161 closing: bool = false,
169162
170 pub fn read(conn: *Connection, buffer: []u8) ReadError!usize {163 read_start: u16 = 0,
171 return switch (conn.protocol) {164 read_end: u16 = 0,
172 .plain => conn.stream.read(buffer),165 read_buf: [buffer_size]u8 = undefined,
173 .tls => conn.tls_client.read(conn.stream, buffer),
174 } catch |err| switch (err) {
175 error.TlsConnectionTruncated, error.TlsRecordOverflow, error.TlsDecodeError, error.TlsBadRecordMac, error.TlsBadLength, error.TlsIllegalParameter, error.TlsUnexpectedMessage => return error.TlsFailure,
176 error.TlsAlert => return error.TlsAlert,
177 error.ConnectionTimedOut => return error.ConnectionTimedOut,
178 error.ConnectionResetByPeer, error.BrokenPipe => return error.ConnectionResetByPeer,
179 else => return error.UnexpectedReadFailure,
180 };
181 }
182166
183 pub fn readAtLeast(conn: *Connection, buffer: []u8, len: usize) ReadError!usize {167 pub fn rawReadAtLeast(conn: *Connection, buffer: []u8, len: usize) ReadError!usize {
184 return switch (conn.protocol) {168 return switch (conn.protocol) {
185 .plain => conn.stream.readAtLeast(buffer, len),169 .plain => conn.stream.readAtLeast(buffer, len),
186 .tls => conn.tls_client.readAtLeast(conn.stream, buffer, len),170 .tls => conn.tls_client.readAtLeast(conn.stream, buffer, len),
...@@ -193,12 +177,70 @@ pub const Connection = struct {...@@ -193,12 +177,70 @@ pub const Connection = struct {
193 };177 };
194 }178 }
195179
180 pub fn fill(conn: *Connection) ReadError!void {
181 if (conn.read_end != conn.read_start) return;
182
183 const nread = try conn.conn.read(conn.read_buf[0..]);
184 if (nread == 0) return error.EndOfStream;
185 conn.read_start = 0;
186 conn.read_end = @intCast(u16, nread);
187 }
188
189 pub fn peek(conn: *Connection) []const u8 {
190 return conn.read_buf[conn.read_start..conn.read_end];
191 }
192
193 pub fn drop(conn: *Connection, num: u16) void {
194 conn.read_start += num;
195 }
196
197 pub fn readAtLeast(conn: *Connection, buffer: []u8, len: usize) ReadError!usize {
198 assert(len <= buffer.len);
199
200 var out_index: u16 = 0;
201 while (out_index < len) {
202 const available_read = conn.read_end - conn.read_start;
203 const available_buffer = buffer.len - out_index;
204
205 if (available_read > available_buffer) { // partially read buffered data
206 @memcpy(buffer[out_index..], conn.read_buf[conn.read_start..][0..available_buffer]);
207 out_index += available_buffer;
208 conn.read_start += available_buffer;
209
210 break;
211 } else if (available_read > 0) { // fully read buffered data
212 @memcpy(buffer[out_index..][0..available_read], conn.read_buf[conn.read_start..]);
213 out_index += available_read;
214 conn.read_start += available_read;
215
216 if (out_index >= len) break;
217 }
218
219 const leftover_buffer = available_buffer - available_read;
220 const leftover_len = len - out_index;
221
222 if (leftover_buffer > conn.read_buf.len) {
223 // skip the buffer if the output is large enough
224 return conn.rawReadAtLeast(buffer[out_index..], leftover_len);
225 }
226
227 try conn.fill();
228 }
229
230 return out_index;
231 }
232
233 pub fn read(conn: *Connection, buffer: []u8) ReadError!usize {
234 return conn.readAtLeast(buffer, 1);
235 }
236
196 pub const ReadError = error{237 pub const ReadError = error{
197 TlsFailure,238 TlsFailure,
198 TlsAlert,239 TlsAlert,
199 ConnectionTimedOut,240 ConnectionTimedOut,
200 ConnectionResetByPeer,241 ConnectionResetByPeer,
201 UnexpectedReadFailure,242 UnexpectedReadFailure,
243 EndOfStream,
202 };244 };
203245
204 pub const Reader = std.io.Reader(*Connection, ReadError, read);246 pub const Reader = std.io.Reader(*Connection, ReadError, read);
...@@ -247,111 +289,10 @@ pub const Connection = struct {...@@ -247,111 +289,10 @@ pub const Connection = struct {
247289
248 conn.stream.close();290 conn.stream.close();
249 }291 }
250};
251
252/// A buffered (and peekable) Connection.
253pub const BufferedConnection = struct {
254 pub const buffer_size = std.crypto.tls.max_ciphertext_record_len;
255
256 conn: Connection,
257 read_buf: [buffer_size]u8 = undefined,
258 read_start: u16 = 0,
259 read_end: u16 = 0,
260
261 write_buf: [buffer_size]u8 = undefined,
262 write_end: u16 = 0,
263
264 pub fn fill(bconn: *BufferedConnection) ReadError!void {
265 if (bconn.read_end != bconn.read_start) return;
266
267 const nread = try bconn.conn.read(bconn.read_buf[0..]);
268 if (nread == 0) return error.EndOfStream;
269 bconn.read_start = 0;
270 bconn.read_end = @intCast(u16, nread);
271 }
272
273 pub fn peek(bconn: *BufferedConnection) []const u8 {
274 return bconn.read_buf[bconn.read_start..bconn.read_end];
275 }
276
277 pub fn clear(bconn: *BufferedConnection, num: u16) void {
278 bconn.read_start += num;
279 }
280
281 pub fn readAtLeast(bconn: *BufferedConnection, buffer: []u8, len: usize) ReadError!usize {
282 var out_index: u16 = 0;
283 while (out_index < len) {
284 const available = bconn.read_end - bconn.read_start;
285 const left = buffer.len - out_index;
286
287 if (available > 0) {
288 const can_read = @intCast(u16, @min(available, left));
289
290 @memcpy(buffer[out_index..][0..can_read], bconn.read_buf[bconn.read_start..][0..can_read]);
291 out_index += can_read;
292 bconn.read_start += can_read;
293
294 continue;
295 }
296
297 if (left > bconn.read_buf.len) {
298 // skip the buffer if the output is large enough
299 return bconn.conn.read(buffer[out_index..]);
300 }
301
302 try bconn.fill();
303 }
304
305 return out_index;
306 }
307
308 pub fn read(bconn: *BufferedConnection, buffer: []u8) ReadError!usize {
309 return bconn.readAtLeast(buffer, 1);
310 }
311292
312 pub const ReadError = Connection.ReadError || error{EndOfStream};293 pub fn deinit(conn: *Connection, client: *const Client) void {
313 pub const Reader = std.io.Reader(*BufferedConnection, ReadError, read);294 conn.close(client);
314295 client.allocator.free(conn.host);
315 pub fn reader(bconn: *BufferedConnection) Reader {
316 return Reader{ .context = bconn };
317 }
318
319 pub fn writeAll(bconn: *BufferedConnection, buffer: []const u8) WriteError!void {
320 if (bconn.write_buf.len - bconn.write_end >= buffer.len) {
321 @memcpy(bconn.write_buf[bconn.write_end..][0..buffer.len], buffer);
322 bconn.write_end += @intCast(u16, buffer.len);
323 } else {
324 try bconn.flush();
325 try bconn.conn.writeAll(buffer);
326 }
327 }
328
329 pub fn write(bconn: *BufferedConnection, buffer: []const u8) WriteError!usize {
330 if (bconn.write_buf.len - bconn.write_end >= buffer.len) {
331 @memcpy(bconn.write_buf[bconn.write_end..][0..buffer.len], buffer);
332 bconn.write_end += @intCast(u16, buffer.len);
333
334 return buffer.len;
335 } else {
336 try bconn.flush();
337 return try bconn.conn.write(buffer);
338 }
339 }
340
341 pub fn flush(bconn: *BufferedConnection) WriteError!void {
342 defer bconn.write_end = 0;
343 return bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
344 }
345
346 pub const WriteError = Connection.WriteError;
347 pub const Writer = std.io.Writer(*BufferedConnection, WriteError, write);
348
349 pub fn writer(bconn: *BufferedConnection) Writer {
350 return Writer{ .context = bconn };
351 }
352
353 pub fn close(bconn: *BufferedConnection, client: *const Client) void {
354 bconn.conn.close(client);
355 }296 }
356};297};
357298
...@@ -585,11 +526,12 @@ pub const Request = struct {...@@ -585,11 +526,12 @@ pub const Request = struct {
585 };526 };
586 }527 }
587528
588 pub const StartError = BufferedConnection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding };529 pub const StartError = Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding };
589530
590 /// Send the request to the server.531 /// Send the request to the server.
591 pub fn start(req: *Request) StartError!void {532 pub fn start(req: *Request) StartError!void {
592 const w = req.connection.data.buffered.writer();533 var buffered = std.io.bufferedWriter(req.connection.data.writer());
534 const w = buffered.writer();
593535
594 try w.writeAll(@tagName(req.method));536 try w.writeAll(@tagName(req.method));
595 try w.writeByte(' ');537 try w.writeByte(' ');
...@@ -662,11 +604,9 @@ pub const Request = struct {...@@ -662,11 +604,9 @@ pub const Request = struct {
662 try w.print("{}", .{req.headers});604 try w.print("{}", .{req.headers});
663605
664 try w.writeAll("\r\n");606 try w.writeAll("\r\n");
665
666 try req.connection.data.buffered.flush();
667 }607 }
668608
669 pub const TransferReadError = BufferedConnection.ReadError || proto.HeadersParser.ReadError;609 pub const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError;
670610
671 pub const TransferReader = std.io.Reader(*Request, TransferReadError, transferRead);611 pub const TransferReader = std.io.Reader(*Request, TransferReadError, transferRead);
672612
...@@ -679,7 +619,7 @@ pub const Request = struct {...@@ -679,7 +619,7 @@ pub const Request = struct {
679619
680 var index: usize = 0;620 var index: usize = 0;
681 while (index == 0) {621 while (index == 0) {
682 const amt = try req.response.parser.read(&req.connection.data.buffered, buf[index..], req.response.skip);622 const amt = try req.response.parser.read(&req.connection.data, buf[index..], req.response.skip);
683 if (amt == 0 and req.response.parser.done) break;623 if (amt == 0 and req.response.parser.done) break;
684 index += amt;624 index += amt;
685 }625 }
...@@ -697,10 +637,10 @@ pub const Request = struct {...@@ -697,10 +637,10 @@ pub const Request = struct {
697 pub fn wait(req: *Request) WaitError!void {637 pub fn wait(req: *Request) WaitError!void {
698 while (true) { // handle redirects638 while (true) { // handle redirects
699 while (true) { // read headers639 while (true) { // read headers
700 try req.connection.data.buffered.fill();640 try req.connection.data.fill();
701641
702 const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.data.buffered.peek());642 const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.data.peek());
703 req.connection.data.buffered.clear(@intCast(u16, nchecked));643 req.connection.data.drop(@intCast(u16, nchecked));
704644
705 if (req.response.parser.state.isContent()) break;645 if (req.response.parser.state.isContent()) break;
706 }646 }
...@@ -816,10 +756,10 @@ pub const Request = struct {...@@ -816,10 +756,10 @@ pub const Request = struct {
816 const has_trail = !req.response.parser.state.isContent();756 const has_trail = !req.response.parser.state.isContent();
817757
818 while (!req.response.parser.state.isContent()) { // read trailing headers758 while (!req.response.parser.state.isContent()) { // read trailing headers
819 try req.connection.data.buffered.fill();759 try req.connection.data.fill();
820760
821 const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.data.buffered.peek());761 const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.data.peek());
822 req.connection.data.buffered.clear(@intCast(u16, nchecked));762 req.connection.data.clear(@intCast(u16, nchecked));
823 }763 }
824764
825 if (has_trail) {765 if (has_trail) {
...@@ -845,7 +785,7 @@ pub const Request = struct {...@@ -845,7 +785,7 @@ pub const Request = struct {
845 return index;785 return index;
846 }786 }
847787
848 pub const WriteError = BufferedConnection.WriteError || error{ NotWriteable, MessageTooLong };788 pub const WriteError = Connection.WriteError || error{ NotWriteable, MessageTooLong };
849789
850 pub const Writer = std.io.Writer(*Request, WriteError, write);790 pub const Writer = std.io.Writer(*Request, WriteError, write);
851791
...@@ -857,16 +797,16 @@ pub const Request = struct {...@@ -857,16 +797,16 @@ pub const Request = struct {
857 pub fn write(req: *Request, bytes: []const u8) WriteError!usize {797 pub fn write(req: *Request, bytes: []const u8) WriteError!usize {
858 switch (req.transfer_encoding) {798 switch (req.transfer_encoding) {
859 .chunked => {799 .chunked => {
860 try req.connection.data.buffered.writer().print("{x}\r\n", .{bytes.len});800 try req.connection.data.writer().print("{x}\r\n", .{bytes.len});
861 try req.connection.data.buffered.writeAll(bytes);801 try req.connection.data.writeAll(bytes);
862 try req.connection.data.buffered.writeAll("\r\n");802 try req.connection.data.writeAll("\r\n");
863803
864 return bytes.len;804 return bytes.len;
865 },805 },
866 .content_length => |*len| {806 .content_length => |*len| {
867 if (len.* < bytes.len) return error.MessageTooLong;807 if (len.* < bytes.len) return error.MessageTooLong;
868808
869 const amt = try req.connection.data.buffered.write(bytes);809 const amt = try req.connection.data.write(bytes);
870 len.* -= amt;810 len.* -= amt;
871 return amt;811 return amt;
872 },812 },
...@@ -886,12 +826,10 @@ pub const Request = struct {...@@ -886,12 +826,10 @@ pub const Request = struct {
886 /// Finish the body of a request. This notifies the server that you have no more data to send.826 /// Finish the body of a request. This notifies the server that you have no more data to send.
887 pub fn finish(req: *Request) FinishError!void {827 pub fn finish(req: *Request) FinishError!void {
888 switch (req.transfer_encoding) {828 switch (req.transfer_encoding) {
889 .chunked => try req.connection.data.buffered.writeAll("0\r\n\r\n"),829 .chunked => try req.connection.data.writeAll("0\r\n\r\n"),
890 .content_length => |len| if (len != 0) return error.MessageNotCompleted,830 .content_length => |len| if (len != 0) return error.MessageNotCompleted,
891 .none => {},831 .none => {},
892 }832 }
893
894 try req.connection.data.buffered.flush();
895 }833 }
896};834};
897835
...@@ -948,11 +886,10 @@ pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol:...@@ -948,11 +886,10 @@ pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol:
948 errdefer stream.close();886 errdefer stream.close();
949887
950 conn.data = .{888 conn.data = .{
951 .buffered = .{ .conn = .{889 .stream = stream,
952 .stream = stream,890 .tls_client = undefined,
953 .tls_client = undefined,891 .protocol = protocol,
954 .protocol = protocol,892
955 } },
956 .host = try client.allocator.dupe(u8, host),893 .host = try client.allocator.dupe(u8, host),
957 .port = port,894 .port = port,
958 };895 };
...@@ -961,13 +898,13 @@ pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol:...@@ -961,13 +898,13 @@ pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol:
961 switch (protocol) {898 switch (protocol) {
962 .plain => {},899 .plain => {},
963 .tls => {900 .tls => {
964 conn.data.buffered.conn.tls_client = try client.allocator.create(std.crypto.tls.Client);901 conn.data.tls_client = try client.allocator.create(std.crypto.tls.Client);
965 errdefer client.allocator.destroy(conn.data.buffered.conn.tls_client);902 errdefer client.allocator.destroy(conn.data.tls_client);
966903
967 conn.data.buffered.conn.tls_client.* = std.crypto.tls.Client.init(stream, client.ca_bundle, host) catch return error.TlsInitializationFailed;904 conn.data.tls_client.* = std.crypto.tls.Client.init(stream, client.ca_bundle, host) catch return error.TlsInitializationFailed;
968 // This is appropriate for HTTPS because the HTTP headers contain905 // This is appropriate for HTTPS because the HTTP headers contain
969 // the content length which is used to detect truncation attacks.906 // the content length which is used to detect truncation attacks.
970 conn.data.buffered.conn.tls_client.allow_truncation_attacks = true;907 conn.data.tls_client.allow_truncation_attacks = true;
971 },908 },
972 }909 }
973910
...@@ -1003,7 +940,7 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio...@@ -1003,7 +940,7 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
1003 }940 }
1004}941}
1005942
1006pub const RequestError = ConnectUnproxiedError || ConnectErrorPartial || Request.StartError || std.fmt.ParseIntError || BufferedConnection.WriteError || error{943pub const RequestError = ConnectUnproxiedError || ConnectErrorPartial || Request.StartError || std.fmt.ParseIntError || Connection.WriteError || error{
1007 UnsupportedUrlScheme,944 UnsupportedUrlScheme,
1008 UriMissingHost,945 UriMissingHost,
1009946
lib/std/http/protocol.zig+10-10
...@@ -641,8 +641,8 @@ const MockBufferedConnection = struct {...@@ -641,8 +641,8 @@ const MockBufferedConnection = struct {
641 return bconn.buf[bconn.start..bconn.end];641 return bconn.buf[bconn.start..bconn.end];
642 }642 }
643643
644 pub fn clear(bconn: *MockBufferedConnection, num: u16) void {644 pub fn drop(conn: *MockBufferedConnection, num: u16) void {
645 bconn.start += num;645 conn.start += num;
646 }646 }
647647
648 pub fn readAtLeast(bconn: *MockBufferedConnection, buffer: []u8, len: usize) ReadError!usize {648 pub fn readAtLeast(bconn: *MockBufferedConnection, buffer: []u8, len: usize) ReadError!usize {
...@@ -760,8 +760,8 @@ test "HeadersParser.read length" {...@@ -760,8 +760,8 @@ test "HeadersParser.read length" {
760 while (true) { // read headers760 while (true) { // read headers
761 try bconn.fill();761 try bconn.fill();
762762
763 const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());763 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
764 bconn.clear(@intCast(u16, nchecked));764 conn.drop(@intCast(u16, nchecked));
765765
766 if (r.state.isContent()) break;766 if (r.state.isContent()) break;
767 }767 }
...@@ -791,8 +791,8 @@ test "HeadersParser.read chunked" {...@@ -791,8 +791,8 @@ test "HeadersParser.read chunked" {
791 while (true) { // read headers791 while (true) { // read headers
792 try bconn.fill();792 try bconn.fill();
793793
794 const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());794 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
795 bconn.clear(@intCast(u16, nchecked));795 conn.drop(@intCast(u16, nchecked));
796796
797 if (r.state.isContent()) break;797 if (r.state.isContent()) break;
798 }798 }
...@@ -821,8 +821,8 @@ test "HeadersParser.read chunked trailer" {...@@ -821,8 +821,8 @@ test "HeadersParser.read chunked trailer" {
821 while (true) { // read headers821 while (true) { // read headers
822 try bconn.fill();822 try bconn.fill();
823823
824 const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());824 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
825 bconn.clear(@intCast(u16, nchecked));825 conn.drop(@intCast(u16, nchecked));
826826
827 if (r.state.isContent()) break;827 if (r.state.isContent()) break;
828 }828 }
...@@ -836,8 +836,8 @@ test "HeadersParser.read chunked trailer" {...@@ -836,8 +836,8 @@ test "HeadersParser.read chunked trailer" {
836 while (true) { // read headers836 while (true) { // read headers
837 try bconn.fill();837 try bconn.fill();
838838
839 const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());839 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
840 bconn.clear(@intCast(u16, nchecked));840 conn.drop(@intCast(u16, nchecked));
841841
842 if (r.state.isContent()) break;842 if (r.state.isContent()) break;
843 }843 }