authorgravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-05-27 07:40:56-05:00
committergravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-06-01 13:43:23-05:00
log8136123aa7cdd6d53c682572405eb6c1d5e0f0a0
tree1588f253155fbbf11719d67dfd229be7ecd70f89
parent6c2f3745564aefa669b336e249888bb7390b3a3f
signaturelock-open Commit is signed but in an unrecognized format.

std.http.Client: collapse BufferedConnection into Connection

2 files changed, 111 insertions(+), 174 deletions(-)

lib/std/http/Client.zig+101-164
......@@ -36,21 +36,7 @@ pub const ConnectionPool = struct {
3636 is_tls: bool,
3737 };
3838
39 pub const StoredConnection = struct {
40 buffered: BufferedConnection,
41 host: []u8,
42 port: u16,
43
44 proxied: bool = false,
45 closing: bool = false,
46
47 pub fn deinit(self: *StoredConnection, client: *Client) void {
48 self.buffered.close(client);
49 client.allocator.free(self.host);
50 }
51 };
52
53 const Queue = std.TailQueue(StoredConnection);
39 const Queue = std.TailQueue(Connection);
5440 pub const Node = Queue.Node;
5541
5642 mutex: std.Thread.Mutex = .{},
......@@ -69,7 +55,7 @@ pub const ConnectionPool = struct {
6955
7056 var next = pool.free.last;
7157 while (next) |node| : (next = node.prev) {
72 if ((node.data.buffered.conn.protocol == .tls) != criteria.is_tls) continue;
58 if ((node.data.protocol == .tls) != criteria.is_tls) continue;
7359 if (node.data.port != criteria.port) continue;
7460 if (!mem.eql(u8, node.data.host, criteria.host)) continue;
7561
......@@ -160,27 +146,25 @@ pub const ConnectionPool = struct {
160146
161147/// An interface to either a plain or TLS connection.
162148pub const Connection = struct {
149 pub const buffer_size = std.crypto.tls.max_ciphertext_record_len;
150 pub const Protocol = enum { plain, tls };
151
163152 stream: net.Stream,
164153 /// undefined unless protocol is tls.
165154 tls_client: *std.crypto.tls.Client,
155
166156 protocol: Protocol,
157 host: []u8,
158 port: u16,
167159
168 pub const Protocol = enum { plain, tls };
160 proxied: bool = false,
161 closing: bool = false,
169162
170 pub fn read(conn: *Connection, buffer: []u8) ReadError!usize {
171 return switch (conn.protocol) {
172 .plain => conn.stream.read(buffer),
173 .tls => conn.tls_client.read(conn.stream, buffer),
174 } catch |err| switch (err) {
175 error.TlsConnectionTruncated, error.TlsRecordOverflow, error.TlsDecodeError, error.TlsBadRecordMac, error.TlsBadLength, error.TlsIllegalParameter, error.TlsUnexpectedMessage => return error.TlsFailure,
176 error.TlsAlert => return error.TlsAlert,
177 error.ConnectionTimedOut => return error.ConnectionTimedOut,
178 error.ConnectionResetByPeer, error.BrokenPipe => return error.ConnectionResetByPeer,
179 else => return error.UnexpectedReadFailure,
180 };
181 }
163 read_start: u16 = 0,
164 read_end: u16 = 0,
165 read_buf: [buffer_size]u8 = undefined,
182166
183 pub fn readAtLeast(conn: *Connection, buffer: []u8, len: usize) ReadError!usize {
167 pub fn rawReadAtLeast(conn: *Connection, buffer: []u8, len: usize) ReadError!usize {
184168 return switch (conn.protocol) {
185169 .plain => conn.stream.readAtLeast(buffer, len),
186170 .tls => conn.tls_client.readAtLeast(conn.stream, buffer, len),
......@@ -193,12 +177,70 @@ pub const Connection = struct {
193177 };
194178 }
195179
180 pub fn fill(conn: *Connection) ReadError!void {
181 if (conn.read_end != conn.read_start) return;
182
183 const nread = try conn.conn.read(conn.read_buf[0..]);
184 if (nread == 0) return error.EndOfStream;
185 conn.read_start = 0;
186 conn.read_end = @intCast(u16, nread);
187 }
188
189 pub fn peek(conn: *Connection) []const u8 {
190 return conn.read_buf[conn.read_start..conn.read_end];
191 }
192
193 pub fn drop(conn: *Connection, num: u16) void {
194 conn.read_start += num;
195 }
196
197 pub fn readAtLeast(conn: *Connection, buffer: []u8, len: usize) ReadError!usize {
198 assert(len <= buffer.len);
199
200 var out_index: u16 = 0;
201 while (out_index < len) {
202 const available_read = conn.read_end - conn.read_start;
203 const available_buffer = buffer.len - out_index;
204
205 if (available_read > available_buffer) { // partially read buffered data
206 @memcpy(buffer[out_index..], conn.read_buf[conn.read_start..][0..available_buffer]);
207 out_index += available_buffer;
208 conn.read_start += available_buffer;
209
210 break;
211 } else if (available_read > 0) { // fully read buffered data
212 @memcpy(buffer[out_index..][0..available_read], conn.read_buf[conn.read_start..]);
213 out_index += available_read;
214 conn.read_start += available_read;
215
216 if (out_index >= len) break;
217 }
218
219 const leftover_buffer = available_buffer - available_read;
220 const leftover_len = len - out_index;
221
222 if (leftover_buffer > conn.read_buf.len) {
223 // skip the buffer if the output is large enough
224 return conn.rawReadAtLeast(buffer[out_index..], leftover_len);
225 }
226
227 try conn.fill();
228 }
229
230 return out_index;
231 }
232
233 pub fn read(conn: *Connection, buffer: []u8) ReadError!usize {
234 return conn.readAtLeast(buffer, 1);
235 }
236
196237 pub const ReadError = error{
197238 TlsFailure,
198239 TlsAlert,
199240 ConnectionTimedOut,
200241 ConnectionResetByPeer,
201242 UnexpectedReadFailure,
243 EndOfStream,
202244 };
203245
204246 pub const Reader = std.io.Reader(*Connection, ReadError, read);
......@@ -247,111 +289,10 @@ pub const Connection = struct {
247289
248290 conn.stream.close();
249291 }
250};
251
252/// A buffered (and peekable) Connection.
253pub const BufferedConnection = struct {
254 pub const buffer_size = std.crypto.tls.max_ciphertext_record_len;
255
256 conn: Connection,
257 read_buf: [buffer_size]u8 = undefined,
258 read_start: u16 = 0,
259 read_end: u16 = 0,
260
261 write_buf: [buffer_size]u8 = undefined,
262 write_end: u16 = 0,
263
264 pub fn fill(bconn: *BufferedConnection) ReadError!void {
265 if (bconn.read_end != bconn.read_start) return;
266
267 const nread = try bconn.conn.read(bconn.read_buf[0..]);
268 if (nread == 0) return error.EndOfStream;
269 bconn.read_start = 0;
270 bconn.read_end = @intCast(u16, nread);
271 }
272
273 pub fn peek(bconn: *BufferedConnection) []const u8 {
274 return bconn.read_buf[bconn.read_start..bconn.read_end];
275 }
276
277 pub fn clear(bconn: *BufferedConnection, num: u16) void {
278 bconn.read_start += num;
279 }
280
281 pub fn readAtLeast(bconn: *BufferedConnection, buffer: []u8, len: usize) ReadError!usize {
282 var out_index: u16 = 0;
283 while (out_index < len) {
284 const available = bconn.read_end - bconn.read_start;
285 const left = buffer.len - out_index;
286
287 if (available > 0) {
288 const can_read = @intCast(u16, @min(available, left));
289
290 @memcpy(buffer[out_index..][0..can_read], bconn.read_buf[bconn.read_start..][0..can_read]);
291 out_index += can_read;
292 bconn.read_start += can_read;
293
294 continue;
295 }
296
297 if (left > bconn.read_buf.len) {
298 // skip the buffer if the output is large enough
299 return bconn.conn.read(buffer[out_index..]);
300 }
301
302 try bconn.fill();
303 }
304
305 return out_index;
306 }
307
308 pub fn read(bconn: *BufferedConnection, buffer: []u8) ReadError!usize {
309 return bconn.readAtLeast(buffer, 1);
310 }
311292
312 pub const ReadError = Connection.ReadError || error{EndOfStream};
313 pub const Reader = std.io.Reader(*BufferedConnection, ReadError, read);
314
315 pub fn reader(bconn: *BufferedConnection) Reader {
316 return Reader{ .context = bconn };
317 }
318
319 pub fn writeAll(bconn: *BufferedConnection, buffer: []const u8) WriteError!void {
320 if (bconn.write_buf.len - bconn.write_end >= buffer.len) {
321 @memcpy(bconn.write_buf[bconn.write_end..][0..buffer.len], buffer);
322 bconn.write_end += @intCast(u16, buffer.len);
323 } else {
324 try bconn.flush();
325 try bconn.conn.writeAll(buffer);
326 }
327 }
328
329 pub fn write(bconn: *BufferedConnection, buffer: []const u8) WriteError!usize {
330 if (bconn.write_buf.len - bconn.write_end >= buffer.len) {
331 @memcpy(bconn.write_buf[bconn.write_end..][0..buffer.len], buffer);
332 bconn.write_end += @intCast(u16, buffer.len);
333
334 return buffer.len;
335 } else {
336 try bconn.flush();
337 return try bconn.conn.write(buffer);
338 }
339 }
340
341 pub fn flush(bconn: *BufferedConnection) WriteError!void {
342 defer bconn.write_end = 0;
343 return bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
344 }
345
346 pub const WriteError = Connection.WriteError;
347 pub const Writer = std.io.Writer(*BufferedConnection, WriteError, write);
348
349 pub fn writer(bconn: *BufferedConnection) Writer {
350 return Writer{ .context = bconn };
351 }
352
353 pub fn close(bconn: *BufferedConnection, client: *const Client) void {
354 bconn.conn.close(client);
293 pub fn deinit(conn: *Connection, client: *const Client) void {
294 conn.close(client);
295 client.allocator.free(conn.host);
355296 }
356297};
357298
......@@ -585,11 +526,12 @@ pub const Request = struct {
585526 };
586527 }
587528
588 pub const StartError = BufferedConnection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding };
529 pub const StartError = Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding };
589530
590531 /// Send the request to the server.
591532 pub fn start(req: *Request) StartError!void {
592 const w = req.connection.data.buffered.writer();
533 var buffered = std.io.bufferedWriter(req.connection.data.writer());
534 const w = buffered.writer();
593535
594536 try w.writeAll(@tagName(req.method));
595537 try w.writeByte(' ');
......@@ -662,11 +604,9 @@ pub const Request = struct {
662604 try w.print("{}", .{req.headers});
663605
664606 try w.writeAll("\r\n");
665
666 try req.connection.data.buffered.flush();
667607 }
668608
669 pub const TransferReadError = BufferedConnection.ReadError || proto.HeadersParser.ReadError;
609 pub const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError;
670610
671611 pub const TransferReader = std.io.Reader(*Request, TransferReadError, transferRead);
672612
......@@ -679,7 +619,7 @@ pub const Request = struct {
679619
680620 var index: usize = 0;
681621 while (index == 0) {
682 const amt = try req.response.parser.read(&req.connection.data.buffered, buf[index..], req.response.skip);
622 const amt = try req.response.parser.read(&req.connection.data, buf[index..], req.response.skip);
683623 if (amt == 0 and req.response.parser.done) break;
684624 index += amt;
685625 }
......@@ -697,10 +637,10 @@ pub const Request = struct {
697637 pub fn wait(req: *Request) WaitError!void {
698638 while (true) { // handle redirects
699639 while (true) { // read headers
700 try req.connection.data.buffered.fill();
640 try req.connection.data.fill();
701641
702 const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.data.buffered.peek());
703 req.connection.data.buffered.clear(@intCast(u16, nchecked));
642 const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.data.peek());
643 req.connection.data.drop(@intCast(u16, nchecked));
704644
705645 if (req.response.parser.state.isContent()) break;
706646 }
......@@ -816,10 +756,10 @@ pub const Request = struct {
816756 const has_trail = !req.response.parser.state.isContent();
817757
818758 while (!req.response.parser.state.isContent()) { // read trailing headers
819 try req.connection.data.buffered.fill();
759 try req.connection.data.fill();
820760
821 const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.data.buffered.peek());
822 req.connection.data.buffered.clear(@intCast(u16, nchecked));
761 const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.data.peek());
762 req.connection.data.clear(@intCast(u16, nchecked));
823763 }
824764
825765 if (has_trail) {
......@@ -845,7 +785,7 @@ pub const Request = struct {
845785 return index;
846786 }
847787
848 pub const WriteError = BufferedConnection.WriteError || error{ NotWriteable, MessageTooLong };
788 pub const WriteError = Connection.WriteError || error{ NotWriteable, MessageTooLong };
849789
850790 pub const Writer = std.io.Writer(*Request, WriteError, write);
851791
......@@ -857,16 +797,16 @@ pub const Request = struct {
857797 pub fn write(req: *Request, bytes: []const u8) WriteError!usize {
858798 switch (req.transfer_encoding) {
859799 .chunked => {
860 try req.connection.data.buffered.writer().print("{x}\r\n", .{bytes.len});
861 try req.connection.data.buffered.writeAll(bytes);
862 try req.connection.data.buffered.writeAll("\r\n");
800 try req.connection.data.writer().print("{x}\r\n", .{bytes.len});
801 try req.connection.data.writeAll(bytes);
802 try req.connection.data.writeAll("\r\n");
863803
864804 return bytes.len;
865805 },
866806 .content_length => |*len| {
867807 if (len.* < bytes.len) return error.MessageTooLong;
868808
869 const amt = try req.connection.data.buffered.write(bytes);
809 const amt = try req.connection.data.write(bytes);
870810 len.* -= amt;
871811 return amt;
872812 },
......@@ -886,12 +826,10 @@ pub const Request = struct {
886826 /// Finish the body of a request. This notifies the server that you have no more data to send.
887827 pub fn finish(req: *Request) FinishError!void {
888828 switch (req.transfer_encoding) {
889 .chunked => try req.connection.data.buffered.writeAll("0\r\n\r\n"),
829 .chunked => try req.connection.data.writeAll("0\r\n\r\n"),
890830 .content_length => |len| if (len != 0) return error.MessageNotCompleted,
891831 .none => {},
892832 }
893
894 try req.connection.data.buffered.flush();
895833 }
896834};
897835
......@@ -948,11 +886,10 @@ pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol:
948886 errdefer stream.close();
949887
950888 conn.data = .{
951 .buffered = .{ .conn = .{
952 .stream = stream,
953 .tls_client = undefined,
954 .protocol = protocol,
955 } },
889 .stream = stream,
890 .tls_client = undefined,
891 .protocol = protocol,
892
956893 .host = try client.allocator.dupe(u8, host),
957894 .port = port,
958895 };
......@@ -961,13 +898,13 @@ pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol:
961898 switch (protocol) {
962899 .plain => {},
963900 .tls => {
964 conn.data.buffered.conn.tls_client = try client.allocator.create(std.crypto.tls.Client);
965 errdefer client.allocator.destroy(conn.data.buffered.conn.tls_client);
901 conn.data.tls_client = try client.allocator.create(std.crypto.tls.Client);
902 errdefer client.allocator.destroy(conn.data.tls_client);
966903
967 conn.data.buffered.conn.tls_client.* = std.crypto.tls.Client.init(stream, client.ca_bundle, host) catch return error.TlsInitializationFailed;
904 conn.data.tls_client.* = std.crypto.tls.Client.init(stream, client.ca_bundle, host) catch return error.TlsInitializationFailed;
968905 // This is appropriate for HTTPS because the HTTP headers contain
969906 // the content length which is used to detect truncation attacks.
970 conn.data.buffered.conn.tls_client.allow_truncation_attacks = true;
907 conn.data.tls_client.allow_truncation_attacks = true;
971908 },
972909 }
973910
......@@ -1003,7 +940,7 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
1003940 }
1004941}
1005942
1006pub const RequestError = ConnectUnproxiedError || ConnectErrorPartial || Request.StartError || std.fmt.ParseIntError || BufferedConnection.WriteError || error{
943pub const RequestError = ConnectUnproxiedError || ConnectErrorPartial || Request.StartError || std.fmt.ParseIntError || Connection.WriteError || error{
1007944 UnsupportedUrlScheme,
1008945 UriMissingHost,
1009946
lib/std/http/protocol.zig+10-10
......@@ -641,8 +641,8 @@ const MockBufferedConnection = struct {
641641 return bconn.buf[bconn.start..bconn.end];
642642 }
643643
644 pub fn clear(bconn: *MockBufferedConnection, num: u16) void {
645 bconn.start += num;
644 pub fn drop(conn: *MockBufferedConnection, num: u16) void {
645 conn.start += num;
646646 }
647647
648648 pub fn readAtLeast(bconn: *MockBufferedConnection, buffer: []u8, len: usize) ReadError!usize {
......@@ -760,8 +760,8 @@ test "HeadersParser.read length" {
760760 while (true) { // read headers
761761 try bconn.fill();
762762
763 const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());
764 bconn.clear(@intCast(u16, nchecked));
763 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
764 conn.drop(@intCast(u16, nchecked));
765765
766766 if (r.state.isContent()) break;
767767 }
......@@ -791,8 +791,8 @@ test "HeadersParser.read chunked" {
791791 while (true) { // read headers
792792 try bconn.fill();
793793
794 const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());
795 bconn.clear(@intCast(u16, nchecked));
794 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
795 conn.drop(@intCast(u16, nchecked));
796796
797797 if (r.state.isContent()) break;
798798 }
......@@ -821,8 +821,8 @@ test "HeadersParser.read chunked trailer" {
821821 while (true) { // read headers
822822 try bconn.fill();
823823
824 const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());
825 bconn.clear(@intCast(u16, nchecked));
824 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
825 conn.drop(@intCast(u16, nchecked));
826826
827827 if (r.state.isContent()) break;
828828 }
......@@ -836,8 +836,8 @@ test "HeadersParser.read chunked trailer" {
836836 while (true) { // read headers
837837 try bconn.fill();
838838
839 const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());
840 bconn.clear(@intCast(u16, nchecked));
839 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
840 conn.drop(@intCast(u16, nchecked));
841841
842842 if (r.state.isContent()) break;
843843 }