Commit fd2f906d1ede2b65ba21eec59137b2d4b676eedc | zig

author	Nameless <truemedian@gmail.com> 2023-03-06 20:13:15-06:00
committer	Nameless <truemedian@gmail.com> 2023-03-09 14:54:26-06:00
log	fd2f906d1ede2b65ba21eec59137b2d4b676eedc
tree	11273d17254e2be90d947f8810e9e5d3e523eb6e
parent	8d86194b6e31788263d2cbdd03e2a8cde4134c37
signature	Commit is signed but in an unrecognized format.

std.http: handle compressed payloads

2 files changed, 496 insertions(+), 267 deletions(-)

lib
- std
  - http.zig
  - http
    Client.zig

lib/std/http.zig+10

...	...	@@ -253,6 +253,16 @@ pub const TransferEncoding = enum {
253	253	gzip,
254	254	};
255	255
	256	pub const Connection = enum {
	257	keep_alive,
	258	close,
	259	};
	260
	261	pub const CustomHeader = struct {
	262	name: []const u8,
	263	value: []const u8,
	264	};
	265
256	266	const std = @import("std.zig");
257	267
258	268	test {

lib/std/http/Client.zig+486-267

...	...	@@ -21,27 +21,51 @@ ca_bundle: std.crypto.Certificate.Bundle = .{},
21	21	/// it will first rescan the system for root certificates.
22	22	next_https_rescan_certs: bool = true,
23	23
24		connection_pool: std.TailQueue(Connection) = .{},
	24	connection_mutex: std.Thread.Mutex = .{},
	25	connection_pool: ConnectionPool = .{},
	26	connection_used: ConnectionPool = .{},
25	27
26	28	const ConnectionPool = std.TailQueue(Connection);
27	29	const ConnectionNode = ConnectionPool.Node;
28	30
	31	/// Acquires an existing connection from the connection pool. This function is threadsafe.
	32	pub fn acquire(client: Client, node: ConnectionNode) void {
	33	client.connection_mutex.lock();
	34	defer client.connection_mutex.unlock();
	35
	36	client.connection_pool.remove(node);
	37	client.connection_used.append(node);
	38	}
	39
	40	/// Tries to release a connection back to the connection pool. This function is threadsafe.
	41	/// If the connection is marked as closing, it will be closed instead.
29	42	pub fn release(client: Client, node: ConnectionNode) void {
30		if (node.data.unusable) return node.data.close(client);
	43	if (node.data.closing) {
	44	node.data.close(client);
	45
	46	return client.allocator.destroy(node);
	47	}
	48
	49	client.connection_mutex.lock();
	50	defer client.connection_mutex.unlock();
31	51
	52	client.connection_used.remove(node);
32	53	client.connection_pool.append(node);
33	54	}
34	55
	56	const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.ReaderRaw);
	57	const GzipDecompressor = std.compress.gzip.Decompress(Request.ReaderRaw);
	58
35	59	pub const Connection = struct {
36	60	stream: net.Stream,
37	61	/// undefined unless protocol is tls.
38		tls_client: std.crypto.tls.Client, // TODO: allocate this, it's currently 16 KB.
	62	tls_client: *std.crypto.tls.Client, // TODO: allocate this, it's currently 16 KB.
39	63	protocol: Protocol,
40	64	host: []u8,
41	65	port: u16,
42	66
43	67	// This connection has been part of a non keepalive request and cannot be added to the pool.
44		unusable: bool = false,
	68	closing: bool = false,
45	69
46	70	pub const Protocol = enum { plain, tls };
47	71
...	...	@@ -59,6 +83,24 @@ pub const Connection = struct {
59	83	}
60	84	}
61	85
	86	pub const ReadError = std.net.Stream.ReadError \|\| error{
	87	TlsConnectionTruncated,
	88	TlsRecordOverflow,
	89	TlsDecodeError,
	90	TlsAlert,
	91	TlsBadRecordMac,
	92	Overflow,
	93	TlsBadLength,
	94	TlsIllegalParameter,
	95	TlsUnexpectedMessage,
	96	};
	97
	98	pub const Reader = std.io.Reader(*Connection, ReadError, read);
	99
	100	pub fn reader(conn: *Connection) Reader {
	101	return Reader{ .context = conn };
	102	}
	103
62	104	pub fn writeAll(conn: *Connection, buffer: []const u8) !void {
63	105	switch (conn.protocol) {
64	106	.plain => return conn.stream.writeAll(buffer),
...	...	@@ -73,10 +115,18 @@ pub const Connection = struct {
73	115	}
74	116	}
75	117
	118	pub const WriteError = std.net.Stream.WriteError \|\| error{};
	119	pub const Writer = std.io.Writer(*Connection, WriteError, write);
	120
	121	pub fn writer(conn: *Connection) Writer {
	122	return Writer{ .context = conn };
	123	}
	124
76	125	pub fn close(conn: Connection, client: const Client) void {
77	126	if (conn.protocol == .tls) {
78	127	// try to cleanly close the TLS connection, for any server that cares.
79	128	_ = conn.tls_client.writeEnd(conn.stream, "", true) catch {};
	129	client.allocator.destroy(conn.tls_client);
80	130	}
81	131
82	132	conn.stream.close();
...	...	@@ -85,10 +135,10 @@ pub const Connection = struct {
85	135	}
86	136	};
87	137
88		/// TODO: emit error.UnexpectedEndOfStream or something like that when the read
89		/// data does not match the content length. This is necessary since HTTPS disables
90		/// close_notify protection on underlying TLS streams.
91	138	pub const Request = struct {
	139	const read_buffer_size = 8192;
	140	const ReadBufferIndex = std.math.IntFittingRange(0, read_buffer_size);
	141
92	142	client: *Client,
93	143	connection: *ConnectionNode,
94	144	redirects_left: u32,
...	...	@@ -97,6 +147,11 @@ pub const Request = struct {
97	147	/// redirects.
98	148	headers: Headers,
99	149
	150	/// Read buffer for the connection. This is used to pull in large amounts of data from the connection even if the user asks for a small amount. This can probably be removed with careful planning.
	151	read_buffer: [read_buffer_size]u8 = undefined,
	152	read_buffer_start: ReadBufferIndex = 0,
	153	read_buffer_len: ReadBufferIndex = 0,
	154
100	155	pub const Response = struct {
101	156	headers: Response.Headers,
102	157	state: State,
...	...	@@ -106,15 +161,24 @@ pub const Request = struct {
106	161	header_bytes: std.ArrayListUnmanaged(u8),
107	162	max_header_bytes: usize,
108	163	next_chunk_length: u64,
109		done: bool,
	164	done: bool = false,
	165
	166	compression: union(enum) {
	167	deflate: DeflateDecompressor,
	168	gzip: GzipDecompressor,
	169	none: void,
	170	} = .none,
110	171
111	172	pub const Headers = struct {
112	173	status: http.Status,
113	174	version: http.Version,
114	175	location: ?[]const u8 = null,
115	176	content_length: ?u64 = null,
116		transfer_encoding: ?http.TransferEncoding = null,
117		connection_close: bool = true,
	177	transfer_encoding: ?http.TransferEncoding = null, // This should only ever be chunked, compression is handled separately.
	178	transfer_compression: ?http.TransferEncoding = null,
	179	connection: http.Connection = .close,
	180
	181	number_of_headers: usize = 0,
118	182
119	183	pub fn parse(bytes: []const u8) !Response.Headers {
120	184	var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
...	...	@@ -137,6 +201,8 @@ pub const Request = struct {
137	201	};
138	202
139	203	while (it.next()) \|line\| {
	204	headers.number_of_headers += 1;
	205
140	206	if (line.len == 0) return error.HttpHeadersInvalid;
141	207	switch (line[0]) {
142	208	' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
...	...	@@ -152,14 +218,65 @@ pub const Request = struct {
152	218	if (headers.content_length != null) return error.HttpHeadersInvalid;
153	219	headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
154	220	} else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
155		if (headers.transfer_encoding != null) return error.HttpHeadersInvalid;
156		headers.transfer_encoding = std.meta.stringToEnum(http.TransferEncoding, header_value) orelse
	221	if (headers.transfer_encoding != null or headers.transfer_compression != null) return error.HttpHeadersInvalid;
	222
	223	// Transfer-Encoding: second, first
	224	// Transfer-Encoding: deflate, chunked
	225	var iter = std.mem.splitBackwards(u8, header_value, ",");
	226
	227	if (iter.next()) \|first\| {
	228	const kind = std.meta.stringToEnum(
	229	http.TransferEncoding,
	230	std.mem.trim(u8, first, " "),
	231	) orelse
	232	return error.HttpTransferEncodingUnsupported;
	233
	234	switch (kind) {
	235	.chunked => headers.transfer_encoding = .chunked,
	236	.compress => headers.transfer_compression = .compress,
	237	.deflate => headers.transfer_compression = .deflate,
	238	.gzip => headers.transfer_compression = .gzip,
	239	}
	240	}
	241
	242	if (iter.next()) \|second\| {
	243	if (headers.transfer_compression != null) return error.HttpTransferEncodingUnsupported;
	244
	245	const kind = std.meta.stringToEnum(
	246	http.TransferEncoding,
	247	std.mem.trim(u8, second, " "),
	248	) orelse
	249	return error.HttpTransferEncodingUnsupported;
	250
	251	switch (kind) {
	252	.chunked => return error.HttpHeadersInvalid, // chunked must come last
	253	.compress => return error.HttpTransferEncodingUnsupported, // compress not supported
	254	.deflate => headers.transfer_compression = .deflate,
	255	.gzip => headers.transfer_compression = .gzip,
	256	}
	257	}
	258
	259	if (iter.next()) \|_\| return error.HttpTransferEncodingUnsupported;
	260	} else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
	261	if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
	262
	263	const kind = std.meta.stringToEnum(
	264	http.TransferEncoding,
	265	std.mem.trim(u8, header_value, " "),
	266	) orelse
157	267	return error.HttpTransferEncodingUnsupported;
	268
	269	switch (kind) {
	270	.chunked => return error.HttpHeadersInvalid, // not transfer encoding
	271	.compress => return error.HttpTransferEncodingUnsupported, // compress not supported
	272	.deflate => headers.transfer_compression = .deflate,
	273	.gzip => headers.transfer_compression = .gzip,
	274	}
158	275	} else if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
159	276	if (std.ascii.eqlIgnoreCase(header_value, "keep-alive")) {
160		headers.connection_close = false;
	277	headers.connection = .keep_alive;
161	278	} else if (std.ascii.eqlIgnoreCase(header_value, "close")) {
162		headers.connection_close = true;
	279	headers.connection = .close;
163	280	} else {
164	281	return error.HttpConnectionHeaderUnsupported;
165	282	}
...	...	@@ -238,7 +355,6 @@ pub const Request = struct {
238	355	.max_header_bytes = max,
239	356	.header_bytes_owned = true,
240	357	.next_chunk_length = undefined,
241		.done = false,
242	358	};
243	359	}
244	360
...	...	@@ -250,7 +366,6 @@ pub const Request = struct {
250	366	.max_header_bytes = buf.len,
251	367	.header_bytes_owned = false,
252	368	.next_chunk_length = undefined,
253		.done = false,
254	369	};
255	370	}
256	371
...	...	@@ -537,10 +652,19 @@ pub const Request = struct {
537	652	}
538	653	};
539	654
	655	pub const RequestTransfer = union(enum) {
	656	content_length: u64,
	657	chunked: void,
	658	none: void,
	659	};
	660
540	661	pub const Headers = struct {
541	662	version: http.Version = .@"HTTP/1.1",
542	663	method: http.Method = .GET,
543		connection_close: bool = false,
	664	connection: http.Connection = .keep_alive,
	665	transfer_encoding: RequestTransfer = .none,
	666
	667	custom: []const http.CustomHeader = &[_]http.CustomHeader{},
544	668	};
545	669
546	670	pub const Options = struct {
...	...	@@ -561,167 +685,131 @@ pub const Request = struct {
561	685	};
562	686	};
563	687
564		/// May be skipped if header strategy is buffer.
	688	/// Frees all resources associated with the request.
565	689	pub fn deinit(req: *Request) void {
	690	switch (req.response.compression) {
	691	.none => {},
	692	.deflate => \|*deflate\| deflate.deinit(),
	693	.gzip => \|*gzip\| gzip.deinit(),
	694	}
	695
566	696	if (req.response.header_bytes_owned) {
567	697	req.response.header_bytes.deinit(req.client.allocator);
568	698	}
	699
	700	if (!req.response.done) {
	701	// If the response wasn't fully read, then we need to close the connection.
	702	req.connection.data.closing = true;
	703	req.client.release(req.connection);
	704	}
	705
569	706	req.* = undefined;
570	707	}
571	708
572		pub const Reader = std.io.Reader(*Request, ReadError, read);
	709	const ReadRawError = Connection.ReadError \|\| std.Uri.ParseError \|\| RequestError \|\| error{
	710	UnexpectedEndOfStream,
	711	TooManyHttpRedirects,
	712	HttpRedirectMissingLocation,
	713	HttpHeadersInvalid,
	714	};
573	715
574		pub fn reader(req: *Request) Reader {
575		return .{ .context = req };
	716	const ReaderRaw = std.io.Reader(*Request, ReadRawError, readRaw);
	717
	718	/// Read from the underlying stream, without decompressing or parsing the headers. Must be called
	719	/// after waitForCompleteHead() has returned successfully.
	720	pub fn readRaw(req: *Request, buffer: []u8) ReadRawError!usize {
	721	assert(req.response.state.isContent());
	722
	723	var index: usize = 0;
	724	while (index == 0) {
	725	const amt = try req.readRawAdvanced(buffer[index..]);
	726	const zero_means_end = req.response.done and req.response.headers.status.class() != .redirect;
	727
	728	if (amt == 0 and zero_means_end) break;
	729	index += amt;
	730	}
	731
	732	return index;
576	733	}
577	734
578		pub fn readAll(req: *Request, buffer: []u8) !usize {
579		return readAtLeast(req, buffer, buffer.len);
	735	fn checkForCompleteHead(req: *Request, buffer: []u8) !usize {
	736	switch (req.response.state) {
	737	.invalid => unreachable,
	738	.start, .seen_r, .seen_rn, .seen_rnr => {},
	739	else => return 0, // No more headers to read.
	740	}
	741
	742	const i = req.response.findHeadersEnd(buffer[0..]);
	743	if (req.response.state == .invalid) return error.HttpHeadersInvalid;
	744
	745	const headers_data = buffer[0..i];
	746	if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
	747	return error.HttpHeadersExceededSizeLimit;
	748	}
	749	try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
	750
	751	if (req.response.state == .finished) {
	752	req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
	753
	754	if (req.response.headers.connection == .keep_alive) {
	755	req.connection.data.closing = false;
	756	} else {
	757	req.connection.data.closing = true;
	758	}
	759
	760	if (req.response.headers.transfer_encoding) \|transfer_encoding\| {
	761	switch (transfer_encoding) {
	762	.chunked => {
	763	req.response.next_chunk_length = 0;
	764	req.response.state = .chunk_size;
	765	},
	766	.compress => unreachable,
	767	.deflate => unreachable,
	768	.gzip => unreachable,
	769	}
	770	} else if (req.response.headers.content_length) \|content_length\| {
	771	req.response.next_chunk_length = content_length;
	772	} else {
	773	req.response.done = true;
	774	}
	775
	776	return i;
	777	}
	778
	779	return 0;
580	780	}
581	781
582		pub const ReadError = net.Stream.ReadError \|\| error{
583		// From HTTP protocol
584		HttpHeadersInvalid,
	782	pub const WaitForCompleteHeadError = ReadRawError \|\| error {
	783	UnexpectedEndOfStream,
	784
585	785	HttpHeadersExceededSizeLimit,
586		HttpRedirectMissingLocation,
587		HttpTransferEncodingUnsupported,
588		HttpConnectionHeaderUnsupported,
589		HttpContentLengthUnknown,
590		TooManyHttpRedirects,
591	786	ShortHttpStatusLine,
592	787	BadHttpVersion,
593	788	HttpHeaderContinuationsUnsupported,
594		UnsupportedUrlScheme,
595		UriMissingHost,
596		UnknownHostName,
597
598		// Network problems
599		NetworkUnreachable,
600		HostLacksNetworkAddresses,
601		TemporaryNameServerFailure,
602		NameServerFailure,
603		ProtocolFamilyNotAvailable,
604		ProtocolNotSupported,
605
606		// System resource problems
607		ProcessFdQuotaExceeded,
608		SystemFdQuotaExceeded,
609		OutOfMemory,
610
611		// TLS problems
612		InsufficientEntropy,
613		TlsConnectionTruncated,
614		TlsRecordOverflow,
615		TlsDecodeError,
616		TlsAlert,
617		TlsBadRecordMac,
618		TlsBadLength,
619		TlsIllegalParameter,
620		TlsUnexpectedMessage,
621		TlsDecryptFailure,
622		CertificateFieldHasInvalidLength,
623		CertificateHostMismatch,
624		CertificatePublicKeyInvalid,
625		CertificateExpired,
626		CertificateFieldHasWrongDataType,
627		CertificateIssuerMismatch,
628		CertificateNotYetValid,
629		CertificateSignatureAlgorithmMismatch,
630		CertificateSignatureAlgorithmUnsupported,
631		CertificateSignatureInvalid,
632		CertificateSignatureInvalidLength,
633		CertificateSignatureNamedCurveUnsupported,
634		CertificateSignatureUnsupportedBitCount,
635		TlsCertificateNotVerified,
636		TlsBadSignatureScheme,
637		TlsBadRsaSignatureBitCount,
638		TlsDecryptError,
639		UnsupportedCertificateVersion,
640		CertificateTimeInvalid,
641		CertificateHasUnrecognizedObjectId,
642		CertificateHasInvalidBitString,
643		CertificateAuthorityBundleTooBig,
644
645		// TODO: convert to higher level errors
646		InvalidFormat,
647		InvalidPort,
648		UnexpectedCharacter,
649		Overflow,
650		InvalidCharacter,
651		AddressFamilyNotSupported,
652		AddressInUse,
653		AddressNotAvailable,
654		ConnectionPending,
655		ConnectionRefused,
656		FileNotFound,
657		PermissionDenied,
658		ServiceUnavailable,
659		SocketTypeNotSupported,
660		FileTooBig,
661		LockViolation,
662		NoSpaceLeft,
663		NotOpenForWriting,
664		InvalidEncoding,
665		IdentityElement,
666		NonCanonical,
667		SignatureVerificationFailed,
668		MessageTooLong,
669		NegativeIntoUnsigned,
670		TargetTooSmall,
671		BufferTooSmall,
672		InvalidSignature,
673		NotSquare,
674		DiskQuota,
675		InvalidEnd,
676		Incomplete,
677		InvalidIpv4Mapping,
678		InvalidIPAddressFormat,
679		BadPathName,
680		DeviceBusy,
681		FileBusy,
682		FileLocksNotSupported,
683		InvalidHandle,
684		InvalidUtf8,
685		NameTooLong,
686		NoDevice,
687		PathAlreadyExists,
688		PipeBusy,
689		SharingViolation,
690		SymLinkLoop,
691		FileSystem,
692		InterfaceNotFound,
693		AlreadyBound,
694		FileDescriptorNotASocket,
695		NetworkSubsystemFailed,
696		NotDir,
697		ReadOnlyFileSystem,
698		Unseekable,
699		MissingEndCertificateMarker,
700		InvalidPadding,
701		EndOfStream,
702		InvalidArgument,
	789	HttpTransferEncodingUnsupported,
	790	HttpConnectionHeaderUnsupported,
703	791	};
704	792
705		pub fn read(req: *Request, buffer: []u8) ReadError!usize {
706		return readAtLeast(req, buffer, 1);
707		}
708
709		pub fn readAtLeast(req: *Request, buffer: []u8, len: usize) !usize {
710		assert(len <= buffer.len);
711		var index: usize = 0;
712		while (index < len) {
713		const amt = try readAdvanced(req, buffer[index..]);
714		const zero_means_end = req.response.done and req.response.headers.status.class() != .redirect;
	793	/// Reads a complete response head. Any leftover data is stored in the request. This function is idempotent.
	794	pub fn waitForCompleteHead(req: *Request) WaitForCompleteHeadError!void {
	795	if (req.response.state.isContent()) return;
715	796
716		if (amt == 0 and zero_means_end) break;
717		index += amt;
	797	while (true) {
	798	const nread = try req.connection.data.read(req.read_buffer[0..]);
	799	const amt = try checkForCompleteHead(req, req.read_buffer[0..nread]);
	800
	801	if (amt != 0) {
	802	req.read_buffer_start = @intCast(ReadBufferIndex, amt);
	803	req.read_buffer_len = @intCast(ReadBufferIndex, nread);
	804	return;
	805	} else if (nread == 0) {
	806	return error.UnexpectedEndOfStream;
	807	}
718	808	}
719		return index;
720	809	}
721	810
722	811	/// This one can return 0 without meaning EOF.
723		/// TODO change to readvAdvanced
724		pub fn readAdvanced(req: *Request, buffer: []u8) !usize {
	812	fn readRawAdvanced(req: *Request, buffer: []u8) !usize {
725	813	if (req.response.done) {
726	814	if (req.response.headers.status.class() == .redirect) {
727	815	if (req.redirects_left == 0) return error.TooManyHttpRedirects;
...	...	@@ -744,82 +832,56 @@ pub const Request = struct {
744	832	}
745	833	}
746	834
747		var in = buffer[0..try req.connection.data.read(buffer)];
	835	// var in: []const u8 = undefined;
	836	if (req.read_buffer_start == req.read_buffer_len) {
	837	const nread = try req.connection.data.read(req.read_buffer[0..]);
	838	if (nread == 0) return error.UnexpectedEndOfStream;
	839
	840	req.read_buffer_start = 0;
	841	req.read_buffer_len = @intCast(ReadBufferIndex, nread);
	842	}
	843
748	844	var out_index: usize = 0;
749	845	while (true) {
750	846	switch (req.response.state) {
751		.invalid => unreachable,
752		.start, .seen_r, .seen_rn, .seen_rnr => {
753		const i = req.response.findHeadersEnd(in);
754		if (req.response.state == .invalid) return error.HttpHeadersInvalid;
755
756		const headers_data = in[0..i];
757		if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
758		return error.HttpHeadersExceededSizeLimit;
759		}
760		try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
761
762		if (req.response.state == .finished) {
763		req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
764
765		if (req.response.headers.connection_close == true) {
766		req.connection.data.unusable = true;
767		} else {
768		req.connection.data.unusable = false;
769		}
770
771		if (req.response.headers.transfer_encoding) \|transfer_encoding\| {
772		switch (transfer_encoding) {
773		.chunked => {
774		req.response.next_chunk_length = 0;
775		req.response.state = .chunk_size;
776		},
777		.compress => return error.HttpTransferEncodingUnsupported,
778		.deflate => return error.HttpTransferEncodingUnsupported,
779		.gzip => return error.HttpTransferEncodingUnsupported,
780		}
781		} else if (req.response.headers.content_length) \|content_length\| {
782		req.response.next_chunk_length = content_length;
783		} else {
784		return error.HttpContentLengthUnknown;
	847	.invalid, .start, .seen_r, .seen_rn, .seen_rnr => unreachable,
	848	.finished => {
	849	// TODO https://github.com/ziglang/zig/issues/14039
	850	const buf_avail = req.read_buffer_len - req.read_buffer_start;
	851	const data_avail = req.response.next_chunk_length;
	852	const out_avail = buffer.len;
	853
	854	if (req.response.state.isContent() and req.response.headers.status.class() == .redirect) {
	855	const can_read = @intCast(usize, @min(buf_avail, data_avail));
	856	req.response.next_chunk_length -= can_read;
	857
	858	if (req.response.next_chunk_length == 0) {
	859	req.client.release(req.connection);
	860	req.connection = undefined;
	861	req.response.done = true;
	862	continue;
785	863	}
786	864
787		in = in[i..];
788		continue;
	865	return 0; // skip over as much data as possible
789	866	}
790	867
791		assert(out_index == 0);
792		return 0;
793		},
794		.finished => {
795		const sub_amt = @intCast(usize, @min(req.response.next_chunk_length, in.len));
796		req.response.next_chunk_length -= sub_amt;
	868	const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
	869	req.response.next_chunk_length -= can_read;
	870
	871	mem.copy(u8, buffer[0..], req.read_buffer[req.read_buffer_start..][0..can_read]);
	872	req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
797	873
798	874	if (req.response.next_chunk_length == 0) {
799	875	req.client.release(req.connection);
800	876	req.connection = undefined;
801
802	877	req.response.done = true;
803		assert(in.len == sub_amt); // TODO: figure out how to not read more than necessary.
804
805		if (req.response.state.isContent() and req.response.headers.status.class() == .redirect) return 0;
806
807		mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
808		return out_index + sub_amt;
809	878	}
810	879
811		if (req.response.state.isContent() and req.response.headers.status.class() == .redirect) return 0;
812
813		if (in.ptr == buffer.ptr) {
814		return sub_amt;
815		} else {
816		mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
817		return out_index + sub_amt;
818		}
	880	return can_read;
819	881	},
820		.chunk_size_prefix_r => switch (in.len) {
	882	.chunk_size_prefix_r => switch (req.read_buffer_len - req.read_buffer_start) {
821	883	0 => return out_index,
822		1 => switch (in[0]) {
	884	1 => switch (req.read_buffer[req.read_buffer_start]) {
823	885	'\r' => {
824	886	req.response.state = .chunk_size_prefix_n;
825	887	return out_index;
...	...	@@ -829,9 +891,9 @@ pub const Request = struct {
829	891	return error.HttpHeadersInvalid;
830	892	},
831	893	},
832		else => switch (int16(in[0..2])) {
	894	else => switch (int16(req.read_buffer[req.read_buffer_start..][0..2])) {
833	895	int16("\r\n") => {
834		in = in[2..];
	896	req.read_buffer_start += 2;
835	897	req.response.state = .chunk_size;
836	898	continue;
837	899	},
...	...	@@ -841,11 +903,11 @@ pub const Request = struct {
841	903	},
842	904	},
843	905	},
844		.chunk_size_prefix_n => switch (in.len) {
	906	.chunk_size_prefix_n => switch (req.read_buffer_len - req.read_buffer_start) {
845	907	0 => return out_index,
846		else => switch (in[0]) {
	908	else => switch (req.read_buffer[req.read_buffer_start]) {
847	909	'\n' => {
848		in = in[1..];
	910	req.read_buffer_start += 1;
849	911	req.response.state = .chunk_size;
850	912	continue;
851	913	},
...	...	@@ -856,7 +918,7 @@ pub const Request = struct {
856	918	},
857	919	},
858	920	.chunk_size, .chunk_r => {
859		const i = req.response.findChunkedLen(in);
	921	const i = req.response.findChunkedLen(req.read_buffer[req.read_buffer_start..req.read_buffer_len]);
860	922	switch (req.response.state) {
861	923	.invalid => return error.HttpHeadersInvalid,
862	924	.chunk_data => {
...	...	@@ -867,7 +929,8 @@ pub const Request = struct {
867	929
868	930	return out_index;
869	931	}
870		in = in[i..];
	932
	933	req.read_buffer_start += @intCast(ReadBufferIndex, i);
871	934	continue;
872	935	},
873	936	.chunk_size => return out_index,
...	...	@@ -876,34 +939,129 @@ pub const Request = struct {
876	939	},
877	940	.chunk_data => {
878	941	// TODO https://github.com/ziglang/zig/issues/14039
879		const sub_amt = @intCast(usize, @min(req.response.next_chunk_length, in.len));
880		req.response.next_chunk_length -= sub_amt;
	942	const buf_avail = req.read_buffer_len - req.read_buffer_start;
	943	const data_avail = req.response.next_chunk_length;
	944	const out_avail = buffer.len - out_index;
	945
	946	if (req.response.state.isContent() and req.response.headers.status.class() == .redirect) {
	947	const can_read = @intCast(usize, @min(buf_avail, data_avail));
	948	req.response.next_chunk_length -= can_read;
	949
	950	if (req.response.next_chunk_length == 0) {
	951	req.client.release(req.connection);
	952	req.connection = undefined;
	953	req.response.done = true;
	954	continue;
	955	}
	956
	957	return 0; // skip over as much data as possible
	958	}
	959
	960	const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
	961	req.response.next_chunk_length -= can_read;
	962
	963	mem.copy(u8, buffer[out_index..], req.read_buffer[req.read_buffer_start..][0..can_read]);
	964	req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
	965	out_index += can_read;
881	966
882	967	if (req.response.next_chunk_length == 0) {
883	968	req.response.state = .chunk_size_prefix_r;
884		in = in[sub_amt..];
885
886		if (req.response.headers.status.class() == .redirect) continue;
887	969
888		mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
889		out_index += sub_amt;
890	970	continue;
891	971	}
892	972
893		if (req.response.headers.status.class() == .redirect) return 0;
894
895		if (in.ptr == buffer.ptr) {
896		return sub_amt;
897		} else {
898		mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
899		out_index += sub_amt;
900		return out_index;
901		}
	973	return out_index;
902	974	},
903	975	}
904	976	}
905	977	}
906	978
	979	pub const ReadError = DeflateDecompressor.Error \|\| GzipDecompressor.Error \|\| WaitForCompleteHeadError \|\| error{
	980	BadHeader,
	981	InvalidCompression,
	982	StreamTooLong,
	983	InvalidWindowSize,
	984	};
	985
	986	pub const Reader = std.io.Reader(*Request, ReadError, read);
	987
	988	pub fn reader(req: *Request) Reader {
	989	return .{ .context = req };
	990	}
	991
	992	pub fn read(req: *Request, buffer: []u8) ReadError!usize {
	993	if (!req.response.state.isContent()) try req.waitForCompleteHead();
	994
	995	if (req.response.compression == .none and req.response.state.isContent()) {
	996	if (req.response.headers.transfer_compression) \|compression\| {
	997	switch (compression) {
	998	.compress => unreachable,
	999	.deflate => req.response.compression = .{
	1000	.deflate = try std.compress.zlib.zlibStream(req.client.allocator, ReaderRaw{ .context = req }),
	1001	},
	1002	.gzip => req.response.compression = .{
	1003	.gzip = try std.compress.gzip.decompress(req.client.allocator, ReaderRaw{ .context = req }),
	1004	},
	1005	.chunked => unreachable,
	1006	}
	1007	}
	1008	}
	1009
	1010	return switch (req.response.compression) {
	1011	.deflate => \|*deflate\| try deflate.read(buffer),
	1012	.gzip => \|*gzip\| try gzip.read(buffer),
	1013	else => try req.readRaw(buffer),
	1014	};
	1015	}
	1016
	1017	pub fn readAll(req: *Request, buffer: []u8) !usize {
	1018	var index: usize = 0;
	1019	while (index < buffer.len) {
	1020	const amt = try read(req, buffer[index..]);
	1021	if (amt == 0) break;
	1022	index += amt;
	1023	}
	1024	return index;
	1025	}
	1026
	1027	pub const WriteError = Connection.WriteError \|\| error{MessageTooLong};
	1028
	1029	pub const Writer = std.io.Writer(*Request, WriteError, write);
	1030
	1031	pub fn writer(req: *Request) Writer {
	1032	return .{ .context = req };
	1033	}
	1034
	1035	/// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.
	1036	pub fn write(req: *Request, bytes: []const u8) !usize {
	1037	switch (req.headers.transfer_encoding) {
	1038	.chunked => {
	1039	try req.connection.data.writer().print("{x}\r\n", .{bytes.len});
	1040	try req.connection.data.writeAll(bytes);
	1041	try req.connection.data.writeAll("\r\n");
	1042
	1043	return bytes.len;
	1044	},
	1045	.content_length => \|*len\| {
	1046	if (len.* < bytes.len) return error.MessageTooLong;
	1047
	1048	const amt = try req.connection.data.write(bytes);
	1049	len.* -= amt;
	1050	return amt;
	1051	},
	1052	.none => return error.NotWriteable,
	1053	}
	1054	}
	1055
	1056	/// Finish the body of a request. This notifies the server that you have no more data to send.
	1057	pub fn finish(req: *Request) !void {
	1058	switch (req.headers.transfer_encoding) {
	1059	.chunked => try req.connection.data.writeAll("0\r\n"),
	1060	.content_length => \|len\| if (len != 0) return error.MessageNotCompleted,
	1061	.none => {},
	1062	}
	1063	}
	1064
907	1065	inline fn int16(array: *const [2]u8) u16 {
908	1066	return @bitCast(u16, array.*);
909	1067	}
...	...	@@ -917,6 +1075,10 @@ pub const Request = struct {
917	1075	}
918	1076
919	1077	test {
	1078	const builtin = @import("builtin");
	1079
	1080	if (builtin.os.tag == .wasi) return error.SkipZigTest;
	1081
920	1082	_ = Response;
921	1083	}
922	1084	};
...	...	@@ -931,23 +1093,39 @@ pub fn deinit(client: *Client) void {
931	1093	client.allocator.destroy(node);
932	1094	}
933	1095
	1096	next = client.connection_used.first;
	1097	while (next) \|node\| {
	1098	next = node.next;
	1099
	1100	node.data.close(client);
	1101
	1102	client.allocator.destroy(node);
	1103	}
	1104
934	1105	client.ca_bundle.deinit(client.allocator);
935	1106	client.* = undefined;
936	1107	}
937	1108
938		pub fn connect(client: Client, host: []const u8, port: u16, protocol: Connection.Protocol) !ConnectionNode {
939		var potential = client.connection_pool.last;
940		while (potential) \|node\| {
941		const same_host = mem.eql(u8, node.data.host, host);
942		const same_port = node.data.port == port;
943		const same_protocol = node.data.protocol == protocol;
	1109	pub const ConnectError = std.mem.Allocator.Error \|\| std.net.TcpConnectToHostError \|\| std.crypto.tls.Client.InitError(std.net.Stream);
944	1110
945		if (same_host and same_port and same_protocol) {
946		client.connection_pool.remove(node);
947		return node;
948		}
	1111	pub fn connect(client: Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!ConnectionNode {
	1112	{ // Search through the connection pool for a potential connection.
	1113	client.connection_mutex.lock();
	1114	defer client.connection_mutex.unlock();
949	1115
950		potential = node.prev;
	1116	var potential = client.connection_pool.last;
	1117	while (potential) \|node\| {
	1118	const same_host = mem.eql(u8, node.data.host, host);
	1119	const same_port = node.data.port == port;
	1120	const same_protocol = node.data.protocol == protocol;
	1121
	1122	if (same_host and same_port and same_protocol) {
	1123	client.acquire(node);
	1124	return node;
	1125	}
	1126
	1127	potential = node.prev;
	1128	}
951	1129	}
952	1130
953	1131	const conn = try client.allocator.create(ConnectionNode);
...	...	@@ -964,17 +1142,35 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
964	1142	switch (protocol) {
965	1143	.plain => {},
966	1144	.tls => {
967		conn.data.tls_client = try std.crypto.tls.Client.init(conn.data.stream, client.ca_bundle, host);
	1145	conn.data.tls_client = try client.allocator.create(std.crypto.tls.Client);
	1146	conn.data.tls_client.* = try std.crypto.tls.Client.init(conn.data.stream, client.ca_bundle, host);
968	1147	// This is appropriate for HTTPS because the HTTP headers contain
969	1148	// the content length which is used to detect truncation attacks.
970	1149	conn.data.tls_client.allow_truncation_attacks = true;
971	1150	},
972	1151	}
973	1152
	1153	{
	1154	client.connection_mutex.lock();
	1155	defer client.connection_mutex.unlock();
	1156
	1157	client.connection_used.append(conn);
	1158	}
	1159
974	1160	return conn;
975	1161	}
976	1162
977		pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Request.Options) !Request {
	1163	pub const RequestError = ConnectError \|\| Connection.WriteError \|\| error{
	1164	UnsupportedUrlScheme,
	1165	UriMissingHost,
	1166
	1167	CertificateAuthorityBundleTooBig,
	1168	InvalidPadding,
	1169	MissingEndCertificateMarker,
	1170	Unseekable,
	1171	};
	1172
	1173	pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Request.Options) RequestError!Request {
978	1174	const protocol: Connection.Protocol = if (mem.eql(u8, uri.scheme, "http"))
979	1175	.plain
980	1176	else if (mem.eql(u8, uri.scheme, "https"))
...	...	@@ -990,8 +1186,13 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req
990	1186	const host = uri.host orelse return error.UriMissingHost;
991	1187
992	1188	if (client.next_https_rescan_certs and protocol == .tls) {
993		try client.ca_bundle.rescan(client.allocator);
994		client.next_https_rescan_certs = false;
	1189	client.connection_mutex.lock(); // TODO: this could be so much better than reusing the connection pool mutex.
	1190	defer client.connection_mutex.unlock();
	1191
	1192	if (client.next_https_rescan_certs) {
	1193	try client.ca_bundle.rescan(client.allocator);
	1194	client.next_https_rescan_certs = false;
	1195	}
995	1196	}
996	1197
997	1198	var req: Request = .{
...	...	@@ -1006,23 +1207,39 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req
1006	1207	};
1007	1208
1008	1209	{
1009		var h = try std.BoundedArray(u8, 1000).init(0);
1010		try h.appendSlice(@tagName(headers.method));
1011		try h.appendSlice(" ");
1012		try h.appendSlice(uri.path);
1013		try h.appendSlice(" ");
1014		try h.appendSlice(@tagName(headers.version));
1015		try h.appendSlice("\r\nHost: ");
1016		try h.appendSlice(host);
1017		if (headers.connection_close) {
1018		try h.appendSlice("\r\nConnection: close");
	1210	var buffered = std.io.bufferedWriter(req.connection.data.writer());
	1211	const writer = buffered.writer();
	1212
	1213	try writer.writeAll(@tagName(headers.method));
	1214	try writer.writeByte(' ');
	1215	try writer.writeAll(uri.path);
	1216	try writer.writeByte(' ');
	1217	try writer.writeAll(@tagName(headers.version));
	1218	try writer.writeAll("\r\nHost: ");
	1219	try writer.writeAll(host);
	1220	if (headers.connection == .close) {
	1221	try writer.writeAll("\r\nConnection: close");
1019	1222	} else {
1020		try h.appendSlice("\r\nConnection: keep-alive");
	1223	try writer.writeAll("\r\nConnection: keep-alive");
1021	1224	}
1022		try h.appendSlice("\r\n\r\n");
	1225	try writer.writeAll("\r\nAccept-Encoding: gzip, deflate");
1023	1226
1024		const header_bytes = h.slice();
1025		try req.connection.data.writeAll(header_bytes);
	1227	switch (headers.transfer_encoding) {
	1228	.chunked => try writer.writeAll("\r\nTransfer-Encoding: chunked"),
	1229	.content_length => \|content_length\| try writer.print("\r\nContent-Length: {d}", .{content_length}),
	1230	.none => {},
	1231	}
	1232
	1233	for (headers.custom) \|header\| {
	1234	try writer.writeAll("\r\n");
	1235	try writer.writeAll(header.name);
	1236	try writer.writeAll(": ");
	1237	try writer.writeAll(header.value);
	1238	}
	1239
	1240	try writer.writeAll("\r\n\r\n");
	1241
	1242	try buffered.flush();
1026	1243	}
1027	1244
1028	1245	return req;
...	...	@@ -1036,5 +1253,7 @@ test {
1036	1253	return error.SkipZigTest;
1037	1254	}
1038	1255
	1256	if (builtin.os.tag == .wasi) return error.SkipZigTest;
	1257
1039	1258	_ = Request;
1040	1259	}