Commit 7c9ed45ac2fd580bbbb4b950ac350dd49fc7600e | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-03-09 22:02:01-05:00
committer	GitHub <noreply@github.com> 2023-03-09 22:02:01-05:00
log	7c9ed45ac2fd580bbbb4b950ac350dd49fc7600e
tree	027efed5e8b03a30b4b7a23e86ad5ca2bd0d75e5
parent	14590e956e06903ac408af57874d3b5a0d697670
parent	524e0cd987a52a60ce1014aa27cd73f99a3b9958
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14762 from truemedian/http-keepalive

std.http: add connection pooling, handle keep-alive and compressed content

8 files changed, 1434 insertions(+), 803 deletions(-)

lib
- std
  - Uri.zig
  - crypto
    tls
    Client.zig
  - http.zig
  - http
    Client.zig
    Client
    Request.zig
    Response.zig
  - net.zig
  - std.zig

lib/std/Uri.zig+96-13

...	@@ -16,15 +16,27 @@ fragment: ?[]const u8,	...	@@ -16,15 +16,27 @@ fragment: ?[]const u8,
16		16
17	/// Applies URI encoding and replaces all reserved characters with their respective %XX code.	17	/// Applies URI encoding and replaces all reserved characters with their respective %XX code.
18	pub fn escapeString(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {	18	pub fn escapeString(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
		19	return escapeStringWithFn(allocator, input, isUnreserved);
		20	}
		21
		22	pub fn escapePath(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
		23	return escapeStringWithFn(allocator, input, isPathChar);
		24	}
		25
		26	pub fn escapeQuery(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
		27	return escapeStringWithFn(allocator, input, isQueryChar);
		28	}
		29
		30	pub fn escapeStringWithFn(allocator: std.mem.Allocator, input: []const u8, comptime keepUnescaped: fn (c: u8) bool) std.mem.Allocator.Error![]const u8 {
19	var outsize: usize = 0;	31	var outsize: usize = 0;
20	for (input) \|c\| {	32	for (input) \|c\| {
21	outsize += if (isUnreserved(c)) @as(usize, 1) else 3;	33	outsize += if (keepUnescaped(c)) @as(usize, 1) else 3;
22	}	34	}
23	var output = try allocator.alloc(u8, outsize);	35	var output = try allocator.alloc(u8, outsize);
24	var outptr: usize = 0;	36	var outptr: usize = 0;
25		37
26	for (input) \|c\| {	38	for (input) \|c\| {
27	if (isUnreserved(c)) {	39	if (keepUnescaped(c)) {
28	output[outptr] = c;	40	output[outptr] = c;
29	outptr += 1;	41	outptr += 1;
30	} else {	42	} else {
...	@@ -94,13 +106,14 @@ pub fn unescapeString(allocator: std.mem.Allocator, input: []const u8) error{Out	...	@@ -94,13 +106,14 @@ pub fn unescapeString(allocator: std.mem.Allocator, input: []const u8) error{Out
94		106
95	pub const ParseError = error{ UnexpectedCharacter, InvalidFormat, InvalidPort };	107	pub const ParseError = error{ UnexpectedCharacter, InvalidFormat, InvalidPort };
96		108
97	/// Parses the URI or returns an error.	109	/// Parses the URI or returns an error. This function is not compliant, but is required to parse
		110	/// some forms of URIs in the wild. Such as HTTP Location headers.
98	/// The return value will contain unescaped strings pointing into the	111	/// The return value will contain unescaped strings pointing into the
99	/// original `text`. Each component that is provided, will be non-`null`.	112	/// original `text`. Each component that is provided, will be non-`null`.
100	pub fn parse(text: []const u8) ParseError!Uri {	113	pub fn parseWithoutScheme(text: []const u8) ParseError!Uri {
101	var reader = SliceReader{ .slice = text };	114	var reader = SliceReader{ .slice = text };
102	var uri = Uri{	115	var uri = Uri{
103	.scheme = reader.readWhile(isSchemeChar),	116	.scheme = "",
104	.user = null,	117	.user = null,
105	.password = null,	118	.password = null,
106	.host = null,	119	.host = null,
...	@@ -110,14 +123,6 @@ pub fn parse(text: []const u8) ParseError!Uri {	...	@@ -110,14 +123,6 @@ pub fn parse(text: []const u8) ParseError!Uri {
110	.fragment = null,	123	.fragment = null,
111	};	124	};
112		125
113	// after the scheme, a ':' must appear
114	if (reader.get()) \|c\| {
115	if (c != ':')
116	return error.UnexpectedCharacter;
117	} else {
118	return error.InvalidFormat;
119	}
120
121	if (reader.peekPrefix("//")) { // authority part	126	if (reader.peekPrefix("//")) { // authority part
122	std.debug.assert(reader.get().? == '/');	127	std.debug.assert(reader.get().? == '/');
123	std.debug.assert(reader.get().? == '/');	128	std.debug.assert(reader.get().? == '/');
...	@@ -179,6 +184,76 @@ pub fn parse(text: []const u8) ParseError!Uri {	...	@@ -179,6 +184,76 @@ pub fn parse(text: []const u8) ParseError!Uri {
179	return uri;	184	return uri;
180	}	185	}
181		186
		187	/// Parses the URI or returns an error.
		188	/// The return value will contain unescaped strings pointing into the
		189	/// original `text`. Each component that is provided, will be non-`null`.
		190	pub fn parse(text: []const u8) ParseError!Uri {
		191	var reader = SliceReader{ .slice = text };
		192	const scheme = reader.readWhile(isSchemeChar);
		193
		194	// after the scheme, a ':' must appear
		195	if (reader.get()) \|c\| {
		196	if (c != ':')
		197	return error.UnexpectedCharacter;
		198	} else {
		199	return error.InvalidFormat;
		200	}
		201
		202	var uri = try parseWithoutScheme(reader.readUntilEof());
		203	uri.scheme = scheme;
		204
		205	return uri;
		206	}
		207
		208	/// Resolves a URI against a base URI, conforming to RFC 3986, Section 5.
		209	/// arena owns any memory allocated by this function.
		210	pub fn resolve(Base: Uri, R: Uri, strict: bool, arena: std.mem.Allocator) !Uri {
		211	var T: Uri = undefined;
		212
		213	if (R.scheme.len > 0 and !((!strict) and (std.mem.eql(u8, R.scheme, Base.scheme)))) {
		214	T.scheme = R.scheme;
		215	T.user = R.user;
		216	T.host = R.host;
		217	T.port = R.port;
		218	T.path = try std.fs.path.resolvePosix(arena, &.{ "/", R.path });
		219	T.query = R.query;
		220	} else {
		221	if (R.host) \|host\| {
		222	T.user = R.user;
		223	T.host = host;
		224	T.port = R.port;
		225	T.path = R.path;
		226	T.path = try std.fs.path.resolvePosix(arena, &.{ "/", R.path });
		227	T.query = R.query;
		228	} else {
		229	if (R.path.len == 0) {
		230	T.path = Base.path;
		231	if (R.query) \|query\| {
		232	T.query = query;
		233	} else {
		234	T.query = Base.query;
		235	}
		236	} else {
		237	if (R.path[0] == '/') {
		238	T.path = try std.fs.path.resolvePosix(arena, &.{ "/", R.path });
		239	} else {
		240	T.path = try std.fs.path.resolvePosix(arena, &.{ "/", Base.path, R.path });
		241	}
		242	T.query = R.query;
		243	}
		244
		245	T.user = Base.user;
		246	T.host = Base.host;
		247	T.port = Base.port;
		248	}
		249	T.scheme = Base.scheme;
		250	}
		251
		252	T.fragment = R.fragment;
		253
		254	return T;
		255	}
		256
182	const SliceReader = struct {	257	const SliceReader = struct {
183	const Self = @This();	258	const Self = @This();
184		259
...	@@ -284,6 +359,14 @@ fn isPathSeparator(c: u8) bool {	...	@@ -284,6 +359,14 @@ fn isPathSeparator(c: u8) bool {
284	};	359	};
285	}	360	}
286		361
		362	fn isPathChar(c: u8) bool {
		363	return isUnreserved(c) or isSubLimit(c) or c == '/' or c == ':' or c == '@';
		364	}
		365
		366	fn isQueryChar(c: u8) bool {
		367	return isPathChar(c) or c == '?';
		368	}
		369
287	fn isQuerySeparator(c: u8) bool {	370	fn isQuerySeparator(c: u8) bool {
288	return switch (c) {	371	return switch (c) {
289	'#' => true,	372	'#' => true,

lib/std/crypto/tls/Client.zig+49-1

...	@@ -88,11 +88,59 @@ pub const StreamInterface = struct {	...	@@ -88,11 +88,59 @@ pub const StreamInterface = struct {
88	}	88	}
89	};	89	};
90		90
		91	pub fn InitError(comptime Stream: type) type {
		92	return std.mem.Allocator.Error \|\| Stream.WriteError \|\| Stream.ReadError \|\| error{
		93	InsufficientEntropy,
		94	DiskQuota,
		95	LockViolation,
		96	NotOpenForWriting,
		97	TlsAlert,
		98	TlsUnexpectedMessage,
		99	TlsIllegalParameter,
		100	TlsDecryptFailure,
		101	TlsRecordOverflow,
		102	TlsBadRecordMac,
		103	CertificateFieldHasInvalidLength,
		104	CertificateHostMismatch,
		105	CertificatePublicKeyInvalid,
		106	CertificateExpired,
		107	CertificateFieldHasWrongDataType,
		108	CertificateIssuerMismatch,
		109	CertificateNotYetValid,
		110	CertificateSignatureAlgorithmMismatch,
		111	CertificateSignatureAlgorithmUnsupported,
		112	CertificateSignatureInvalid,
		113	CertificateSignatureInvalidLength,
		114	CertificateSignatureNamedCurveUnsupported,
		115	CertificateSignatureUnsupportedBitCount,
		116	TlsCertificateNotVerified,
		117	TlsBadSignatureScheme,
		118	TlsBadRsaSignatureBitCount,
		119	InvalidEncoding,
		120	IdentityElement,
		121	SignatureVerificationFailed,
		122	TlsDecryptError,
		123	TlsConnectionTruncated,
		124	TlsDecodeError,
		125	UnsupportedCertificateVersion,
		126	CertificateTimeInvalid,
		127	CertificateHasUnrecognizedObjectId,
		128	CertificateHasInvalidBitString,
		129	MessageTooLong,
		130	NegativeIntoUnsigned,
		131	TargetTooSmall,
		132	BufferTooSmall,
		133	InvalidSignature,
		134	NotSquare,
		135	NonCanonical,
		136	};
		137	}
		138
91	/// Initiates a TLS handshake and establishes a TLSv1.3 session with `stream`, which	139	/// Initiates a TLS handshake and establishes a TLSv1.3 session with `stream`, which
92	/// must conform to `StreamInterface`.	140	/// must conform to `StreamInterface`.
93	///	141	///
94	/// `host` is only borrowed during this function call.	142	/// `host` is only borrowed during this function call.
95	pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) !Client {	143	pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) InitError(@TypeOf(stream))!Client {
96	const host_len = @intCast(u16, host.len);	144	const host_len = @intCast(u16, host.len);
97		145
98	var random_buffer: [128]u8 = undefined;	146	var random_buffer: [128]u8 = undefined;

lib/std/http.zig+15

...	@@ -248,9 +248,24 @@ pub const Status = enum(u10) {	...	@@ -248,9 +248,24 @@ pub const Status = enum(u10) {
248		248
249	pub const TransferEncoding = enum {	249	pub const TransferEncoding = enum {
250	chunked,	250	chunked,
		251	// compression is intentionally omitted here, as std.http.Client stores it as content-encoding
		252	};
		253
		254	pub const ContentEncoding = enum {
251	compress,	255	compress,
252	deflate,	256	deflate,
253	gzip,	257	gzip,
		258	zstd,
		259	};
		260
		261	pub const Connection = enum {
		262	keep_alive,
		263	close,
		264	};
		265
		266	pub const CustomHeader = struct {
		267	name: []const u8,
		268	value: []const u8,
254	};	269	};
255		270
256	const std = @import("std.zig");	271	const std = @import("std.zig");

lib/std/http/Client.zig+247-786

...	@@ -13,6 +13,12 @@ const Uri = std.Uri;	...	@@ -13,6 +13,12 @@ const Uri = std.Uri;
13	const Allocator = std.mem.Allocator;	13	const Allocator = std.mem.Allocator;
14	const testing = std.testing;	14	const testing = std.testing;
15		15
		16	pub const Request = @import("Client/Request.zig");
		17	pub const Response = @import("Client/Response.zig");
		18
		19	pub const default_connection_pool_size = 32;
		20	const connection_pool_size = std.options.http_connection_pool_size;
		21
16	/// Used for tcpConnectToHost and storing HTTP headers when an externally	22	/// Used for tcpConnectToHost and storing HTTP headers when an externally
17	/// managed buffer is not provided.	23	/// managed buffer is not provided.
18	allocator: Allocator,	24	allocator: Allocator,
...	@@ -21,854 +27,256 @@ ca_bundle: std.crypto.Certificate.Bundle = .{},	...	@@ -21,854 +27,256 @@ ca_bundle: std.crypto.Certificate.Bundle = .{},
21	/// it will first rescan the system for root certificates.	27	/// it will first rescan the system for root certificates.
22	next_https_rescan_certs: bool = true,	28	next_https_rescan_certs: bool = true,
23		29
24	pub const Connection = struct {	30	connection_pool: ConnectionPool = .{},
25	stream: net.Stream,
26	/// undefined unless protocol is tls.
27	tls_client: std.crypto.tls.Client,
28	protocol: Protocol,
29		31
30	pub const Protocol = enum { plain, tls };	32	pub const ConnectionPool = struct {
		33	pub const Criteria = struct {
		34	host: []const u8,
		35	port: u16,
		36	is_tls: bool,
		37	};
31		38
32	pub fn read(conn: *Connection, buffer: []u8) !usize {	39	const Queue = std.TailQueue(Connection);
33	switch (conn.protocol) {	40	pub const Node = Queue.Node;
34	.plain => return conn.stream.read(buffer),	41
35	.tls => return conn.tls_client.read(conn.stream, buffer),	42	mutex: std.Thread.Mutex = .{},
		43	used: Queue = .{},
		44	free: Queue = .{},
		45	free_len: usize = 0,
		46	free_size: usize = default_connection_pool_size,
		47
		48	/// Finds and acquires a connection from the connection pool matching the criteria. This function is threadsafe.
		49	/// If no connection is found, null is returned.
		50	pub fn findConnection(pool: ConnectionPool, criteria: Criteria) ?Node {
		51	pool.mutex.lock();
		52	defer pool.mutex.unlock();
		53
		54	var next = pool.free.last;
		55	while (next) \|node\| : (next = node.prev) {
		56	if ((node.data.protocol == .tls) != criteria.is_tls) continue;
		57	if (node.data.port != criteria.port) continue;
		58	if (std.mem.eql(u8, node.data.host, criteria.host)) continue;
		59
		60	pool.acquireUnsafe(node);
		61	return node;
36	}	62	}
37	}
38		63
39	pub fn readAtLeast(conn: *Connection, buffer: []u8, len: usize) !usize {	64	return null;
40	switch (conn.protocol) {
41	.plain => return conn.stream.readAtLeast(buffer, len),
42	.tls => return conn.tls_client.readAtLeast(conn.stream, buffer, len),
43	}
44	}	65	}
45		66
46	pub fn writeAll(conn: *Connection, buffer: []const u8) !void {	67	/// Acquires an existing connection from the connection pool. This function is not threadsafe.
47	switch (conn.protocol) {	68	pub fn acquireUnsafe(pool: ConnectionPool, node: Node) void {
48	.plain => return conn.stream.writeAll(buffer),	69	pool.free.remove(node);
49	.tls => return conn.tls_client.writeAll(conn.stream, buffer),	70	pool.free_len -= 1;
50	}	71
		72	pool.used.append(node);
51	}	73	}
52		74
53	pub fn write(conn: *Connection, buffer: []const u8) !usize {	75	/// Acquires an existing connection from the connection pool. This function is threadsafe.
54	switch (conn.protocol) {	76	pub fn acquire(pool: ConnectionPool, node: Node) void {
55	.plain => return conn.stream.write(buffer),	77	pool.mutex.lock();
56	.tls => return conn.tls_client.write(conn.stream, buffer),	78	defer pool.mutex.unlock();
57	}	79
		80	return pool.acquireUnsafe(node);
58	}	81	}
59	};
60		82
61	/// TODO: emit error.UnexpectedEndOfStream or something like that when the read	83	/// Tries to release a connection back to the connection pool. This function is threadsafe.
62	/// data does not match the content length. This is necessary since HTTPS disables	84	/// If the connection is marked as closing, it will be closed instead.
63	/// close_notify protection on underlying TLS streams.	85	pub fn release(pool: ConnectionPool, client: Client, node: *Node) void {
64	pub const Request = struct {	86	pool.mutex.lock();
65	client: *Client,	87	defer pool.mutex.unlock();
66	connection: Connection,
67	redirects_left: u32,
68	response: Response,
69	/// These are stored in Request so that they are available when following
70	/// redirects.
71	headers: Headers,
72
73	pub const Response = struct {
74	headers: Response.Headers,
75	state: State,
76	header_bytes_owned: bool,
77	/// This could either be a fixed buffer provided by the API user or it
78	/// could be our own array list.
79	header_bytes: std.ArrayListUnmanaged(u8),
80	max_header_bytes: usize,
81	next_chunk_length: u64,
82
83	pub const Headers = struct {
84	status: http.Status,
85	version: http.Version,
86	location: ?[]const u8 = null,
87	content_length: ?u64 = null,
88	transfer_encoding: ?http.TransferEncoding = null,
89
90	pub fn parse(bytes: []const u8) !Response.Headers {
91	var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
92
93	const first_line = it.first();
94	if (first_line.len < 12)
95	return error.ShortHttpStatusLine;
96
97	const version: http.Version = switch (int64(first_line[0..8])) {
98	int64("HTTP/1.0") => .@"HTTP/1.0",
99	int64("HTTP/1.1") => .@"HTTP/1.1",
100	else => return error.BadHttpVersion,
101	};
102	if (first_line[8] != ' ') return error.HttpHeadersInvalid;
103	const status = @intToEnum(http.Status, parseInt3(first_line[9..12].*));
104
105	var headers: Response.Headers = .{
106	.version = version,
107	.status = status,
108	};
109
110	while (it.next()) \|line\| {
111	if (line.len == 0) return error.HttpHeadersInvalid;
112	switch (line[0]) {
113	' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
114	else => {},
115	}
116	var line_it = mem.split(u8, line, ": ");
117	const header_name = line_it.first();
118	const header_value = line_it.rest();
119	if (std.ascii.eqlIgnoreCase(header_name, "location")) {
120	if (headers.location != null) return error.HttpHeadersInvalid;
121	headers.location = header_value;
122	} else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
123	if (headers.content_length != null) return error.HttpHeadersInvalid;
124	headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
125	} else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
126	if (headers.transfer_encoding != null) return error.HttpHeadersInvalid;
127	headers.transfer_encoding = std.meta.stringToEnum(http.TransferEncoding, header_value) orelse
128	return error.HttpTransferEncodingUnsupported;
129	}
130	}
131
132	return headers;
133	}
134
135	test "parse headers" {
136	const example =
137	"HTTP/1.1 301 Moved Permanently\r\n" ++
138	"Location: https://www.example.com/\r\n" ++
139	"Content-Type: text/html; charset=UTF-8\r\n" ++
140	"Content-Length: 220\r\n\r\n";
141	const parsed = try Response.Headers.parse(example);
142	try testing.expectEqual(http.Version.@"HTTP/1.1", parsed.version);
143	try testing.expectEqual(http.Status.moved_permanently, parsed.status);
144	try testing.expectEqualStrings("https://www.example.com/", parsed.location orelse
145	return error.TestFailed);
146	try testing.expectEqual(@as(?u64, 220), parsed.content_length);
147	}
148
149	test "header continuation" {
150	const example =
151	"HTTP/1.0 200 OK\r\n" ++
152	"Content-Type: text/html;\r\n charset=UTF-8\r\n" ++
153	"Content-Length: 220\r\n\r\n";
154	try testing.expectError(
155	error.HttpHeaderContinuationsUnsupported,
156	Response.Headers.parse(example),
157	);
158	}
159
160	test "extra content length" {
161	const example =
162	"HTTP/1.0 200 OK\r\n" ++
163	"Content-Length: 220\r\n" ++
164	"Content-Type: text/html; charset=UTF-8\r\n" ++
165	"content-length: 220\r\n\r\n";
166	try testing.expectError(
167	error.HttpHeadersInvalid,
168	Response.Headers.parse(example),
169	);
170	}
171	};
172
173	pub const State = enum {
174	/// Begin header parsing states.
175	invalid,
176	start,
177	seen_r,
178	seen_rn,
179	seen_rnr,
180	finished,
181	/// Begin transfer-encoding: chunked parsing states.
182	chunk_size_prefix_r,
183	chunk_size_prefix_n,
184	chunk_size,
185	chunk_r,
186	chunk_data,
187
188	pub fn zeroMeansEnd(state: State) bool {
189	return switch (state) {
190	.finished, .chunk_data => true,
191	else => false,
192	};
193	}
194	};
195
196	pub fn initDynamic(max: usize) Response {
197	return .{
198	.state = .start,
199	.headers = undefined,
200	.header_bytes = .{},
201	.max_header_bytes = max,
202	.header_bytes_owned = true,
203	.next_chunk_length = undefined,
204	};
205	}
206		88
207	pub fn initStatic(buf: []u8) Response {	89	pool.used.remove(node);
208	return .{
209	.state = .start,
210	.headers = undefined,
211	.header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
212	.max_header_bytes = buf.len,
213	.header_bytes_owned = false,
214	.next_chunk_length = undefined,
215	};
216	}
217		90
218	/// Returns how many bytes are part of HTTP headers. Always less than or	91	if (node.data.closing) {
219	/// equal to bytes.len. If the amount returned is less than bytes.len, it	92	node.data.close(client);
220	/// means the headers ended and the first byte after the double \r\n\r\n is
221	/// located at `bytes[result]`.
222	pub fn findHeadersEnd(r: *Response, bytes: []const u8) usize {
223	var index: usize = 0;
224
225	// TODO: https://github.com/ziglang/zig/issues/8220
226	state: while (true) {
227	switch (r.state) {
228	.invalid => unreachable,
229	.finished => unreachable,
230	.start => while (true) {
231	switch (bytes.len - index) {
232	0 => return index,
233	1 => {
234	if (bytes[index] == '\r')
235	r.state = .seen_r;
236	return index + 1;
237	},
238	2 => {
239	if (int16(bytes[index..][0..2]) == int16("\r\n")) {
240	r.state = .seen_rn;
241	} else if (bytes[index + 1] == '\r') {
242	r.state = .seen_r;
243	}
244	return index + 2;
245	},
246	3 => {
247	if (int16(bytes[index..][0..2]) == int16("\r\n") and
248	bytes[index + 2] == '\r')
249	{
250	r.state = .seen_rnr;
251	} else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n")) {
252	r.state = .seen_rn;
253	} else if (bytes[index + 2] == '\r') {
254	r.state = .seen_r;
255	}
256	return index + 3;
257	},
258	4...15 => {
259	if (int32(bytes[index..][0..4]) == int32("\r\n\r\n")) {
260	r.state = .finished;
261	return index + 4;
262	} else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n") and
263	bytes[index + 3] == '\r')
264	{
265	r.state = .seen_rnr;
266	index += 4;
267	continue :state;
268	} else if (int16(bytes[index + 2 ..][0..2]) == int16("\r\n")) {
269	r.state = .seen_rn;
270	index += 4;
271	continue :state;
272	} else if (bytes[index + 3] == '\r') {
273	r.state = .seen_r;
274	index += 4;
275	continue :state;
276	}
277	index += 4;
278	continue;
279	},
280	else => {
281	const chunk = bytes[index..][0..16];
282	const v: @Vector(16, u8) = chunk.*;
283	const matches_r = v == @splat(16, @as(u8, '\r'));
284	const iota = std.simd.iota(u8, 16);
285	const default = @splat(16, @as(u8, 16));
286	const sub_index = @reduce(.Min, @select(u8, matches_r, iota, default));
287	switch (sub_index) {
288	0...12 => {
289	index += sub_index + 4;
290	if (int32(chunk[sub_index..][0..4]) == int32("\r\n\r\n")) {
291	r.state = .finished;
292	return index;
293	}
294	continue;
295	},
296	13 => {
297	index += 16;
298	if (int16(chunk[14..][0..2]) == int16("\n\r")) {
299	r.state = .seen_rnr;
300	continue :state;
301	}
302	continue;
303	},
304	14 => {
305	index += 16;
306	if (chunk[15] == '\n') {
307	r.state = .seen_rn;
308	continue :state;
309	}
310	continue;
311	},
312	15 => {
313	r.state = .seen_r;
314	index += 16;
315	continue :state;
316	},
317	16 => {
318	index += 16;
319	continue;
320	},
321	else => unreachable,
322	}
323	},
324	}
325	},
326
327	.seen_r => switch (bytes.len - index) {
328	0 => return index,
329	1 => {
330	switch (bytes[index]) {
331	'\n' => r.state = .seen_rn,
332	'\r' => r.state = .seen_r,
333	else => r.state = .start,
334	}
335	return index + 1;
336	},
337	2 => {
338	if (int16(bytes[index..][0..2]) == int16("\n\r")) {
339	r.state = .seen_rnr;
340	return index + 2;
341	}
342	r.state = .start;
343	return index + 2;
344	},
345	else => {
346	if (int16(bytes[index..][0..2]) == int16("\n\r") and
347	bytes[index + 2] == '\n')
348	{
349	r.state = .finished;
350	return index + 3;
351	}
352	index += 3;
353	r.state = .start;
354	continue :state;
355	},
356	},
357	.seen_rn => switch (bytes.len - index) {
358	0 => return index,
359	1 => {
360	switch (bytes[index]) {
361	'\r' => r.state = .seen_rnr,
362	else => r.state = .start,
363	}
364	return index + 1;
365	},
366	else => {
367	if (int16(bytes[index..][0..2]) == int16("\r\n")) {
368	r.state = .finished;
369	return index + 2;
370	}
371	index += 2;
372	r.state = .start;
373	continue :state;
374	},
375	},
376	.seen_rnr => switch (bytes.len - index) {
377	0 => return index,
378	else => {
379	if (bytes[index] == '\n') {
380	r.state = .finished;
381	return index + 1;
382	}
383	index += 1;
384	r.state = .start;
385	continue :state;
386	},
387	},
388	.chunk_size_prefix_r => unreachable,
389	.chunk_size_prefix_n => unreachable,
390	.chunk_size => unreachable,
391	.chunk_r => unreachable,
392	.chunk_data => unreachable,
393	}
394
395	return index;
396	}
397	}
398		93
399	pub fn findChunkedLen(r: *Response, bytes: []const u8) usize {	94	return client.allocator.destroy(node);
400	var i: usize = 0;
401	if (r.state == .chunk_size) {
402	while (i < bytes.len) : (i += 1) {
403	const digit = switch (bytes[i]) {
404	'0'...'9' => \|b\| b - '0',
405	'A'...'Z' => \|b\| b - 'A' + 10,
406	'a'...'z' => \|b\| b - 'a' + 10,
407	'\r' => {
408	r.state = .chunk_r;
409	i += 1;
410	break;
411	},
412	else => {
413	r.state = .invalid;
414	return i;
415	},
416	};
417	const mul = @mulWithOverflow(r.next_chunk_length, 16);
418	if (mul[1] != 0) {
419	r.state = .invalid;
420	return i;
421	}
422	const add = @addWithOverflow(mul[0], digit);
423	if (add[1] != 0) {
424	r.state = .invalid;
425	return i;
426	}
427	r.next_chunk_length = add[0];
428	} else {
429	return i;
430	}
431	}
432	assert(r.state == .chunk_r);
433	if (i == bytes.len) return i;
434
435	if (bytes[i] == '\n') {
436	r.state = .chunk_data;
437	return i + 1;
438	} else {
439	r.state = .invalid;
440	return i;
441	}
442	}	95	}
443		96
444	fn parseInt3(nnn: @Vector(3, u8)) u10 {	97	if (pool.free_len + 1 >= pool.free_size) {
445	const zero: @Vector(3, u8) = .{ '0', '0', '0' };	98	const popped = pool.free.popFirst() orelse unreachable;
446	const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
447	return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
448	}
449		99
450	test parseInt3 {	100	popped.data.close(client);
451	const expectEqual = std.testing.expectEqual;
452	try expectEqual(@as(u10, 0), parseInt3("000".*));
453	try expectEqual(@as(u10, 418), parseInt3("418".*));
454	try expectEqual(@as(u10, 999), parseInt3("999".*));
455	}
456		101
457	test "find headers end basic" {	102	return client.allocator.destroy(popped);
458	var buffer: [1]u8 = undefined;
459	var r = Response.initStatic(&buffer);
460	try testing.expectEqual(@as(usize, 10), r.findHeadersEnd("HTTP/1.1 4"));
461	try testing.expectEqual(@as(usize, 2), r.findHeadersEnd("18"));
462	try testing.expectEqual(@as(usize, 8), r.findHeadersEnd(" lol\r\n\r\nblah blah"));
463	}	103	}
464		104
465	test "find headers end vectorized" {	105	pool.free.append(node);
466	var buffer: [1]u8 = undefined;	106	pool.free_len += 1;
467	var r = Response.initStatic(&buffer);	107	}
468	const example =
469	"HTTP/1.1 301 Moved Permanently\r\n" ++
470	"Location: https://www.example.com/\r\n" ++
471	"Content-Type: text/html; charset=UTF-8\r\n" ++
472	"Content-Length: 220\r\n" ++
473	"\r\ncontent";
474	try testing.expectEqual(@as(usize, 131), r.findHeadersEnd(example));
475	}
476		108
477	test "find headers end bug" {	109	/// Adds a newly created node to the pool of used connections. This function is threadsafe.
478	var buffer: [1]u8 = undefined;	110	pub fn addUsed(pool: ConnectionPool, node: Node) void {
479	var r = Response.initStatic(&buffer);	111	pool.mutex.lock();
480	const trail = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";	112	defer pool.mutex.unlock();
481	const example =
482	"HTTP/1.1 200 OK\r\n" ++
483	"Access-Control-Allow-Origin: https://render.githubusercontent.com\r\n" ++
484	"content-disposition: attachment; filename=zig-0.10.0.tar.gz\r\n" ++
485	"Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'; sandbox\r\n" ++
486	"Content-Type: application/x-gzip\r\n" ++
487	"ETag: \"bfae0af6b01c7c0d89eb667cb5f0e65265968aeebda2689177e6b26acd3155ca\"\r\n" ++
488	"Strict-Transport-Security: max-age=31536000\r\n" ++
489	"Vary: Authorization,Accept-Encoding,Origin\r\n" ++
490	"X-Content-Type-Options: nosniff\r\n" ++
491	"X-Frame-Options: deny\r\n" ++
492	"X-XSS-Protection: 1; mode=block\r\n" ++
493	"Date: Fri, 06 Jan 2023 22:26:22 GMT\r\n" ++
494	"Transfer-Encoding: chunked\r\n" ++
495	"X-GitHub-Request-Id: 89C6:17E9:A7C9E:124B51:63B8A00E\r\n" ++
496	"connection: close\r\n\r\n" ++ trail;
497	try testing.expectEqual(@as(usize, example.len - trail.len), r.findHeadersEnd(example));
498	}
499	};
500		113
501	pub const Headers = struct {	114	pool.used.append(node);
502	version: http.Version = .@"HTTP/1.1",	115	}
503	method: http.Method = .GET,
504	};
505		116
506	pub const Options = struct {	117	pub fn deinit(pool: ConnectionPool, client: Client) void {
507	max_redirects: u32 = 3,	118	pool.mutex.lock();
508	header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
509
510	pub const HeaderStrategy = union(enum) {
511	/// In this case, the client's Allocator will be used to store the
512	/// entire HTTP header. This value is the maximum total size of
513	/// HTTP headers allowed, otherwise
514	/// error.HttpHeadersExceededSizeLimit is returned from read().
515	dynamic: usize,
516	/// This is used to store the entire HTTP header. If the HTTP
517	/// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
518	/// is returned from read(). When this is used, `error.OutOfMemory`
519	/// cannot be returned from `read()`.
520	static: []u8,
521	};
522	};
523		119
524	/// May be skipped if header strategy is buffer.	120	var next = pool.free.first;
525	pub fn deinit(req: *Request) void {	121	while (next) \|node\| {
526	if (req.response.header_bytes_owned) {	122	defer client.allocator.destroy(node);
527	req.response.header_bytes.deinit(req.client.allocator);	123	next = node.next;
		124
		125	node.data.close(client);
528	}	126	}
529	req.* = undefined;	127
		128	next = pool.used.first;
		129	while (next) \|node\| {
		130	defer client.allocator.destroy(node);
		131	next = node.next;
		132
		133	node.data.close(client);
		134	}
		135
		136	pool.* = undefined;
530	}	137	}
		138	};
		139
		140	pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.ReaderRaw);
		141	pub const GzipDecompressor = std.compress.gzip.Decompress(Request.ReaderRaw);
		142	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.ReaderRaw, .{});
531		143
532	pub const Reader = std.io.Reader(*Request, ReadError, read);	144	pub const Connection = struct {
		145	stream: net.Stream,
		146	/// undefined unless protocol is tls.
		147	tls_client: *std.crypto.tls.Client, // TODO: allocate this, it's currently 16 KB.
		148	protocol: Protocol,
		149	host: []u8,
		150	port: u16,
533		151
534	pub fn reader(req: *Request) Reader {	152	// This connection has been part of a non keepalive request and cannot be added to the pool.
535	return .{ .context = req };	153	closing: bool = false,
		154
		155	pub const Protocol = enum { plain, tls };
		156
		157	pub fn read(conn: *Connection, buffer: []u8) !usize {
		158	switch (conn.protocol) {
		159	.plain => return conn.stream.read(buffer),
		160	.tls => return conn.tls_client.read(conn.stream, buffer),
		161	}
536	}	162	}
537		163
538	pub fn readAll(req: *Request, buffer: []u8) !usize {	164	pub fn readAtLeast(conn: *Connection, buffer: []u8, len: usize) !usize {
539	return readAtLeast(req, buffer, buffer.len);	165	switch (conn.protocol) {
		166	.plain => return conn.stream.readAtLeast(buffer, len),
		167	.tls => return conn.tls_client.readAtLeast(conn.stream, buffer, len),
		168	}
540	}	169	}
541		170
542	pub const ReadError = net.Stream.ReadError \|\| error{	171	pub const ReadError = net.Stream.ReadError \|\| error{
543	// From HTTP protocol
544	HttpHeadersInvalid,
545	HttpHeadersExceededSizeLimit,
546	HttpRedirectMissingLocation,
547	HttpTransferEncodingUnsupported,
548	HttpContentLengthUnknown,
549	TooManyHttpRedirects,
550	ShortHttpStatusLine,
551	BadHttpVersion,
552	HttpHeaderContinuationsUnsupported,
553	UnsupportedUrlScheme,
554	UriMissingHost,
555	UnknownHostName,
556
557	// Network problems
558	NetworkUnreachable,
559	HostLacksNetworkAddresses,
560	TemporaryNameServerFailure,
561	NameServerFailure,
562	ProtocolFamilyNotAvailable,
563	ProtocolNotSupported,
564
565	// System resource problems
566	ProcessFdQuotaExceeded,
567	SystemFdQuotaExceeded,
568	OutOfMemory,
569
570	// TLS problems
571	InsufficientEntropy,
572	TlsConnectionTruncated,	172	TlsConnectionTruncated,
573	TlsRecordOverflow,	173	TlsRecordOverflow,
574	TlsDecodeError,	174	TlsDecodeError,
575	TlsAlert,	175	TlsAlert,
576	TlsBadRecordMac,	176	TlsBadRecordMac,
		177	Overflow,
577	TlsBadLength,	178	TlsBadLength,
578	TlsIllegalParameter,	179	TlsIllegalParameter,
579	TlsUnexpectedMessage,	180	TlsUnexpectedMessage,
580	TlsDecryptFailure,
581	CertificateFieldHasInvalidLength,
582	CertificateHostMismatch,
583	CertificatePublicKeyInvalid,
584	CertificateExpired,
585	CertificateFieldHasWrongDataType,
586	CertificateIssuerMismatch,
587	CertificateNotYetValid,
588	CertificateSignatureAlgorithmMismatch,
589	CertificateSignatureAlgorithmUnsupported,
590	CertificateSignatureInvalid,
591	CertificateSignatureInvalidLength,
592	CertificateSignatureNamedCurveUnsupported,
593	CertificateSignatureUnsupportedBitCount,
594	TlsCertificateNotVerified,
595	TlsBadSignatureScheme,
596	TlsBadRsaSignatureBitCount,
597	TlsDecryptError,
598	UnsupportedCertificateVersion,
599	CertificateTimeInvalid,
600	CertificateHasUnrecognizedObjectId,
601	CertificateHasInvalidBitString,
602	CertificateAuthorityBundleTooBig,
603
604	// TODO: convert to higher level errors
605	InvalidFormat,
606	InvalidPort,
607	UnexpectedCharacter,
608	Overflow,
609	InvalidCharacter,
610	AddressFamilyNotSupported,
611	AddressInUse,
612	AddressNotAvailable,
613	ConnectionPending,
614	ConnectionRefused,
615	FileNotFound,
616	PermissionDenied,
617	ServiceUnavailable,
618	SocketTypeNotSupported,
619	FileTooBig,
620	LockViolation,
621	NoSpaceLeft,
622	NotOpenForWriting,
623	InvalidEncoding,
624	IdentityElement,
625	NonCanonical,
626	SignatureVerificationFailed,
627	MessageTooLong,
628	NegativeIntoUnsigned,
629	TargetTooSmall,
630	BufferTooSmall,
631	InvalidSignature,
632	NotSquare,
633	DiskQuota,
634	InvalidEnd,
635	Incomplete,
636	InvalidIpv4Mapping,
637	InvalidIPAddressFormat,
638	BadPathName,
639	DeviceBusy,
640	FileBusy,
641	FileLocksNotSupported,
642	InvalidHandle,
643	InvalidUtf8,
644	NameTooLong,
645	NoDevice,
646	PathAlreadyExists,
647	PipeBusy,
648	SharingViolation,
649	SymLinkLoop,
650	FileSystem,
651	InterfaceNotFound,
652	AlreadyBound,
653	FileDescriptorNotASocket,
654	NetworkSubsystemFailed,
655	NotDir,
656	ReadOnlyFileSystem,
657	Unseekable,
658	MissingEndCertificateMarker,
659	InvalidPadding,
660	EndOfStream,
661	InvalidArgument,
662	};	181	};
663		182
664	pub fn read(req: *Request, buffer: []u8) ReadError!usize {	183	pub const Reader = std.io.Reader(*Connection, ReadError, read);
665	return readAtLeast(req, buffer, 1);	184
		185	pub fn reader(conn: *Connection) Reader {
		186	return Reader{ .context = conn };
666	}	187	}
667		188
668	pub fn readAtLeast(req: *Request, buffer: []u8, len: usize) !usize {	189	pub fn writeAll(conn: *Connection, buffer: []const u8) !void {
669	assert(len <= buffer.len);	190	switch (conn.protocol) {
670	var index: usize = 0;	191	.plain => return conn.stream.writeAll(buffer),
671	while (index < len) {	192	.tls => return conn.tls_client.writeAll(conn.stream, buffer),
672	const zero_means_end = req.response.state.zeroMeansEnd();
673	const amt = try readAdvanced(req, buffer[index..]);
674	if (amt == 0 and zero_means_end) break;
675	index += amt;
676	}	193	}
677	return index;
678	}	194	}
679		195
680	/// This one can return 0 without meaning EOF.	196	pub fn write(conn: *Connection, buffer: []const u8) !usize {
681	/// TODO change to readvAdvanced	197	switch (conn.protocol) {
682	pub fn readAdvanced(req: *Request, buffer: []u8) !usize {	198	.plain => return conn.stream.write(buffer),
683	var in = buffer[0..try req.connection.read(buffer)];	199	.tls => return conn.tls_client.write(conn.stream, buffer),
684	var out_index: usize = 0;
685	while (true) {
686	switch (req.response.state) {
687	.invalid => unreachable,
688	.start, .seen_r, .seen_rn, .seen_rnr => {
689	const i = req.response.findHeadersEnd(in);
690	if (req.response.state == .invalid) return error.HttpHeadersInvalid;
691
692	const headers_data = in[0..i];
693	if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
694	return error.HttpHeadersExceededSizeLimit;
695	}
696	try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
697
698	if (req.response.state == .finished) {
699	req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
700
701	if (req.response.headers.status.class() == .redirect) {
702	if (req.redirects_left == 0) return error.TooManyHttpRedirects;
703	const location = req.response.headers.location orelse
704	return error.HttpRedirectMissingLocation;
705	const new_url = try std.Uri.parse(location);
706	const new_req = try req.client.request(new_url, req.headers, .{
707	.max_redirects = req.redirects_left - 1,
708	.header_strategy = if (req.response.header_bytes_owned) .{
709	.dynamic = req.response.max_header_bytes,
710	} else .{
711	.static = req.response.header_bytes.unusedCapacitySlice(),
712	},
713	});
714	req.deinit();
715	req.* = new_req;
716	assert(out_index == 0);
717	in = buffer[0..try req.connection.read(buffer)];
718	continue;
719	}
720
721	if (req.response.headers.transfer_encoding) \|transfer_encoding\| {
722	switch (transfer_encoding) {
723	.chunked => {
724	req.response.next_chunk_length = 0;
725	req.response.state = .chunk_size;
726	},
727	.compress => return error.HttpTransferEncodingUnsupported,
728	.deflate => return error.HttpTransferEncodingUnsupported,
729	.gzip => return error.HttpTransferEncodingUnsupported,
730	}
731	} else if (req.response.headers.content_length) \|content_length\| {
732	req.response.next_chunk_length = content_length;
733	} else {
734	return error.HttpContentLengthUnknown;
735	}
736
737	in = in[i..];
738	continue;
739	}
740
741	assert(out_index == 0);
742	return 0;
743	},
744	.finished => {
745	if (in.ptr == buffer.ptr) {
746	return in.len;
747	} else {
748	mem.copy(u8, buffer[out_index..], in);
749	return out_index + in.len;
750	}
751	},
752	.chunk_size_prefix_r => switch (in.len) {
753	0 => return out_index,
754	1 => switch (in[0]) {
755	'\r' => {
756	req.response.state = .chunk_size_prefix_n;
757	return out_index;
758	},
759	else => {
760	req.response.state = .invalid;
761	return error.HttpHeadersInvalid;
762	},
763	},
764	else => switch (int16(in[0..2])) {
765	int16("\r\n") => {
766	in = in[2..];
767	req.response.state = .chunk_size;
768	continue;
769	},
770	else => {
771	req.response.state = .invalid;
772	return error.HttpHeadersInvalid;
773	},
774	},
775	},
776	.chunk_size_prefix_n => switch (in.len) {
777	0 => return out_index,
778	else => switch (in[0]) {
779	'\n' => {
780	in = in[1..];
781	req.response.state = .chunk_size;
782	continue;
783	},
784	else => {
785	req.response.state = .invalid;
786	return error.HttpHeadersInvalid;
787	},
788	},
789	},
790	.chunk_size, .chunk_r => {
791	const i = req.response.findChunkedLen(in);
792	switch (req.response.state) {
793	.invalid => return error.HttpHeadersInvalid,
794	.chunk_data => {
795	if (req.response.next_chunk_length == 0) {
796	req.response.state = .start;
797	return out_index;
798	}
799	in = in[i..];
800	continue;
801	},
802	.chunk_size => return out_index,
803	else => unreachable,
804	}
805	},
806	.chunk_data => {
807	// TODO https://github.com/ziglang/zig/issues/14039
808	const sub_amt = @intCast(usize, @min(req.response.next_chunk_length, in.len));
809	req.response.next_chunk_length -= sub_amt;
810	if (req.response.next_chunk_length > 0) {
811	if (in.ptr == buffer.ptr) {
812	return sub_amt;
813	} else {
814	mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
815	out_index += sub_amt;
816	return out_index;
817	}
818	}
819	mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
820	out_index += sub_amt;
821	req.response.state = .chunk_size_prefix_r;
822	in = in[sub_amt..];
823	continue;
824	},
825	}
826	}	200	}
827	}	201	}
828		202
829	inline fn int16(array: *const [2]u8) u16 {	203	pub const WriteError = net.Stream.WriteError \|\| error{};
830	return @bitCast(u16, array.*);	204	pub const Writer = std.io.Writer(*Connection, WriteError, write);
831	}
832		205
833	inline fn int32(array: *const [4]u8) u32 {	206	pub fn writer(conn: *Connection) Writer {
834	return @bitCast(u32, array.*);	207	return Writer{ .context = conn };
835	}	208	}
836		209
837	inline fn int64(array: *const [8]u8) u64 {	210	pub fn close(conn: Connection, client: const Client) void {
838	return @bitCast(u64, array.*);	211	if (conn.protocol == .tls) {
839	}	212	// try to cleanly close the TLS connection, for any server that cares.
		213	_ = conn.tls_client.writeEnd(conn.stream, "", true) catch {};
		214	client.allocator.destroy(conn.tls_client);
		215	}
840		216
841	test {	217	conn.stream.close();
842	_ = Response;	218
		219	client.allocator.free(conn.host);
843	}	220	}
844	};	221	};
845		222
846	pub fn deinit(client: *Client) void {	223	pub fn deinit(client: *Client) void {
		224	client.connection_pool.deinit(client);
		225
847	client.ca_bundle.deinit(client.allocator);	226	client.ca_bundle.deinit(client.allocator);
848	client.* = undefined;	227	client.* = undefined;
849	}	228	}
850		229
851	pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) !Connection {	230	pub const ConnectError = std.mem.Allocator.Error \|\| net.TcpConnectToHostError \|\| std.crypto.tls.Client.InitError(net.Stream);
852	var conn: Connection = .{	231
		232	pub fn connect(client: Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!ConnectionPool.Node {
		233	if (client.connection_pool.findConnection(.{
		234	.host = host,
		235	.port = port,
		236	.is_tls = protocol == .tls,
		237	})) \|node\|
		238	return node;
		239
		240	const conn = try client.allocator.create(ConnectionPool.Node);
		241	errdefer client.allocator.destroy(conn);
		242	conn.* = .{ .data = undefined };
		243
		244	conn.data = .{
853	.stream = try net.tcpConnectToHost(client.allocator, host, port),	245	.stream = try net.tcpConnectToHost(client.allocator, host, port),
854	.tls_client = undefined,	246	.tls_client = undefined,
855	.protocol = protocol,	247	.protocol = protocol,
		248	.host = try client.allocator.dupe(u8, host),
		249	.port = port,
856	};	250	};
857		251
858	switch (protocol) {	252	switch (protocol) {
859	.plain => {},	253	.plain => {},
860	.tls => {	254	.tls => {
861	conn.tls_client = try std.crypto.tls.Client.init(conn.stream, client.ca_bundle, host);	255	conn.data.tls_client = try client.allocator.create(std.crypto.tls.Client);
		256	conn.data.tls_client.* = try std.crypto.tls.Client.init(conn.data.stream, client.ca_bundle, host);
862	// This is appropriate for HTTPS because the HTTP headers contain	257	// This is appropriate for HTTPS because the HTTP headers contain
863	// the content length which is used to detect truncation attacks.	258	// the content length which is used to detect truncation attacks.
864	conn.tls_client.allow_truncation_attacks = true;	259	conn.data.tls_client.allow_truncation_attacks = true;
865	},	260	},
866	}	261	}
867		262
		263	client.connection_pool.addUsed(conn);
		264
868	return conn;	265	return conn;
869	}	266	}
870		267
871	pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Request.Options) !Request {	268	pub const RequestError = ConnectError \|\| Connection.WriteError \|\| error{
		269	UnsupportedUrlScheme,
		270	UriMissingHost,
		271
		272	CertificateAuthorityBundleTooBig,
		273	InvalidPadding,
		274	MissingEndCertificateMarker,
		275	Unseekable,
		276	EndOfStream,
		277	};
		278
		279	pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Request.Options) RequestError!Request {
872	const protocol: Connection.Protocol = if (mem.eql(u8, uri.scheme, "http"))	280	const protocol: Connection.Protocol = if (mem.eql(u8, uri.scheme, "http"))
873	.plain	281	.plain
874	else if (mem.eql(u8, uri.scheme, "https"))	282	else if (mem.eql(u8, uri.scheme, "https"))
...	@@ -884,34 +292,85 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req	...	@@ -884,34 +292,85 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req
884	const host = uri.host orelse return error.UriMissingHost;	292	const host = uri.host orelse return error.UriMissingHost;
885		293
886	if (client.next_https_rescan_certs and protocol == .tls) {	294	if (client.next_https_rescan_certs and protocol == .tls) {
887	try client.ca_bundle.rescan(client.allocator);	295	client.connection_pool.mutex.lock(); // TODO: this could be so much better than reusing the connection pool mutex.
888	client.next_https_rescan_certs = false;	296	defer client.connection_pool.mutex.unlock();
		297
		298	if (client.next_https_rescan_certs) {
		299	try client.ca_bundle.rescan(client.allocator);
		300	client.next_https_rescan_certs = false;
		301	}
889	}	302	}
890		303
891	var req: Request = .{	304	var req: Request = .{
		305	.uri = uri,
892	.client = client,	306	.client = client,
893	.headers = headers,	307	.headers = headers,
894	.connection = try client.connect(host, port, protocol),	308	.connection = try client.connect(host, port, protocol),
895	.redirects_left = options.max_redirects,	309	.redirects_left = options.max_redirects,
		310	.handle_redirects = options.handle_redirects,
		311	.compression_init = false,
896	.response = switch (options.header_strategy) {	312	.response = switch (options.header_strategy) {
897	.dynamic => \|max\| Request.Response.initDynamic(max),	313	.dynamic => \|max\| Response.initDynamic(max),
898	.static => \|buf\| Request.Response.initStatic(buf),	314	.static => \|buf\| Response.initStatic(buf),
899	},	315	},
		316	.arena = undefined,
900	};	317	};
901		318
		319	req.arena = std.heap.ArenaAllocator.init(client.allocator);
		320
902	{	321	{
903	var h = try std.BoundedArray(u8, 1000).init(0);	322	var buffered = std.io.bufferedWriter(req.connection.data.writer());
904	try h.appendSlice(@tagName(headers.method));	323	const writer = buffered.writer();
905	try h.appendSlice(" ");	324
906	try h.appendSlice(uri.path);	325	const escaped_path = try Uri.escapePath(client.allocator, uri.path);
907	try h.appendSlice(" ");	326	defer client.allocator.free(escaped_path);
908	try h.appendSlice(@tagName(headers.version));	327
909	try h.appendSlice("\r\nHost: ");	328	const escaped_query = if (uri.query) \|q\| try Uri.escapeQuery(client.allocator, q) else null;
910	try h.appendSlice(host);	329	defer if (escaped_query) \|q\| client.allocator.free(q);
911	try h.appendSlice("\r\nConnection: close\r\n\r\n");	330
912		331	const escaped_fragment = if (uri.fragment) \|f\| try Uri.escapeQuery(client.allocator, f) else null;
913	const header_bytes = h.slice();	332	defer if (escaped_fragment) \|f\| client.allocator.free(f);
914	try req.connection.writeAll(header_bytes);	333
		334	try writer.writeAll(@tagName(headers.method));
		335	try writer.writeByte(' ');
		336	try writer.writeAll(escaped_path);
		337	if (escaped_query) \|q\| {
		338	try writer.writeByte('?');
		339	try writer.writeAll(q);
		340	}
		341	if (escaped_fragment) \|f\| {
		342	try writer.writeByte('#');
		343	try writer.writeAll(f);
		344	}
		345	try writer.writeByte(' ');
		346	try writer.writeAll(@tagName(headers.version));
		347	try writer.writeAll("\r\nHost: ");
		348	try writer.writeAll(host);
		349	try writer.writeAll("\r\nUser-Agent: ");
		350	try writer.writeAll(headers.user_agent);
		351	if (headers.connection == .close) {
		352	try writer.writeAll("\r\nConnection: close");
		353	} else {
		354	try writer.writeAll("\r\nConnection: keep-alive");
		355	}
		356	try writer.writeAll("\r\nAccept-Encoding: gzip, deflate, zstd");
		357
		358	switch (headers.transfer_encoding) {
		359	.chunked => try writer.writeAll("\r\nTransfer-Encoding: chunked"),
		360	.content_length => \|content_length\| try writer.print("\r\nContent-Length: {d}", .{content_length}),
		361	.none => {},
		362	}
		363
		364	for (headers.custom) \|header\| {
		365	try writer.writeAll("\r\n");
		366	try writer.writeAll(header.name);
		367	try writer.writeAll(": ");
		368	try writer.writeAll(header.value);
		369	}
		370
		371	try writer.writeAll("\r\n\r\n");
		372
		373	try buffered.flush();
915	}	374	}
916		375
917	return req;	376	return req;
...	@@ -925,5 +384,7 @@ test {	...	@@ -925,5 +384,7 @@ test {
925	return error.SkipZigTest;	384	return error.SkipZigTest;
926	}	385	}
927		386
		387	if (builtin.os.tag == .wasi) return error.SkipZigTest;
		388
928	_ = Request;	389	_ = Request;
929	}	390	}

lib/std/http/Client/Request.zig created+482

...	@@ -0,0 +1,482 @@
1	const std = @import("std");
2	const http = std.http;
3	const Uri = std.Uri;
4	const mem = std.mem;
5	const assert = std.debug.assert;
6
7	const Client = @import("../Client.zig");
8	const Connection = Client.Connection;
9	const ConnectionNode = Client.ConnectionPool.Node;
10	const Response = @import("Response.zig");
11
12	const Request = @This();
13
14	const read_buffer_size = 8192;
15	const ReadBufferIndex = std.math.IntFittingRange(0, read_buffer_size);
16
17	uri: Uri,
18	client: *Client,
19	connection: *ConnectionNode,
20	response: Response,
21	/// These are stored in Request so that they are available when following
22	/// redirects.
23	headers: Headers,
24
25	redirects_left: u32,
26	handle_redirects: bool,
27	compression_init: bool,
28
29	/// Used as a allocator for resolving redirects locations.
30	arena: std.heap.ArenaAllocator,
31
32	/// Read buffer for the connection. This is used to pull in large amounts of data from the connection even if the user asks for a small amount. This can probably be removed with careful planning.
33	read_buffer: [read_buffer_size]u8 = undefined,
34	read_buffer_start: ReadBufferIndex = 0,
35	read_buffer_len: ReadBufferIndex = 0,
36
37	pub const RequestTransfer = union(enum) {
38	content_length: u64,
39	chunked: void,
40	none: void,
41	};
42
43	pub const Headers = struct {
44	version: http.Version = .@"HTTP/1.1",
45	method: http.Method = .GET,
46	user_agent: []const u8 = "zig (std.http)",
47	connection: http.Connection = .keep_alive,
48	transfer_encoding: RequestTransfer = .none,
49
50	custom: []const http.CustomHeader = &[_]http.CustomHeader{},
51	};
52
53	pub const Options = struct {
54	handle_redirects: bool = true,
55	max_redirects: u32 = 3,
56	header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
57
58	pub const HeaderStrategy = union(enum) {
59	/// In this case, the client's Allocator will be used to store the
60	/// entire HTTP header. This value is the maximum total size of
61	/// HTTP headers allowed, otherwise
62	/// error.HttpHeadersExceededSizeLimit is returned from read().
63	dynamic: usize,
64	/// This is used to store the entire HTTP header. If the HTTP
65	/// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
66	/// is returned from read(). When this is used, `error.OutOfMemory`
67	/// cannot be returned from `read()`.
68	static: []u8,
69	};
70	};
71
72	/// Frees all resources associated with the request.
73	pub fn deinit(req: *Request) void {
74	switch (req.response.compression) {
75	.none => {},
76	.deflate => \|*deflate\| deflate.deinit(),
77	.gzip => \|*gzip\| gzip.deinit(),
78	.zstd => \|*zstd\| zstd.deinit(),
79	}
80
81	if (req.response.header_bytes_owned) {
82	req.response.header_bytes.deinit(req.client.allocator);
83	}
84
85	if (!req.response.done) {
86	// If the response wasn't fully read, then we need to close the connection.
87	req.connection.data.closing = true;
88	req.client.connection_pool.release(req.client, req.connection);
89	}
90
91	req.arena.deinit();
92	req.* = undefined;
93	}
94
95	pub const ReadRawError = Connection.ReadError \|\| Uri.ParseError \|\| Client.RequestError \|\| error{
96	UnexpectedEndOfStream,
97	TooManyHttpRedirects,
98	HttpRedirectMissingLocation,
99	HttpHeadersInvalid,
100	};
101
102	pub const ReaderRaw = std.io.Reader(*Request, ReadRawError, readRaw);
103
104	/// Read from the underlying stream, without decompressing or parsing the headers. Must be called
105	/// after waitForCompleteHead() has returned successfully.
106	pub fn readRaw(req: *Request, buffer: []u8) ReadRawError!usize {
107	assert(req.response.state.isContent());
108
109	var index: usize = 0;
110	while (index == 0) {
111	const amt = try req.readRawAdvanced(buffer[index..]);
112	if (amt == 0 and req.response.done) break;
113	index += amt;
114	}
115
116	return index;
117	}
118
119	fn checkForCompleteHead(req: *Request, buffer: []u8) !usize {
120	switch (req.response.state) {
121	.invalid => unreachable,
122	.start, .seen_r, .seen_rn, .seen_rnr => {},
123	else => return 0, // No more headers to read.
124	}
125
126	const i = req.response.findHeadersEnd(buffer[0..]);
127	if (req.response.state == .invalid) return error.HttpHeadersInvalid;
128
129	const headers_data = buffer[0..i];
130	if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
131	return error.HttpHeadersExceededSizeLimit;
132	}
133	try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
134
135	if (req.response.state == .finished) {
136	req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
137
138	if (req.response.headers.upgrade) \|_\| {
139	req.connection.data.closing = false;
140	req.response.done = true;
141	return i;
142	}
143
144	if (req.response.headers.connection == .keep_alive) {
145	req.connection.data.closing = false;
146	} else {
147	req.connection.data.closing = true;
148	}
149
150	if (req.response.headers.transfer_encoding) \|transfer_encoding\| {
151	switch (transfer_encoding) {
152	.chunked => {
153	req.response.next_chunk_length = 0;
154	req.response.state = .chunk_size;
155	},
156	}
157	} else if (req.response.headers.content_length) \|content_length\| {
158	req.response.next_chunk_length = content_length;
159
160	if (content_length == 0) req.response.done = true;
161	} else {
162	req.response.done = true;
163	}
164
165	return i;
166	}
167
168	return 0;
169	}
170
171	pub const WaitForCompleteHeadError = ReadRawError \|\| error{
172	UnexpectedEndOfStream,
173
174	HttpHeadersExceededSizeLimit,
175	ShortHttpStatusLine,
176	BadHttpVersion,
177	HttpHeaderContinuationsUnsupported,
178	HttpTransferEncodingUnsupported,
179	HttpConnectionHeaderUnsupported,
180	};
181
182	/// Reads a complete response head. Any leftover data is stored in the request. This function is idempotent.
183	pub fn waitForCompleteHead(req: *Request) WaitForCompleteHeadError!void {
184	if (req.response.state.isContent()) return;
185
186	while (true) {
187	const nread = try req.connection.data.read(req.read_buffer[0..]);
188	const amt = try checkForCompleteHead(req, req.read_buffer[0..nread]);
189
190	if (amt != 0) {
191	req.read_buffer_start = @intCast(ReadBufferIndex, amt);
192	req.read_buffer_len = @intCast(ReadBufferIndex, nread);
193	return;
194	} else if (nread == 0) {
195	return error.UnexpectedEndOfStream;
196	}
197	}
198	}
199
200	/// This one can return 0 without meaning EOF.
201	fn readRawAdvanced(req: *Request, buffer: []u8) !usize {
202	assert(req.response.state.isContent());
203	if (req.response.done) return 0;
204
205	// var in: []const u8 = undefined;
206	if (req.read_buffer_start == req.read_buffer_len) {
207	const nread = try req.connection.data.read(req.read_buffer[0..]);
208	if (nread == 0) return error.UnexpectedEndOfStream;
209
210	req.read_buffer_start = 0;
211	req.read_buffer_len = @intCast(ReadBufferIndex, nread);
212	}
213
214	var out_index: usize = 0;
215	while (true) {
216	switch (req.response.state) {
217	.invalid, .start, .seen_r, .seen_rn, .seen_rnr => unreachable,
218	.finished => {
219	// TODO https://github.com/ziglang/zig/issues/14039
220	const buf_avail = req.read_buffer_len - req.read_buffer_start;
221	const data_avail = req.response.next_chunk_length;
222	const out_avail = buffer.len;
223
224	if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
225	const can_read = @intCast(usize, @min(buf_avail, data_avail));
226	req.response.next_chunk_length -= can_read;
227
228	if (req.response.next_chunk_length == 0) {
229	req.client.connection_pool.release(req.client, req.connection);
230	req.connection = undefined;
231	req.response.done = true;
232	}
233
234	return 0; // skip over as much data as possible
235	}
236
237	const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
238	req.response.next_chunk_length -= can_read;
239
240	mem.copy(u8, buffer[0..], req.read_buffer[req.read_buffer_start..][0..can_read]);
241	req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
242
243	if (req.response.next_chunk_length == 0) {
244	req.client.connection_pool.release(req.client, req.connection);
245	req.connection = undefined;
246	req.response.done = true;
247	}
248
249	return can_read;
250	},
251	.chunk_size_prefix_r => switch (req.read_buffer_len - req.read_buffer_start) {
252	0 => return out_index,
253	1 => switch (req.read_buffer[req.read_buffer_start]) {
254	'\r' => {
255	req.response.state = .chunk_size_prefix_n;
256	return out_index;
257	},
258	else => {
259	req.response.state = .invalid;
260	return error.HttpHeadersInvalid;
261	},
262	},
263	else => switch (int16(req.read_buffer[req.read_buffer_start..][0..2])) {
264	int16("\r\n") => {
265	req.read_buffer_start += 2;
266	req.response.state = .chunk_size;
267	continue;
268	},
269	else => {
270	req.response.state = .invalid;
271	return error.HttpHeadersInvalid;
272	},
273	},
274	},
275	.chunk_size_prefix_n => switch (req.read_buffer_len - req.read_buffer_start) {
276	0 => return out_index,
277	else => switch (req.read_buffer[req.read_buffer_start]) {
278	'\n' => {
279	req.read_buffer_start += 1;
280	req.response.state = .chunk_size;
281	continue;
282	},
283	else => {
284	req.response.state = .invalid;
285	return error.HttpHeadersInvalid;
286	},
287	},
288	},
289	.chunk_size, .chunk_r => {
290	const i = req.response.findChunkedLen(req.read_buffer[req.read_buffer_start..req.read_buffer_len]);
291	switch (req.response.state) {
292	.invalid => return error.HttpHeadersInvalid,
293	.chunk_data => {
294	if (req.response.next_chunk_length == 0) {
295	req.response.done = true;
296	req.client.connection_pool.release(req.client, req.connection);
297	req.connection = undefined;
298
299	return out_index;
300	}
301
302	req.read_buffer_start += @intCast(ReadBufferIndex, i);
303	continue;
304	},
305	.chunk_size => return out_index,
306	else => unreachable,
307	}
308	},
309	.chunk_data => {
310	// TODO https://github.com/ziglang/zig/issues/14039
311	const buf_avail = req.read_buffer_len - req.read_buffer_start;
312	const data_avail = req.response.next_chunk_length;
313	const out_avail = buffer.len - out_index;
314
315	if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
316	const can_read = @intCast(usize, @min(buf_avail, data_avail));
317	req.response.next_chunk_length -= can_read;
318
319	if (req.response.next_chunk_length == 0) {
320	req.client.connection_pool.release(req.client, req.connection);
321	req.connection = undefined;
322	req.response.done = true;
323	continue;
324	}
325
326	return 0; // skip over as much data as possible
327	}
328
329	const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
330	req.response.next_chunk_length -= can_read;
331
332	mem.copy(u8, buffer[out_index..], req.read_buffer[req.read_buffer_start..][0..can_read]);
333	req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
334	out_index += can_read;
335
336	if (req.response.next_chunk_length == 0) {
337	req.response.state = .chunk_size_prefix_r;
338
339	continue;
340	}
341
342	return out_index;
343	},
344	}
345	}
346	}
347
348	pub const ReadError = Client.DeflateDecompressor.Error \|\| Client.GzipDecompressor.Error \|\| Client.ZstdDecompressor.Error \|\| WaitForCompleteHeadError \|\| error{ BadHeader, InvalidCompression, StreamTooLong, InvalidWindowSize, CompressionNotSupported };
349
350	pub const Reader = std.io.Reader(*Request, ReadError, read);
351
352	pub fn reader(req: *Request) Reader {
353	return .{ .context = req };
354	}
355
356	pub fn read(req: *Request, buffer: []u8) ReadError!usize {
357	while (true) {
358	if (!req.response.state.isContent()) try req.waitForCompleteHead();
359
360	if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
361	assert(try req.readRaw(buffer) == 0);
362
363	if (req.redirects_left == 0) return error.TooManyHttpRedirects;
364
365	const location = req.response.headers.location orelse
366	return error.HttpRedirectMissingLocation;
367	const new_url = Uri.parse(location) catch try Uri.parseWithoutScheme(location);
368
369	var new_arena = std.heap.ArenaAllocator.init(req.client.allocator);
370	const resolved_url = try req.uri.resolve(new_url, false, new_arena.allocator());
371	errdefer new_arena.deinit();
372
373	req.arena.deinit();
374	req.arena = new_arena;
375
376	const new_req = try req.client.request(resolved_url, req.headers, .{
377	.max_redirects = req.redirects_left - 1,
378	.header_strategy = if (req.response.header_bytes_owned) .{
379	.dynamic = req.response.max_header_bytes,
380	} else .{
381	.static = req.response.header_bytes.unusedCapacitySlice(),
382	},
383	});
384	req.deinit();
385	req.* = new_req;
386	} else {
387	break;
388	}
389	}
390
391	if (req.response.compression == .none) {
392	if (req.response.headers.transfer_compression) \|compression\| {
393	switch (compression) {
394	.compress => return error.CompressionNotSupported,
395	.deflate => req.response.compression = .{
396	.deflate = try std.compress.zlib.zlibStream(req.client.allocator, ReaderRaw{ .context = req }),
397	},
398	.gzip => req.response.compression = .{
399	.gzip = try std.compress.gzip.decompress(req.client.allocator, ReaderRaw{ .context = req }),
400	},
401	.zstd => req.response.compression = .{
402	.zstd = std.compress.zstd.decompressStream(req.client.allocator, ReaderRaw{ .context = req }),
403	},
404	}
405	}
406	}
407
408	return switch (req.response.compression) {
409	.deflate => \|*deflate\| try deflate.read(buffer),
410	.gzip => \|*gzip\| try gzip.read(buffer),
411	.zstd => \|*zstd\| try zstd.read(buffer),
412	else => try req.readRaw(buffer),
413	};
414	}
415
416	pub fn readAll(req: *Request, buffer: []u8) !usize {
417	var index: usize = 0;
418	while (index < buffer.len) {
419	const amt = try read(req, buffer[index..]);
420	if (amt == 0) break;
421	index += amt;
422	}
423	return index;
424	}
425
426	pub const WriteError = Connection.WriteError \|\| error{MessageTooLong};
427
428	pub const Writer = std.io.Writer(*Request, WriteError, write);
429
430	pub fn writer(req: *Request) Writer {
431	return .{ .context = req };
432	}
433
434	/// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.
435	pub fn write(req: *Request, bytes: []const u8) !usize {
436	switch (req.headers.transfer_encoding) {
437	.chunked => {
438	try req.connection.data.writer().print("{x}\r\n", .{bytes.len});
439	try req.connection.data.writeAll(bytes);
440	try req.connection.data.writeAll("\r\n");
441
442	return bytes.len;
443	},
444	.content_length => \|*len\| {
445	if (len.* < bytes.len) return error.MessageTooLong;
446
447	const amt = try req.connection.data.write(bytes);
448	len.* -= amt;
449	return amt;
450	},
451	.none => return error.NotWriteable,
452	}
453	}
454
455	/// Finish the body of a request. This notifies the server that you have no more data to send.
456	pub fn finish(req: *Request) !void {
457	switch (req.headers.transfer_encoding) {
458	.chunked => try req.connection.data.writeAll("0\r\n"),
459	.content_length => \|len\| if (len != 0) return error.MessageNotCompleted,
460	.none => {},
461	}
462	}
463
464	inline fn int16(array: *const [2]u8) u16 {
465	return @bitCast(u16, array.*);
466	}
467
468	inline fn int32(array: *const [4]u8) u32 {
469	return @bitCast(u32, array.*);
470	}
471
472	inline fn int64(array: *const [8]u8) u64 {
473	return @bitCast(u64, array.*);
474	}
475
476	test {
477	const builtin = @import("builtin");
478
479	if (builtin.os.tag == .wasi) return error.SkipZigTest;
480
481	_ = Response;
482	}

lib/std/http/Client/Response.zig created+509

...	@@ -0,0 +1,509 @@
1	const std = @import("std");
2	const http = std.http;
3	const mem = std.mem;
4	const testing = std.testing;
5	const assert = std.debug.assert;
6
7	const Client = @import("../Client.zig");
8	const Response = @This();
9
10	headers: Headers,
11	state: State,
12	header_bytes_owned: bool,
13	/// This could either be a fixed buffer provided by the API user or it
14	/// could be our own array list.
15	header_bytes: std.ArrayListUnmanaged(u8),
16	max_header_bytes: usize,
17	next_chunk_length: u64,
18	done: bool = false,
19
20	compression: union(enum) {
21	deflate: Client.DeflateDecompressor,
22	gzip: Client.GzipDecompressor,
23	zstd: Client.ZstdDecompressor,
24	none: void,
25	} = .none,
26
27	pub const Headers = struct {
28	status: http.Status,
29	version: http.Version,
30	location: ?[]const u8 = null,
31	content_length: ?u64 = null,
32	transfer_encoding: ?http.TransferEncoding = null,
33	transfer_compression: ?http.ContentEncoding = null,
34	connection: http.Connection = .close,
35	upgrade: ?[]const u8 = null,
36
37	number_of_headers: usize = 0,
38
39	pub fn parse(bytes: []const u8) !Headers {
40	var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
41
42	const first_line = it.first();
43	if (first_line.len < 12)
44	return error.ShortHttpStatusLine;
45
46	const version: http.Version = switch (int64(first_line[0..8])) {
47	int64("HTTP/1.0") => .@"HTTP/1.0",
48	int64("HTTP/1.1") => .@"HTTP/1.1",
49	else => return error.BadHttpVersion,
50	};
51	if (first_line[8] != ' ') return error.HttpHeadersInvalid;
52	const status = @intToEnum(http.Status, parseInt3(first_line[9..12].*));
53
54	var headers: Headers = .{
55	.version = version,
56	.status = status,
57	};
58
59	while (it.next()) \|line\| {
60	headers.number_of_headers += 1;
61
62	if (line.len == 0) return error.HttpHeadersInvalid;
63	switch (line[0]) {
64	' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
65	else => {},
66	}
67	var line_it = mem.split(u8, line, ": ");
68	const header_name = line_it.first();
69	const header_value = line_it.rest();
70	if (std.ascii.eqlIgnoreCase(header_name, "location")) {
71	if (headers.location != null) return error.HttpHeadersInvalid;
72	headers.location = header_value;
73	} else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
74	if (headers.content_length != null) return error.HttpHeadersInvalid;
75	headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
76	} else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
77	if (headers.transfer_encoding != null or headers.transfer_compression != null) return error.HttpHeadersInvalid;
78
79	// Transfer-Encoding: second, first
80	// Transfer-Encoding: deflate, chunked
81	var iter = std.mem.splitBackwards(u8, header_value, ",");
82
83	if (iter.next()) \|first\| {
84	const trimmed = std.mem.trim(u8, first, " ");
85
86	if (std.meta.stringToEnum(http.TransferEncoding, trimmed)) \|te\| {
87	headers.transfer_encoding = te;
88	} else if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
89	headers.transfer_compression = ce;
90	} else {
91	return error.HttpTransferEncodingUnsupported;
92	}
93	}
94
95	if (iter.next()) \|second\| {
96	if (headers.transfer_compression != null) return error.HttpTransferEncodingUnsupported;
97
98	const trimmed = std.mem.trim(u8, second, " ");
99
100	if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
101	headers.transfer_compression = ce;
102	} else {
103	return error.HttpTransferEncodingUnsupported;
104	}
105	}
106
107	if (iter.next()) \|_\| return error.HttpTransferEncodingUnsupported;
108	} else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
109	if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
110
111	const trimmed = std.mem.trim(u8, header_value, " ");
112
113	if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
114	headers.transfer_compression = ce;
115	} else {
116	return error.HttpTransferEncodingUnsupported;
117	}
118	} else if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
119	if (std.ascii.eqlIgnoreCase(header_value, "keep-alive")) {
120	headers.connection = .keep_alive;
121	} else if (std.ascii.eqlIgnoreCase(header_value, "close")) {
122	headers.connection = .close;
123	} else {
124	return error.HttpConnectionHeaderUnsupported;
125	}
126	} else if (std.ascii.eqlIgnoreCase(header_name, "upgrade")) {
127	headers.upgrade = header_value;
128	}
129	}
130
131	return headers;
132	}
133
134	test "parse headers" {
135	const example =
136	"HTTP/1.1 301 Moved Permanently\r\n" ++
137	"Location: https://www.example.com/\r\n" ++
138	"Content-Type: text/html; charset=UTF-8\r\n" ++
139	"Content-Length: 220\r\n\r\n";
140	const parsed = try Headers.parse(example);
141	try testing.expectEqual(http.Version.@"HTTP/1.1", parsed.version);
142	try testing.expectEqual(http.Status.moved_permanently, parsed.status);
143	try testing.expectEqualStrings("https://www.example.com/", parsed.location orelse
144	return error.TestFailed);
145	try testing.expectEqual(@as(?u64, 220), parsed.content_length);
146	}
147
148	test "header continuation" {
149	const example =
150	"HTTP/1.0 200 OK\r\n" ++
151	"Content-Type: text/html;\r\n charset=UTF-8\r\n" ++
152	"Content-Length: 220\r\n\r\n";
153	try testing.expectError(
154	error.HttpHeaderContinuationsUnsupported,
155	Headers.parse(example),
156	);
157	}
158
159	test "extra content length" {
160	const example =
161	"HTTP/1.0 200 OK\r\n" ++
162	"Content-Length: 220\r\n" ++
163	"Content-Type: text/html; charset=UTF-8\r\n" ++
164	"content-length: 220\r\n\r\n";
165	try testing.expectError(
166	error.HttpHeadersInvalid,
167	Headers.parse(example),
168	);
169	}
170	};
171
172	inline fn int16(array: *const [2]u8) u16 {
173	return @bitCast(u16, array.*);
174	}
175
176	inline fn int32(array: *const [4]u8) u32 {
177	return @bitCast(u32, array.*);
178	}
179
180	inline fn int64(array: *const [8]u8) u64 {
181	return @bitCast(u64, array.*);
182	}
183
184	pub const State = enum {
185	/// Begin header parsing states.
186	invalid,
187	start,
188	seen_r,
189	seen_rn,
190	seen_rnr,
191	finished,
192	/// Begin transfer-encoding: chunked parsing states.
193	chunk_size_prefix_r,
194	chunk_size_prefix_n,
195	chunk_size,
196	chunk_r,
197	chunk_data,
198
199	pub fn isContent(self: State) bool {
200	return switch (self) {
201	.invalid, .start, .seen_r, .seen_rn, .seen_rnr => false,
202	.finished, .chunk_size_prefix_r, .chunk_size_prefix_n, .chunk_size, .chunk_r, .chunk_data => true,
203	};
204	}
205	};
206
207	pub fn initDynamic(max: usize) Response {
208	return .{
209	.state = .start,
210	.headers = undefined,
211	.header_bytes = .{},
212	.max_header_bytes = max,
213	.header_bytes_owned = true,
214	.next_chunk_length = undefined,
215	};
216	}
217
218	pub fn initStatic(buf: []u8) Response {
219	return .{
220	.state = .start,
221	.headers = undefined,
222	.header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
223	.max_header_bytes = buf.len,
224	.header_bytes_owned = false,
225	.next_chunk_length = undefined,
226	};
227	}
228
229	/// Returns how many bytes are part of HTTP headers. Always less than or
230	/// equal to bytes.len. If the amount returned is less than bytes.len, it
231	/// means the headers ended and the first byte after the double \r\n\r\n is
232	/// located at `bytes[result]`.
233	pub fn findHeadersEnd(r: *Response, bytes: []const u8) usize {
234	var index: usize = 0;
235
236	// TODO: https://github.com/ziglang/zig/issues/8220
237	state: while (true) {
238	switch (r.state) {
239	.invalid => unreachable,
240	.finished => unreachable,
241	.start => while (true) {
242	switch (bytes.len - index) {
243	0 => return index,
244	1 => {
245	if (bytes[index] == '\r')
246	r.state = .seen_r;
247	return index + 1;
248	},
249	2 => {
250	if (int16(bytes[index..][0..2]) == int16("\r\n")) {
251	r.state = .seen_rn;
252	} else if (bytes[index + 1] == '\r') {
253	r.state = .seen_r;
254	}
255	return index + 2;
256	},
257	3 => {
258	if (int16(bytes[index..][0..2]) == int16("\r\n") and
259	bytes[index + 2] == '\r')
260	{
261	r.state = .seen_rnr;
262	} else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n")) {
263	r.state = .seen_rn;
264	} else if (bytes[index + 2] == '\r') {
265	r.state = .seen_r;
266	}
267	return index + 3;
268	},
269	4...15 => {
270	if (int32(bytes[index..][0..4]) == int32("\r\n\r\n")) {
271	r.state = .finished;
272	return index + 4;
273	} else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n") and
274	bytes[index + 3] == '\r')
275	{
276	r.state = .seen_rnr;
277	index += 4;
278	continue :state;
279	} else if (int16(bytes[index + 2 ..][0..2]) == int16("\r\n")) {
280	r.state = .seen_rn;
281	index += 4;
282	continue :state;
283	} else if (bytes[index + 3] == '\r') {
284	r.state = .seen_r;
285	index += 4;
286	continue :state;
287	}
288	index += 4;
289	continue;
290	},
291	else => {
292	const chunk = bytes[index..][0..16];
293	const v: @Vector(16, u8) = chunk.*;
294	const matches_r = v == @splat(16, @as(u8, '\r'));
295	const iota = std.simd.iota(u8, 16);
296	const default = @splat(16, @as(u8, 16));
297	const sub_index = @reduce(.Min, @select(u8, matches_r, iota, default));
298	switch (sub_index) {
299	0...12 => {
300	index += sub_index + 4;
301	if (int32(chunk[sub_index..][0..4]) == int32("\r\n\r\n")) {
302	r.state = .finished;
303	return index;
304	}
305	continue;
306	},
307	13 => {
308	index += 16;
309	if (int16(chunk[14..][0..2]) == int16("\n\r")) {
310	r.state = .seen_rnr;
311	continue :state;
312	}
313	continue;
314	},
315	14 => {
316	index += 16;
317	if (chunk[15] == '\n') {
318	r.state = .seen_rn;
319	continue :state;
320	}
321	continue;
322	},
323	15 => {
324	r.state = .seen_r;
325	index += 16;
326	continue :state;
327	},
328	16 => {
329	index += 16;
330	continue;
331	},
332	else => unreachable,
333	}
334	},
335	}
336	},
337
338	.seen_r => switch (bytes.len - index) {
339	0 => return index,
340	1 => {
341	switch (bytes[index]) {
342	'\n' => r.state = .seen_rn,
343	'\r' => r.state = .seen_r,
344	else => r.state = .start,
345	}
346	return index + 1;
347	},
348	2 => {
349	if (int16(bytes[index..][0..2]) == int16("\n\r")) {
350	r.state = .seen_rnr;
351	return index + 2;
352	}
353	r.state = .start;
354	return index + 2;
355	},
356	else => {
357	if (int16(bytes[index..][0..2]) == int16("\n\r") and
358	bytes[index + 2] == '\n')
359	{
360	r.state = .finished;
361	return index + 3;
362	}
363	index += 3;
364	r.state = .start;
365	continue :state;
366	},
367	},
368	.seen_rn => switch (bytes.len - index) {
369	0 => return index,
370	1 => {
371	switch (bytes[index]) {
372	'\r' => r.state = .seen_rnr,
373	else => r.state = .start,
374	}
375	return index + 1;
376	},
377	else => {
378	if (int16(bytes[index..][0..2]) == int16("\r\n")) {
379	r.state = .finished;
380	return index + 2;
381	}
382	index += 2;
383	r.state = .start;
384	continue :state;
385	},
386	},
387	.seen_rnr => switch (bytes.len - index) {
388	0 => return index,
389	else => {
390	if (bytes[index] == '\n') {
391	r.state = .finished;
392	return index + 1;
393	}
394	index += 1;
395	r.state = .start;
396	continue :state;
397	},
398	},
399	.chunk_size_prefix_r => unreachable,
400	.chunk_size_prefix_n => unreachable,
401	.chunk_size => unreachable,
402	.chunk_r => unreachable,
403	.chunk_data => unreachable,
404	}
405
406	return index;
407	}
408	}
409
410	pub fn findChunkedLen(r: *Response, bytes: []const u8) usize {
411	var i: usize = 0;
412	if (r.state == .chunk_size) {
413	while (i < bytes.len) : (i += 1) {
414	const digit = switch (bytes[i]) {
415	'0'...'9' => \|b\| b - '0',
416	'A'...'Z' => \|b\| b - 'A' + 10,
417	'a'...'z' => \|b\| b - 'a' + 10,
418	'\r' => {
419	r.state = .chunk_r;
420	i += 1;
421	break;
422	},
423	else => {
424	r.state = .invalid;
425	return i;
426	},
427	};
428	const mul = @mulWithOverflow(r.next_chunk_length, 16);
429	if (mul[1] != 0) {
430	r.state = .invalid;
431	return i;
432	}
433	const add = @addWithOverflow(mul[0], digit);
434	if (add[1] != 0) {
435	r.state = .invalid;
436	return i;
437	}
438	r.next_chunk_length = add[0];
439	} else {
440	return i;
441	}
442	}
443	assert(r.state == .chunk_r);
444	if (i == bytes.len) return i;
445
446	if (bytes[i] == '\n') {
447	r.state = .chunk_data;
448	return i + 1;
449	} else {
450	r.state = .invalid;
451	return i;
452	}
453	}
454
455	fn parseInt3(nnn: @Vector(3, u8)) u10 {
456	const zero: @Vector(3, u8) = .{ '0', '0', '0' };
457	const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
458	return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
459	}
460
461	test parseInt3 {
462	const expectEqual = std.testing.expectEqual;
463	try expectEqual(@as(u10, 0), parseInt3("000".*));
464	try expectEqual(@as(u10, 418), parseInt3("418".*));
465	try expectEqual(@as(u10, 999), parseInt3("999".*));
466	}
467
468	test "find headers end basic" {
469	var buffer: [1]u8 = undefined;
470	var r = Response.initStatic(&buffer);
471	try testing.expectEqual(@as(usize, 10), r.findHeadersEnd("HTTP/1.1 4"));
472	try testing.expectEqual(@as(usize, 2), r.findHeadersEnd("18"));
473	try testing.expectEqual(@as(usize, 8), r.findHeadersEnd(" lol\r\n\r\nblah blah"));
474	}
475
476	test "find headers end vectorized" {
477	var buffer: [1]u8 = undefined;
478	var r = Response.initStatic(&buffer);
479	const example =
480	"HTTP/1.1 301 Moved Permanently\r\n" ++
481	"Location: https://www.example.com/\r\n" ++
482	"Content-Type: text/html; charset=UTF-8\r\n" ++
483	"Content-Length: 220\r\n" ++
484	"\r\ncontent";
485	try testing.expectEqual(@as(usize, 131), r.findHeadersEnd(example));
486	}
487
488	test "find headers end bug" {
489	var buffer: [1]u8 = undefined;
490	var r = Response.initStatic(&buffer);
491	const trail = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
492	const example =
493	"HTTP/1.1 200 OK\r\n" ++
494	"Access-Control-Allow-Origin: https://render.githubusercontent.com\r\n" ++
495	"content-disposition: attachment; filename=zig-0.10.0.tar.gz\r\n" ++
496	"Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'; sandbox\r\n" ++
497	"Content-Type: application/x-gzip\r\n" ++
498	"ETag: \"bfae0af6b01c7c0d89eb667cb5f0e65265968aeebda2689177e6b26acd3155ca\"\r\n" ++
499	"Strict-Transport-Security: max-age=31536000\r\n" ++
500	"Vary: Authorization,Accept-Encoding,Origin\r\n" ++
501	"X-Content-Type-Options: nosniff\r\n" ++
502	"X-Frame-Options: deny\r\n" ++
503	"X-XSS-Protection: 1; mode=block\r\n" ++
504	"Date: Fri, 06 Jan 2023 22:26:22 GMT\r\n" ++
505	"Transfer-Encoding: chunked\r\n" ++
506	"X-GitHub-Request-Id: 89C6:17E9:A7C9E:124B51:63B8A00E\r\n" ++
507	"connection: close\r\n\r\n" ++ trail;
508	try testing.expectEqual(@as(usize, example.len - trail.len), r.findHeadersEnd(example));
509	}

lib/std/net.zig+31-3

...	@@ -702,8 +702,10 @@ pub const AddressList = struct {	...	@@ -702,8 +702,10 @@ pub const AddressList = struct {
702	}	702	}
703	};	703	};
704		704
		705	pub const TcpConnectToHostError = GetAddressListError \|\| TcpConnectToAddressError;
		706
705	/// All memory allocated with `allocator` will be freed before this function returns.	707	/// All memory allocated with `allocator` will be freed before this function returns.
706	pub fn tcpConnectToHost(allocator: mem.Allocator, name: []const u8, port: u16) !Stream {	708	pub fn tcpConnectToHost(allocator: mem.Allocator, name: []const u8, port: u16) TcpConnectToHostError!Stream {
707	const list = try getAddressList(allocator, name, port);	709	const list = try getAddressList(allocator, name, port);
708	defer list.deinit();	710	defer list.deinit();
709		711
...	@@ -720,7 +722,9 @@ pub fn tcpConnectToHost(allocator: mem.Allocator, name: []const u8, port: u16) !	...	@@ -720,7 +722,9 @@ pub fn tcpConnectToHost(allocator: mem.Allocator, name: []const u8, port: u16) !
720	return std.os.ConnectError.ConnectionRefused;	722	return std.os.ConnectError.ConnectionRefused;
721	}	723	}
722		724
723	pub fn tcpConnectToAddress(address: Address) !Stream {	725	pub const TcpConnectToAddressError = std.os.SocketError \|\| std.os.ConnectError;
		726
		727	pub fn tcpConnectToAddress(address: Address) TcpConnectToAddressError!Stream {
724	const nonblock = if (std.io.is_async) os.SOCK.NONBLOCK else 0;	728	const nonblock = if (std.io.is_async) os.SOCK.NONBLOCK else 0;
725	const sock_flags = os.SOCK.STREAM \| nonblock \|	729	const sock_flags = os.SOCK.STREAM \| nonblock \|
726	(if (builtin.target.os.tag == .windows) 0 else os.SOCK.CLOEXEC);	730	(if (builtin.target.os.tag == .windows) 0 else os.SOCK.CLOEXEC);
...	@@ -737,8 +741,32 @@ pub fn tcpConnectToAddress(address: Address) !Stream {	...	@@ -737,8 +741,32 @@ pub fn tcpConnectToAddress(address: Address) !Stream {
737	return Stream{ .handle = sockfd };	741	return Stream{ .handle = sockfd };
738	}	742	}
739		743
		744	const GetAddressListError = std.mem.Allocator.Error \|\| std.fs.File.OpenError \|\| std.fs.File.ReadError \|\| std.os.SocketError \|\| std.os.BindError \|\| error{
		745	// TODO: break this up into error sets from the various underlying functions
		746
		747	TemporaryNameServerFailure,
		748	NameServerFailure,
		749	AddressFamilyNotSupported,
		750	UnknownHostName,
		751	ServiceUnavailable,
		752	Unexpected,
		753
		754	HostLacksNetworkAddresses,
		755
		756	InvalidCharacter,
		757	InvalidEnd,
		758	NonCanonical,
		759	Overflow,
		760	Incomplete,
		761	InvalidIpv4Mapping,
		762	InvalidIPAddressFormat,
		763
		764	InterfaceNotFound,
		765	FileSystem,
		766	};
		767
740	/// Call `AddressList.deinit` on the result.	768	/// Call `AddressList.deinit` on the result.
741	pub fn getAddressList(allocator: mem.Allocator, name: []const u8, port: u16) !*AddressList {	769	pub fn getAddressList(allocator: mem.Allocator, name: []const u8, port: u16) GetAddressListError!*AddressList {
742	const result = blk: {	770	const result = blk: {
743	var arena = std.heap.ArenaAllocator.init(allocator);	771	var arena = std.heap.ArenaAllocator.init(allocator);
744	errdefer arena.deinit();	772	errdefer arena.deinit();

lib/std/std.zig+5

 @@ -185,6 +185,11 @@ pub const options = struct {
         options_override.keep_sigpipe
     else
         false;
+    pub const http_connection_pool_size = if (@hasDecl(options_override, "http_connection_pool_size"))
+        options_override.http_connection_pool_size
+    else
+        http.Client.default_connection_pool_size;
 };
 // This forces the start.zig file to be imported, and the comptime logic inside that