Commit 7c9ed45ac2fd580bbbb4b950ac350dd49fc7600e | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-03-09 22:02:01-05:00
committer	GitHub <noreply@github.com> 2023-03-09 22:02:01-05:00
log	7c9ed45ac2fd580bbbb4b950ac350dd49fc7600e
tree	027efed5e8b03a30b4b7a23e86ad5ca2bd0d75e5
parent	14590e956e06903ac408af57874d3b5a0d697670
parent	524e0cd987a52a60ce1014aa27cd73f99a3b9958
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14762 from truemedian/http-keepalive

std.http: add connection pooling, handle keep-alive and compressed content

8 files changed, 1434 insertions(+), 803 deletions(-)

lib
- std
  - Uri.zig
  - crypto
    tls
    Client.zig
  - http.zig
  - http
    Client.zig
    Client
    Request.zig
    Response.zig
  - net.zig
  - std.zig

lib/std/Uri.zig+96-13

...	...	@@ -16,15 +16,27 @@ fragment: ?[]const u8,
16	16
17	17	/// Applies URI encoding and replaces all reserved characters with their respective %XX code.
18	18	pub fn escapeString(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
	19	return escapeStringWithFn(allocator, input, isUnreserved);
	20	}
	21
	22	pub fn escapePath(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
	23	return escapeStringWithFn(allocator, input, isPathChar);
	24	}
	25
	26	pub fn escapeQuery(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
	27	return escapeStringWithFn(allocator, input, isQueryChar);
	28	}
	29
	30	pub fn escapeStringWithFn(allocator: std.mem.Allocator, input: []const u8, comptime keepUnescaped: fn (c: u8) bool) std.mem.Allocator.Error![]const u8 {
19	31	var outsize: usize = 0;
20	32	for (input) \|c\| {
21		outsize += if (isUnreserved(c)) @as(usize, 1) else 3;
	33	outsize += if (keepUnescaped(c)) @as(usize, 1) else 3;
22	34	}
23	35	var output = try allocator.alloc(u8, outsize);
24	36	var outptr: usize = 0;
25	37
26	38	for (input) \|c\| {
27		if (isUnreserved(c)) {
	39	if (keepUnescaped(c)) {
28	40	output[outptr] = c;
29	41	outptr += 1;
30	42	} else {
...	...	@@ -94,13 +106,14 @@ pub fn unescapeString(allocator: std.mem.Allocator, input: []const u8) error{Out
94	106
95	107	pub const ParseError = error{ UnexpectedCharacter, InvalidFormat, InvalidPort };
96	108
97		/// Parses the URI or returns an error.
	109	/// Parses the URI or returns an error. This function is not compliant, but is required to parse
	110	/// some forms of URIs in the wild. Such as HTTP Location headers.
98	111	/// The return value will contain unescaped strings pointing into the
99	112	/// original `text`. Each component that is provided, will be non-`null`.
100		pub fn parse(text: []const u8) ParseError!Uri {
	113	pub fn parseWithoutScheme(text: []const u8) ParseError!Uri {
101	114	var reader = SliceReader{ .slice = text };
102	115	var uri = Uri{
103		.scheme = reader.readWhile(isSchemeChar),
	116	.scheme = "",
104	117	.user = null,
105	118	.password = null,
106	119	.host = null,
...	...	@@ -110,14 +123,6 @@ pub fn parse(text: []const u8) ParseError!Uri {
110	123	.fragment = null,
111	124	};
112	125
113		// after the scheme, a ':' must appear
114		if (reader.get()) \|c\| {
115		if (c != ':')
116		return error.UnexpectedCharacter;
117		} else {
118		return error.InvalidFormat;
119		}
120
121	126	if (reader.peekPrefix("//")) { // authority part
122	127	std.debug.assert(reader.get().? == '/');
123	128	std.debug.assert(reader.get().? == '/');
...	...	@@ -179,6 +184,76 @@ pub fn parse(text: []const u8) ParseError!Uri {
179	184	return uri;
180	185	}
181	186
	187	/// Parses the URI or returns an error.
	188	/// The return value will contain unescaped strings pointing into the
	189	/// original `text`. Each component that is provided, will be non-`null`.
	190	pub fn parse(text: []const u8) ParseError!Uri {
	191	var reader = SliceReader{ .slice = text };
	192	const scheme = reader.readWhile(isSchemeChar);
	193
	194	// after the scheme, a ':' must appear
	195	if (reader.get()) \|c\| {
	196	if (c != ':')
	197	return error.UnexpectedCharacter;
	198	} else {
	199	return error.InvalidFormat;
	200	}
	201
	202	var uri = try parseWithoutScheme(reader.readUntilEof());
	203	uri.scheme = scheme;
	204
	205	return uri;
	206	}
	207
	208	/// Resolves a URI against a base URI, conforming to RFC 3986, Section 5.
	209	/// arena owns any memory allocated by this function.
	210	pub fn resolve(Base: Uri, R: Uri, strict: bool, arena: std.mem.Allocator) !Uri {
	211	var T: Uri = undefined;
	212
	213	if (R.scheme.len > 0 and !((!strict) and (std.mem.eql(u8, R.scheme, Base.scheme)))) {
	214	T.scheme = R.scheme;
	215	T.user = R.user;
	216	T.host = R.host;
	217	T.port = R.port;
	218	T.path = try std.fs.path.resolvePosix(arena, &.{ "/", R.path });
	219	T.query = R.query;
	220	} else {
	221	if (R.host) \|host\| {
	222	T.user = R.user;
	223	T.host = host;
	224	T.port = R.port;
	225	T.path = R.path;
	226	T.path = try std.fs.path.resolvePosix(arena, &.{ "/", R.path });
	227	T.query = R.query;
	228	} else {
	229	if (R.path.len == 0) {
	230	T.path = Base.path;
	231	if (R.query) \|query\| {
	232	T.query = query;
	233	} else {
	234	T.query = Base.query;
	235	}
	236	} else {
	237	if (R.path[0] == '/') {
	238	T.path = try std.fs.path.resolvePosix(arena, &.{ "/", R.path });
	239	} else {
	240	T.path = try std.fs.path.resolvePosix(arena, &.{ "/", Base.path, R.path });
	241	}
	242	T.query = R.query;
	243	}
	244
	245	T.user = Base.user;
	246	T.host = Base.host;
	247	T.port = Base.port;
	248	}
	249	T.scheme = Base.scheme;
	250	}
	251
	252	T.fragment = R.fragment;
	253
	254	return T;
	255	}
	256
182	257	const SliceReader = struct {
183	258	const Self = @This();
184	259
...	...	@@ -284,6 +359,14 @@ fn isPathSeparator(c: u8) bool {
284	359	};
285	360	}
286	361
	362	fn isPathChar(c: u8) bool {
	363	return isUnreserved(c) or isSubLimit(c) or c == '/' or c == ':' or c == '@';
	364	}
	365
	366	fn isQueryChar(c: u8) bool {
	367	return isPathChar(c) or c == '?';
	368	}
	369
287	370	fn isQuerySeparator(c: u8) bool {
288	371	return switch (c) {
289	372	'#' => true,

lib/std/crypto/tls/Client.zig+49-1

...	...	@@ -88,11 +88,59 @@ pub const StreamInterface = struct {
88	88	}
89	89	};
90	90
	91	pub fn InitError(comptime Stream: type) type {
	92	return std.mem.Allocator.Error \|\| Stream.WriteError \|\| Stream.ReadError \|\| error{
	93	InsufficientEntropy,
	94	DiskQuota,
	95	LockViolation,
	96	NotOpenForWriting,
	97	TlsAlert,
	98	TlsUnexpectedMessage,
	99	TlsIllegalParameter,
	100	TlsDecryptFailure,
	101	TlsRecordOverflow,
	102	TlsBadRecordMac,
	103	CertificateFieldHasInvalidLength,
	104	CertificateHostMismatch,
	105	CertificatePublicKeyInvalid,
	106	CertificateExpired,
	107	CertificateFieldHasWrongDataType,
	108	CertificateIssuerMismatch,
	109	CertificateNotYetValid,
	110	CertificateSignatureAlgorithmMismatch,
	111	CertificateSignatureAlgorithmUnsupported,
	112	CertificateSignatureInvalid,
	113	CertificateSignatureInvalidLength,
	114	CertificateSignatureNamedCurveUnsupported,
	115	CertificateSignatureUnsupportedBitCount,
	116	TlsCertificateNotVerified,
	117	TlsBadSignatureScheme,
	118	TlsBadRsaSignatureBitCount,
	119	InvalidEncoding,
	120	IdentityElement,
	121	SignatureVerificationFailed,
	122	TlsDecryptError,
	123	TlsConnectionTruncated,
	124	TlsDecodeError,
	125	UnsupportedCertificateVersion,
	126	CertificateTimeInvalid,
	127	CertificateHasUnrecognizedObjectId,
	128	CertificateHasInvalidBitString,
	129	MessageTooLong,
	130	NegativeIntoUnsigned,
	131	TargetTooSmall,
	132	BufferTooSmall,
	133	InvalidSignature,
	134	NotSquare,
	135	NonCanonical,
	136	};
	137	}
	138
91	139	/// Initiates a TLS handshake and establishes a TLSv1.3 session with `stream`, which
92	140	/// must conform to `StreamInterface`.
93	141	///
94	142	/// `host` is only borrowed during this function call.
95		pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) !Client {
	143	pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) InitError(@TypeOf(stream))!Client {
96	144	const host_len = @intCast(u16, host.len);
97	145
98	146	var random_buffer: [128]u8 = undefined;

lib/std/http.zig+15

...	...	@@ -248,9 +248,24 @@ pub const Status = enum(u10) {
248	248
249	249	pub const TransferEncoding = enum {
250	250	chunked,
	251	// compression is intentionally omitted here, as std.http.Client stores it as content-encoding
	252	};
	253
	254	pub const ContentEncoding = enum {
251	255	compress,
252	256	deflate,
253	257	gzip,
	258	zstd,
	259	};
	260
	261	pub const Connection = enum {
	262	keep_alive,
	263	close,
	264	};
	265
	266	pub const CustomHeader = struct {
	267	name: []const u8,
	268	value: []const u8,
254	269	};
255	270
256	271	const std = @import("std.zig");

lib/std/http/Client.zig+247-786

...	...	@@ -13,6 +13,12 @@ const Uri = std.Uri;
13	13	const Allocator = std.mem.Allocator;
14	14	const testing = std.testing;
15	15
	16	pub const Request = @import("Client/Request.zig");
	17	pub const Response = @import("Client/Response.zig");
	18
	19	pub const default_connection_pool_size = 32;
	20	const connection_pool_size = std.options.http_connection_pool_size;
	21
16	22	/// Used for tcpConnectToHost and storing HTTP headers when an externally
17	23	/// managed buffer is not provided.
18	24	allocator: Allocator,
...	...	@@ -21,854 +27,256 @@ ca_bundle: std.crypto.Certificate.Bundle = .{},
21	27	/// it will first rescan the system for root certificates.
22	28	next_https_rescan_certs: bool = true,
23	29
24		pub const Connection = struct {
25		stream: net.Stream,
26		/// undefined unless protocol is tls.
27		tls_client: std.crypto.tls.Client,
28		protocol: Protocol,
	30	connection_pool: ConnectionPool = .{},
29	31
30		pub const Protocol = enum { plain, tls };
	32	pub const ConnectionPool = struct {
	33	pub const Criteria = struct {
	34	host: []const u8,
	35	port: u16,
	36	is_tls: bool,
	37	};
31	38
32		pub fn read(conn: *Connection, buffer: []u8) !usize {
33		switch (conn.protocol) {
34		.plain => return conn.stream.read(buffer),
35		.tls => return conn.tls_client.read(conn.stream, buffer),
	39	const Queue = std.TailQueue(Connection);
	40	pub const Node = Queue.Node;
	41
	42	mutex: std.Thread.Mutex = .{},
	43	used: Queue = .{},
	44	free: Queue = .{},
	45	free_len: usize = 0,
	46	free_size: usize = default_connection_pool_size,
	47
	48	/// Finds and acquires a connection from the connection pool matching the criteria. This function is threadsafe.
	49	/// If no connection is found, null is returned.
	50	pub fn findConnection(pool: ConnectionPool, criteria: Criteria) ?Node {
	51	pool.mutex.lock();
	52	defer pool.mutex.unlock();
	53
	54	var next = pool.free.last;
	55	while (next) \|node\| : (next = node.prev) {
	56	if ((node.data.protocol == .tls) != criteria.is_tls) continue;
	57	if (node.data.port != criteria.port) continue;
	58	if (std.mem.eql(u8, node.data.host, criteria.host)) continue;
	59
	60	pool.acquireUnsafe(node);
	61	return node;
36	62	}
37		}
38	63
39		pub fn readAtLeast(conn: *Connection, buffer: []u8, len: usize) !usize {
40		switch (conn.protocol) {
41		.plain => return conn.stream.readAtLeast(buffer, len),
42		.tls => return conn.tls_client.readAtLeast(conn.stream, buffer, len),
43		}
	64	return null;
44	65	}
45	66
46		pub fn writeAll(conn: *Connection, buffer: []const u8) !void {
47		switch (conn.protocol) {
48		.plain => return conn.stream.writeAll(buffer),
49		.tls => return conn.tls_client.writeAll(conn.stream, buffer),
50		}
	67	/// Acquires an existing connection from the connection pool. This function is not threadsafe.
	68	pub fn acquireUnsafe(pool: ConnectionPool, node: Node) void {
	69	pool.free.remove(node);
	70	pool.free_len -= 1;
	71
	72	pool.used.append(node);
51	73	}
52	74
53		pub fn write(conn: *Connection, buffer: []const u8) !usize {
54		switch (conn.protocol) {
55		.plain => return conn.stream.write(buffer),
56		.tls => return conn.tls_client.write(conn.stream, buffer),
57		}
	75	/// Acquires an existing connection from the connection pool. This function is threadsafe.
	76	pub fn acquire(pool: ConnectionPool, node: Node) void {
	77	pool.mutex.lock();
	78	defer pool.mutex.unlock();
	79
	80	return pool.acquireUnsafe(node);
58	81	}
59		};
60	82
61		/// TODO: emit error.UnexpectedEndOfStream or something like that when the read
62		/// data does not match the content length. This is necessary since HTTPS disables
63		/// close_notify protection on underlying TLS streams.
64		pub const Request = struct {
65		client: *Client,
66		connection: Connection,
67		redirects_left: u32,
68		response: Response,
69		/// These are stored in Request so that they are available when following
70		/// redirects.
71		headers: Headers,
72
73		pub const Response = struct {
74		headers: Response.Headers,
75		state: State,
76		header_bytes_owned: bool,
77		/// This could either be a fixed buffer provided by the API user or it
78		/// could be our own array list.
79		header_bytes: std.ArrayListUnmanaged(u8),
80		max_header_bytes: usize,
81		next_chunk_length: u64,
82
83		pub const Headers = struct {
84		status: http.Status,
85		version: http.Version,
86		location: ?[]const u8 = null,
87		content_length: ?u64 = null,
88		transfer_encoding: ?http.TransferEncoding = null,
89
90		pub fn parse(bytes: []const u8) !Response.Headers {
91		var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
92
93		const first_line = it.first();
94		if (first_line.len < 12)
95		return error.ShortHttpStatusLine;
96
97		const version: http.Version = switch (int64(first_line[0..8])) {
98		int64("HTTP/1.0") => .@"HTTP/1.0",
99		int64("HTTP/1.1") => .@"HTTP/1.1",
100		else => return error.BadHttpVersion,
101		};
102		if (first_line[8] != ' ') return error.HttpHeadersInvalid;
103		const status = @intToEnum(http.Status, parseInt3(first_line[9..12].*));
104
105		var headers: Response.Headers = .{
106		.version = version,
107		.status = status,
108		};
109
110		while (it.next()) \|line\| {
111		if (line.len == 0) return error.HttpHeadersInvalid;
112		switch (line[0]) {
113		' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
114		else => {},
115		}
116		var line_it = mem.split(u8, line, ": ");
117		const header_name = line_it.first();
118		const header_value = line_it.rest();
119		if (std.ascii.eqlIgnoreCase(header_name, "location")) {
120		if (headers.location != null) return error.HttpHeadersInvalid;
121		headers.location = header_value;
122		} else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
123		if (headers.content_length != null) return error.HttpHeadersInvalid;
124		headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
125		} else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
126		if (headers.transfer_encoding != null) return error.HttpHeadersInvalid;
127		headers.transfer_encoding = std.meta.stringToEnum(http.TransferEncoding, header_value) orelse
128		return error.HttpTransferEncodingUnsupported;
129		}
130		}
131
132		return headers;
133		}
134
135		test "parse headers" {
136		const example =
137		"HTTP/1.1 301 Moved Permanently\r\n" ++
138		"Location: https://www.example.com/\r\n" ++
139		"Content-Type: text/html; charset=UTF-8\r\n" ++
140		"Content-Length: 220\r\n\r\n";
141		const parsed = try Response.Headers.parse(example);
142		try testing.expectEqual(http.Version.@"HTTP/1.1", parsed.version);
143		try testing.expectEqual(http.Status.moved_permanently, parsed.status);
144		try testing.expectEqualStrings("https://www.example.com/", parsed.location orelse
145		return error.TestFailed);
146		try testing.expectEqual(@as(?u64, 220), parsed.content_length);
147		}
148
149		test "header continuation" {
150		const example =
151		"HTTP/1.0 200 OK\r\n" ++
152		"Content-Type: text/html;\r\n charset=UTF-8\r\n" ++
153		"Content-Length: 220\r\n\r\n";
154		try testing.expectError(
155		error.HttpHeaderContinuationsUnsupported,
156		Response.Headers.parse(example),
157		);
158		}
159
160		test "extra content length" {
161		const example =
162		"HTTP/1.0 200 OK\r\n" ++
163		"Content-Length: 220\r\n" ++
164		"Content-Type: text/html; charset=UTF-8\r\n" ++
165		"content-length: 220\r\n\r\n";
166		try testing.expectError(
167		error.HttpHeadersInvalid,
168		Response.Headers.parse(example),
169		);
170		}
171		};
172
173		pub const State = enum {
174		/// Begin header parsing states.
175		invalid,
176		start,
177		seen_r,
178		seen_rn,
179		seen_rnr,
180		finished,
181		/// Begin transfer-encoding: chunked parsing states.
182		chunk_size_prefix_r,
183		chunk_size_prefix_n,
184		chunk_size,
185		chunk_r,
186		chunk_data,
187
188		pub fn zeroMeansEnd(state: State) bool {
189		return switch (state) {
190		.finished, .chunk_data => true,
191		else => false,
192		};
193		}
194		};
195
196		pub fn initDynamic(max: usize) Response {
197		return .{
198		.state = .start,
199		.headers = undefined,
200		.header_bytes = .{},
201		.max_header_bytes = max,
202		.header_bytes_owned = true,
203		.next_chunk_length = undefined,
204		};
205		}
	83	/// Tries to release a connection back to the connection pool. This function is threadsafe.
	84	/// If the connection is marked as closing, it will be closed instead.
	85	pub fn release(pool: ConnectionPool, client: Client, node: *Node) void {
	86	pool.mutex.lock();
	87	defer pool.mutex.unlock();
206	88
207		pub fn initStatic(buf: []u8) Response {
208		return .{
209		.state = .start,
210		.headers = undefined,
211		.header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
212		.max_header_bytes = buf.len,
213		.header_bytes_owned = false,
214		.next_chunk_length = undefined,
215		};
216		}
	89	pool.used.remove(node);
217	90
218		/// Returns how many bytes are part of HTTP headers. Always less than or
219		/// equal to bytes.len. If the amount returned is less than bytes.len, it
220		/// means the headers ended and the first byte after the double \r\n\r\n is
221		/// located at `bytes[result]`.
222		pub fn findHeadersEnd(r: *Response, bytes: []const u8) usize {
223		var index: usize = 0;
224
225		// TODO: https://github.com/ziglang/zig/issues/8220
226		state: while (true) {
227		switch (r.state) {
228		.invalid => unreachable,
229		.finished => unreachable,
230		.start => while (true) {
231		switch (bytes.len - index) {
232		0 => return index,
233		1 => {
234		if (bytes[index] == '\r')
235		r.state = .seen_r;
236		return index + 1;
237		},
238		2 => {
239		if (int16(bytes[index..][0..2]) == int16("\r\n")) {
240		r.state = .seen_rn;
241		} else if (bytes[index + 1] == '\r') {
242		r.state = .seen_r;
243		}
244		return index + 2;
245		},
246		3 => {
247		if (int16(bytes[index..][0..2]) == int16("\r\n") and
248		bytes[index + 2] == '\r')
249		{
250		r.state = .seen_rnr;
251		} else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n")) {
252		r.state = .seen_rn;
253		} else if (bytes[index + 2] == '\r') {
254		r.state = .seen_r;
255		}
256		return index + 3;
257		},
258		4...15 => {
259		if (int32(bytes[index..][0..4]) == int32("\r\n\r\n")) {
260		r.state = .finished;
261		return index + 4;
262		} else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n") and
263		bytes[index + 3] == '\r')
264		{
265		r.state = .seen_rnr;
266		index += 4;
267		continue :state;
268		} else if (int16(bytes[index + 2 ..][0..2]) == int16("\r\n")) {
269		r.state = .seen_rn;
270		index += 4;
271		continue :state;
272		} else if (bytes[index + 3] == '\r') {
273		r.state = .seen_r;
274		index += 4;
275		continue :state;
276		}
277		index += 4;
278		continue;
279		},
280		else => {
281		const chunk = bytes[index..][0..16];
282		const v: @Vector(16, u8) = chunk.*;
283		const matches_r = v == @splat(16, @as(u8, '\r'));
284		const iota = std.simd.iota(u8, 16);
285		const default = @splat(16, @as(u8, 16));
286		const sub_index = @reduce(.Min, @select(u8, matches_r, iota, default));
287		switch (sub_index) {
288		0...12 => {
289		index += sub_index + 4;
290		if (int32(chunk[sub_index..][0..4]) == int32("\r\n\r\n")) {
291		r.state = .finished;
292		return index;
293		}
294		continue;
295		},
296		13 => {
297		index += 16;
298		if (int16(chunk[14..][0..2]) == int16("\n\r")) {
299		r.state = .seen_rnr;
300		continue :state;
301		}
302		continue;
303		},
304		14 => {
305		index += 16;
306		if (chunk[15] == '\n') {
307		r.state = .seen_rn;
308		continue :state;
309		}
310		continue;
311		},
312		15 => {
313		r.state = .seen_r;
314		index += 16;
315		continue :state;
316		},
317		16 => {
318		index += 16;
319		continue;
320		},
321		else => unreachable,
322		}
323		},
324		}
325		},
326
327		.seen_r => switch (bytes.len - index) {
328		0 => return index,
329		1 => {
330		switch (bytes[index]) {
331		'\n' => r.state = .seen_rn,
332		'\r' => r.state = .seen_r,
333		else => r.state = .start,
334		}
335		return index + 1;
336		},
337		2 => {
338		if (int16(bytes[index..][0..2]) == int16("\n\r")) {
339		r.state = .seen_rnr;
340		return index + 2;
341		}
342		r.state = .start;
343		return index + 2;
344		},
345		else => {
346		if (int16(bytes[index..][0..2]) == int16("\n\r") and
347		bytes[index + 2] == '\n')
348		{
349		r.state = .finished;
350		return index + 3;
351		}
352		index += 3;
353		r.state = .start;
354		continue :state;
355		},
356		},
357		.seen_rn => switch (bytes.len - index) {
358		0 => return index,
359		1 => {
360		switch (bytes[index]) {
361		'\r' => r.state = .seen_rnr,
362		else => r.state = .start,
363		}
364		return index + 1;
365		},
366		else => {
367		if (int16(bytes[index..][0..2]) == int16("\r\n")) {
368		r.state = .finished;
369		return index + 2;
370		}
371		index += 2;
372		r.state = .start;
373		continue :state;
374		},
375		},
376		.seen_rnr => switch (bytes.len - index) {
377		0 => return index,
378		else => {
379		if (bytes[index] == '\n') {
380		r.state = .finished;
381		return index + 1;
382		}
383		index += 1;
384		r.state = .start;
385		continue :state;
386		},
387		},
388		.chunk_size_prefix_r => unreachable,
389		.chunk_size_prefix_n => unreachable,
390		.chunk_size => unreachable,
391		.chunk_r => unreachable,
392		.chunk_data => unreachable,
393		}
394
395		return index;
396		}
397		}
	91	if (node.data.closing) {
	92	node.data.close(client);
398	93
399		pub fn findChunkedLen(r: *Response, bytes: []const u8) usize {
400		var i: usize = 0;
401		if (r.state == .chunk_size) {
402		while (i < bytes.len) : (i += 1) {
403		const digit = switch (bytes[i]) {
404		'0'...'9' => \|b\| b - '0',
405		'A'...'Z' => \|b\| b - 'A' + 10,
406		'a'...'z' => \|b\| b - 'a' + 10,
407		'\r' => {
408		r.state = .chunk_r;
409		i += 1;
410		break;
411		},
412		else => {
413		r.state = .invalid;
414		return i;
415		},
416		};
417		const mul = @mulWithOverflow(r.next_chunk_length, 16);
418		if (mul[1] != 0) {
419		r.state = .invalid;
420		return i;
421		}
422		const add = @addWithOverflow(mul[0], digit);
423		if (add[1] != 0) {
424		r.state = .invalid;
425		return i;
426		}
427		r.next_chunk_length = add[0];
428		} else {
429		return i;
430		}
431		}
432		assert(r.state == .chunk_r);
433		if (i == bytes.len) return i;
434
435		if (bytes[i] == '\n') {
436		r.state = .chunk_data;
437		return i + 1;
438		} else {
439		r.state = .invalid;
440		return i;
441		}
	94	return client.allocator.destroy(node);
442	95	}
443	96
444		fn parseInt3(nnn: @Vector(3, u8)) u10 {
445		const zero: @Vector(3, u8) = .{ '0', '0', '0' };
446		const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
447		return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
448		}
	97	if (pool.free_len + 1 >= pool.free_size) {
	98	const popped = pool.free.popFirst() orelse unreachable;
449	99
450		test parseInt3 {
451		const expectEqual = std.testing.expectEqual;
452		try expectEqual(@as(u10, 0), parseInt3("000".*));
453		try expectEqual(@as(u10, 418), parseInt3("418".*));
454		try expectEqual(@as(u10, 999), parseInt3("999".*));
455		}
	100	popped.data.close(client);
456	101
457		test "find headers end basic" {
458		var buffer: [1]u8 = undefined;
459		var r = Response.initStatic(&buffer);
460		try testing.expectEqual(@as(usize, 10), r.findHeadersEnd("HTTP/1.1 4"));
461		try testing.expectEqual(@as(usize, 2), r.findHeadersEnd("18"));
462		try testing.expectEqual(@as(usize, 8), r.findHeadersEnd(" lol\r\n\r\nblah blah"));
	102	return client.allocator.destroy(popped);
463	103	}
464	104
465		test "find headers end vectorized" {
466		var buffer: [1]u8 = undefined;
467		var r = Response.initStatic(&buffer);
468		const example =
469		"HTTP/1.1 301 Moved Permanently\r\n" ++
470		"Location: https://www.example.com/\r\n" ++
471		"Content-Type: text/html; charset=UTF-8\r\n" ++
472		"Content-Length: 220\r\n" ++
473		"\r\ncontent";
474		try testing.expectEqual(@as(usize, 131), r.findHeadersEnd(example));
475		}
	105	pool.free.append(node);
	106	pool.free_len += 1;
	107	}
476	108
477		test "find headers end bug" {
478		var buffer: [1]u8 = undefined;
479		var r = Response.initStatic(&buffer);
480		const trail = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
481		const example =
482		"HTTP/1.1 200 OK\r\n" ++
483		"Access-Control-Allow-Origin: https://render.githubusercontent.com\r\n" ++
484		"content-disposition: attachment; filename=zig-0.10.0.tar.gz\r\n" ++
485		"Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'; sandbox\r\n" ++
486		"Content-Type: application/x-gzip\r\n" ++
487		"ETag: \"bfae0af6b01c7c0d89eb667cb5f0e65265968aeebda2689177e6b26acd3155ca\"\r\n" ++
488		"Strict-Transport-Security: max-age=31536000\r\n" ++
489		"Vary: Authorization,Accept-Encoding,Origin\r\n" ++
490		"X-Content-Type-Options: nosniff\r\n" ++
491		"X-Frame-Options: deny\r\n" ++
492		"X-XSS-Protection: 1; mode=block\r\n" ++
493		"Date: Fri, 06 Jan 2023 22:26:22 GMT\r\n" ++
494		"Transfer-Encoding: chunked\r\n" ++
495		"X-GitHub-Request-Id: 89C6:17E9:A7C9E:124B51:63B8A00E\r\n" ++
496		"connection: close\r\n\r\n" ++ trail;
497		try testing.expectEqual(@as(usize, example.len - trail.len), r.findHeadersEnd(example));
498		}
499		};
	109	/// Adds a newly created node to the pool of used connections. This function is threadsafe.
	110	pub fn addUsed(pool: ConnectionPool, node: Node) void {
	111	pool.mutex.lock();
	112	defer pool.mutex.unlock();
500	113
501		pub const Headers = struct {
502		version: http.Version = .@"HTTP/1.1",
503		method: http.Method = .GET,
504		};
	114	pool.used.append(node);
	115	}
505	116
506		pub const Options = struct {
507		max_redirects: u32 = 3,
508		header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
509
510		pub const HeaderStrategy = union(enum) {
511		/// In this case, the client's Allocator will be used to store the
512		/// entire HTTP header. This value is the maximum total size of
513		/// HTTP headers allowed, otherwise
514		/// error.HttpHeadersExceededSizeLimit is returned from read().
515		dynamic: usize,
516		/// This is used to store the entire HTTP header. If the HTTP
517		/// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
518		/// is returned from read(). When this is used, `error.OutOfMemory`
519		/// cannot be returned from `read()`.
520		static: []u8,
521		};
522		};
	117	pub fn deinit(pool: ConnectionPool, client: Client) void {
	118	pool.mutex.lock();
523	119
524		/// May be skipped if header strategy is buffer.
525		pub fn deinit(req: *Request) void {
526		if (req.response.header_bytes_owned) {
527		req.response.header_bytes.deinit(req.client.allocator);
	120	var next = pool.free.first;
	121	while (next) \|node\| {
	122	defer client.allocator.destroy(node);
	123	next = node.next;
	124
	125	node.data.close(client);
528	126	}
529		req.* = undefined;
	127
	128	next = pool.used.first;
	129	while (next) \|node\| {
	130	defer client.allocator.destroy(node);
	131	next = node.next;
	132
	133	node.data.close(client);
	134	}
	135
	136	pool.* = undefined;
530	137	}
	138	};
	139
	140	pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.ReaderRaw);
	141	pub const GzipDecompressor = std.compress.gzip.Decompress(Request.ReaderRaw);
	142	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.ReaderRaw, .{});
531	143
532		pub const Reader = std.io.Reader(*Request, ReadError, read);
	144	pub const Connection = struct {
	145	stream: net.Stream,
	146	/// undefined unless protocol is tls.
	147	tls_client: *std.crypto.tls.Client, // TODO: allocate this, it's currently 16 KB.
	148	protocol: Protocol,
	149	host: []u8,
	150	port: u16,
533	151
534		pub fn reader(req: *Request) Reader {
535		return .{ .context = req };
	152	// This connection has been part of a non keepalive request and cannot be added to the pool.
	153	closing: bool = false,
	154
	155	pub const Protocol = enum { plain, tls };
	156
	157	pub fn read(conn: *Connection, buffer: []u8) !usize {
	158	switch (conn.protocol) {
	159	.plain => return conn.stream.read(buffer),
	160	.tls => return conn.tls_client.read(conn.stream, buffer),
	161	}
536	162	}
537	163
538		pub fn readAll(req: *Request, buffer: []u8) !usize {
539		return readAtLeast(req, buffer, buffer.len);
	164	pub fn readAtLeast(conn: *Connection, buffer: []u8, len: usize) !usize {
	165	switch (conn.protocol) {
	166	.plain => return conn.stream.readAtLeast(buffer, len),
	167	.tls => return conn.tls_client.readAtLeast(conn.stream, buffer, len),
	168	}
540	169	}
541	170
542	171	pub const ReadError = net.Stream.ReadError \|\| error{
543		// From HTTP protocol
544		HttpHeadersInvalid,
545		HttpHeadersExceededSizeLimit,
546		HttpRedirectMissingLocation,
547		HttpTransferEncodingUnsupported,
548		HttpContentLengthUnknown,
549		TooManyHttpRedirects,
550		ShortHttpStatusLine,
551		BadHttpVersion,
552		HttpHeaderContinuationsUnsupported,
553		UnsupportedUrlScheme,
554		UriMissingHost,
555		UnknownHostName,
556
557		// Network problems
558		NetworkUnreachable,
559		HostLacksNetworkAddresses,
560		TemporaryNameServerFailure,
561		NameServerFailure,
562		ProtocolFamilyNotAvailable,
563		ProtocolNotSupported,
564
565		// System resource problems
566		ProcessFdQuotaExceeded,
567		SystemFdQuotaExceeded,
568		OutOfMemory,
569
570		// TLS problems
571		InsufficientEntropy,
572	172	TlsConnectionTruncated,
573	173	TlsRecordOverflow,
574	174	TlsDecodeError,
575	175	TlsAlert,
576	176	TlsBadRecordMac,
	177	Overflow,
577	178	TlsBadLength,
578	179	TlsIllegalParameter,
579	180	TlsUnexpectedMessage,
580		TlsDecryptFailure,
581		CertificateFieldHasInvalidLength,
582		CertificateHostMismatch,
583		CertificatePublicKeyInvalid,
584		CertificateExpired,
585		CertificateFieldHasWrongDataType,
586		CertificateIssuerMismatch,
587		CertificateNotYetValid,
588		CertificateSignatureAlgorithmMismatch,
589		CertificateSignatureAlgorithmUnsupported,
590		CertificateSignatureInvalid,
591		CertificateSignatureInvalidLength,
592		CertificateSignatureNamedCurveUnsupported,
593		CertificateSignatureUnsupportedBitCount,
594		TlsCertificateNotVerified,
595		TlsBadSignatureScheme,
596		TlsBadRsaSignatureBitCount,
597		TlsDecryptError,
598		UnsupportedCertificateVersion,
599		CertificateTimeInvalid,
600		CertificateHasUnrecognizedObjectId,
601		CertificateHasInvalidBitString,
602		CertificateAuthorityBundleTooBig,
603
604		// TODO: convert to higher level errors
605		InvalidFormat,
606		InvalidPort,
607		UnexpectedCharacter,
608		Overflow,
609		InvalidCharacter,
610		AddressFamilyNotSupported,
611		AddressInUse,
612		AddressNotAvailable,
613		ConnectionPending,
614		ConnectionRefused,
615		FileNotFound,
616		PermissionDenied,
617		ServiceUnavailable,
618		SocketTypeNotSupported,
619		FileTooBig,
620		LockViolation,
621		NoSpaceLeft,
622		NotOpenForWriting,
623		InvalidEncoding,
624		IdentityElement,
625		NonCanonical,
626		SignatureVerificationFailed,
627		MessageTooLong,
628		NegativeIntoUnsigned,
629		TargetTooSmall,
630		BufferTooSmall,
631		InvalidSignature,
632		NotSquare,
633		DiskQuota,
634		InvalidEnd,
635		Incomplete,
636		InvalidIpv4Mapping,
637		InvalidIPAddressFormat,
638		BadPathName,
639		DeviceBusy,
640		FileBusy,
641		FileLocksNotSupported,
642		InvalidHandle,
643		InvalidUtf8,
644		NameTooLong,
645		NoDevice,
646		PathAlreadyExists,
647		PipeBusy,
648		SharingViolation,
649		SymLinkLoop,
650		FileSystem,
651		InterfaceNotFound,
652		AlreadyBound,
653		FileDescriptorNotASocket,
654		NetworkSubsystemFailed,
655		NotDir,
656		ReadOnlyFileSystem,
657		Unseekable,
658		MissingEndCertificateMarker,
659		InvalidPadding,
660		EndOfStream,
661		InvalidArgument,
662	181	};
663	182
664		pub fn read(req: *Request, buffer: []u8) ReadError!usize {
665		return readAtLeast(req, buffer, 1);
	183	pub const Reader = std.io.Reader(*Connection, ReadError, read);
	184
	185	pub fn reader(conn: *Connection) Reader {
	186	return Reader{ .context = conn };
666	187	}
667	188
668		pub fn readAtLeast(req: *Request, buffer: []u8, len: usize) !usize {
669		assert(len <= buffer.len);
670		var index: usize = 0;
671		while (index < len) {
672		const zero_means_end = req.response.state.zeroMeansEnd();
673		const amt = try readAdvanced(req, buffer[index..]);
674		if (amt == 0 and zero_means_end) break;
675		index += amt;
	189	pub fn writeAll(conn: *Connection, buffer: []const u8) !void {
	190	switch (conn.protocol) {
	191	.plain => return conn.stream.writeAll(buffer),
	192	.tls => return conn.tls_client.writeAll(conn.stream, buffer),
676	193	}
677		return index;
678	194	}
679	195
680		/// This one can return 0 without meaning EOF.
681		/// TODO change to readvAdvanced
682		pub fn readAdvanced(req: *Request, buffer: []u8) !usize {
683		var in = buffer[0..try req.connection.read(buffer)];
684		var out_index: usize = 0;
685		while (true) {
686		switch (req.response.state) {
687		.invalid => unreachable,
688		.start, .seen_r, .seen_rn, .seen_rnr => {
689		const i = req.response.findHeadersEnd(in);
690		if (req.response.state == .invalid) return error.HttpHeadersInvalid;
691
692		const headers_data = in[0..i];
693		if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
694		return error.HttpHeadersExceededSizeLimit;
695		}
696		try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
697
698		if (req.response.state == .finished) {
699		req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
700
701		if (req.response.headers.status.class() == .redirect) {
702		if (req.redirects_left == 0) return error.TooManyHttpRedirects;
703		const location = req.response.headers.location orelse
704		return error.HttpRedirectMissingLocation;
705		const new_url = try std.Uri.parse(location);
706		const new_req = try req.client.request(new_url, req.headers, .{
707		.max_redirects = req.redirects_left - 1,
708		.header_strategy = if (req.response.header_bytes_owned) .{
709		.dynamic = req.response.max_header_bytes,
710		} else .{
711		.static = req.response.header_bytes.unusedCapacitySlice(),
712		},
713		});
714		req.deinit();
715		req.* = new_req;
716		assert(out_index == 0);
717		in = buffer[0..try req.connection.read(buffer)];
718		continue;
719		}
720
721		if (req.response.headers.transfer_encoding) \|transfer_encoding\| {
722		switch (transfer_encoding) {
723		.chunked => {
724		req.response.next_chunk_length = 0;
725		req.response.state = .chunk_size;
726		},
727		.compress => return error.HttpTransferEncodingUnsupported,
728		.deflate => return error.HttpTransferEncodingUnsupported,
729		.gzip => return error.HttpTransferEncodingUnsupported,
730		}
731		} else if (req.response.headers.content_length) \|content_length\| {
732		req.response.next_chunk_length = content_length;
733		} else {
734		return error.HttpContentLengthUnknown;
735		}
736
737		in = in[i..];
738		continue;
739		}
740
741		assert(out_index == 0);
742		return 0;
743		},
744		.finished => {
745		if (in.ptr == buffer.ptr) {
746		return in.len;
747		} else {
748		mem.copy(u8, buffer[out_index..], in);
749		return out_index + in.len;
750		}
751		},
752		.chunk_size_prefix_r => switch (in.len) {
753		0 => return out_index,
754		1 => switch (in[0]) {
755		'\r' => {
756		req.response.state = .chunk_size_prefix_n;
757		return out_index;
758		},
759		else => {
760		req.response.state = .invalid;
761		return error.HttpHeadersInvalid;
762		},
763		},
764		else => switch (int16(in[0..2])) {
765		int16("\r\n") => {
766		in = in[2..];
767		req.response.state = .chunk_size;
768		continue;
769		},
770		else => {
771		req.response.state = .invalid;
772		return error.HttpHeadersInvalid;
773		},
774		},
775		},
776		.chunk_size_prefix_n => switch (in.len) {
777		0 => return out_index,
778		else => switch (in[0]) {
779		'\n' => {
780		in = in[1..];
781		req.response.state = .chunk_size;
782		continue;
783		},
784		else => {
785		req.response.state = .invalid;
786		return error.HttpHeadersInvalid;
787		},
788		},
789		},
790		.chunk_size, .chunk_r => {
791		const i = req.response.findChunkedLen(in);
792		switch (req.response.state) {
793		.invalid => return error.HttpHeadersInvalid,
794		.chunk_data => {
795		if (req.response.next_chunk_length == 0) {
796		req.response.state = .start;
797		return out_index;
798		}
799		in = in[i..];
800		continue;
801		},
802		.chunk_size => return out_index,
803		else => unreachable,
804		}
805		},
806		.chunk_data => {
807		// TODO https://github.com/ziglang/zig/issues/14039
808		const sub_amt = @intCast(usize, @min(req.response.next_chunk_length, in.len));
809		req.response.next_chunk_length -= sub_amt;
810		if (req.response.next_chunk_length > 0) {
811		if (in.ptr == buffer.ptr) {
812		return sub_amt;
813		} else {
814		mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
815		out_index += sub_amt;
816		return out_index;
817		}
818		}
819		mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
820		out_index += sub_amt;
821		req.response.state = .chunk_size_prefix_r;
822		in = in[sub_amt..];
823		continue;
824		},
825		}
	196	pub fn write(conn: *Connection, buffer: []const u8) !usize {
	197	switch (conn.protocol) {
	198	.plain => return conn.stream.write(buffer),
	199	.tls => return conn.tls_client.write(conn.stream, buffer),
826	200	}
827	201	}
828	202
829		inline fn int16(array: *const [2]u8) u16 {
830		return @bitCast(u16, array.*);
831		}
	203	pub const WriteError = net.Stream.WriteError \|\| error{};
	204	pub const Writer = std.io.Writer(*Connection, WriteError, write);
832	205
833		inline fn int32(array: *const [4]u8) u32 {
834		return @bitCast(u32, array.*);
	206	pub fn writer(conn: *Connection) Writer {
	207	return Writer{ .context = conn };
835	208	}
836	209
837		inline fn int64(array: *const [8]u8) u64 {
838		return @bitCast(u64, array.*);
839		}
	210	pub fn close(conn: Connection, client: const Client) void {
	211	if (conn.protocol == .tls) {
	212	// try to cleanly close the TLS connection, for any server that cares.
	213	_ = conn.tls_client.writeEnd(conn.stream, "", true) catch {};
	214	client.allocator.destroy(conn.tls_client);
	215	}
840	216
841		test {
842		_ = Response;
	217	conn.stream.close();
	218
	219	client.allocator.free(conn.host);
843	220	}
844	221	};
845	222
846	223	pub fn deinit(client: *Client) void {
	224	client.connection_pool.deinit(client);
	225
847	226	client.ca_bundle.deinit(client.allocator);
848	227	client.* = undefined;
849	228	}
850	229
851		pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) !Connection {
852		var conn: Connection = .{
	230	pub const ConnectError = std.mem.Allocator.Error \|\| net.TcpConnectToHostError \|\| std.crypto.tls.Client.InitError(net.Stream);
	231
	232	pub fn connect(client: Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!ConnectionPool.Node {
	233	if (client.connection_pool.findConnection(.{
	234	.host = host,
	235	.port = port,
	236	.is_tls = protocol == .tls,
	237	})) \|node\|
	238	return node;
	239
	240	const conn = try client.allocator.create(ConnectionPool.Node);
	241	errdefer client.allocator.destroy(conn);
	242	conn.* = .{ .data = undefined };
	243
	244	conn.data = .{
853	245	.stream = try net.tcpConnectToHost(client.allocator, host, port),
854	246	.tls_client = undefined,
855	247	.protocol = protocol,
	248	.host = try client.allocator.dupe(u8, host),
	249	.port = port,
856	250	};
857	251
858	252	switch (protocol) {
859	253	.plain => {},
860	254	.tls => {
861		conn.tls_client = try std.crypto.tls.Client.init(conn.stream, client.ca_bundle, host);
	255	conn.data.tls_client = try client.allocator.create(std.crypto.tls.Client);
	256	conn.data.tls_client.* = try std.crypto.tls.Client.init(conn.data.stream, client.ca_bundle, host);
862	257	// This is appropriate for HTTPS because the HTTP headers contain
863	258	// the content length which is used to detect truncation attacks.
864		conn.tls_client.allow_truncation_attacks = true;
	259	conn.data.tls_client.allow_truncation_attacks = true;
865	260	},
866	261	}
867	262
	263	client.connection_pool.addUsed(conn);
	264
868	265	return conn;
869	266	}
870	267
871		pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Request.Options) !Request {
	268	pub const RequestError = ConnectError \|\| Connection.WriteError \|\| error{
	269	UnsupportedUrlScheme,
	270	UriMissingHost,
	271
	272	CertificateAuthorityBundleTooBig,
	273	InvalidPadding,
	274	MissingEndCertificateMarker,
	275	Unseekable,
	276	EndOfStream,
	277	};
	278
	279	pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Request.Options) RequestError!Request {
872	280	const protocol: Connection.Protocol = if (mem.eql(u8, uri.scheme, "http"))
873	281	.plain
874	282	else if (mem.eql(u8, uri.scheme, "https"))
...	...	@@ -884,34 +292,85 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req
884	292	const host = uri.host orelse return error.UriMissingHost;
885	293
886	294	if (client.next_https_rescan_certs and protocol == .tls) {
887		try client.ca_bundle.rescan(client.allocator);
888		client.next_https_rescan_certs = false;
	295	client.connection_pool.mutex.lock(); // TODO: this could be so much better than reusing the connection pool mutex.
	296	defer client.connection_pool.mutex.unlock();
	297
	298	if (client.next_https_rescan_certs) {
	299	try client.ca_bundle.rescan(client.allocator);
	300	client.next_https_rescan_certs = false;
	301	}
889	302	}
890	303
891	304	var req: Request = .{
	305	.uri = uri,
892	306	.client = client,
893	307	.headers = headers,
894	308	.connection = try client.connect(host, port, protocol),
895	309	.redirects_left = options.max_redirects,
	310	.handle_redirects = options.handle_redirects,
	311	.compression_init = false,
896	312	.response = switch (options.header_strategy) {
897		.dynamic => \|max\| Request.Response.initDynamic(max),
898		.static => \|buf\| Request.Response.initStatic(buf),
	313	.dynamic => \|max\| Response.initDynamic(max),
	314	.static => \|buf\| Response.initStatic(buf),
899	315	},
	316	.arena = undefined,
900	317	};
901	318
	319	req.arena = std.heap.ArenaAllocator.init(client.allocator);
	320
902	321	{
903		var h = try std.BoundedArray(u8, 1000).init(0);
904		try h.appendSlice(@tagName(headers.method));
905		try h.appendSlice(" ");
906		try h.appendSlice(uri.path);
907		try h.appendSlice(" ");
908		try h.appendSlice(@tagName(headers.version));
909		try h.appendSlice("\r\nHost: ");
910		try h.appendSlice(host);
911		try h.appendSlice("\r\nConnection: close\r\n\r\n");
912
913		const header_bytes = h.slice();
914		try req.connection.writeAll(header_bytes);
	322	var buffered = std.io.bufferedWriter(req.connection.data.writer());
	323	const writer = buffered.writer();
	324
	325	const escaped_path = try Uri.escapePath(client.allocator, uri.path);
	326	defer client.allocator.free(escaped_path);
	327
	328	const escaped_query = if (uri.query) \|q\| try Uri.escapeQuery(client.allocator, q) else null;
	329	defer if (escaped_query) \|q\| client.allocator.free(q);
	330
	331	const escaped_fragment = if (uri.fragment) \|f\| try Uri.escapeQuery(client.allocator, f) else null;
	332	defer if (escaped_fragment) \|f\| client.allocator.free(f);
	333
	334	try writer.writeAll(@tagName(headers.method));
	335	try writer.writeByte(' ');
	336	try writer.writeAll(escaped_path);
	337	if (escaped_query) \|q\| {
	338	try writer.writeByte('?');
	339	try writer.writeAll(q);
	340	}
	341	if (escaped_fragment) \|f\| {
	342	try writer.writeByte('#');
	343	try writer.writeAll(f);
	344	}
	345	try writer.writeByte(' ');
	346	try writer.writeAll(@tagName(headers.version));
	347	try writer.writeAll("\r\nHost: ");
	348	try writer.writeAll(host);
	349	try writer.writeAll("\r\nUser-Agent: ");
	350	try writer.writeAll(headers.user_agent);
	351	if (headers.connection == .close) {
	352	try writer.writeAll("\r\nConnection: close");
	353	} else {
	354	try writer.writeAll("\r\nConnection: keep-alive");
	355	}
	356	try writer.writeAll("\r\nAccept-Encoding: gzip, deflate, zstd");
	357
	358	switch (headers.transfer_encoding) {
	359	.chunked => try writer.writeAll("\r\nTransfer-Encoding: chunked"),
	360	.content_length => \|content_length\| try writer.print("\r\nContent-Length: {d}", .{content_length}),
	361	.none => {},
	362	}
	363
	364	for (headers.custom) \|header\| {
	365	try writer.writeAll("\r\n");
	366	try writer.writeAll(header.name);
	367	try writer.writeAll(": ");
	368	try writer.writeAll(header.value);
	369	}
	370
	371	try writer.writeAll("\r\n\r\n");
	372
	373	try buffered.flush();
915	374	}
916	375
917	376	return req;
...	...	@@ -925,5 +384,7 @@ test {
925	384	return error.SkipZigTest;
926	385	}
927	386
	387	if (builtin.os.tag == .wasi) return error.SkipZigTest;
	388
928	389	_ = Request;
929	390	}

lib/std/http/Client/Request.zig created+482

...	...	@@ -0,0 +1,482 @@
	1	const std = @import("std");
	2	const http = std.http;
	3	const Uri = std.Uri;
	4	const mem = std.mem;
	5	const assert = std.debug.assert;
	6
	7	const Client = @import("../Client.zig");
	8	const Connection = Client.Connection;
	9	const ConnectionNode = Client.ConnectionPool.Node;
	10	const Response = @import("Response.zig");
	11
	12	const Request = @This();
	13
	14	const read_buffer_size = 8192;
	15	const ReadBufferIndex = std.math.IntFittingRange(0, read_buffer_size);
	16
	17	uri: Uri,
	18	client: *Client,
	19	connection: *ConnectionNode,
	20	response: Response,
	21	/// These are stored in Request so that they are available when following
	22	/// redirects.
	23	headers: Headers,
	24
	25	redirects_left: u32,
	26	handle_redirects: bool,
	27	compression_init: bool,
	28
	29	/// Used as a allocator for resolving redirects locations.
	30	arena: std.heap.ArenaAllocator,
	31
	32	/// Read buffer for the connection. This is used to pull in large amounts of data from the connection even if the user asks for a small amount. This can probably be removed with careful planning.
	33	read_buffer: [read_buffer_size]u8 = undefined,
	34	read_buffer_start: ReadBufferIndex = 0,
	35	read_buffer_len: ReadBufferIndex = 0,
	36
	37	pub const RequestTransfer = union(enum) {
	38	content_length: u64,
	39	chunked: void,
	40	none: void,
	41	};
	42
	43	pub const Headers = struct {
	44	version: http.Version = .@"HTTP/1.1",
	45	method: http.Method = .GET,
	46	user_agent: []const u8 = "zig (std.http)",
	47	connection: http.Connection = .keep_alive,
	48	transfer_encoding: RequestTransfer = .none,
	49
	50	custom: []const http.CustomHeader = &[_]http.CustomHeader{},
	51	};
	52
	53	pub const Options = struct {
	54	handle_redirects: bool = true,
	55	max_redirects: u32 = 3,
	56	header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
	57
	58	pub const HeaderStrategy = union(enum) {
	59	/// In this case, the client's Allocator will be used to store the
	60	/// entire HTTP header. This value is the maximum total size of
	61	/// HTTP headers allowed, otherwise
	62	/// error.HttpHeadersExceededSizeLimit is returned from read().
	63	dynamic: usize,
	64	/// This is used to store the entire HTTP header. If the HTTP
	65	/// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
	66	/// is returned from read(). When this is used, `error.OutOfMemory`
	67	/// cannot be returned from `read()`.
	68	static: []u8,
	69	};
	70	};
	71
	72	/// Frees all resources associated with the request.
	73	pub fn deinit(req: *Request) void {
	74	switch (req.response.compression) {
	75	.none => {},
	76	.deflate => \|*deflate\| deflate.deinit(),
	77	.gzip => \|*gzip\| gzip.deinit(),
	78	.zstd => \|*zstd\| zstd.deinit(),
	79	}
	80
	81	if (req.response.header_bytes_owned) {
	82	req.response.header_bytes.deinit(req.client.allocator);
	83	}
	84
	85	if (!req.response.done) {
	86	// If the response wasn't fully read, then we need to close the connection.
	87	req.connection.data.closing = true;
	88	req.client.connection_pool.release(req.client, req.connection);
	89	}
	90
	91	req.arena.deinit();
	92	req.* = undefined;
	93	}
	94
	95	pub const ReadRawError = Connection.ReadError \|\| Uri.ParseError \|\| Client.RequestError \|\| error{
	96	UnexpectedEndOfStream,
	97	TooManyHttpRedirects,
	98	HttpRedirectMissingLocation,
	99	HttpHeadersInvalid,
	100	};
	101
	102	pub const ReaderRaw = std.io.Reader(*Request, ReadRawError, readRaw);
	103
	104	/// Read from the underlying stream, without decompressing or parsing the headers. Must be called
	105	/// after waitForCompleteHead() has returned successfully.
	106	pub fn readRaw(req: *Request, buffer: []u8) ReadRawError!usize {
	107	assert(req.response.state.isContent());
	108
	109	var index: usize = 0;
	110	while (index == 0) {
	111	const amt = try req.readRawAdvanced(buffer[index..]);
	112	if (amt == 0 and req.response.done) break;
	113	index += amt;
	114	}
	115
	116	return index;
	117	}
	118
	119	fn checkForCompleteHead(req: *Request, buffer: []u8) !usize {
	120	switch (req.response.state) {
	121	.invalid => unreachable,
	122	.start, .seen_r, .seen_rn, .seen_rnr => {},
	123	else => return 0, // No more headers to read.
	124	}
	125
	126	const i = req.response.findHeadersEnd(buffer[0..]);
	127	if (req.response.state == .invalid) return error.HttpHeadersInvalid;
	128
	129	const headers_data = buffer[0..i];
	130	if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
	131	return error.HttpHeadersExceededSizeLimit;
	132	}
	133	try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
	134
	135	if (req.response.state == .finished) {
	136	req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
	137
	138	if (req.response.headers.upgrade) \|_\| {
	139	req.connection.data.closing = false;
	140	req.response.done = true;
	141	return i;
	142	}
	143
	144	if (req.response.headers.connection == .keep_alive) {
	145	req.connection.data.closing = false;
	146	} else {
	147	req.connection.data.closing = true;
	148	}
	149
	150	if (req.response.headers.transfer_encoding) \|transfer_encoding\| {
	151	switch (transfer_encoding) {
	152	.chunked => {
	153	req.response.next_chunk_length = 0;
	154	req.response.state = .chunk_size;
	155	},
	156	}
	157	} else if (req.response.headers.content_length) \|content_length\| {
	158	req.response.next_chunk_length = content_length;
	159
	160	if (content_length == 0) req.response.done = true;
	161	} else {
	162	req.response.done = true;
	163	}
	164
	165	return i;
	166	}
	167
	168	return 0;
	169	}
	170
	171	pub const WaitForCompleteHeadError = ReadRawError \|\| error{
	172	UnexpectedEndOfStream,
	173
	174	HttpHeadersExceededSizeLimit,
	175	ShortHttpStatusLine,
	176	BadHttpVersion,
	177	HttpHeaderContinuationsUnsupported,
	178	HttpTransferEncodingUnsupported,
	179	HttpConnectionHeaderUnsupported,
	180	};
	181
	182	/// Reads a complete response head. Any leftover data is stored in the request. This function is idempotent.
	183	pub fn waitForCompleteHead(req: *Request) WaitForCompleteHeadError!void {
	184	if (req.response.state.isContent()) return;
	185
	186	while (true) {
	187	const nread = try req.connection.data.read(req.read_buffer[0..]);
	188	const amt = try checkForCompleteHead(req, req.read_buffer[0..nread]);
	189
	190	if (amt != 0) {
	191	req.read_buffer_start = @intCast(ReadBufferIndex, amt);
	192	req.read_buffer_len = @intCast(ReadBufferIndex, nread);
	193	return;
	194	} else if (nread == 0) {
	195	return error.UnexpectedEndOfStream;
	196	}
	197	}
	198	}
	199
	200	/// This one can return 0 without meaning EOF.
	201	fn readRawAdvanced(req: *Request, buffer: []u8) !usize {
	202	assert(req.response.state.isContent());
	203	if (req.response.done) return 0;
	204
	205	// var in: []const u8 = undefined;
	206	if (req.read_buffer_start == req.read_buffer_len) {
	207	const nread = try req.connection.data.read(req.read_buffer[0..]);
	208	if (nread == 0) return error.UnexpectedEndOfStream;
	209
	210	req.read_buffer_start = 0;
	211	req.read_buffer_len = @intCast(ReadBufferIndex, nread);
	212	}
	213
	214	var out_index: usize = 0;
	215	while (true) {
	216	switch (req.response.state) {
	217	.invalid, .start, .seen_r, .seen_rn, .seen_rnr => unreachable,
	218	.finished => {
	219	// TODO https://github.com/ziglang/zig/issues/14039
	220	const buf_avail = req.read_buffer_len - req.read_buffer_start;
	221	const data_avail = req.response.next_chunk_length;
	222	const out_avail = buffer.len;
	223
	224	if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
	225	const can_read = @intCast(usize, @min(buf_avail, data_avail));
	226	req.response.next_chunk_length -= can_read;
	227
	228	if (req.response.next_chunk_length == 0) {
	229	req.client.connection_pool.release(req.client, req.connection);
	230	req.connection = undefined;
	231	req.response.done = true;
	232	}
	233
	234	return 0; // skip over as much data as possible
	235	}
	236
	237	const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
	238	req.response.next_chunk_length -= can_read;
	239
	240	mem.copy(u8, buffer[0..], req.read_buffer[req.read_buffer_start..][0..can_read]);
	241	req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
	242
	243	if (req.response.next_chunk_length == 0) {
	244	req.client.connection_pool.release(req.client, req.connection);
	245	req.connection = undefined;
	246	req.response.done = true;
	247	}
	248
	249	return can_read;
	250	},
	251	.chunk_size_prefix_r => switch (req.read_buffer_len - req.read_buffer_start) {
	252	0 => return out_index,
	253	1 => switch (req.read_buffer[req.read_buffer_start]) {
	254	'\r' => {
	255	req.response.state = .chunk_size_prefix_n;
	256	return out_index;
	257	},
	258	else => {
	259	req.response.state = .invalid;
	260	return error.HttpHeadersInvalid;
	261	},
	262	},
	263	else => switch (int16(req.read_buffer[req.read_buffer_start..][0..2])) {
	264	int16("\r\n") => {
	265	req.read_buffer_start += 2;
	266	req.response.state = .chunk_size;
	267	continue;
	268	},
	269	else => {
	270	req.response.state = .invalid;
	271	return error.HttpHeadersInvalid;
	272	},
	273	},
	274	},
	275	.chunk_size_prefix_n => switch (req.read_buffer_len - req.read_buffer_start) {
	276	0 => return out_index,
	277	else => switch (req.read_buffer[req.read_buffer_start]) {
	278	'\n' => {
	279	req.read_buffer_start += 1;
	280	req.response.state = .chunk_size;
	281	continue;
	282	},
	283	else => {
	284	req.response.state = .invalid;
	285	return error.HttpHeadersInvalid;
	286	},
	287	},
	288	},
	289	.chunk_size, .chunk_r => {
	290	const i = req.response.findChunkedLen(req.read_buffer[req.read_buffer_start..req.read_buffer_len]);
	291	switch (req.response.state) {
	292	.invalid => return error.HttpHeadersInvalid,
	293	.chunk_data => {
	294	if (req.response.next_chunk_length == 0) {
	295	req.response.done = true;
	296	req.client.connection_pool.release(req.client, req.connection);
	297	req.connection = undefined;
	298
	299	return out_index;
	300	}
	301
	302	req.read_buffer_start += @intCast(ReadBufferIndex, i);
	303	continue;
	304	},
	305	.chunk_size => return out_index,
	306	else => unreachable,
	307	}
	308	},
	309	.chunk_data => {
	310	// TODO https://github.com/ziglang/zig/issues/14039
	311	const buf_avail = req.read_buffer_len - req.read_buffer_start;
	312	const data_avail = req.response.next_chunk_length;
	313	const out_avail = buffer.len - out_index;
	314
	315	if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
	316	const can_read = @intCast(usize, @min(buf_avail, data_avail));
	317	req.response.next_chunk_length -= can_read;
	318
	319	if (req.response.next_chunk_length == 0) {
	320	req.client.connection_pool.release(req.client, req.connection);
	321	req.connection = undefined;
	322	req.response.done = true;
	323	continue;
	324	}
	325
	326	return 0; // skip over as much data as possible
	327	}
	328
	329	const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
	330	req.response.next_chunk_length -= can_read;
	331
	332	mem.copy(u8, buffer[out_index..], req.read_buffer[req.read_buffer_start..][0..can_read]);
	333	req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
	334	out_index += can_read;
	335
	336	if (req.response.next_chunk_length == 0) {
	337	req.response.state = .chunk_size_prefix_r;
	338
	339	continue;
	340	}
	341
	342	return out_index;
	343	},
	344	}
	345	}
	346	}
	347
	348	pub const ReadError = Client.DeflateDecompressor.Error \|\| Client.GzipDecompressor.Error \|\| Client.ZstdDecompressor.Error \|\| WaitForCompleteHeadError \|\| error{ BadHeader, InvalidCompression, StreamTooLong, InvalidWindowSize, CompressionNotSupported };
	349
	350	pub const Reader = std.io.Reader(*Request, ReadError, read);
	351
	352	pub fn reader(req: *Request) Reader {
	353	return .{ .context = req };
	354	}
	355
	356	pub fn read(req: *Request, buffer: []u8) ReadError!usize {
	357	while (true) {
	358	if (!req.response.state.isContent()) try req.waitForCompleteHead();
	359
	360	if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
	361	assert(try req.readRaw(buffer) == 0);
	362
	363	if (req.redirects_left == 0) return error.TooManyHttpRedirects;
	364
	365	const location = req.response.headers.location orelse
	366	return error.HttpRedirectMissingLocation;
	367	const new_url = Uri.parse(location) catch try Uri.parseWithoutScheme(location);
	368
	369	var new_arena = std.heap.ArenaAllocator.init(req.client.allocator);
	370	const resolved_url = try req.uri.resolve(new_url, false, new_arena.allocator());
	371	errdefer new_arena.deinit();
	372
	373	req.arena.deinit();
	374	req.arena = new_arena;
	375
	376	const new_req = try req.client.request(resolved_url, req.headers, .{
	377	.max_redirects = req.redirects_left - 1,
	378	.header_strategy = if (req.response.header_bytes_owned) .{
	379	.dynamic = req.response.max_header_bytes,
	380	} else .{
	381	.static = req.response.header_bytes.unusedCapacitySlice(),
	382	},
	383	});
	384	req.deinit();
	385	req.* = new_req;
	386	} else {
	387	break;
	388	}
	389	}
	390
	391	if (req.response.compression == .none) {
	392	if (req.response.headers.transfer_compression) \|compression\| {
	393	switch (compression) {
	394	.compress => return error.CompressionNotSupported,
	395	.deflate => req.response.compression = .{
	396	.deflate = try std.compress.zlib.zlibStream(req.client.allocator, ReaderRaw{ .context = req }),
	397	},
	398	.gzip => req.response.compression = .{
	399	.gzip = try std.compress.gzip.decompress(req.client.allocator, ReaderRaw{ .context = req }),
	400	},
	401	.zstd => req.response.compression = .{
	402	.zstd = std.compress.zstd.decompressStream(req.client.allocator, ReaderRaw{ .context = req }),
	403	},
	404	}
	405	}
	406	}
	407
	408	return switch (req.response.compression) {
	409	.deflate => \|*deflate\| try deflate.read(buffer),
	410	.gzip => \|*gzip\| try gzip.read(buffer),
	411	.zstd => \|*zstd\| try zstd.read(buffer),
	412	else => try req.readRaw(buffer),
	413	};
	414	}
	415
	416	pub fn readAll(req: *Request, buffer: []u8) !usize {
	417	var index: usize = 0;
	418	while (index < buffer.len) {
	419	const amt = try read(req, buffer[index..]);
	420	if (amt == 0) break;
	421	index += amt;
	422	}
	423	return index;
	424	}
	425
	426	pub const WriteError = Connection.WriteError \|\| error{MessageTooLong};
	427
	428	pub const Writer = std.io.Writer(*Request, WriteError, write);
	429
	430	pub fn writer(req: *Request) Writer {
	431	return .{ .context = req };
	432	}
	433
	434	/// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.
	435	pub fn write(req: *Request, bytes: []const u8) !usize {
	436	switch (req.headers.transfer_encoding) {
	437	.chunked => {
	438	try req.connection.data.writer().print("{x}\r\n", .{bytes.len});
	439	try req.connection.data.writeAll(bytes);
	440	try req.connection.data.writeAll("\r\n");
	441
	442	return bytes.len;
	443	},
	444	.content_length => \|*len\| {
	445	if (len.* < bytes.len) return error.MessageTooLong;
	446
	447	const amt = try req.connection.data.write(bytes);
	448	len.* -= amt;
	449	return amt;
	450	},
	451	.none => return error.NotWriteable,
	452	}
	453	}
	454
	455	/// Finish the body of a request. This notifies the server that you have no more data to send.
	456	pub fn finish(req: *Request) !void {
	457	switch (req.headers.transfer_encoding) {
	458	.chunked => try req.connection.data.writeAll("0\r\n"),
	459	.content_length => \|len\| if (len != 0) return error.MessageNotCompleted,
	460	.none => {},
	461	}
	462	}
	463
	464	inline fn int16(array: *const [2]u8) u16 {
	465	return @bitCast(u16, array.*);
	466	}
	467
	468	inline fn int32(array: *const [4]u8) u32 {
	469	return @bitCast(u32, array.*);
	470	}
	471
	472	inline fn int64(array: *const [8]u8) u64 {
	473	return @bitCast(u64, array.*);
	474	}
	475
	476	test {
	477	const builtin = @import("builtin");
	478
	479	if (builtin.os.tag == .wasi) return error.SkipZigTest;
	480
	481	_ = Response;
	482	}

lib/std/http/Client/Response.zig created+509

...	...	@@ -0,0 +1,509 @@
	1	const std = @import("std");
	2	const http = std.http;
	3	const mem = std.mem;
	4	const testing = std.testing;
	5	const assert = std.debug.assert;
	6
	7	const Client = @import("../Client.zig");
	8	const Response = @This();
	9
	10	headers: Headers,
	11	state: State,
	12	header_bytes_owned: bool,
	13	/// This could either be a fixed buffer provided by the API user or it
	14	/// could be our own array list.
	15	header_bytes: std.ArrayListUnmanaged(u8),
	16	max_header_bytes: usize,
	17	next_chunk_length: u64,
	18	done: bool = false,
	19
	20	compression: union(enum) {
	21	deflate: Client.DeflateDecompressor,
	22	gzip: Client.GzipDecompressor,
	23	zstd: Client.ZstdDecompressor,
	24	none: void,
	25	} = .none,
	26
	27	pub const Headers = struct {
	28	status: http.Status,
	29	version: http.Version,
	30	location: ?[]const u8 = null,
	31	content_length: ?u64 = null,
	32	transfer_encoding: ?http.TransferEncoding = null,
	33	transfer_compression: ?http.ContentEncoding = null,
	34	connection: http.Connection = .close,
	35	upgrade: ?[]const u8 = null,
	36
	37	number_of_headers: usize = 0,
	38
	39	pub fn parse(bytes: []const u8) !Headers {
	40	var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
	41
	42	const first_line = it.first();
	43	if (first_line.len < 12)
	44	return error.ShortHttpStatusLine;
	45
	46	const version: http.Version = switch (int64(first_line[0..8])) {
	47	int64("HTTP/1.0") => .@"HTTP/1.0",
	48	int64("HTTP/1.1") => .@"HTTP/1.1",
	49	else => return error.BadHttpVersion,
	50	};
	51	if (first_line[8] != ' ') return error.HttpHeadersInvalid;
	52	const status = @intToEnum(http.Status, parseInt3(first_line[9..12].*));
	53
	54	var headers: Headers = .{
	55	.version = version,
	56	.status = status,
	57	};
	58
	59	while (it.next()) \|line\| {
	60	headers.number_of_headers += 1;
	61
	62	if (line.len == 0) return error.HttpHeadersInvalid;
	63	switch (line[0]) {
	64	' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
	65	else => {},
	66	}
	67	var line_it = mem.split(u8, line, ": ");
	68	const header_name = line_it.first();
	69	const header_value = line_it.rest();
	70	if (std.ascii.eqlIgnoreCase(header_name, "location")) {
	71	if (headers.location != null) return error.HttpHeadersInvalid;
	72	headers.location = header_value;
	73	} else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
	74	if (headers.content_length != null) return error.HttpHeadersInvalid;
	75	headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
	76	} else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
	77	if (headers.transfer_encoding != null or headers.transfer_compression != null) return error.HttpHeadersInvalid;
	78
	79	// Transfer-Encoding: second, first
	80	// Transfer-Encoding: deflate, chunked
	81	var iter = std.mem.splitBackwards(u8, header_value, ",");
	82
	83	if (iter.next()) \|first\| {
	84	const trimmed = std.mem.trim(u8, first, " ");
	85
	86	if (std.meta.stringToEnum(http.TransferEncoding, trimmed)) \|te\| {
	87	headers.transfer_encoding = te;
	88	} else if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
	89	headers.transfer_compression = ce;
	90	} else {
	91	return error.HttpTransferEncodingUnsupported;
	92	}
	93	}
	94
	95	if (iter.next()) \|second\| {
	96	if (headers.transfer_compression != null) return error.HttpTransferEncodingUnsupported;
	97
	98	const trimmed = std.mem.trim(u8, second, " ");
	99
	100	if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
	101	headers.transfer_compression = ce;
	102	} else {
	103	return error.HttpTransferEncodingUnsupported;
	104	}
	105	}
	106
	107	if (iter.next()) \|_\| return error.HttpTransferEncodingUnsupported;
	108	} else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
	109	if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
	110
	111	const trimmed = std.mem.trim(u8, header_value, " ");
	112
	113	if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
	114	headers.transfer_compression = ce;
	115	} else {
	116	return error.HttpTransferEncodingUnsupported;
	117	}
	118	} else if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
	119	if (std.ascii.eqlIgnoreCase(header_value, "keep-alive")) {
	120	headers.connection = .keep_alive;
	121	} else if (std.ascii.eqlIgnoreCase(header_value, "close")) {
	122	headers.connection = .close;
	123	} else {
	124	return error.HttpConnectionHeaderUnsupported;
	125	}
	126	} else if (std.ascii.eqlIgnoreCase(header_name, "upgrade")) {
	127	headers.upgrade = header_value;
	128	}
	129	}
	130
	131	return headers;
	132	}
	133
	134	test "parse headers" {
	135	const example =
	136	"HTTP/1.1 301 Moved Permanently\r\n" ++
	137	"Location: https://www.example.com/\r\n" ++
	138	"Content-Type: text/html; charset=UTF-8\r\n" ++
	139	"Content-Length: 220\r\n\r\n";
	140	const parsed = try Headers.parse(example);
	141	try testing.expectEqual(http.Version.@"HTTP/1.1", parsed.version);
	142	try testing.expectEqual(http.Status.moved_permanently, parsed.status);
	143	try testing.expectEqualStrings("https://www.example.com/", parsed.location orelse
	144	return error.TestFailed);
	145	try testing.expectEqual(@as(?u64, 220), parsed.content_length);
	146	}
	147
	148	test "header continuation" {
	149	const example =
	150	"HTTP/1.0 200 OK\r\n" ++
	151	"Content-Type: text/html;\r\n charset=UTF-8\r\n" ++
	152	"Content-Length: 220\r\n\r\n";
	153	try testing.expectError(
	154	error.HttpHeaderContinuationsUnsupported,
	155	Headers.parse(example),
	156	);
	157	}
	158
	159	test "extra content length" {
	160	const example =
	161	"HTTP/1.0 200 OK\r\n" ++
	162	"Content-Length: 220\r\n" ++
	163	"Content-Type: text/html; charset=UTF-8\r\n" ++
	164	"content-length: 220\r\n\r\n";
	165	try testing.expectError(
	166	error.HttpHeadersInvalid,
	167	Headers.parse(example),
	168	);
	169	}
	170	};
	171
	172	inline fn int16(array: *const [2]u8) u16 {
	173	return @bitCast(u16, array.*);
	174	}
	175
	176	inline fn int32(array: *const [4]u8) u32 {
	177	return @bitCast(u32, array.*);
	178	}
	179
	180	inline fn int64(array: *const [8]u8) u64 {
	181	return @bitCast(u64, array.*);
	182	}
	183
	184	pub const State = enum {
	185	/// Begin header parsing states.
	186	invalid,
	187	start,
	188	seen_r,
	189	seen_rn,
	190	seen_rnr,
	191	finished,
	192	/// Begin transfer-encoding: chunked parsing states.
	193	chunk_size_prefix_r,
	194	chunk_size_prefix_n,
	195	chunk_size,
	196	chunk_r,
	197	chunk_data,
	198
	199	pub fn isContent(self: State) bool {
	200	return switch (self) {
	201	.invalid, .start, .seen_r, .seen_rn, .seen_rnr => false,
	202	.finished, .chunk_size_prefix_r, .chunk_size_prefix_n, .chunk_size, .chunk_r, .chunk_data => true,
	203	};
	204	}
	205	};
	206
	207	pub fn initDynamic(max: usize) Response {
	208	return .{
	209	.state = .start,
	210	.headers = undefined,
	211	.header_bytes = .{},
	212	.max_header_bytes = max,
	213	.header_bytes_owned = true,
	214	.next_chunk_length = undefined,
	215	};
	216	}
	217
	218	pub fn initStatic(buf: []u8) Response {
	219	return .{
	220	.state = .start,
	221	.headers = undefined,
	222	.header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
	223	.max_header_bytes = buf.len,
	224	.header_bytes_owned = false,
	225	.next_chunk_length = undefined,
	226	};
	227	}
	228
	229	/// Returns how many bytes are part of HTTP headers. Always less than or
	230	/// equal to bytes.len. If the amount returned is less than bytes.len, it
	231	/// means the headers ended and the first byte after the double \r\n\r\n is
	232	/// located at `bytes[result]`.
	233	pub fn findHeadersEnd(r: *Response, bytes: []const u8) usize {
	234	var index: usize = 0;
	235
	236	// TODO: https://github.com/ziglang/zig/issues/8220
	237	state: while (true) {
	238	switch (r.state) {
	239	.invalid => unreachable,
	240	.finished => unreachable,
	241	.start => while (true) {
	242	switch (bytes.len - index) {
	243	0 => return index,
	244	1 => {
	245	if (bytes[index] == '\r')
	246	r.state = .seen_r;
	247	return index + 1;
	248	},
	249	2 => {
	250	if (int16(bytes[index..][0..2]) == int16("\r\n")) {
	251	r.state = .seen_rn;
	252	} else if (bytes[index + 1] == '\r') {
	253	r.state = .seen_r;
	254	}
	255	return index + 2;
	256	},
	257	3 => {
	258	if (int16(bytes[index..][0..2]) == int16("\r\n") and
	259	bytes[index + 2] == '\r')
	260	{
	261	r.state = .seen_rnr;
	262	} else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n")) {
	263	r.state = .seen_rn;
	264	} else if (bytes[index + 2] == '\r') {
	265	r.state = .seen_r;
	266	}
	267	return index + 3;
	268	},
	269	4...15 => {
	270	if (int32(bytes[index..][0..4]) == int32("\r\n\r\n")) {
	271	r.state = .finished;
	272	return index + 4;
	273	} else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n") and
	274	bytes[index + 3] == '\r')
	275	{
	276	r.state = .seen_rnr;
	277	index += 4;
	278	continue :state;
	279	} else if (int16(bytes[index + 2 ..][0..2]) == int16("\r\n")) {
	280	r.state = .seen_rn;
	281	index += 4;
	282	continue :state;
	283	} else if (bytes[index + 3] == '\r') {
	284	r.state = .seen_r;
	285	index += 4;
	286	continue :state;
	287	}
	288	index += 4;
	289	continue;
	290	},
	291	else => {
	292	const chunk = bytes[index..][0..16];
	293	const v: @Vector(16, u8) = chunk.*;
	294	const matches_r = v == @splat(16, @as(u8, '\r'));
	295	const iota = std.simd.iota(u8, 16);
	296	const default = @splat(16, @as(u8, 16));
	297	const sub_index = @reduce(.Min, @select(u8, matches_r, iota, default));
	298	switch (sub_index) {
	299	0...12 => {
	300	index += sub_index + 4;
	301	if (int32(chunk[sub_index..][0..4]) == int32("\r\n\r\n")) {
	302	r.state = .finished;
	303	return index;
	304	}
	305	continue;
	306	},
	307	13 => {
	308	index += 16;
	309	if (int16(chunk[14..][0..2]) == int16("\n\r")) {
	310	r.state = .seen_rnr;
	311	continue :state;
	312	}
	313	continue;
	314	},
	315	14 => {
	316	index += 16;
	317	if (chunk[15] == '\n') {
	318	r.state = .seen_rn;
	319	continue :state;
	320	}
	321	continue;
	322	},
	323	15 => {
	324	r.state = .seen_r;
	325	index += 16;
	326	continue :state;
	327	},
	328	16 => {
	329	index += 16;
	330	continue;
	331	},
	332	else => unreachable,
	333	}
	334	},
	335	}
	336	},
	337
	338	.seen_r => switch (bytes.len - index) {
	339	0 => return index,
	340	1 => {
	341	switch (bytes[index]) {
	342	'\n' => r.state = .seen_rn,
	343	'\r' => r.state = .seen_r,
	344	else => r.state = .start,
	345	}
	346	return index + 1;
	347	},
	348	2 => {
	349	if (int16(bytes[index..][0..2]) == int16("\n\r")) {
	350	r.state = .seen_rnr;
	351	return index + 2;
	352	}
	353	r.state = .start;
	354	return index + 2;
	355	},
	356	else => {
	357	if (int16(bytes[index..][0..2]) == int16("\n\r") and
	358	bytes[index + 2] == '\n')
	359	{
	360	r.state = .finished;
	361	return index + 3;
	362	}
	363	index += 3;
	364	r.state = .start;
	365	continue :state;
	366	},
	367	},
	368	.seen_rn => switch (bytes.len - index) {
	369	0 => return index,
	370	1 => {
	371	switch (bytes[index]) {
	372	'\r' => r.state = .seen_rnr,
	373	else => r.state = .start,
	374	}
	375	return index + 1;
	376	},
	377	else => {
	378	if (int16(bytes[index..][0..2]) == int16("\r\n")) {
	379	r.state = .finished;
	380	return index + 2;
	381	}
	382	index += 2;
	383	r.state = .start;
	384	continue :state;
	385	},
	386	},
	387	.seen_rnr => switch (bytes.len - index) {
	388	0 => return index,
	389	else => {
	390	if (bytes[index] == '\n') {
	391	r.state = .finished;
	392	return index + 1;
	393	}
	394	index += 1;
	395	r.state = .start;
	396	continue :state;
	397	},
	398	},
	399	.chunk_size_prefix_r => unreachable,
	400	.chunk_size_prefix_n => unreachable,
	401	.chunk_size => unreachable,
	402	.chunk_r => unreachable,
	403	.chunk_data => unreachable,
	404	}
	405
	406	return index;
	407	}
	408	}
	409
	410	pub fn findChunkedLen(r: *Response, bytes: []const u8) usize {
	411	var i: usize = 0;
	412	if (r.state == .chunk_size) {
	413	while (i < bytes.len) : (i += 1) {
	414	const digit = switch (bytes[i]) {
	415	'0'...'9' => \|b\| b - '0',
	416	'A'...'Z' => \|b\| b - 'A' + 10,
	417	'a'...'z' => \|b\| b - 'a' + 10,
	418	'\r' => {
	419	r.state = .chunk_r;
	420	i += 1;
	421	break;
	422	},
	423	else => {
	424	r.state = .invalid;
	425	return i;
	426	},
	427	};
	428	const mul = @mulWithOverflow(r.next_chunk_length, 16);
	429	if (mul[1] != 0) {
	430	r.state = .invalid;
	431	return i;
	432	}
	433	const add = @addWithOverflow(mul[0], digit);
	434	if (add[1] != 0) {
	435	r.state = .invalid;
	436	return i;
	437	}
	438	r.next_chunk_length = add[0];
	439	} else {
	440	return i;
	441	}
	442	}
	443	assert(r.state == .chunk_r);
	444	if (i == bytes.len) return i;
	445
	446	if (bytes[i] == '\n') {
	447	r.state = .chunk_data;
	448	return i + 1;
	449	} else {
	450	r.state = .invalid;
	451	return i;
	452	}
	453	}
	454
	455	fn parseInt3(nnn: @Vector(3, u8)) u10 {
	456	const zero: @Vector(3, u8) = .{ '0', '0', '0' };
	457	const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
	458	return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
	459	}
	460
	461	test parseInt3 {
	462	const expectEqual = std.testing.expectEqual;
	463	try expectEqual(@as(u10, 0), parseInt3("000".*));
	464	try expectEqual(@as(u10, 418), parseInt3("418".*));
	465	try expectEqual(@as(u10, 999), parseInt3("999".*));
	466	}
	467
	468	test "find headers end basic" {
	469	var buffer: [1]u8 = undefined;
	470	var r = Response.initStatic(&buffer);
	471	try testing.expectEqual(@as(usize, 10), r.findHeadersEnd("HTTP/1.1 4"));
	472	try testing.expectEqual(@as(usize, 2), r.findHeadersEnd("18"));
	473	try testing.expectEqual(@as(usize, 8), r.findHeadersEnd(" lol\r\n\r\nblah blah"));
	474	}
	475
	476	test "find headers end vectorized" {
	477	var buffer: [1]u8 = undefined;
	478	var r = Response.initStatic(&buffer);
	479	const example =
	480	"HTTP/1.1 301 Moved Permanently\r\n" ++
	481	"Location: https://www.example.com/\r\n" ++
	482	"Content-Type: text/html; charset=UTF-8\r\n" ++
	483	"Content-Length: 220\r\n" ++
	484	"\r\ncontent";
	485	try testing.expectEqual(@as(usize, 131), r.findHeadersEnd(example));
	486	}
	487
	488	test "find headers end bug" {
	489	var buffer: [1]u8 = undefined;
	490	var r = Response.initStatic(&buffer);
	491	const trail = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
	492	const example =
	493	"HTTP/1.1 200 OK\r\n" ++
	494	"Access-Control-Allow-Origin: https://render.githubusercontent.com\r\n" ++
	495	"content-disposition: attachment; filename=zig-0.10.0.tar.gz\r\n" ++
	496	"Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'; sandbox\r\n" ++
	497	"Content-Type: application/x-gzip\r\n" ++
	498	"ETag: \"bfae0af6b01c7c0d89eb667cb5f0e65265968aeebda2689177e6b26acd3155ca\"\r\n" ++
	499	"Strict-Transport-Security: max-age=31536000\r\n" ++
	500	"Vary: Authorization,Accept-Encoding,Origin\r\n" ++
	501	"X-Content-Type-Options: nosniff\r\n" ++
	502	"X-Frame-Options: deny\r\n" ++
	503	"X-XSS-Protection: 1; mode=block\r\n" ++
	504	"Date: Fri, 06 Jan 2023 22:26:22 GMT\r\n" ++
	505	"Transfer-Encoding: chunked\r\n" ++
	506	"X-GitHub-Request-Id: 89C6:17E9:A7C9E:124B51:63B8A00E\r\n" ++
	507	"connection: close\r\n\r\n" ++ trail;
	508	try testing.expectEqual(@as(usize, example.len - trail.len), r.findHeadersEnd(example));
	509	}

lib/std/net.zig+31-3

...	...	@@ -702,8 +702,10 @@ pub const AddressList = struct {
702	702	}
703	703	};
704	704
	705	pub const TcpConnectToHostError = GetAddressListError \|\| TcpConnectToAddressError;
	706
705	707	/// All memory allocated with `allocator` will be freed before this function returns.
706		pub fn tcpConnectToHost(allocator: mem.Allocator, name: []const u8, port: u16) !Stream {
	708	pub fn tcpConnectToHost(allocator: mem.Allocator, name: []const u8, port: u16) TcpConnectToHostError!Stream {
707	709	const list = try getAddressList(allocator, name, port);
708	710	defer list.deinit();
709	711
...	...	@@ -720,7 +722,9 @@ pub fn tcpConnectToHost(allocator: mem.Allocator, name: []const u8, port: u16) !
720	722	return std.os.ConnectError.ConnectionRefused;
721	723	}
722	724
723		pub fn tcpConnectToAddress(address: Address) !Stream {
	725	pub const TcpConnectToAddressError = std.os.SocketError \|\| std.os.ConnectError;
	726
	727	pub fn tcpConnectToAddress(address: Address) TcpConnectToAddressError!Stream {
724	728	const nonblock = if (std.io.is_async) os.SOCK.NONBLOCK else 0;
725	729	const sock_flags = os.SOCK.STREAM \| nonblock \|
726	730	(if (builtin.target.os.tag == .windows) 0 else os.SOCK.CLOEXEC);
...	...	@@ -737,8 +741,32 @@ pub fn tcpConnectToAddress(address: Address) !Stream {
737	741	return Stream{ .handle = sockfd };
738	742	}
739	743
	744	const GetAddressListError = std.mem.Allocator.Error \|\| std.fs.File.OpenError \|\| std.fs.File.ReadError \|\| std.os.SocketError \|\| std.os.BindError \|\| error{
	745	// TODO: break this up into error sets from the various underlying functions
	746
	747	TemporaryNameServerFailure,
	748	NameServerFailure,
	749	AddressFamilyNotSupported,
	750	UnknownHostName,
	751	ServiceUnavailable,
	752	Unexpected,
	753
	754	HostLacksNetworkAddresses,
	755
	756	InvalidCharacter,
	757	InvalidEnd,
	758	NonCanonical,
	759	Overflow,
	760	Incomplete,
	761	InvalidIpv4Mapping,
	762	InvalidIPAddressFormat,
	763
	764	InterfaceNotFound,
	765	FileSystem,
	766	};
	767
740	768	/// Call `AddressList.deinit` on the result.
741		pub fn getAddressList(allocator: mem.Allocator, name: []const u8, port: u16) !*AddressList {
	769	pub fn getAddressList(allocator: mem.Allocator, name: []const u8, port: u16) GetAddressListError!*AddressList {
742	770	const result = blk: {
743	771	var arena = std.heap.ArenaAllocator.init(allocator);
744	772	errdefer arena.deinit();

lib/std/std.zig+5

...	...	@@ -185,6 +185,11 @@ pub const options = struct {
185	185	options_override.keep_sigpipe
186	186	else
187	187	false;
	188
	189	pub const http_connection_pool_size = if (@hasDecl(options_override, "http_connection_pool_size"))
	190	options_override.http_connection_pool_size
	191	else
	192	http.Client.default_connection_pool_size;
188	193	};
189	194
190	195	// This forces the start.zig file to be imported, and the comptime logic inside that