Commit 2ee328995a70c5c446f24c5593e0fad760e6d839 | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-04-09 10:44:52-04:00
committer	GitHub <noreply@github.com> 2023-04-09 10:44:52-04:00
log	2ee328995a70c5c446f24c5593e0fad760e6d839
tree	0e547171b7790ffd182fc298d384ef614571e97e
parent	c22a30ac99b9a2b92d9a8e926b9bf0c9dbc3d14e
parent	7f9a4625fda0b1a33177cdd66819f0a061c6b2da
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15123 from truemedian/http-server

std.http: add http server

7 files changed, 2205 insertions(+), 1112 deletions(-)

lib
- std
  - http.zig
  - http
    Client.zig
    Client
    Request.zig
    Response.zig
    Server.zig
    protocol.zig
src
- Package.zig

lib/std/http.zig+2

...	@@ -1,4 +1,6 @@	...	@@ -1,4 +1,6 @@
1	pub const Client = @import("http/Client.zig");	1	pub const Client = @import("http/Client.zig");
		2	pub const Server = @import("http/Server.zig");
		3	pub const protocol = @import("http/protocol.zig");
2		4
3	pub const Version = enum {	5	pub const Version = enum {
4	@"HTTP/1.0",	6	@"HTTP/1.0",

lib/std/http/Client.zig+759-121

...	@@ -1,49 +1,105 @@	...	@@ -1,49 +1,105 @@
1	//! TODO: send connection: keep-alive and LRU cache a configurable number of	1	//! Connecting and opening requests are threadsafe. Individual requests are not.
2	//! open connections to skip DNS and TLS handshake for subsequent requests.
3	//!
4	//! This API is not thread safe.
5		2
6	const std = @import("../std.zig");	3	const std = @import("../std.zig");
7	const mem = std.mem;	4	const testing = std.testing;
8	const assert = std.debug.assert;
9	const http = std.http;	5	const http = std.http;
		6	const mem = std.mem;
10	const net = std.net;	7	const net = std.net;
11	const Client = @This();
12	const Uri = std.Uri;	8	const Uri = std.Uri;
13	const Allocator = std.mem.Allocator;	9	const Allocator = mem.Allocator;
14	const testing = std.testing;	10	const assert = std.debug.assert;
15		11
16	pub const Request = @import("Client/Request.zig");	12	const Client = @This();
17	pub const Response = @import("Client/Response.zig");	13	const proto = @import("protocol.zig");
18		14
19	pub const default_connection_pool_size = 32;	15	pub const default_connection_pool_size = 32;
20	const connection_pool_size = std.options.http_connection_pool_size;	16	pub const connection_pool_size = std.options.http_connection_pool_size;
21		17
22	/// Used for tcpConnectToHost and storing HTTP headers when an externally
23	/// managed buffer is not provided.
24	allocator: Allocator,	18	allocator: Allocator,
25	ca_bundle: std.crypto.Certificate.Bundle = .{},	19	ca_bundle: std.crypto.Certificate.Bundle = .{},
		20	ca_bundle_mutex: std.Thread.Mutex = .{},
26	/// When this is `true`, the next time this client performs an HTTPS request,	21	/// When this is `true`, the next time this client performs an HTTPS request,
27	/// it will first rescan the system for root certificates.	22	/// it will first rescan the system for root certificates.
28	next_https_rescan_certs: bool = true,	23	next_https_rescan_certs: bool = true,
29		24
		25	/// The pool of connections that can be reused (and currently in use).
30	connection_pool: ConnectionPool = .{},	26	connection_pool: ConnectionPool = .{},
31		27
		28	/// The last error that occurred on this client. This is not threadsafe, do not expect it to be completely accurate.
		29	last_error: ?ExtraError = null,
		30
		31	pub const ExtraError = union(enum) {
		32	fn impliedErrorSet(comptime f: anytype) type {
		33	const set = @typeInfo(@typeInfo(@TypeOf(f)).Fn.return_type.?).ErrorUnion.error_set;
		34	if (@typeName(set)[0] != '@') @compileError(@typeName(f) ++ " doesn't have an implied error set any more.");
		35	return set;
		36	}
		37
		38	// There's apparently a dependency loop with using Client.DeflateDecompressor.
		39	const FakeTransferError = proto.HeadersParser.ReadError \|\| error{ReadFailed};
		40	const FakeTransferReader = std.io.Reader(void, FakeTransferError, fakeRead);
		41	fn fakeRead(ctx: void, buf: []u8) FakeTransferError!usize {
		42	_ = .{ buf, ctx };
		43	return 0;
		44	}
		45
		46	const FakeDeflateDecompressor = std.compress.zlib.ZlibStream(FakeTransferReader);
		47	const FakeGzipDecompressor = std.compress.gzip.Decompress(FakeTransferReader);
		48	const FakeZstdDecompressor = std.compress.zstd.DecompressStream(FakeTransferReader, .{});
		49
		50	pub const TcpConnectError = std.net.TcpConnectToHostError;
		51	pub const TlsError = std.crypto.tls.Client.InitError(net.Stream);
		52	pub const WriteError = BufferedConnection.WriteError;
		53	pub const ReadError = BufferedConnection.ReadError \|\| error{HttpChunkInvalid};
		54	pub const CaBundleError = impliedErrorSet(std.crypto.Certificate.Bundle.rescan);
		55
		56	pub const ZlibInitError = error{ BadHeader, InvalidCompression, InvalidWindowSize, Unsupported, EndOfStream, OutOfMemory } \|\| Request.TransferReadError;
		57	pub const GzipInitError = error{ BadHeader, InvalidCompression, OutOfMemory, WrongChecksum, EndOfStream, StreamTooLong } \|\| Request.TransferReadError;
		58	// pub const DecompressError = Client.DeflateDecompressor.Error \|\| Client.GzipDecompressor.Error \|\| Client.ZstdDecompressor.Error;
		59	pub const DecompressError = FakeDeflateDecompressor.Error \|\| FakeGzipDecompressor.Error \|\| FakeZstdDecompressor.Error;
		60
		61	zlib_init: ZlibInitError, // error.CompressionInitializationFailed
		62	gzip_init: GzipInitError, // error.CompressionInitializationFailed
		63	connect: TcpConnectError, // error.ConnectionFailed
		64	ca_bundle: CaBundleError, // error.CertificateAuthorityBundleFailed
		65	tls: TlsError, // error.TlsInitializationFailed
		66	write: WriteError, // error.WriteFailed
		67	read: ReadError, // error.ReadFailed
		68	decompress: DecompressError, // error.ReadFailed
		69	};
		70
		71	/// A set of linked lists of connections that can be reused.
32	pub const ConnectionPool = struct {	72	pub const ConnectionPool = struct {
		73	/// The criteria for a connection to be considered a match.
33	pub const Criteria = struct {	74	pub const Criteria = struct {
34	host: []const u8,	75	host: []const u8,
35	port: u16,	76	port: u16,
36	is_tls: bool,	77	is_tls: bool,
37	};	78	};
38		79
39	const Queue = std.TailQueue(Connection);	80	pub const StoredConnection = struct {
		81	buffered: BufferedConnection,
		82	host: []u8,
		83	port: u16,
		84
		85	closing: bool = false,
		86
		87	pub fn deinit(self: StoredConnection, client: Client) void {
		88	self.buffered.close(client);
		89	client.allocator.free(self.host);
		90	}
		91	};
		92
		93	const Queue = std.TailQueue(StoredConnection);
40	pub const Node = Queue.Node;	94	pub const Node = Queue.Node;
41		95
42	mutex: std.Thread.Mutex = .{},	96	mutex: std.Thread.Mutex = .{},
		97	/// Open connections that are currently in use.
43	used: Queue = .{},	98	used: Queue = .{},
		99	/// Open connections that are not currently in use.
44	free: Queue = .{},	100	free: Queue = .{},
45	free_len: usize = 0,	101	free_len: usize = 0,
46	free_size: usize = default_connection_pool_size,	102	free_size: usize = connection_pool_size,
47		103
48	/// Finds and acquires a connection from the connection pool matching the criteria. This function is threadsafe.	104	/// Finds and acquires a connection from the connection pool matching the criteria. This function is threadsafe.
49	/// If no connection is found, null is returned.	105	/// If no connection is found, null is returned.
...	@@ -53,9 +109,9 @@ pub const ConnectionPool = struct {	...	@@ -53,9 +109,9 @@ pub const ConnectionPool = struct {
53		109
54	var next = pool.free.last;	110	var next = pool.free.last;
55	while (next) \|node\| : (next = node.prev) {	111	while (next) \|node\| : (next = node.prev) {
56	if ((node.data.protocol == .tls) != criteria.is_tls) continue;	112	if ((node.data.buffered.conn.protocol == .tls) != criteria.is_tls) continue;
57	if (node.data.port != criteria.port) continue;	113	if (node.data.port != criteria.port) continue;
58	if (std.mem.eql(u8, node.data.host, criteria.host)) continue;	114	if (mem.eql(u8, node.data.host, criteria.host)) continue;
59		115
60	pool.acquireUnsafe(node);	116	pool.acquireUnsafe(node);
61	return node;	117	return node;
...	@@ -89,7 +145,7 @@ pub const ConnectionPool = struct {	...	@@ -89,7 +145,7 @@ pub const ConnectionPool = struct {
89	pool.used.remove(node);	145	pool.used.remove(node);
90		146
91	if (node.data.closing) {	147	if (node.data.closing) {
92	node.data.close(client);	148	node.data.deinit(client);
93		149
94	return client.allocator.destroy(node);	150	return client.allocator.destroy(node);
95	}	151	}
...	@@ -97,7 +153,7 @@ pub const ConnectionPool = struct {	...	@@ -97,7 +153,7 @@ pub const ConnectionPool = struct {
97	if (pool.free_len + 1 >= pool.free_size) {	153	if (pool.free_len + 1 >= pool.free_size) {
98	const popped = pool.free.popFirst() orelse unreachable;	154	const popped = pool.free.popFirst() orelse unreachable;
99		155
100	popped.data.close(client);	156	popped.data.deinit(client);
101		157
102	return client.allocator.destroy(popped);	158	return client.allocator.destroy(popped);
103	}	159	}
...	@@ -122,7 +178,7 @@ pub const ConnectionPool = struct {	...	@@ -122,7 +178,7 @@ pub const ConnectionPool = struct {
122	defer client.allocator.destroy(node);	178	defer client.allocator.destroy(node);
123	next = node.next;	179	next = node.next;
124		180
125	node.data.close(client);	181	node.data.deinit(client);
126	}	182	}
127		183
128	next = pool.used.first;	184	next = pool.used.first;
...	@@ -130,27 +186,19 @@ pub const ConnectionPool = struct {	...	@@ -130,27 +186,19 @@ pub const ConnectionPool = struct {
130	defer client.allocator.destroy(node);	186	defer client.allocator.destroy(node);
131	next = node.next;	187	next = node.next;
132		188
133	node.data.close(client);	189	node.data.deinit(client);
134	}	190	}
135		191
136	pool.* = undefined;	192	pool.* = undefined;
137	}	193	}
138	};	194	};
139		195
140	pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.ReaderRaw);	196	/// An interface to either a plain or TLS connection.
141	pub const GzipDecompressor = std.compress.gzip.Decompress(Request.ReaderRaw);
142	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.ReaderRaw, .{});
143
144	pub const Connection = struct {	197	pub const Connection = struct {
145	stream: net.Stream,	198	stream: net.Stream,
146	/// undefined unless protocol is tls.	199	/// undefined unless protocol is tls.
147	tls_client: *std.crypto.tls.Client, // TODO: allocate this, it's currently 16 KB.	200	tls_client: *std.crypto.tls.Client,
148	protocol: Protocol,	201	protocol: Protocol,
149	host: []u8,
150	port: u16,
151
152	// This connection has been part of a non keepalive request and cannot be added to the pool.
153	closing: bool = false,
154		202
155	pub const Protocol = enum { plain, tls };	203	pub const Protocol = enum { plain, tls };
156		204
...	@@ -215,11 +263,611 @@ pub const Connection = struct {	...	@@ -215,11 +263,611 @@ pub const Connection = struct {
215	}	263	}
216		264
217	conn.stream.close();	265	conn.stream.close();
		266	}
		267	};
		268
		269	/// A buffered (and peekable) Connection.
		270	pub const BufferedConnection = struct {
		271	pub const buffer_size = 0x2000;
		272
		273	conn: Connection,
		274	buf: [buffer_size]u8 = undefined,
		275	start: u16 = 0,
		276	end: u16 = 0,
		277
		278	pub fn fill(bconn: *BufferedConnection) ReadError!void {
		279	if (bconn.end != bconn.start) return;
		280
		281	const nread = try bconn.conn.read(bconn.buf[0..]);
		282	if (nread == 0) return error.EndOfStream;
		283	bconn.start = 0;
		284	bconn.end = @truncate(u16, nread);
		285	}
		286
		287	pub fn peek(bconn: *BufferedConnection) []const u8 {
		288	return bconn.buf[bconn.start..bconn.end];
		289	}
		290
		291	pub fn clear(bconn: *BufferedConnection, num: u16) void {
		292	bconn.start += num;
		293	}
		294
		295	pub fn readAtLeast(bconn: *BufferedConnection, buffer: []u8, len: usize) ReadError!usize {
		296	var out_index: u16 = 0;
		297	while (out_index < len) {
		298	const available = bconn.end - bconn.start;
		299	const left = buffer.len - out_index;
		300
		301	if (available > 0) {
		302	const can_read = @truncate(u16, @min(available, left));
		303
		304	std.mem.copy(u8, buffer[out_index..], bconn.buf[bconn.start..][0..can_read]);
		305	out_index += can_read;
		306	bconn.start += can_read;
		307
		308	continue;
		309	}
		310
		311	if (left > bconn.buf.len) {
		312	// skip the buffer if the output is large enough
		313	return bconn.conn.read(buffer[out_index..]);
		314	}
		315
		316	try bconn.fill();
		317	}
		318
		319	return out_index;
		320	}
		321
		322	pub fn read(bconn: *BufferedConnection, buffer: []u8) ReadError!usize {
		323	return bconn.readAtLeast(buffer, 1);
		324	}
		325
		326	pub const ReadError = Connection.ReadError \|\| error{EndOfStream};
		327	pub const Reader = std.io.Reader(*BufferedConnection, ReadError, read);
		328
		329	pub fn reader(bconn: *BufferedConnection) Reader {
		330	return Reader{ .context = bconn };
		331	}
		332
		333	pub fn writeAll(bconn: *BufferedConnection, buffer: []const u8) WriteError!void {
		334	return bconn.conn.writeAll(buffer);
		335	}
		336
		337	pub fn write(bconn: *BufferedConnection, buffer: []const u8) WriteError!usize {
		338	return bconn.conn.write(buffer);
		339	}
		340
		341	pub const WriteError = Connection.WriteError;
		342	pub const Writer = std.io.Writer(*BufferedConnection, WriteError, write);
		343
		344	pub fn writer(bconn: *BufferedConnection) Writer {
		345	return Writer{ .context = bconn };
		346	}
		347
		348	pub fn close(bconn: BufferedConnection, client: const Client) void {
		349	bconn.conn.close(client);
		350	}
		351	};
		352
		353	/// The mode of transport for requests.
		354	pub const RequestTransfer = union(enum) {
		355	content_length: u64,
		356	chunked: void,
		357	none: void,
		358	};
		359
		360	/// The decompressor for response messages.
		361	pub const Compression = union(enum) {
		362	pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.TransferReader);
		363	pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader);
		364	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});
		365
		366	deflate: DeflateDecompressor,
		367	gzip: GzipDecompressor,
		368	zstd: ZstdDecompressor,
		369	none: void,
		370	};
		371
		372	/// A HTTP response originating from a server.
		373	pub const Response = struct {
		374	pub const Headers = struct {
		375	status: http.Status,
		376	version: http.Version,
		377	location: ?[]const u8 = null,
		378	content_length: ?u64 = null,
		379	transfer_encoding: ?http.TransferEncoding = null,
		380	transfer_compression: ?http.ContentEncoding = null,
		381	connection: http.Connection = .close,
		382	upgrade: ?[]const u8 = null,
		383
		384	pub const ParseError = error{
		385	ShortHttpStatusLine,
		386	BadHttpVersion,
		387	HttpHeadersInvalid,
		388	HttpHeaderContinuationsUnsupported,
		389	HttpTransferEncodingUnsupported,
		390	HttpConnectionHeaderUnsupported,
		391	InvalidContentLength,
		392	CompressionNotSupported,
		393	};
		394
		395	pub fn parse(bytes: []const u8) ParseError!Headers {
		396	var it = mem.tokenize(u8, bytes[0 .. bytes.len - 4], "\r\n");
		397
		398	const first_line = it.next() orelse return error.HttpHeadersInvalid;
		399	if (first_line.len < 12)
		400	return error.ShortHttpStatusLine;
		401
		402	const version: http.Version = switch (int64(first_line[0..8])) {
		403	int64("HTTP/1.0") => .@"HTTP/1.0",
		404	int64("HTTP/1.1") => .@"HTTP/1.1",
		405	else => return error.BadHttpVersion,
		406	};
		407	if (first_line[8] != ' ') return error.HttpHeadersInvalid;
		408	const status = @intToEnum(http.Status, parseInt3(first_line[9..12].*));
		409
		410	var headers: Headers = .{
		411	.version = version,
		412	.status = status,
		413	};
		414
		415	while (it.next()) \|line\| {
		416	if (line.len == 0) return error.HttpHeadersInvalid;
		417	switch (line[0]) {
		418	' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
		419	else => {},
		420	}
		421
		422	var line_it = mem.tokenize(u8, line, ": ");
		423	const header_name = line_it.next() orelse return error.HttpHeadersInvalid;
		424	const header_value = line_it.rest();
		425	if (std.ascii.eqlIgnoreCase(header_name, "location")) {
		426	if (headers.location != null) return error.HttpHeadersInvalid;
		427	headers.location = header_value;
		428	} else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
		429	if (headers.content_length != null) return error.HttpHeadersInvalid;
		430	headers.content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength;
		431	} else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
		432	// Transfer-Encoding: second, first
		433	// Transfer-Encoding: deflate, chunked
		434	var iter = mem.splitBackwards(u8, header_value, ",");
		435
		436	if (iter.next()) \|first\| {
		437	const trimmed = mem.trim(u8, first, " ");
		438
		439	if (std.meta.stringToEnum(http.TransferEncoding, trimmed)) \|te\| {
		440	if (headers.transfer_encoding != null) return error.HttpHeadersInvalid;
		441	headers.transfer_encoding = te;
		442	} else if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
		443	if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
		444	headers.transfer_compression = ce;
		445	} else {
		446	return error.HttpTransferEncodingUnsupported;
		447	}
		448	}
		449
		450	if (iter.next()) \|second\| {
		451	if (headers.transfer_compression != null) return error.HttpTransferEncodingUnsupported;
		452
		453	const trimmed = mem.trim(u8, second, " ");
		454
		455	if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
		456	headers.transfer_compression = ce;
		457	} else {
		458	return error.HttpTransferEncodingUnsupported;
		459	}
		460	}
		461
		462	if (iter.next()) \|_\| return error.HttpTransferEncodingUnsupported;
		463	} else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
		464	if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
		465
		466	const trimmed = mem.trim(u8, header_value, " ");
		467
		468	if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
		469	headers.transfer_compression = ce;
		470	} else {
		471	return error.HttpTransferEncodingUnsupported;
		472	}
		473	} else if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
		474	if (std.ascii.eqlIgnoreCase(header_value, "keep-alive")) {
		475	headers.connection = .keep_alive;
		476	} else if (std.ascii.eqlIgnoreCase(header_value, "close")) {
		477	headers.connection = .close;
		478	} else {
		479	return error.HttpConnectionHeaderUnsupported;
		480	}
		481	} else if (std.ascii.eqlIgnoreCase(header_name, "upgrade")) {
		482	headers.upgrade = header_value;
		483	}
		484	}
		485
		486	return headers;
		487	}
		488
		489	inline fn int64(array: *const [8]u8) u64 {
		490	return @bitCast(u64, array.*);
		491	}
		492
		493	fn parseInt3(nnn: @Vector(3, u8)) u10 {
		494	const zero: @Vector(3, u8) = .{ '0', '0', '0' };
		495	const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
		496	return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
		497	}
		498
		499	test parseInt3 {
		500	const expectEqual = testing.expectEqual;
		501	try expectEqual(@as(u10, 0), parseInt3("000".*));
		502	try expectEqual(@as(u10, 418), parseInt3("418".*));
		503	try expectEqual(@as(u10, 999), parseInt3("999".*));
		504	}
		505	};
		506
		507	headers: Headers = undefined,
		508	parser: proto.HeadersParser,
		509	compression: Compression = .none,
		510	skip: bool = false,
		511	};
		512
		513	/// A HTTP request that has been sent.
		514	///
		515	/// Order of operations: request[ -> write -> finish] -> do -> read
		516	pub const Request = struct {
		517	pub const Headers = struct {
		518	version: http.Version = .@"HTTP/1.1",
		519	method: http.Method = .GET,
		520	user_agent: []const u8 = "zig (std.http)",
		521	connection: http.Connection = .keep_alive,
		522	transfer_encoding: RequestTransfer = .none,
		523
		524	custom: []const http.CustomHeader = &[_]http.CustomHeader{},
		525	};
		526
		527	uri: Uri,
		528	client: *Client,
		529	connection: *ConnectionPool.Node,
		530	/// These are stored in Request so that they are available when following
		531	/// redirects.
		532	headers: Headers,
		533
		534	redirects_left: u32,
		535	handle_redirects: bool,
		536
		537	response: Response,
		538
		539	/// Used as a allocator for resolving redirects locations.
		540	arena: std.heap.ArenaAllocator,
		541
		542	/// Frees all resources associated with the request.
		543	pub fn deinit(req: *Request) void {
		544	switch (req.response.compression) {
		545	.none => {},
		546	.deflate => \|*deflate\| deflate.deinit(),
		547	.gzip => \|*gzip\| gzip.deinit(),
		548	.zstd => \|*zstd\| zstd.deinit(),
		549	}
		550
		551	if (req.response.parser.header_bytes_owned) {
		552	req.response.parser.header_bytes.deinit(req.client.allocator);
		553	}
		554
		555	if (!req.response.parser.done) {
		556	// If the response wasn't fully read, then we need to close the connection.
		557	req.connection.data.closing = true;
		558	req.client.connection_pool.release(req.client, req.connection);
		559	}
		560
		561	req.arena.deinit();
		562	req.* = undefined;
		563	}
218		564
219	client.allocator.free(conn.host);	565	pub fn start(req: *Request, uri: Uri, headers: Headers) !void {
		566	var buffered = std.io.bufferedWriter(req.connection.data.buffered.writer());
		567	const w = buffered.writer();
		568
		569	const escaped_path = try Uri.escapePath(req.client.allocator, uri.path);
		570	defer req.client.allocator.free(escaped_path);
		571
		572	const escaped_query = if (uri.query) \|q\| try Uri.escapeQuery(req.client.allocator, q) else null;
		573	defer if (escaped_query) \|q\| req.client.allocator.free(q);
		574
		575	const escaped_fragment = if (uri.fragment) \|f\| try Uri.escapeQuery(req.client.allocator, f) else null;
		576	defer if (escaped_fragment) \|f\| req.client.allocator.free(f);
		577
		578	try w.writeAll(@tagName(headers.method));
		579	try w.writeByte(' ');
		580	if (escaped_path.len == 0) {
		581	try w.writeByte('/');
		582	} else {
		583	try w.writeAll(escaped_path);
		584	}
		585	if (escaped_query) \|q\| {
		586	try w.writeByte('?');
		587	try w.writeAll(q);
		588	}
		589	if (escaped_fragment) \|f\| {
		590	try w.writeByte('#');
		591	try w.writeAll(f);
		592	}
		593	try w.writeByte(' ');
		594	try w.writeAll(@tagName(headers.version));
		595	try w.writeAll("\r\nHost: ");
		596	try w.writeAll(uri.host.?);
		597	try w.writeAll("\r\nUser-Agent: ");
		598	try w.writeAll(headers.user_agent);
		599	if (headers.connection == .close) {
		600	try w.writeAll("\r\nConnection: close");
		601	} else {
		602	try w.writeAll("\r\nConnection: keep-alive");
		603	}
		604	try w.writeAll("\r\nAccept-Encoding: gzip, deflate, zstd");
		605	try w.writeAll("\r\nTE: gzip, deflate"); // TODO: add trailers when someone finds a nice way to integrate them without completely invalidating all pointers to headers.
		606
		607	switch (headers.transfer_encoding) {
		608	.chunked => try w.writeAll("\r\nTransfer-Encoding: chunked"),
		609	.content_length => \|content_length\| try w.print("\r\nContent-Length: {d}", .{content_length}),
		610	.none => {},
		611	}
		612
		613	for (headers.custom) \|header\| {
		614	try w.writeAll("\r\n");
		615	try w.writeAll(header.name);
		616	try w.writeAll(": ");
		617	try w.writeAll(header.value);
		618	}
		619
		620	try w.writeAll("\r\n\r\n");
		621
		622	try buffered.flush();
		623	}
		624
		625	pub const TransferReadError = proto.HeadersParser.ReadError \|\| error{ReadFailed};
		626
		627	pub const TransferReader = std.io.Reader(*Request, TransferReadError, transferRead);
		628
		629	pub fn transferReader(req: *Request) TransferReader {
		630	return .{ .context = req };
		631	}
		632
		633	pub fn transferRead(req: *Request, buf: []u8) TransferReadError!usize {
		634	if (req.response.parser.done) return 0;
		635
		636	var index: usize = 0;
		637	while (index == 0) {
		638	const amt = req.response.parser.read(&req.connection.data.buffered, buf[index..], req.response.skip) catch \|err\| {
		639	req.client.last_error = .{ .read = err };
		640	return error.ReadFailed;
		641	};
		642	if (amt == 0 and req.response.parser.done) break;
		643	index += amt;
		644	}
		645
		646	return index;
		647	}
		648
		649	pub const DoError = RequestError \|\| TransferReadError \|\| proto.HeadersParser.CheckCompleteHeadError \|\| Response.Headers.ParseError \|\| Uri.ParseError \|\| error{ TooManyHttpRedirects, HttpRedirectMissingLocation, CompressionInitializationFailed };
		650
		651	/// Waits for a response from the server and parses any headers that are sent.
		652	/// This function will block until the final response is received.
		653	///
		654	/// If `handle_redirects` is true, then this function will automatically follow
		655	/// redirects.
		656	pub fn do(req: *Request) DoError!void {
		657	while (true) { // handle redirects
		658	while (true) { // read headers
		659	req.connection.data.buffered.fill() catch \|err\| {
		660	req.client.last_error = .{ .read = err };
		661	return error.ReadFailed;
		662	};
		663
		664	const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.data.buffered.peek());
		665	req.connection.data.buffered.clear(@intCast(u16, nchecked));
		666
		667	if (req.response.parser.state.isContent()) break;
		668	}
		669
		670	req.response.headers = try Response.Headers.parse(req.response.parser.header_bytes.items);
		671
		672	if (req.response.headers.status == .switching_protocols) {
		673	req.connection.data.closing = false;
		674	req.response.parser.done = true;
		675	}
		676
		677	if (req.headers.connection == .keep_alive and req.response.headers.connection == .keep_alive) {
		678	req.connection.data.closing = false;
		679	} else {
		680	req.connection.data.closing = true;
		681	}
		682
		683	if (req.response.headers.transfer_encoding) \|te\| {
		684	switch (te) {
		685	.chunked => {
		686	req.response.parser.next_chunk_length = 0;
		687	req.response.parser.state = .chunk_head_size;
		688	},
		689	}
		690	} else if (req.response.headers.content_length) \|cl\| {
		691	req.response.parser.next_chunk_length = cl;
		692
		693	if (cl == 0) req.response.parser.done = true;
		694	} else {
		695	req.response.parser.done = true;
		696	}
		697
		698	if (req.response.headers.status.class() == .redirect and req.handle_redirects) {
		699	req.response.skip = true;
		700
		701	const empty = @as([*]u8, undefined)[0..0];
		702	assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary
		703
		704	if (req.redirects_left == 0) return error.TooManyHttpRedirects;
		705
		706	const location = req.response.headers.location orelse
		707	return error.HttpRedirectMissingLocation;
		708	const new_url = Uri.parse(location) catch try Uri.parseWithoutScheme(location);
		709
		710	var new_arena = std.heap.ArenaAllocator.init(req.client.allocator);
		711	const resolved_url = try req.uri.resolve(new_url, false, new_arena.allocator());
		712	errdefer new_arena.deinit();
		713
		714	req.arena.deinit();
		715	req.arena = new_arena;
		716
		717	const new_req = try req.client.request(resolved_url, req.headers, .{
		718	.max_redirects = req.redirects_left - 1,
		719	.header_strategy = if (req.response.parser.header_bytes_owned) .{
		720	.dynamic = req.response.parser.max_header_bytes,
		721	} else .{
		722	.static = req.response.parser.header_bytes.items.ptr[0..req.response.parser.max_header_bytes],
		723	},
		724	});
		725	req.deinit();
		726	req.* = new_req;
		727	} else {
		728	req.response.skip = false;
		729	if (!req.response.parser.done) {
		730	if (req.response.headers.transfer_compression) \|tc\| switch (tc) {
		731	.compress => return error.CompressionNotSupported,
		732	.deflate => req.response.compression = .{
		733	.deflate = std.compress.zlib.zlibStream(req.client.allocator, req.transferReader()) catch \|err\| {
		734	req.client.last_error = .{ .zlib_init = err };
		735	return error.CompressionInitializationFailed;
		736	},
		737	},
		738	.gzip => req.response.compression = .{
		739	.gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch \|err\| {
		740	req.client.last_error = .{ .gzip_init = err };
		741	return error.CompressionInitializationFailed;
		742	},
		743	},
		744	.zstd => req.response.compression = .{
		745	.zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()),
		746	},
		747	};
		748	}
		749
		750	break;
		751	}
		752	}
		753	}
		754
		755	pub const ReadError = TransferReadError \|\| proto.HeadersParser.CheckCompleteHeadError;
		756
		757	pub const Reader = std.io.Reader(*Request, ReadError, read);
		758
		759	pub fn reader(req: *Request) Reader {
		760	return .{ .context = req };
		761	}
		762
		763	/// Reads data from the response body. Must be called after `do`.
		764	pub fn read(req: *Request, buffer: []u8) ReadError!usize {
		765	while (true) {
		766	const out_index = switch (req.response.compression) {
		767	.deflate => \|*deflate\| deflate.read(buffer) catch \|err\| {
		768	req.client.last_error = .{ .decompress = err };
		769	err catch {};
		770	return error.ReadFailed;
		771	},
		772	.gzip => \|*gzip\| gzip.read(buffer) catch \|err\| {
		773	req.client.last_error = .{ .decompress = err };
		774	err catch {};
		775	return error.ReadFailed;
		776	},
		777	.zstd => \|*zstd\| zstd.read(buffer) catch \|err\| {
		778	req.client.last_error = .{ .decompress = err };
		779	err catch {};
		780	return error.ReadFailed;
		781	},
		782	else => try req.transferRead(buffer),
		783	};
		784
		785	if (out_index == 0) {
		786	while (!req.response.parser.state.isContent()) { // read trailing headers
		787	req.connection.data.buffered.fill() catch \|err\| {
		788	req.client.last_error = .{ .read = err };
		789	return error.ReadFailed;
		790	};
		791
		792	const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.data.buffered.peek());
		793	req.connection.data.buffered.clear(@intCast(u16, nchecked));
		794	}
		795	}
		796
		797	return out_index;
		798	}
		799	}
		800
		801	/// Reads data from the response body. Must be called after `do`.
		802	pub fn readAll(req: *Request, buffer: []u8) !usize {
		803	var index: usize = 0;
		804	while (index < buffer.len) {
		805	const amt = read(req, buffer[index..]) catch \|err\| {
		806	req.client.last_error = .{ .read = err };
		807	return error.ReadFailed;
		808	};
		809	if (amt == 0) break;
		810	index += amt;
		811	}
		812	return index;
		813	}
		814
		815	pub const WriteError = error{ WriteFailed, NotWriteable, MessageTooLong };
		816
		817	pub const Writer = std.io.Writer(*Request, WriteError, write);
		818
		819	pub fn writer(req: *Request) Writer {
		820	return .{ .context = req };
		821	}
		822
		823	/// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.
		824	pub fn write(req: *Request, bytes: []const u8) WriteError!usize {
		825	switch (req.headers.transfer_encoding) {
		826	.chunked => {
		827	req.connection.data.conn.writer().print("{x}\r\n", .{bytes.len}) catch \|err\| {
		828	req.client.last_error = .{ .write = err };
		829	return error.WriteFailed;
		830	};
		831	req.connection.data.conn.writeAll(bytes) catch \|err\| {
		832	req.client.last_error = .{ .write = err };
		833	return error.WriteFailed;
		834	};
		835	req.connection.data.conn.writeAll("\r\n") catch \|err\| {
		836	req.client.last_error = .{ .write = err };
		837	return error.WriteFailed;
		838	};
		839
		840	return bytes.len;
		841	},
		842	.content_length => \|*len\| {
		843	if (len.* < bytes.len) return error.MessageTooLong;
		844
		845	const amt = req.connection.data.conn.write(bytes) catch \|err\| {
		846	req.client.last_error = .{ .write = err };
		847	return error.WriteFailed;
		848	};
		849	len.* -= amt;
		850	return amt;
		851	},
		852	.none => return error.NotWriteable,
		853	}
		854	}
		855
		856	/// Finish the body of a request. This notifies the server that you have no more data to send.
		857	pub fn finish(req: *Request) !void {
		858	switch (req.headers.transfer_encoding) {
		859	.chunked => req.connection.data.conn.writeAll("0\r\n") catch \|err\| {
		860	req.client.last_error = .{ .write = err };
		861	return error.WriteFailed;
		862	},
		863	.content_length => \|len\| if (len != 0) return error.MessageNotCompleted,
		864	.none => {},
		865	}
220	}	866	}
221	};	867	};
222		868
		869	/// Release all associated resources with the client.
		870	/// TODO: currently leaks all request allocated data
223	pub fn deinit(client: *Client) void {	871	pub fn deinit(client: *Client) void {
224	client.connection_pool.deinit(client);	872	client.connection_pool.deinit(client);
225		873
...	@@ -227,8 +875,10 @@ pub fn deinit(client: *Client) void {	...	@@ -227,8 +875,10 @@ pub fn deinit(client: *Client) void {
227	client.* = undefined;	875	client.* = undefined;
228	}	876	}
229		877
230	pub const ConnectError = std.mem.Allocator.Error \|\| net.TcpConnectToHostError \|\| std.crypto.tls.Client.InitError(net.Stream);	878	pub const ConnectError = Allocator.Error \|\| error{ ConnectionFailed, TlsInitializationFailed };
231		879
		880	/// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open.
		881	/// This function is threadsafe.
232	pub fn connect(client: Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!ConnectionPool.Node {	882	pub fn connect(client: Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!ConnectionPool.Node {
233	if (client.connection_pool.findConnection(.{	883	if (client.connection_pool.findConnection(.{
234	.host = host,	884	.host = host,
...	@@ -241,22 +891,36 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio	...	@@ -241,22 +891,36 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
241	errdefer client.allocator.destroy(conn);	891	errdefer client.allocator.destroy(conn);
242	conn.* = .{ .data = undefined };	892	conn.* = .{ .data = undefined };
243		893
		894	const stream = net.tcpConnectToHost(client.allocator, host, port) catch \|err\| {
		895	client.last_error = .{ .connect = err };
		896	return error.ConnectionFailed;
		897	};
		898	errdefer stream.close();
		899
244	conn.data = .{	900	conn.data = .{
245	.stream = try net.tcpConnectToHost(client.allocator, host, port),	901	.buffered = .{ .conn = .{
246	.tls_client = undefined,	902	.stream = stream,
247	.protocol = protocol,	903	.tls_client = undefined,
		904	.protocol = protocol,
		905	} },
248	.host = try client.allocator.dupe(u8, host),	906	.host = try client.allocator.dupe(u8, host),
249	.port = port,	907	.port = port,
250	};	908	};
		909	errdefer client.allocator.free(conn.data.host);
251		910
252	switch (protocol) {	911	switch (protocol) {
253	.plain => {},	912	.plain => {},
254	.tls => {	913	.tls => {
255	conn.data.tls_client = try client.allocator.create(std.crypto.tls.Client);	914	conn.data.buffered.conn.tls_client = try client.allocator.create(std.crypto.tls.Client);
256	conn.data.tls_client.* = try std.crypto.tls.Client.init(conn.data.stream, client.ca_bundle, host);	915	errdefer client.allocator.destroy(conn.data.buffered.conn.tls_client);
		916
		917	conn.data.buffered.conn.tls_client.* = std.crypto.tls.Client.init(stream, client.ca_bundle, host) catch \|err\| {
		918	client.last_error = .{ .tls = err };
		919	return error.TlsInitializationFailed;
		920	};
257	// This is appropriate for HTTPS because the HTTP headers contain	921	// This is appropriate for HTTPS because the HTTP headers contain
258	// the content length which is used to detect truncation attacks.	922	// the content length which is used to detect truncation attacks.
259	conn.data.tls_client.allow_truncation_attacks = true;	923	conn.data.buffered.conn.tls_client.allow_truncation_attacks = true;
260	},	924	},
261	}	925	}
262		926
...	@@ -265,24 +929,44 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio	...	@@ -265,24 +929,44 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
265	return conn;	929	return conn;
266	}	930	}
267		931
268	pub const RequestError = ConnectError \|\| Connection.WriteError \|\| error{	932	pub const RequestError = ConnectError \|\| error{
269	UnsupportedUrlScheme,	933	UnsupportedUrlScheme,
270	UriMissingHost,	934	UriMissingHost,
271		935
272	CertificateAuthorityBundleTooBig,	936	CertificateAuthorityBundleFailed,
273	InvalidPadding,	937	WriteFailed,
274	MissingEndCertificateMarker,	938	};
275	Unseekable,	939
276	EndOfStream,	940	pub const Options = struct {
		941	handle_redirects: bool = true,
		942	max_redirects: u32 = 3,
		943	header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
		944
		945	pub const HeaderStrategy = union(enum) {
		946	/// In this case, the client's Allocator will be used to store the
		947	/// entire HTTP header. This value is the maximum total size of
		948	/// HTTP headers allowed, otherwise
		949	/// error.HttpHeadersExceededSizeLimit is returned from read().
		950	dynamic: usize,
		951	/// This is used to store the entire HTTP header. If the HTTP
		952	/// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
		953	/// is returned from read(). When this is used, `error.OutOfMemory`
		954	/// cannot be returned from `read()`.
		955	static: []u8,
		956	};
277	};	957	};
278		958
279	pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Request.Options) RequestError!Request {	959	pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{
280	const protocol: Connection.Protocol = if (mem.eql(u8, uri.scheme, "http"))	960	.{ "http", .plain },
281	.plain	961	.{ "ws", .plain },
282	else if (mem.eql(u8, uri.scheme, "https"))	962	.{ "https", .tls },
283	.tls	963	.{ "wss", .tls },
284	else	964	});
285	return error.UnsupportedUrlScheme;	965
		966	/// Form and send a http request to a server.
		967	/// This function is threadsafe.
		968	pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Options) RequestError!Request {
		969	const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;
286		970
287	const port: u16 = uri.port orelse switch (protocol) {	971	const port: u16 = uri.port orelse switch (protocol) {
288	.plain => 80,	972	.plain => 80,
...	@@ -291,91 +975,45 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req	...	@@ -291,91 +975,45 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req
291		975
292	const host = uri.host orelse return error.UriMissingHost;	976	const host = uri.host orelse return error.UriMissingHost;
293		977
294	if (client.next_https_rescan_certs and protocol == .tls) {	978	if (protocol == .tls and @atomicLoad(bool, &client.next_https_rescan_certs, .Acquire)) {
295	client.connection_pool.mutex.lock(); // TODO: this could be so much better than reusing the connection pool mutex.	979	client.ca_bundle_mutex.lock();
296	defer client.connection_pool.mutex.unlock();	980	defer client.ca_bundle_mutex.unlock();
297		981
298	if (client.next_https_rescan_certs) {	982	if (client.next_https_rescan_certs) {
299	try client.ca_bundle.rescan(client.allocator);	983	client.ca_bundle.rescan(client.allocator) catch \|err\| {
300	client.next_https_rescan_certs = false;	984	client.last_error = .{ .ca_bundle = err };
		985	return error.CertificateAuthorityBundleFailed;
		986	};
		987	@atomicStore(bool, &client.next_https_rescan_certs, false, .Release);
301	}	988	}
302	}	989	}
303		990
304	var req: Request = .{	991	var req: Request = .{
305	.uri = uri,	992	.uri = uri,
306	.client = client,	993	.client = client,
307	.headers = headers,
308	.connection = try client.connect(host, port, protocol),	994	.connection = try client.connect(host, port, protocol),
		995	.headers = headers,
309	.redirects_left = options.max_redirects,	996	.redirects_left = options.max_redirects,
310	.handle_redirects = options.handle_redirects,	997	.handle_redirects = options.handle_redirects,
311	.compression_init = false,	998	.response = .{
312	.response = switch (options.header_strategy) {	999	.parser = switch (options.header_strategy) {
313	.dynamic => \|max\| Response.initDynamic(max),	1000	.dynamic => \|max\| proto.HeadersParser.initDynamic(max),
314	.static => \|buf\| Response.initStatic(buf),	1001	.static => \|buf\| proto.HeadersParser.initStatic(buf),
		1002	},
315	},	1003	},
316	.arena = undefined,	1004	.arena = undefined,
317	};	1005	};
		1006	errdefer req.deinit();
318		1007
319	req.arena = std.heap.ArenaAllocator.init(client.allocator);	1008	req.arena = std.heap.ArenaAllocator.init(client.allocator);
320		1009
321	{	1010	req.start(uri, headers) catch \|err\| {
322	var buffered = std.io.bufferedWriter(req.connection.data.writer());	1011	if (err == error.OutOfMemory) return error.OutOfMemory;
323	const writer = buffered.writer();	1012	const err_casted = @errSetCast(BufferedConnection.WriteError, err);
324
325	const escaped_path = try Uri.escapePath(client.allocator, uri.path);
326	defer client.allocator.free(escaped_path);
327
328	const escaped_query = if (uri.query) \|q\| try Uri.escapeQuery(client.allocator, q) else null;
329	defer if (escaped_query) \|q\| client.allocator.free(q);
330		1013
331	const escaped_fragment = if (uri.fragment) \|f\| try Uri.escapeQuery(client.allocator, f) else null;	1014	client.last_error = .{ .write = err_casted };
332	defer if (escaped_fragment) \|f\| client.allocator.free(f);	1015	return error.WriteFailed;
333		1016	};
334	try writer.writeAll(@tagName(headers.method));
335	try writer.writeByte(' ');
336	if (escaped_path.len == 0) {
337	try writer.writeByte('/');
338	} else {
339	try writer.writeAll(escaped_path);
340	}
341	if (escaped_query) \|q\| {
342	try writer.writeByte('?');
343	try writer.writeAll(q);
344	}
345	if (escaped_fragment) \|f\| {
346	try writer.writeByte('#');
347	try writer.writeAll(f);
348	}
349	try writer.writeByte(' ');
350	try writer.writeAll(@tagName(headers.version));
351	try writer.writeAll("\r\nHost: ");
352	try writer.writeAll(host);
353	try writer.writeAll("\r\nUser-Agent: ");
354	try writer.writeAll(headers.user_agent);
355	if (headers.connection == .close) {
356	try writer.writeAll("\r\nConnection: close");
357	} else {
358	try writer.writeAll("\r\nConnection: keep-alive");
359	}
360	try writer.writeAll("\r\nAccept-Encoding: gzip, deflate, zstd");
361
362	switch (headers.transfer_encoding) {
363	.chunked => try writer.writeAll("\r\nTransfer-Encoding: chunked"),
364	.content_length => \|content_length\| try writer.print("\r\nContent-Length: {d}", .{content_length}),
365	.none => {},
366	}
367
368	for (headers.custom) \|header\| {
369	try writer.writeAll("\r\n");
370	try writer.writeAll(header.name);
371	try writer.writeAll(": ");
372	try writer.writeAll(header.value);
373	}
374
375	try writer.writeAll("\r\n\r\n");
376
377	try buffered.flush();
378	}
379		1017
380	return req;	1018	return req;
381	}	1019	}
...	@@ -390,5 +1028,5 @@ test {	...	@@ -390,5 +1028,5 @@ test {
390		1028
391	if (builtin.os.tag == .wasi) return error.SkipZigTest;	1029	if (builtin.os.tag == .wasi) return error.SkipZigTest;
392		1030
393	_ = Request;	1031	std.testing.refAllDecls(@This());
394	}	1032	}

lib/std/http/Client/Request.zig deleted-482

...	@@ -1,482 +0,0 @@
1	const std = @import("std");
2	const http = std.http;
3	const Uri = std.Uri;
4	const mem = std.mem;
5	const assert = std.debug.assert;
6
7	const Client = @import("../Client.zig");
8	const Connection = Client.Connection;
9	const ConnectionNode = Client.ConnectionPool.Node;
10	const Response = @import("Response.zig");
11
12	const Request = @This();
13
14	const read_buffer_size = 8192;
15	const ReadBufferIndex = std.math.IntFittingRange(0, read_buffer_size);
16
17	uri: Uri,
18	client: *Client,
19	connection: *ConnectionNode,
20	response: Response,
21	/// These are stored in Request so that they are available when following
22	/// redirects.
23	headers: Headers,
24
25	redirects_left: u32,
26	handle_redirects: bool,
27	compression_init: bool,
28
29	/// Used as a allocator for resolving redirects locations.
30	arena: std.heap.ArenaAllocator,
31
32	/// Read buffer for the connection. This is used to pull in large amounts of data from the connection even if the user asks for a small amount. This can probably be removed with careful planning.
33	read_buffer: [read_buffer_size]u8 = undefined,
34	read_buffer_start: ReadBufferIndex = 0,
35	read_buffer_len: ReadBufferIndex = 0,
36
37	pub const RequestTransfer = union(enum) {
38	content_length: u64,
39	chunked: void,
40	none: void,
41	};
42
43	pub const Headers = struct {
44	version: http.Version = .@"HTTP/1.1",
45	method: http.Method = .GET,
46	user_agent: []const u8 = "zig (std.http)",
47	connection: http.Connection = .keep_alive,
48	transfer_encoding: RequestTransfer = .none,
49
50	custom: []const http.CustomHeader = &[_]http.CustomHeader{},
51	};
52
53	pub const Options = struct {
54	handle_redirects: bool = true,
55	max_redirects: u32 = 3,
56	header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
57
58	pub const HeaderStrategy = union(enum) {
59	/// In this case, the client's Allocator will be used to store the
60	/// entire HTTP header. This value is the maximum total size of
61	/// HTTP headers allowed, otherwise
62	/// error.HttpHeadersExceededSizeLimit is returned from read().
63	dynamic: usize,
64	/// This is used to store the entire HTTP header. If the HTTP
65	/// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
66	/// is returned from read(). When this is used, `error.OutOfMemory`
67	/// cannot be returned from `read()`.
68	static: []u8,
69	};
70	};
71
72	/// Frees all resources associated with the request.
73	pub fn deinit(req: *Request) void {
74	switch (req.response.compression) {
75	.none => {},
76	.deflate => \|*deflate\| deflate.deinit(),
77	.gzip => \|*gzip\| gzip.deinit(),
78	.zstd => \|*zstd\| zstd.deinit(),
79	}
80
81	if (req.response.header_bytes_owned) {
82	req.response.header_bytes.deinit(req.client.allocator);
83	}
84
85	if (!req.response.done) {
86	// If the response wasn't fully read, then we need to close the connection.
87	req.connection.data.closing = true;
88	req.client.connection_pool.release(req.client, req.connection);
89	}
90
91	req.arena.deinit();
92	req.* = undefined;
93	}
94
95	pub const ReadRawError = Connection.ReadError \|\| Uri.ParseError \|\| Client.RequestError \|\| error{
96	UnexpectedEndOfStream,
97	TooManyHttpRedirects,
98	HttpRedirectMissingLocation,
99	HttpHeadersInvalid,
100	};
101
102	pub const ReaderRaw = std.io.Reader(*Request, ReadRawError, readRaw);
103
104	/// Read from the underlying stream, without decompressing or parsing the headers. Must be called
105	/// after waitForCompleteHead() has returned successfully.
106	pub fn readRaw(req: *Request, buffer: []u8) ReadRawError!usize {
107	assert(req.response.state.isContent());
108
109	var index: usize = 0;
110	while (index == 0) {
111	const amt = try req.readRawAdvanced(buffer[index..]);
112	if (amt == 0 and req.response.done) break;
113	index += amt;
114	}
115
116	return index;
117	}
118
119	fn checkForCompleteHead(req: *Request, buffer: []u8) !usize {
120	switch (req.response.state) {
121	.invalid => unreachable,
122	.start, .seen_r, .seen_rn, .seen_rnr => {},
123	else => return 0, // No more headers to read.
124	}
125
126	const i = req.response.findHeadersEnd(buffer[0..]);
127	if (req.response.state == .invalid) return error.HttpHeadersInvalid;
128
129	const headers_data = buffer[0..i];
130	if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
131	return error.HttpHeadersExceededSizeLimit;
132	}
133	try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
134
135	if (req.response.state == .finished) {
136	req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
137
138	if (req.response.headers.upgrade) \|_\| {
139	req.connection.data.closing = false;
140	req.response.done = true;
141	return i;
142	}
143
144	if (req.response.headers.connection == .keep_alive) {
145	req.connection.data.closing = false;
146	} else {
147	req.connection.data.closing = true;
148	}
149
150	if (req.response.headers.transfer_encoding) \|transfer_encoding\| {
151	switch (transfer_encoding) {
152	.chunked => {
153	req.response.next_chunk_length = 0;
154	req.response.state = .chunk_size;
155	},
156	}
157	} else if (req.response.headers.content_length) \|content_length\| {
158	req.response.next_chunk_length = content_length;
159
160	if (content_length == 0) req.response.done = true;
161	} else {
162	req.response.done = true;
163	}
164
165	return i;
166	}
167
168	return 0;
169	}
170
171	pub const WaitForCompleteHeadError = ReadRawError \|\| error{
172	UnexpectedEndOfStream,
173
174	HttpHeadersExceededSizeLimit,
175	ShortHttpStatusLine,
176	BadHttpVersion,
177	HttpHeaderContinuationsUnsupported,
178	HttpTransferEncodingUnsupported,
179	HttpConnectionHeaderUnsupported,
180	};
181
182	/// Reads a complete response head. Any leftover data is stored in the request. This function is idempotent.
183	pub fn waitForCompleteHead(req: *Request) WaitForCompleteHeadError!void {
184	if (req.response.state.isContent()) return;
185
186	while (true) {
187	const nread = try req.connection.data.read(req.read_buffer[0..]);
188	const amt = try checkForCompleteHead(req, req.read_buffer[0..nread]);
189
190	if (amt != 0) {
191	req.read_buffer_start = @intCast(ReadBufferIndex, amt);
192	req.read_buffer_len = @intCast(ReadBufferIndex, nread);
193	return;
194	} else if (nread == 0) {
195	return error.UnexpectedEndOfStream;
196	}
197	}
198	}
199
200	/// This one can return 0 without meaning EOF.
201	fn readRawAdvanced(req: *Request, buffer: []u8) !usize {
202	assert(req.response.state.isContent());
203	if (req.response.done) return 0;
204
205	// var in: []const u8 = undefined;
206	if (req.read_buffer_start == req.read_buffer_len) {
207	const nread = try req.connection.data.read(req.read_buffer[0..]);
208	if (nread == 0) return error.UnexpectedEndOfStream;
209
210	req.read_buffer_start = 0;
211	req.read_buffer_len = @intCast(ReadBufferIndex, nread);
212	}
213
214	var out_index: usize = 0;
215	while (true) {
216	switch (req.response.state) {
217	.invalid, .start, .seen_r, .seen_rn, .seen_rnr => unreachable,
218	.finished => {
219	// TODO https://github.com/ziglang/zig/issues/14039
220	const buf_avail = req.read_buffer_len - req.read_buffer_start;
221	const data_avail = req.response.next_chunk_length;
222	const out_avail = buffer.len;
223
224	if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
225	const can_read = @intCast(usize, @min(buf_avail, data_avail));
226	req.response.next_chunk_length -= can_read;
227
228	if (req.response.next_chunk_length == 0) {
229	req.client.connection_pool.release(req.client, req.connection);
230	req.connection = undefined;
231	req.response.done = true;
232	}
233
234	return 0; // skip over as much data as possible
235	}
236
237	const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
238	req.response.next_chunk_length -= can_read;
239
240	mem.copy(u8, buffer[0..], req.read_buffer[req.read_buffer_start..][0..can_read]);
241	req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
242
243	if (req.response.next_chunk_length == 0) {
244	req.client.connection_pool.release(req.client, req.connection);
245	req.connection = undefined;
246	req.response.done = true;
247	}
248
249	return can_read;
250	},
251	.chunk_size_prefix_r => switch (req.read_buffer_len - req.read_buffer_start) {
252	0 => return out_index,
253	1 => switch (req.read_buffer[req.read_buffer_start]) {
254	'\r' => {
255	req.response.state = .chunk_size_prefix_n;
256	return out_index;
257	},
258	else => {
259	req.response.state = .invalid;
260	return error.HttpHeadersInvalid;
261	},
262	},
263	else => switch (int16(req.read_buffer[req.read_buffer_start..][0..2])) {
264	int16("\r\n") => {
265	req.read_buffer_start += 2;
266	req.response.state = .chunk_size;
267	continue;
268	},
269	else => {
270	req.response.state = .invalid;
271	return error.HttpHeadersInvalid;
272	},
273	},
274	},
275	.chunk_size_prefix_n => switch (req.read_buffer_len - req.read_buffer_start) {
276	0 => return out_index,
277	else => switch (req.read_buffer[req.read_buffer_start]) {
278	'\n' => {
279	req.read_buffer_start += 1;
280	req.response.state = .chunk_size;
281	continue;
282	},
283	else => {
284	req.response.state = .invalid;
285	return error.HttpHeadersInvalid;
286	},
287	},
288	},
289	.chunk_size, .chunk_r => {
290	const i = req.response.findChunkedLen(req.read_buffer[req.read_buffer_start..req.read_buffer_len]);
291	switch (req.response.state) {
292	.invalid => return error.HttpHeadersInvalid,
293	.chunk_data => {
294	if (req.response.next_chunk_length == 0) {
295	req.response.done = true;
296	req.client.connection_pool.release(req.client, req.connection);
297	req.connection = undefined;
298
299	return out_index;
300	}
301
302	req.read_buffer_start += @intCast(ReadBufferIndex, i);
303	continue;
304	},
305	.chunk_size => return out_index,
306	else => unreachable,
307	}
308	},
309	.chunk_data => {
310	// TODO https://github.com/ziglang/zig/issues/14039
311	const buf_avail = req.read_buffer_len - req.read_buffer_start;
312	const data_avail = req.response.next_chunk_length;
313	const out_avail = buffer.len - out_index;
314
315	if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
316	const can_read = @intCast(usize, @min(buf_avail, data_avail));
317	req.response.next_chunk_length -= can_read;
318
319	if (req.response.next_chunk_length == 0) {
320	req.client.connection_pool.release(req.client, req.connection);
321	req.connection = undefined;
322	req.response.done = true;
323	continue;
324	}
325
326	return 0; // skip over as much data as possible
327	}
328
329	const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
330	req.response.next_chunk_length -= can_read;
331
332	mem.copy(u8, buffer[out_index..], req.read_buffer[req.read_buffer_start..][0..can_read]);
333	req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
334	out_index += can_read;
335
336	if (req.response.next_chunk_length == 0) {
337	req.response.state = .chunk_size_prefix_r;
338
339	continue;
340	}
341
342	return out_index;
343	},
344	}
345	}
346	}
347
348	pub const ReadError = Client.DeflateDecompressor.Error \|\| Client.GzipDecompressor.Error \|\| Client.ZstdDecompressor.Error \|\| WaitForCompleteHeadError \|\| error{ BadHeader, InvalidCompression, StreamTooLong, InvalidWindowSize, CompressionNotSupported };
349
350	pub const Reader = std.io.Reader(*Request, ReadError, read);
351
352	pub fn reader(req: *Request) Reader {
353	return .{ .context = req };
354	}
355
356	pub fn read(req: *Request, buffer: []u8) ReadError!usize {
357	while (true) {
358	if (!req.response.state.isContent()) try req.waitForCompleteHead();
359
360	if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
361	assert(try req.readRaw(buffer) == 0);
362
363	if (req.redirects_left == 0) return error.TooManyHttpRedirects;
364
365	const location = req.response.headers.location orelse
366	return error.HttpRedirectMissingLocation;
367	const new_url = Uri.parse(location) catch try Uri.parseWithoutScheme(location);
368
369	var new_arena = std.heap.ArenaAllocator.init(req.client.allocator);
370	const resolved_url = try req.uri.resolve(new_url, false, new_arena.allocator());
371	errdefer new_arena.deinit();
372
373	req.arena.deinit();
374	req.arena = new_arena;
375
376	const new_req = try req.client.request(resolved_url, req.headers, .{
377	.max_redirects = req.redirects_left - 1,
378	.header_strategy = if (req.response.header_bytes_owned) .{
379	.dynamic = req.response.max_header_bytes,
380	} else .{
381	.static = req.response.header_bytes.unusedCapacitySlice(),
382	},
383	});
384	req.deinit();
385	req.* = new_req;
386	} else {
387	break;
388	}
389	}
390
391	if (req.response.compression == .none) {
392	if (req.response.headers.transfer_compression) \|compression\| {
393	switch (compression) {
394	.compress => return error.CompressionNotSupported,
395	.deflate => req.response.compression = .{
396	.deflate = try std.compress.zlib.zlibStream(req.client.allocator, ReaderRaw{ .context = req }),
397	},
398	.gzip => req.response.compression = .{
399	.gzip = try std.compress.gzip.decompress(req.client.allocator, ReaderRaw{ .context = req }),
400	},
401	.zstd => req.response.compression = .{
402	.zstd = std.compress.zstd.decompressStream(req.client.allocator, ReaderRaw{ .context = req }),
403	},
404	}
405	}
406	}
407
408	return switch (req.response.compression) {
409	.deflate => \|*deflate\| try deflate.read(buffer),
410	.gzip => \|*gzip\| try gzip.read(buffer),
411	.zstd => \|*zstd\| try zstd.read(buffer),
412	else => try req.readRaw(buffer),
413	};
414	}
415
416	pub fn readAll(req: *Request, buffer: []u8) !usize {
417	var index: usize = 0;
418	while (index < buffer.len) {
419	const amt = try read(req, buffer[index..]);
420	if (amt == 0) break;
421	index += amt;
422	}
423	return index;
424	}
425
426	pub const WriteError = Connection.WriteError \|\| error{MessageTooLong};
427
428	pub const Writer = std.io.Writer(*Request, WriteError, write);
429
430	pub fn writer(req: *Request) Writer {
431	return .{ .context = req };
432	}
433
434	/// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.
435	pub fn write(req: *Request, bytes: []const u8) !usize {
436	switch (req.headers.transfer_encoding) {
437	.chunked => {
438	try req.connection.data.writer().print("{x}\r\n", .{bytes.len});
439	try req.connection.data.writeAll(bytes);
440	try req.connection.data.writeAll("\r\n");
441
442	return bytes.len;
443	},
444	.content_length => \|*len\| {
445	if (len.* < bytes.len) return error.MessageTooLong;
446
447	const amt = try req.connection.data.write(bytes);
448	len.* -= amt;
449	return amt;
450	},
451	.none => return error.NotWriteable,
452	}
453	}
454
455	/// Finish the body of a request. This notifies the server that you have no more data to send.
456	pub fn finish(req: *Request) !void {
457	switch (req.headers.transfer_encoding) {
458	.chunked => try req.connection.data.writeAll("0\r\n"),
459	.content_length => \|len\| if (len != 0) return error.MessageNotCompleted,
460	.none => {},
461	}
462	}
463
464	inline fn int16(array: *const [2]u8) u16 {
465	return @bitCast(u16, array.*);
466	}
467
468	inline fn int32(array: *const [4]u8) u32 {
469	return @bitCast(u32, array.*);
470	}
471
472	inline fn int64(array: *const [8]u8) u64 {
473	return @bitCast(u64, array.*);
474	}
475
476	test {
477	const builtin = @import("builtin");
478
479	if (builtin.os.tag == .wasi) return error.SkipZigTest;
480
481	_ = Response;
482	}

lib/std/http/Client/Response.zig deleted-509

...	@@ -1,509 +0,0 @@
1	const std = @import("std");
2	const http = std.http;
3	const mem = std.mem;
4	const testing = std.testing;
5	const assert = std.debug.assert;
6
7	const Client = @import("../Client.zig");
8	const Response = @This();
9
10	headers: Headers,
11	state: State,
12	header_bytes_owned: bool,
13	/// This could either be a fixed buffer provided by the API user or it
14	/// could be our own array list.
15	header_bytes: std.ArrayListUnmanaged(u8),
16	max_header_bytes: usize,
17	next_chunk_length: u64,
18	done: bool = false,
19
20	compression: union(enum) {
21	deflate: Client.DeflateDecompressor,
22	gzip: Client.GzipDecompressor,
23	zstd: Client.ZstdDecompressor,
24	none: void,
25	} = .none,
26
27	pub const Headers = struct {
28	status: http.Status,
29	version: http.Version,
30	location: ?[]const u8 = null,
31	content_length: ?u64 = null,
32	transfer_encoding: ?http.TransferEncoding = null,
33	transfer_compression: ?http.ContentEncoding = null,
34	connection: http.Connection = .close,
35	upgrade: ?[]const u8 = null,
36
37	number_of_headers: usize = 0,
38
39	pub fn parse(bytes: []const u8) !Headers {
40	var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
41
42	const first_line = it.first();
43	if (first_line.len < 12)
44	return error.ShortHttpStatusLine;
45
46	const version: http.Version = switch (int64(first_line[0..8])) {
47	int64("HTTP/1.0") => .@"HTTP/1.0",
48	int64("HTTP/1.1") => .@"HTTP/1.1",
49	else => return error.BadHttpVersion,
50	};
51	if (first_line[8] != ' ') return error.HttpHeadersInvalid;
52	const status = @intToEnum(http.Status, parseInt3(first_line[9..12].*));
53
54	var headers: Headers = .{
55	.version = version,
56	.status = status,
57	};
58
59	while (it.next()) \|line\| {
60	headers.number_of_headers += 1;
61
62	if (line.len == 0) return error.HttpHeadersInvalid;
63	switch (line[0]) {
64	' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
65	else => {},
66	}
67	var line_it = mem.split(u8, line, ": ");
68	const header_name = line_it.first();
69	const header_value = line_it.rest();
70	if (std.ascii.eqlIgnoreCase(header_name, "location")) {
71	if (headers.location != null) return error.HttpHeadersInvalid;
72	headers.location = header_value;
73	} else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
74	if (headers.content_length != null) return error.HttpHeadersInvalid;
75	headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
76	} else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
77	// Transfer-Encoding: second, first
78	// Transfer-Encoding: deflate, chunked
79	var iter = std.mem.splitBackwards(u8, header_value, ",");
80
81	if (iter.next()) \|first\| {
82	const trimmed = std.mem.trim(u8, first, " ");
83
84	if (std.meta.stringToEnum(http.TransferEncoding, trimmed)) \|te\| {
85	if (headers.transfer_encoding != null) return error.HttpHeadersInvalid;
86	headers.transfer_encoding = te;
87	} else if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
88	if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
89	headers.transfer_compression = ce;
90	} else {
91	return error.HttpTransferEncodingUnsupported;
92	}
93	}
94
95	if (iter.next()) \|second\| {
96	if (headers.transfer_compression != null) return error.HttpTransferEncodingUnsupported;
97
98	const trimmed = std.mem.trim(u8, second, " ");
99
100	if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
101	headers.transfer_compression = ce;
102	} else {
103	return error.HttpTransferEncodingUnsupported;
104	}
105	}
106
107	if (iter.next()) \|_\| return error.HttpTransferEncodingUnsupported;
108	} else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
109	if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
110
111	const trimmed = std.mem.trim(u8, header_value, " ");
112
113	if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
114	headers.transfer_compression = ce;
115	} else {
116	return error.HttpTransferEncodingUnsupported;
117	}
118	} else if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
119	if (std.ascii.eqlIgnoreCase(header_value, "keep-alive")) {
120	headers.connection = .keep_alive;
121	} else if (std.ascii.eqlIgnoreCase(header_value, "close")) {
122	headers.connection = .close;
123	} else {
124	return error.HttpConnectionHeaderUnsupported;
125	}
126	} else if (std.ascii.eqlIgnoreCase(header_name, "upgrade")) {
127	headers.upgrade = header_value;
128	}
129	}
130
131	return headers;
132	}
133
134	test "parse headers" {
135	const example =
136	"HTTP/1.1 301 Moved Permanently\r\n" ++
137	"Location: https://www.example.com/\r\n" ++
138	"Content-Type: text/html; charset=UTF-8\r\n" ++
139	"Content-Length: 220\r\n\r\n";
140	const parsed = try Headers.parse(example);
141	try testing.expectEqual(http.Version.@"HTTP/1.1", parsed.version);
142	try testing.expectEqual(http.Status.moved_permanently, parsed.status);
143	try testing.expectEqualStrings("https://www.example.com/", parsed.location orelse
144	return error.TestFailed);
145	try testing.expectEqual(@as(?u64, 220), parsed.content_length);
146	}
147
148	test "header continuation" {
149	const example =
150	"HTTP/1.0 200 OK\r\n" ++
151	"Content-Type: text/html;\r\n charset=UTF-8\r\n" ++
152	"Content-Length: 220\r\n\r\n";
153	try testing.expectError(
154	error.HttpHeaderContinuationsUnsupported,
155	Headers.parse(example),
156	);
157	}
158
159	test "extra content length" {
160	const example =
161	"HTTP/1.0 200 OK\r\n" ++
162	"Content-Length: 220\r\n" ++
163	"Content-Type: text/html; charset=UTF-8\r\n" ++
164	"content-length: 220\r\n\r\n";
165	try testing.expectError(
166	error.HttpHeadersInvalid,
167	Headers.parse(example),
168	);
169	}
170	};
171
172	inline fn int16(array: *const [2]u8) u16 {
173	return @bitCast(u16, array.*);
174	}
175
176	inline fn int32(array: *const [4]u8) u32 {
177	return @bitCast(u32, array.*);
178	}
179
180	inline fn int64(array: *const [8]u8) u64 {
181	return @bitCast(u64, array.*);
182	}
183
184	pub const State = enum {
185	/// Begin header parsing states.
186	invalid,
187	start,
188	seen_r,
189	seen_rn,
190	seen_rnr,
191	finished,
192	/// Begin transfer-encoding: chunked parsing states.
193	chunk_size_prefix_r,
194	chunk_size_prefix_n,
195	chunk_size,
196	chunk_r,
197	chunk_data,
198
199	pub fn isContent(self: State) bool {
200	return switch (self) {
201	.invalid, .start, .seen_r, .seen_rn, .seen_rnr => false,
202	.finished, .chunk_size_prefix_r, .chunk_size_prefix_n, .chunk_size, .chunk_r, .chunk_data => true,
203	};
204	}
205	};
206
207	pub fn initDynamic(max: usize) Response {
208	return .{
209	.state = .start,
210	.headers = undefined,
211	.header_bytes = .{},
212	.max_header_bytes = max,
213	.header_bytes_owned = true,
214	.next_chunk_length = undefined,
215	};
216	}
217
218	pub fn initStatic(buf: []u8) Response {
219	return .{
220	.state = .start,
221	.headers = undefined,
222	.header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
223	.max_header_bytes = buf.len,
224	.header_bytes_owned = false,
225	.next_chunk_length = undefined,
226	};
227	}
228
229	/// Returns how many bytes are part of HTTP headers. Always less than or
230	/// equal to bytes.len. If the amount returned is less than bytes.len, it
231	/// means the headers ended and the first byte after the double \r\n\r\n is
232	/// located at `bytes[result]`.
233	pub fn findHeadersEnd(r: *Response, bytes: []const u8) usize {
234	var index: usize = 0;
235
236	// TODO: https://github.com/ziglang/zig/issues/8220
237	state: while (true) {
238	switch (r.state) {
239	.invalid => unreachable,
240	.finished => unreachable,
241	.start => while (true) {
242	switch (bytes.len - index) {
243	0 => return index,
244	1 => {
245	if (bytes[index] == '\r')
246	r.state = .seen_r;
247	return index + 1;
248	},
249	2 => {
250	if (int16(bytes[index..][0..2]) == int16("\r\n")) {
251	r.state = .seen_rn;
252	} else if (bytes[index + 1] == '\r') {
253	r.state = .seen_r;
254	}
255	return index + 2;
256	},
257	3 => {
258	if (int16(bytes[index..][0..2]) == int16("\r\n") and
259	bytes[index + 2] == '\r')
260	{
261	r.state = .seen_rnr;
262	} else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n")) {
263	r.state = .seen_rn;
264	} else if (bytes[index + 2] == '\r') {
265	r.state = .seen_r;
266	}
267	return index + 3;
268	},
269	4...15 => {
270	if (int32(bytes[index..][0..4]) == int32("\r\n\r\n")) {
271	r.state = .finished;
272	return index + 4;
273	} else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n") and
274	bytes[index + 3] == '\r')
275	{
276	r.state = .seen_rnr;
277	index += 4;
278	continue :state;
279	} else if (int16(bytes[index + 2 ..][0..2]) == int16("\r\n")) {
280	r.state = .seen_rn;
281	index += 4;
282	continue :state;
283	} else if (bytes[index + 3] == '\r') {
284	r.state = .seen_r;
285	index += 4;
286	continue :state;
287	}
288	index += 4;
289	continue;
290	},
291	else => {
292	const chunk = bytes[index..][0..16];
293	const v: @Vector(16, u8) = chunk.*;
294	const matches_r = v == @splat(16, @as(u8, '\r'));
295	const iota = std.simd.iota(u8, 16);
296	const default = @splat(16, @as(u8, 16));
297	const sub_index = @reduce(.Min, @select(u8, matches_r, iota, default));
298	switch (sub_index) {
299	0...12 => {
300	index += sub_index + 4;
301	if (int32(chunk[sub_index..][0..4]) == int32("\r\n\r\n")) {
302	r.state = .finished;
303	return index;
304	}
305	continue;
306	},
307	13 => {
308	index += 16;
309	if (int16(chunk[14..][0..2]) == int16("\n\r")) {
310	r.state = .seen_rnr;
311	continue :state;
312	}
313	continue;
314	},
315	14 => {
316	index += 16;
317	if (chunk[15] == '\n') {
318	r.state = .seen_rn;
319	continue :state;
320	}
321	continue;
322	},
323	15 => {
324	r.state = .seen_r;
325	index += 16;
326	continue :state;
327	},
328	16 => {
329	index += 16;
330	continue;
331	},
332	else => unreachable,
333	}
334	},
335	}
336	},
337
338	.seen_r => switch (bytes.len - index) {
339	0 => return index,
340	1 => {
341	switch (bytes[index]) {
342	'\n' => r.state = .seen_rn,
343	'\r' => r.state = .seen_r,
344	else => r.state = .start,
345	}
346	return index + 1;
347	},
348	2 => {
349	if (int16(bytes[index..][0..2]) == int16("\n\r")) {
350	r.state = .seen_rnr;
351	return index + 2;
352	}
353	r.state = .start;
354	return index + 2;
355	},
356	else => {
357	if (int16(bytes[index..][0..2]) == int16("\n\r") and
358	bytes[index + 2] == '\n')
359	{
360	r.state = .finished;
361	return index + 3;
362	}
363	index += 3;
364	r.state = .start;
365	continue :state;
366	},
367	},
368	.seen_rn => switch (bytes.len - index) {
369	0 => return index,
370	1 => {
371	switch (bytes[index]) {
372	'\r' => r.state = .seen_rnr,
373	else => r.state = .start,
374	}
375	return index + 1;
376	},
377	else => {
378	if (int16(bytes[index..][0..2]) == int16("\r\n")) {
379	r.state = .finished;
380	return index + 2;
381	}
382	index += 2;
383	r.state = .start;
384	continue :state;
385	},
386	},
387	.seen_rnr => switch (bytes.len - index) {
388	0 => return index,
389	else => {
390	if (bytes[index] == '\n') {
391	r.state = .finished;
392	return index + 1;
393	}
394	index += 1;
395	r.state = .start;
396	continue :state;
397	},
398	},
399	.chunk_size_prefix_r => unreachable,
400	.chunk_size_prefix_n => unreachable,
401	.chunk_size => unreachable,
402	.chunk_r => unreachable,
403	.chunk_data => unreachable,
404	}
405
406	return index;
407	}
408	}
409
410	pub fn findChunkedLen(r: *Response, bytes: []const u8) usize {
411	var i: usize = 0;
412	if (r.state == .chunk_size) {
413	while (i < bytes.len) : (i += 1) {
414	const digit = switch (bytes[i]) {
415	'0'...'9' => \|b\| b - '0',
416	'A'...'Z' => \|b\| b - 'A' + 10,
417	'a'...'z' => \|b\| b - 'a' + 10,
418	'\r' => {
419	r.state = .chunk_r;
420	i += 1;
421	break;
422	},
423	else => {
424	r.state = .invalid;
425	return i;
426	},
427	};
428	const mul = @mulWithOverflow(r.next_chunk_length, 16);
429	if (mul[1] != 0) {
430	r.state = .invalid;
431	return i;
432	}
433	const add = @addWithOverflow(mul[0], digit);
434	if (add[1] != 0) {
435	r.state = .invalid;
436	return i;
437	}
438	r.next_chunk_length = add[0];
439	} else {
440	return i;
441	}
442	}
443	assert(r.state == .chunk_r);
444	if (i == bytes.len) return i;
445
446	if (bytes[i] == '\n') {
447	r.state = .chunk_data;
448	return i + 1;
449	} else {
450	r.state = .invalid;
451	return i;
452	}
453	}
454
455	fn parseInt3(nnn: @Vector(3, u8)) u10 {
456	const zero: @Vector(3, u8) = .{ '0', '0', '0' };
457	const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
458	return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
459	}
460
461	test parseInt3 {
462	const expectEqual = std.testing.expectEqual;
463	try expectEqual(@as(u10, 0), parseInt3("000".*));
464	try expectEqual(@as(u10, 418), parseInt3("418".*));
465	try expectEqual(@as(u10, 999), parseInt3("999".*));
466	}
467
468	test "find headers end basic" {
469	var buffer: [1]u8 = undefined;
470	var r = Response.initStatic(&buffer);
471	try testing.expectEqual(@as(usize, 10), r.findHeadersEnd("HTTP/1.1 4"));
472	try testing.expectEqual(@as(usize, 2), r.findHeadersEnd("18"));
473	try testing.expectEqual(@as(usize, 8), r.findHeadersEnd(" lol\r\n\r\nblah blah"));
474	}
475
476	test "find headers end vectorized" {
477	var buffer: [1]u8 = undefined;
478	var r = Response.initStatic(&buffer);
479	const example =
480	"HTTP/1.1 301 Moved Permanently\r\n" ++
481	"Location: https://www.example.com/\r\n" ++
482	"Content-Type: text/html; charset=UTF-8\r\n" ++
483	"Content-Length: 220\r\n" ++
484	"\r\ncontent";
485	try testing.expectEqual(@as(usize, 131), r.findHeadersEnd(example));
486	}
487
488	test "find headers end bug" {
489	var buffer: [1]u8 = undefined;
490	var r = Response.initStatic(&buffer);
491	const trail = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
492	const example =
493	"HTTP/1.1 200 OK\r\n" ++
494	"Access-Control-Allow-Origin: https://render.githubusercontent.com\r\n" ++
495	"content-disposition: attachment; filename=zig-0.10.0.tar.gz\r\n" ++
496	"Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'; sandbox\r\n" ++
497	"Content-Type: application/x-gzip\r\n" ++
498	"ETag: \"bfae0af6b01c7c0d89eb667cb5f0e65265968aeebda2689177e6b26acd3155ca\"\r\n" ++
499	"Strict-Transport-Security: max-age=31536000\r\n" ++
500	"Vary: Authorization,Accept-Encoding,Origin\r\n" ++
501	"X-Content-Type-Options: nosniff\r\n" ++
502	"X-Frame-Options: deny\r\n" ++
503	"X-XSS-Protection: 1; mode=block\r\n" ++
504	"Date: Fri, 06 Jan 2023 22:26:22 GMT\r\n" ++
505	"Transfer-Encoding: chunked\r\n" ++
506	"X-GitHub-Request-Id: 89C6:17E9:A7C9E:124B51:63B8A00E\r\n" ++
507	"connection: close\r\n\r\n" ++ trail;
508	try testing.expectEqual(@as(usize, example.len - trail.len), r.findHeadersEnd(example));
509	}

lib/std/http/Server.zig created+600

...	@@ -0,0 +1,600 @@
1	const std = @import("../std.zig");
2	const testing = std.testing;
3	const http = std.http;
4	const mem = std.mem;
5	const net = std.net;
6	const Uri = std.Uri;
7	const Allocator = mem.Allocator;
8	const assert = std.debug.assert;
9
10	const Server = @This();
11	const proto = @import("protocol.zig");
12
13	allocator: Allocator,
14
15	socket: net.StreamServer,
16
17	/// An interface to either a plain or TLS connection.
18	pub const Connection = struct {
19	stream: net.Stream,
20	protocol: Protocol,
21
22	closing: bool = true,
23
24	pub const Protocol = enum { plain };
25
26	pub fn read(conn: *Connection, buffer: []u8) !usize {
27	switch (conn.protocol) {
28	.plain => return conn.stream.read(buffer),
29	// .tls => return conn.tls_client.read(conn.stream, buffer),
30	}
31	}
32
33	pub fn readAtLeast(conn: *Connection, buffer: []u8, len: usize) !usize {
34	switch (conn.protocol) {
35	.plain => return conn.stream.readAtLeast(buffer, len),
36	// .tls => return conn.tls_client.readAtLeast(conn.stream, buffer, len),
37	}
38	}
39
40	pub const ReadError = net.Stream.ReadError;
41
42	pub const Reader = std.io.Reader(*Connection, ReadError, read);
43
44	pub fn reader(conn: *Connection) Reader {
45	return Reader{ .context = conn };
46	}
47
48	pub fn writeAll(conn: *Connection, buffer: []const u8) !void {
49	switch (conn.protocol) {
50	.plain => return conn.stream.writeAll(buffer),
51	// .tls => return conn.tls_client.writeAll(conn.stream, buffer),
52	}
53	}
54
55	pub fn write(conn: *Connection, buffer: []const u8) !usize {
56	switch (conn.protocol) {
57	.plain => return conn.stream.write(buffer),
58	// .tls => return conn.tls_client.write(conn.stream, buffer),
59	}
60	}
61
62	pub const WriteError = net.Stream.WriteError \|\| error{};
63	pub const Writer = std.io.Writer(*Connection, WriteError, write);
64
65	pub fn writer(conn: *Connection) Writer {
66	return Writer{ .context = conn };
67	}
68
69	pub fn close(conn: *Connection) void {
70	conn.stream.close();
71	}
72	};
73
74	/// A buffered (and peekable) Connection.
75	pub const BufferedConnection = struct {
76	pub const buffer_size = 0x2000;
77
78	conn: Connection,
79	buf: [buffer_size]u8 = undefined,
80	start: u16 = 0,
81	end: u16 = 0,
82
83	pub fn fill(bconn: *BufferedConnection) ReadError!void {
84	if (bconn.end != bconn.start) return;
85
86	const nread = try bconn.conn.read(bconn.buf[0..]);
87	if (nread == 0) return error.EndOfStream;
88	bconn.start = 0;
89	bconn.end = @truncate(u16, nread);
90	}
91
92	pub fn peek(bconn: *BufferedConnection) []const u8 {
93	return bconn.buf[bconn.start..bconn.end];
94	}
95
96	pub fn clear(bconn: *BufferedConnection, num: u16) void {
97	bconn.start += num;
98	}
99
100	pub fn readAtLeast(bconn: *BufferedConnection, buffer: []u8, len: usize) ReadError!usize {
101	var out_index: u16 = 0;
102	while (out_index < len) {
103	const available = bconn.end - bconn.start;
104	const left = buffer.len - out_index;
105
106	if (available > 0) {
107	const can_read = @truncate(u16, @min(available, left));
108
109	std.mem.copy(u8, buffer[out_index..], bconn.buf[bconn.start..][0..can_read]);
110	out_index += can_read;
111	bconn.start += can_read;
112
113	continue;
114	}
115
116	if (left > bconn.buf.len) {
117	// skip the buffer if the output is large enough
118	return bconn.conn.read(buffer[out_index..]);
119	}
120
121	try bconn.fill();
122	}
123
124	return out_index;
125	}
126
127	pub fn read(bconn: *BufferedConnection, buffer: []u8) ReadError!usize {
128	return bconn.readAtLeast(buffer, 1);
129	}
130
131	pub const ReadError = Connection.ReadError \|\| error{EndOfStream};
132	pub const Reader = std.io.Reader(*BufferedConnection, ReadError, read);
133
134	pub fn reader(bconn: *BufferedConnection) Reader {
135	return Reader{ .context = bconn };
136	}
137
138	pub fn writeAll(bconn: *BufferedConnection, buffer: []const u8) WriteError!void {
139	return bconn.conn.writeAll(buffer);
140	}
141
142	pub fn write(bconn: *BufferedConnection, buffer: []const u8) WriteError!usize {
143	return bconn.conn.write(buffer);
144	}
145
146	pub const WriteError = Connection.WriteError;
147	pub const Writer = std.io.Writer(*BufferedConnection, WriteError, write);
148
149	pub fn writer(bconn: *BufferedConnection) Writer {
150	return Writer{ .context = bconn };
151	}
152
153	pub fn close(bconn: *BufferedConnection) void {
154	bconn.conn.close();
155	}
156	};
157
158	/// A HTTP request originating from a client.
159	pub const Request = struct {
160	pub const Headers = struct {
161	method: http.Method,
162	target: []const u8,
163	version: http.Version,
164	content_length: ?u64 = null,
165	transfer_encoding: ?http.TransferEncoding = null,
166	transfer_compression: ?http.ContentEncoding = null,
167	connection: http.Connection = .close,
168	host: ?[]const u8 = null,
169
170	pub const ParseError = error{
171	ShortHttpStatusLine,
172	BadHttpVersion,
173	UnknownHttpMethod,
174	HttpHeadersInvalid,
175	HttpHeaderContinuationsUnsupported,
176	HttpTransferEncodingUnsupported,
177	HttpConnectionHeaderUnsupported,
178	InvalidCharacter,
179	};
180
181	pub fn parse(bytes: []const u8) !Headers {
182	var it = mem.tokenize(u8, bytes[0 .. bytes.len - 4], "\r\n");
183
184	const first_line = it.next() orelse return error.HttpHeadersInvalid;
185	if (first_line.len < 10)
186	return error.ShortHttpStatusLine;
187
188	const method_end = mem.indexOfScalar(u8, first_line, ' ') orelse return error.HttpHeadersInvalid;
189	const method_str = first_line[0..method_end];
190	const method = std.meta.stringToEnum(http.Method, method_str) orelse return error.UnknownHttpMethod;
191
192	const version_start = mem.lastIndexOfScalar(u8, first_line, ' ') orelse return error.HttpHeadersInvalid;
193	if (version_start == method_end) return error.HttpHeadersInvalid;
194
195	const version_str = first_line[version_start + 1 ..];
196	if (version_str.len != 8) return error.HttpHeadersInvalid;
197	const version: http.Version = switch (int64(version_str[0..8])) {
198	int64("HTTP/1.0") => .@"HTTP/1.0",
199	int64("HTTP/1.1") => .@"HTTP/1.1",
200	else => return error.BadHttpVersion,
201	};
202
203	const target = first_line[method_end + 1 .. version_start];
204
205	var headers: Headers = .{
206	.method = method,
207	.target = target,
208	.version = version,
209	};
210
211	while (it.next()) \|line\| {
212	if (line.len == 0) return error.HttpHeadersInvalid;
213	switch (line[0]) {
214	' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
215	else => {},
216	}
217
218	var line_it = mem.tokenize(u8, line, ": ");
219	const header_name = line_it.next() orelse return error.HttpHeadersInvalid;
220	const header_value = line_it.rest();
221	if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
222	if (headers.content_length != null) return error.HttpHeadersInvalid;
223	headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
224	} else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
225	// Transfer-Encoding: second, first
226	// Transfer-Encoding: deflate, chunked
227	var iter = mem.splitBackwards(u8, header_value, ",");
228
229	if (iter.next()) \|first\| {
230	const trimmed = mem.trim(u8, first, " ");
231
232	if (std.meta.stringToEnum(http.TransferEncoding, trimmed)) \|te\| {
233	if (headers.transfer_encoding != null) return error.HttpHeadersInvalid;
234	headers.transfer_encoding = te;
235	} else if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
236	if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
237	headers.transfer_compression = ce;
238	} else {
239	return error.HttpTransferEncodingUnsupported;
240	}
241	}
242
243	if (iter.next()) \|second\| {
244	if (headers.transfer_compression != null) return error.HttpTransferEncodingUnsupported;
245
246	const trimmed = mem.trim(u8, second, " ");
247
248	if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
249	headers.transfer_compression = ce;
250	} else {
251	return error.HttpTransferEncodingUnsupported;
252	}
253	}
254
255	if (iter.next()) \|_\| return error.HttpTransferEncodingUnsupported;
256	} else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
257	if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
258
259	const trimmed = mem.trim(u8, header_value, " ");
260
261	if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) \|ce\| {
262	headers.transfer_compression = ce;
263	} else {
264	return error.HttpTransferEncodingUnsupported;
265	}
266	} else if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
267	if (std.ascii.eqlIgnoreCase(header_value, "keep-alive")) {
268	headers.connection = .keep_alive;
269	} else if (std.ascii.eqlIgnoreCase(header_value, "close")) {
270	headers.connection = .close;
271	} else {
272	return error.HttpConnectionHeaderUnsupported;
273	}
274	} else if (std.ascii.eqlIgnoreCase(header_name, "host")) {
275	headers.host = header_value;
276	}
277	}
278
279	return headers;
280	}
281
282	inline fn int64(array: *const [8]u8) u64 {
283	return @bitCast(u64, array.*);
284	}
285	};
286
287	headers: Headers = undefined,
288	parser: proto.HeadersParser,
289	compression: Compression = .none,
290	};
291
292	/// A HTTP response waiting to be sent.
293	///
294	/// [/ <----------------------------------- \]
295	/// Order of operations: accept -> wait -> do [ -> write -> finish][ -> reset /]
296	/// \ -> read /
297	pub const Response = struct {
298	pub const Headers = struct {
299	version: http.Version = .@"HTTP/1.1",
300	status: http.Status = .ok,
301	reason: ?[]const u8 = null,
302
303	server: ?[]const u8 = "zig (std.http)",
304	connection: http.Connection = .keep_alive,
305	transfer_encoding: RequestTransfer = .none,
306
307	custom: []const http.CustomHeader = &[_]http.CustomHeader{},
308	};
309
310	server: *Server,
311	address: net.Address,
312	connection: BufferedConnection,
313
314	headers: Headers = .{},
315	request: Request,
316
317	/// Reset this response to its initial state. This must be called before handling a second request on the same connection.
318	pub fn reset(res: *Response) void {
319	switch (res.request.compression) {
320	.none => {},
321	.deflate => \|*deflate\| deflate.deinit(),
322	.gzip => \|*gzip\| gzip.deinit(),
323	.zstd => \|*zstd\| zstd.deinit(),
324	}
325
326	if (!res.request.parser.done) {
327	// If the response wasn't fully read, then we need to close the connection.
328	res.connection.conn.closing = true;
329	}
330
331	if (res.connection.conn.closing) {
332	res.connection.close();
333
334	if (res.request.parser.header_bytes_owned) {
335	res.request.parser.header_bytes.deinit(res.server.allocator);
336	}
337
338	res.* = undefined;
339	} else {
340	res.request.parser.reset();
341	}
342	}
343
344	/// Send the response headers.
345	pub fn do(res: *Response) !void {
346	var buffered = std.io.bufferedWriter(res.connection.writer());
347	const w = buffered.writer();
348
349	try w.writeAll(@tagName(res.headers.version));
350	try w.writeByte(' ');
351	try w.print("{d}", .{@enumToInt(res.headers.status)});
352	try w.writeByte(' ');
353	if (res.headers.reason) \|reason\| {
354	try w.writeAll(reason);
355	} else if (res.headers.status.phrase()) \|phrase\| {
356	try w.writeAll(phrase);
357	}
358
359	if (res.headers.server) \|server\| {
360	try w.writeAll("\r\nServer: ");
361	try w.writeAll(server);
362	}
363
364	if (res.headers.connection == .close) {
365	try w.writeAll("\r\nConnection: close");
366	} else {
367	try w.writeAll("\r\nConnection: keep-alive");
368	}
369
370	switch (res.headers.transfer_encoding) {
371	.chunked => try w.writeAll("\r\nTransfer-Encoding: chunked"),
372	.content_length => \|content_length\| try w.print("\r\nContent-Length: {d}", .{content_length}),
373	.none => {},
374	}
375
376	for (res.headers.custom) \|header\| {
377	try w.writeAll("\r\n");
378	try w.writeAll(header.name);
379	try w.writeAll(": ");
380	try w.writeAll(header.value);
381	}
382
383	try w.writeAll("\r\n\r\n");
384
385	try buffered.flush();
386	}
387
388	pub const TransferReadError = BufferedConnection.ReadError \|\| proto.HeadersParser.ReadError;
389
390	pub const TransferReader = std.io.Reader(*Response, TransferReadError, transferRead);
391
392	pub fn transferReader(res: *Response) TransferReader {
393	return .{ .context = res };
394	}
395
396	pub fn transferRead(res: *Response, buf: []u8) TransferReadError!usize {
397	if (res.request.parser.isComplete()) return 0;
398
399	var index: usize = 0;
400	while (index == 0) {
401	const amt = try res.request.parser.read(&res.connection, buf[index..], false);
402	if (amt == 0 and res.request.parser.isComplete()) break;
403	index += amt;
404	}
405
406	return index;
407	}
408
409	pub const WaitForCompleteHeadError = BufferedConnection.ReadError \|\| proto.HeadersParser.WaitForCompleteHeadError \|\| Request.Headers.ParseError \|\| error{ BadHeader, InvalidCompression, StreamTooLong, InvalidWindowSize } \|\| error{CompressionNotSupported};
410
411	/// Wait for the client to send a complete request head.
412	pub fn wait(res: *Response) !void {
413	while (true) {
414	try res.connection.fill();
415
416	const nchecked = try res.request.parser.checkCompleteHead(res.server.allocator, res.connection.peek());
417	res.connection.clear(@intCast(u16, nchecked));
418
419	if (res.request.parser.state.isContent()) break;
420	}
421
422	res.request.headers = try Request.Headers.parse(res.request.parser.header_bytes.items);
423
424	if (res.headers.connection == .keep_alive and res.request.headers.connection == .keep_alive) {
425	res.connection.conn.closing = false;
426	} else {
427	res.connection.conn.closing = true;
428	}
429
430	if (res.request.headers.transfer_encoding) \|te\| {
431	switch (te) {
432	.chunked => {
433	res.request.parser.next_chunk_length = 0;
434	res.request.parser.state = .chunk_head_size;
435	},
436	}
437	} else if (res.request.headers.content_length) \|cl\| {
438	res.request.parser.next_chunk_length = cl;
439
440	if (cl == 0) res.request.parser.done = true;
441	} else {
442	res.request.parser.done = true;
443	}
444
445	if (!res.request.parser.done) {
446	if (res.request.headers.transfer_compression) \|tc\| switch (tc) {
447	.compress => return error.CompressionNotSupported,
448	.deflate => res.request.compression = .{
449	.deflate = try std.compress.zlib.zlibStream(res.server.allocator, res.transferReader()),
450	},
451	.gzip => res.request.compression = .{
452	.gzip = try std.compress.gzip.decompress(res.server.allocator, res.transferReader()),
453	},
454	.zstd => res.request.compression = .{
455	.zstd = std.compress.zstd.decompressStream(res.server.allocator, res.transferReader()),
456	},
457	};
458	}
459	}
460
461	pub const ReadError = Compression.DeflateDecompressor.Error \|\| Compression.GzipDecompressor.Error \|\| Compression.ZstdDecompressor.Error \|\| WaitForCompleteHeadError;
462
463	pub const Reader = std.io.Reader(*Response, ReadError, read);
464
465	pub fn reader(res: *Response) Reader {
466	return .{ .context = res };
467	}
468
469	pub fn read(res: *Response, buffer: []u8) ReadError!usize {
470	return switch (res.request.compression) {
471	.deflate => \|*deflate\| try deflate.read(buffer),
472	.gzip => \|*gzip\| try gzip.read(buffer),
473	.zstd => \|*zstd\| try zstd.read(buffer),
474	else => try res.transferRead(buffer),
475	};
476	}
477
478	pub fn readAll(res: *Response, buffer: []u8) !usize {
479	var index: usize = 0;
480	while (index < buffer.len) {
481	const amt = try read(res, buffer[index..]);
482	if (amt == 0) break;
483	index += amt;
484	}
485	return index;
486	}
487
488	pub const WriteError = BufferedConnection.WriteError \|\| error{ NotWriteable, MessageTooLong };
489
490	pub const Writer = std.io.Writer(*Response, WriteError, write);
491
492	pub fn writer(res: *Response) Writer {
493	return .{ .context = res };
494	}
495
496	/// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.
497	pub fn write(res: *Response, bytes: []const u8) WriteError!usize {
498	switch (res.headers.transfer_encoding) {
499	.chunked => {
500	try res.connection.writer().print("{x}\r\n", .{bytes.len});
501	try res.connection.writeAll(bytes);
502	try res.connection.writeAll("\r\n");
503
504	return bytes.len;
505	},
506	.content_length => \|*len\| {
507	if (len.* < bytes.len) return error.MessageTooLong;
508
509	const amt = try res.connection.write(bytes);
510	len.* -= amt;
511	return amt;
512	},
513	.none => return error.NotWriteable,
514	}
515	}
516
517	/// Finish the body of a request. This notifies the server that you have no more data to send.
518	pub fn finish(res: *Response) !void {
519	switch (res.headers.transfer_encoding) {
520	.chunked => try res.connection.writeAll("0\r\n"),
521	.content_length => \|len\| if (len != 0) return error.MessageNotCompleted,
522	.none => {},
523	}
524	}
525	};
526
527	/// The mode of transport for responses.
528	pub const RequestTransfer = union(enum) {
529	content_length: u64,
530	chunked: void,
531	none: void,
532	};
533
534	/// The decompressor for request messages.
535	pub const Compression = union(enum) {
536	pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Response.TransferReader);
537	pub const GzipDecompressor = std.compress.gzip.Decompress(Response.TransferReader);
538	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});
539
540	deflate: DeflateDecompressor,
541	gzip: GzipDecompressor,
542	zstd: ZstdDecompressor,
543	none: void,
544	};
545
546	pub fn init(allocator: Allocator, options: net.StreamServer.Options) Server {
547	return .{
548	.allocator = allocator,
549	.socket = net.StreamServer.init(options),
550	};
551	}
552
553	pub fn deinit(server: *Server) void {
554	server.socket.deinit();
555	}
556
557	pub const ListenError = std.os.SocketError \|\| std.os.BindError \|\| std.os.ListenError \|\| std.os.SetSockOptError \|\| std.os.GetSockNameError;
558
559	/// Start the HTTP server listening on the given address.
560	pub fn listen(server: *Server, address: net.Address) !void {
561	try server.socket.listen(address);
562	}
563
564	pub const AcceptError = net.StreamServer.AcceptError \|\| Allocator.Error;
565
566	pub const HeaderStrategy = union(enum) {
567	/// In this case, the client's Allocator will be used to store the
568	/// entire HTTP header. This value is the maximum total size of
569	/// HTTP headers allowed, otherwise
570	/// error.HttpHeadersExceededSizeLimit is returned from read().
571	dynamic: usize,
572	/// This is used to store the entire HTTP header. If the HTTP
573	/// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
574	/// is returned from read(). When this is used, `error.OutOfMemory`
575	/// cannot be returned from `read()`.
576	static: []u8,
577	};
578
579	/// Accept a new connection and allocate a Response for it.
580	pub fn accept(server: Server, options: HeaderStrategy) AcceptError!Response {
581	const in = try server.socket.accept();
582
583	const res = try server.allocator.create(Response);
584	res.* = .{
585	.server = server,
586	.address = in.address,
587	.connection = .{ .conn = .{
588	.stream = in.stream,
589	.protocol = .plain,
590	} },
591	.request = .{
592	.parser = switch (options) {
593	.dynamic => \|max\| proto.HeadersParser.initDynamic(max),
594	.static => \|buf\| proto.HeadersParser.initStatic(buf),
595	},
596	},
597	};
598
599	return res;
600	}

lib/std/http/protocol.zig created+842

...	@@ -0,0 +1,842 @@
1	const std = @import("std");
2	const testing = std.testing;
3	const mem = std.mem;
4
5	const assert = std.debug.assert;
6
7	pub const State = enum {
8	/// Begin header parsing states.
9	invalid,
10	start,
11	seen_n,
12	seen_r,
13	seen_rn,
14	seen_rnr,
15	finished,
16	/// Begin transfer-encoding: chunked parsing states.
17	chunk_head_size,
18	chunk_head_ext,
19	chunk_head_r,
20	chunk_data,
21	chunk_data_suffix,
22	chunk_data_suffix_r,
23
24	/// Returns true if the parser is in a content state (ie. not waiting for more headers).
25	pub fn isContent(self: State) bool {
26	return switch (self) {
27	.invalid, .start, .seen_n, .seen_r, .seen_rn, .seen_rnr => false,
28	.finished, .chunk_head_size, .chunk_head_ext, .chunk_head_r, .chunk_data, .chunk_data_suffix, .chunk_data_suffix_r => true,
29	};
30	}
31	};
32
33	pub const HeadersParser = struct {
34	state: State = .start,
35	/// Whether or not `header_bytes` is allocated or was provided as a fixed buffer.
36	header_bytes_owned: bool,
37	/// Either a fixed buffer of len `max_header_bytes` or a dynamic buffer that can grow up to `max_header_bytes`.
38	/// Pointers into this buffer are not stable until after a message is complete.
39	header_bytes: std.ArrayListUnmanaged(u8),
40	/// The maximum allowed size of `header_bytes`.
41	max_header_bytes: usize,
42	next_chunk_length: u64 = 0,
43	/// Whether this parser is done parsing a complete message.
44	/// A message is only done when the entire payload has been read.
45	done: bool = false,
46
47	/// Initializes the parser with a dynamically growing header buffer of up to `max` bytes.
48	pub fn initDynamic(max: usize) HeadersParser {
49	return .{
50	.header_bytes = .{},
51	.max_header_bytes = max,
52	.header_bytes_owned = true,
53	};
54	}
55
56	/// Initializes the parser with a provided buffer `buf`.
57	pub fn initStatic(buf: []u8) HeadersParser {
58	return .{
59	.header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
60	.max_header_bytes = buf.len,
61	.header_bytes_owned = false,
62	};
63	}
64
65	/// Completely resets the parser to it's initial state.
66	/// This must be called after a message is complete.
67	pub fn reset(r: *HeadersParser) void {
68	assert(r.done); // The message must be completely read before reset, otherwise the parser is in an invalid state.
69
70	r.header_bytes.clearRetainingCapacity();
71
72	r.* = .{
73	.header_bytes = r.header_bytes,
74	.max_header_bytes = r.max_header_bytes,
75	.header_bytes_owned = r.header_bytes_owned,
76	};
77	}
78
79	/// Returns the number of bytes consumed by headers. This is always less than or equal to `bytes.len`.
80	/// You should check `r.state.isContent()` after this to check if the headers are done.
81	///
82	/// If the amount returned is less than `bytes.len`, you may assume that the parser is in a content state and the
83	/// first byte of content is located at `bytes[result]`.
84	pub fn findHeadersEnd(r: *HeadersParser, bytes: []const u8) u32 {
85	const vector_len: comptime_int = comptime std.math.max(std.simd.suggestVectorSize(u8) orelse 1, 8);
86	const len = @intCast(u32, bytes.len);
87	var index: u32 = 0;
88
89	while (true) {
90	switch (r.state) {
91	.invalid => unreachable,
92	.finished => return index,
93	.start => switch (len - index) {
94	0 => return index,
95	1 => {
96	switch (bytes[index]) {
97	'\r' => r.state = .seen_r,
98	'\n' => r.state = .seen_n,
99	else => {},
100	}
101
102	return index + 1;
103	},
104	2 => {
105	const b16 = int16(bytes[index..][0..2]);
106	const b8 = intShift(u8, b16);
107
108	switch (b8) {
109	'\r' => r.state = .seen_r,
110	'\n' => r.state = .seen_n,
111	else => {},
112	}
113
114	switch (b16) {
115	int16("\r\n") => r.state = .seen_rn,
116	int16("\n\n") => r.state = .finished,
117	else => {},
118	}
119
120	return index + 2;
121	},
122	3 => {
123	const b24 = int24(bytes[index..][0..3]);
124	const b16 = intShift(u16, b24);
125	const b8 = intShift(u8, b24);
126
127	switch (b8) {
128	'\r' => r.state = .seen_r,
129	'\n' => r.state = .seen_n,
130	else => {},
131	}
132
133	switch (b16) {
134	int16("\r\n") => r.state = .seen_rn,
135	int16("\n\n") => r.state = .finished,
136	else => {},
137	}
138
139	switch (b24) {
140	int24("\r\n\r") => r.state = .seen_rnr,
141	else => {},
142	}
143
144	return index + 3;
145	},
146	4...vector_len - 1 => {
147	const b32 = int32(bytes[index..][0..4]);
148	const b24 = intShift(u24, b32);
149	const b16 = intShift(u16, b32);
150	const b8 = intShift(u8, b32);
151
152	switch (b8) {
153	'\r' => r.state = .seen_r,
154	'\n' => r.state = .seen_n,
155	else => {},
156	}
157
158	switch (b16) {
159	int16("\r\n") => r.state = .seen_rn,
160	int16("\n\n") => r.state = .finished,
161	else => {},
162	}
163
164	switch (b24) {
165	int24("\r\n\r") => r.state = .seen_rnr,
166	else => {},
167	}
168
169	switch (b32) {
170	int32("\r\n\r\n") => r.state = .finished,
171	else => {},
172	}
173
174	index += 4;
175	continue;
176	},
177	else => {
178	const Vector = @Vector(vector_len, u8);
179	// const BoolVector = @Vector(vector_len, bool);
180	const BitVector = @Vector(vector_len, u1);
181	const SizeVector = @Vector(vector_len, u8);
182
183	const chunk = bytes[index..][0..vector_len];
184	const v: Vector = chunk.*;
185	const matches_r = @bitCast(BitVector, v == @splat(vector_len, @as(u8, '\r')));
186	const matches_n = @bitCast(BitVector, v == @splat(vector_len, @as(u8, '\n')));
187	const matches_or: SizeVector = matches_r \| matches_n;
188
189	const matches = @reduce(.Add, matches_or);
190	switch (matches) {
191	0 => {},
192	1 => switch (chunk[vector_len - 1]) {
193	'\r' => r.state = .seen_r,
194	'\n' => r.state = .seen_n,
195	else => {},
196	},
197	2 => {
198	const b16 = int16(chunk[vector_len - 2 ..][0..2]);
199	const b8 = intShift(u8, b16);
200
201	switch (b8) {
202	'\r' => r.state = .seen_r,
203	'\n' => r.state = .seen_n,
204	else => {},
205	}
206
207	switch (b16) {
208	int16("\r\n") => r.state = .seen_rn,
209	int16("\n\n") => r.state = .finished,
210	else => {},
211	}
212	},
213	3 => {
214	const b24 = int24(chunk[vector_len - 3 ..][0..3]);
215	const b16 = intShift(u16, b24);
216	const b8 = intShift(u8, b24);
217
218	switch (b8) {
219	'\r' => r.state = .seen_r,
220	'\n' => r.state = .seen_n,
221	else => {},
222	}
223
224	switch (b16) {
225	int16("\r\n") => r.state = .seen_rn,
226	int16("\n\n") => r.state = .finished,
227	else => {},
228	}
229
230	switch (b24) {
231	int24("\r\n\r") => r.state = .seen_rnr,
232	else => {},
233	}
234	},
235	4...vector_len => {
236	inline for (0..vector_len - 3) \|i_usize\| {
237	const i = @truncate(u32, i_usize);
238
239	const b32 = int32(chunk[i..][0..4]);
240	const b16 = intShift(u16, b32);
241
242	if (b32 == int32("\r\n\r\n")) {
243	r.state = .finished;
244	return index + i + 4;
245	} else if (b16 == int16("\n\n")) {
246	r.state = .finished;
247	return index + i + 2;
248	}
249	}
250
251	const b24 = int24(chunk[vector_len - 3 ..][0..3]);
252	const b16 = intShift(u16, b24);
253	const b8 = intShift(u8, b24);
254
255	switch (b8) {
256	'\r' => r.state = .seen_r,
257	'\n' => r.state = .seen_n,
258	else => {},
259	}
260
261	switch (b16) {
262	int16("\r\n") => r.state = .seen_rn,
263	int16("\n\n") => r.state = .finished,
264	else => {},
265	}
266
267	switch (b24) {
268	int24("\r\n\r") => r.state = .seen_rnr,
269	else => {},
270	}
271	},
272	else => unreachable,
273	}
274
275	index += vector_len;
276	continue;
277	},
278	},
279	.seen_n => switch (len - index) {
280	0 => return index,
281	else => {
282	switch (bytes[index]) {
283	'\n' => r.state = .finished,
284	else => r.state = .start,
285	}
286
287	index += 1;
288	continue;
289	},
290	},
291	.seen_r => switch (len - index) {
292	0 => return index,
293	1 => {
294	switch (bytes[index]) {
295	'\n' => r.state = .seen_rn,
296	'\r' => r.state = .seen_r,
297	else => r.state = .start,
298	}
299
300	return index + 1;
301	},
302	2 => {
303	const b16 = int16(bytes[index..][0..2]);
304	const b8 = intShift(u8, b16);
305
306	switch (b8) {
307	'\r' => r.state = .seen_r,
308	'\n' => r.state = .seen_rn,
309	else => r.state = .start,
310	}
311
312	switch (b16) {
313	int16("\r\n") => r.state = .seen_rn,
314	int16("\n\r") => r.state = .seen_rnr,
315	int16("\n\n") => r.state = .finished,
316	else => {},
317	}
318
319	return index + 2;
320	},
321	else => {
322	const b24 = int24(bytes[index..][0..3]);
323	const b16 = intShift(u16, b24);
324	const b8 = intShift(u8, b24);
325
326	switch (b8) {
327	'\r' => r.state = .seen_r,
328	'\n' => r.state = .seen_n,
329	else => r.state = .start,
330	}
331
332	switch (b16) {
333	int16("\r\n") => r.state = .seen_rn,
334	int16("\n\n") => r.state = .finished,
335	else => {},
336	}
337
338	switch (b24) {
339	int24("\n\r\n") => r.state = .finished,
340	else => {},
341	}
342
343	index += 3;
344	continue;
345	},
346	},
347	.seen_rn => switch (len - index) {
348	0 => return index,
349	1 => {
350	switch (bytes[index]) {
351	'\r' => r.state = .seen_rnr,
352	'\n' => r.state = .seen_n,
353	else => r.state = .start,
354	}
355
356	return index + 1;
357	},
358	else => {
359	const b16 = int16(bytes[index..][0..2]);
360	const b8 = intShift(u8, b16);
361
362	switch (b8) {
363	'\r' => r.state = .seen_rnr,
364	'\n' => r.state = .seen_n,
365	else => r.state = .start,
366	}
367
368	switch (b16) {
369	int16("\r\n") => r.state = .finished,
370	int16("\n\n") => r.state = .finished,
371	else => {},
372	}
373
374	index += 2;
375	continue;
376	},
377	},
378	.seen_rnr => switch (len - index) {
379	0 => return index,
380	else => {
381	switch (bytes[index]) {
382	'\n' => r.state = .finished,
383	else => r.state = .start,
384	}
385
386	index += 1;
387	continue;
388	},
389	},
390	.chunk_head_size => unreachable,
391	.chunk_head_ext => unreachable,
392	.chunk_head_r => unreachable,
393	.chunk_data => unreachable,
394	.chunk_data_suffix => unreachable,
395	.chunk_data_suffix_r => unreachable,
396	}
397
398	return index;
399	}
400	}
401
402	/// Returns the number of bytes consumed by the chunk size. This is always less than or equal to `bytes.len`.
403	/// You should check `r.state == .chunk_data` after this to check if the chunk size has been fully parsed.
404	///
405	/// If the amount returned is less than `bytes.len`, you may assume that the parser is in the `chunk_data` state
406	/// and that the first byte of the chunk is at `bytes[result]`.
407	pub fn findChunkedLen(r: *HeadersParser, bytes: []const u8) u32 {
408	const len = @intCast(u32, bytes.len);
409
410	for (bytes[0..], 0..) \|c, i\| {
411	const index = @intCast(u32, i);
412	switch (r.state) {
413	.chunk_data_suffix => switch (c) {
414	'\r' => r.state = .chunk_data_suffix_r,
415	'\n' => r.state = .chunk_head_size,
416	else => {
417	r.state = .invalid;
418	return index;
419	},
420	},
421	.chunk_data_suffix_r => switch (c) {
422	'\n' => r.state = .chunk_head_size,
423	else => {
424	r.state = .invalid;
425	return index;
426	},
427	},
428	.chunk_head_size => {
429	const digit = switch (c) {
430	'0'...'9' => \|b\| b - '0',
431	'A'...'Z' => \|b\| b - 'A' + 10,
432	'a'...'z' => \|b\| b - 'a' + 10,
433	'\r' => {
434	r.state = .chunk_head_r;
435	continue;
436	},
437	'\n' => {
438	r.state = .chunk_data;
439	return index + 1;
440	},
441	else => {
442	r.state = .chunk_head_ext;
443	continue;
444	},
445	};
446
447	const new_len = r.next_chunk_length *% 16 +% digit;
448	if (new_len <= r.next_chunk_length and r.next_chunk_length != 0) {
449	r.state = .invalid;
450	return index;
451	}
452
453	r.next_chunk_length = new_len;
454	},
455	.chunk_head_ext => switch (c) {
456	'\r' => r.state = .chunk_head_r,
457	'\n' => {
458	r.state = .chunk_data;
459	return index + 1;
460	},
461	else => continue,
462	},
463	.chunk_head_r => switch (c) {
464	'\n' => {
465	r.state = .chunk_data;
466	return index + 1;
467	},
468	else => {
469	r.state = .invalid;
470	return index;
471	},
472	},
473	else => unreachable,
474	}
475	}
476
477	return len;
478	}
479
480	/// Returns whether or not the parser has finished parsing a complete message. A message is only complete after the
481	/// entire body has been read and any trailing headers have been parsed.
482	pub fn isComplete(r: *HeadersParser) bool {
483	return r.done and r.state == .finished;
484	}
485
486	pub const CheckCompleteHeadError = mem.Allocator.Error \|\| error{HttpHeadersExceededSizeLimit};
487
488	/// Pushes `in` into the parser. Returns the number of bytes consumed by the header. Any header bytes are appended
489	/// to the `header_bytes` buffer.
490	///
491	/// This function only uses `allocator` if `r.header_bytes_owned` is true, and may be undefined otherwise.
492	pub fn checkCompleteHead(r: *HeadersParser, allocator: std.mem.Allocator, in: []const u8) CheckCompleteHeadError!u32 {
493	if (r.state.isContent()) return 0;
494
495	const i = r.findHeadersEnd(in);
496	const data = in[0..i];
497	if (r.header_bytes.items.len + data.len > r.max_header_bytes) {
498	return error.HttpHeadersExceededSizeLimit;
499	} else {
500	if (r.header_bytes_owned) try r.header_bytes.ensureUnusedCapacity(allocator, data.len);
501
502	r.header_bytes.appendSliceAssumeCapacity(data);
503	}
504
505	return i;
506	}
507
508	pub const ReadError = error{
509	HttpChunkInvalid,
510	};
511
512	/// Reads the body of the message into `buffer`. Returns the number of bytes placed in the buffer.
513	///
514	/// If `skip` is true, the buffer will be unused and the body will be skipped.
515	///
516	/// See `std.http.Client.BufferedConnection for an example of `bconn`.
517	pub fn read(r: *HeadersParser, bconn: anytype, buffer: []u8, skip: bool) !usize {
518	assert(r.state.isContent());
519	if (r.done) return 0;
520
521	var out_index: usize = 0;
522	while (true) {
523	switch (r.state) {
524	.invalid, .start, .seen_n, .seen_r, .seen_rn, .seen_rnr => unreachable,
525	.finished => {
526	const data_avail = r.next_chunk_length;
527
528	if (skip) {
529	try bconn.fill();
530
531	const nread = @min(bconn.peek().len, data_avail);
532	bconn.clear(@intCast(u16, nread));
533	r.next_chunk_length -= nread;
534
535	if (r.next_chunk_length == 0) r.done = true;
536
537	return 0;
538	} else {
539	const out_avail = buffer.len;
540
541	const can_read = @intCast(usize, @min(data_avail, out_avail));
542	const nread = try bconn.read(buffer[0..can_read]);
543	r.next_chunk_length -= nread;
544
545	if (r.next_chunk_length == 0) r.done = true;
546
547	return nread;
548	}
549	},
550	.chunk_data_suffix, .chunk_data_suffix_r, .chunk_head_size, .chunk_head_ext, .chunk_head_r => {
551	try bconn.fill();
552
553	const i = r.findChunkedLen(bconn.peek());
554	bconn.clear(@intCast(u16, i));
555
556	switch (r.state) {
557	.invalid => return error.HttpChunkInvalid,
558	.chunk_data => if (r.next_chunk_length == 0) {
559	// The trailer section is formatted identically to the header section.
560	r.state = .seen_rn;
561	r.done = true;
562
563	return out_index;
564	},
565	else => return out_index,
566	}
567
568	continue;
569	},
570	.chunk_data => {
571	const data_avail = r.next_chunk_length;
572	const out_avail = buffer.len - out_index;
573
574	if (skip) {
575	try bconn.fill();
576
577	const nread = @min(bconn.peek().len, data_avail);
578	bconn.clear(@intCast(u16, nread));
579	r.next_chunk_length -= nread;
580	} else {
581	const can_read = @intCast(usize, @min(data_avail, out_avail));
582	const nread = try bconn.read(buffer[out_index..][0..can_read]);
583	r.next_chunk_length -= nread;
584	out_index += nread;
585	}
586
587	if (r.next_chunk_length == 0) {
588	r.state = .chunk_data_suffix;
589	continue;
590	}
591
592	return out_index;
593	},
594	}
595	}
596	}
597	};
598
599	inline fn int16(array: *const [2]u8) u16 {
600	return @bitCast(u16, array.*);
601	}
602
603	inline fn int24(array: *const [3]u8) u24 {
604	return @bitCast(u24, array.*);
605	}
606
607	inline fn int32(array: *const [4]u8) u32 {
608	return @bitCast(u32, array.*);
609	}
610
611	inline fn intShift(comptime T: type, x: anytype) T {
612	switch (@import("builtin").cpu.arch.endian()) {
613	.Little => return @truncate(T, x >> (@bitSizeOf(@TypeOf(x)) - @bitSizeOf(T))),
614	.Big => return @truncate(T, x),
615	}
616	}
617
618	/// A buffered (and peekable) Connection.
619	const MockBufferedConnection = struct {
620	pub const buffer_size = 0x2000;
621
622	conn: std.io.FixedBufferStream([]const u8),
623	buf: [buffer_size]u8 = undefined,
624	start: u16 = 0,
625	end: u16 = 0,
626
627	pub fn fill(bconn: *MockBufferedConnection) ReadError!void {
628	if (bconn.end != bconn.start) return;
629
630	const nread = try bconn.conn.read(bconn.buf[0..]);
631	if (nread == 0) return error.EndOfStream;
632	bconn.start = 0;
633	bconn.end = @truncate(u16, nread);
634	}
635
636	pub fn peek(bconn: *MockBufferedConnection) []const u8 {
637	return bconn.buf[bconn.start..bconn.end];
638	}
639
640	pub fn clear(bconn: *MockBufferedConnection, num: u16) void {
641	bconn.start += num;
642	}
643
644	pub fn readAtLeast(bconn: *MockBufferedConnection, buffer: []u8, len: usize) ReadError!usize {
645	var out_index: u16 = 0;
646	while (out_index < len) {
647	const available = bconn.end - bconn.start;
648	const left = buffer.len - out_index;
649
650	if (available > 0) {
651	const can_read = @truncate(u16, @min(available, left));
652
653	std.mem.copy(u8, buffer[out_index..], bconn.buf[bconn.start..][0..can_read]);
654	out_index += can_read;
655	bconn.start += can_read;
656
657	continue;
658	}
659
660	if (left > bconn.buf.len) {
661	// skip the buffer if the output is large enough
662	return bconn.conn.read(buffer[out_index..]);
663	}
664
665	try bconn.fill();
666	}
667
668	return out_index;
669	}
670
671	pub fn read(bconn: *MockBufferedConnection, buffer: []u8) ReadError!usize {
672	return bconn.readAtLeast(buffer, 1);
673	}
674
675	pub const ReadError = std.io.FixedBufferStream([]const u8).ReadError \|\| error{EndOfStream};
676	pub const Reader = std.io.Reader(*MockBufferedConnection, ReadError, read);
677
678	pub fn reader(bconn: *MockBufferedConnection) Reader {
679	return Reader{ .context = bconn };
680	}
681
682	pub fn writeAll(bconn: *MockBufferedConnection, buffer: []const u8) WriteError!void {
683	return bconn.conn.writeAll(buffer);
684	}
685
686	pub fn write(bconn: *MockBufferedConnection, buffer: []const u8) WriteError!usize {
687	return bconn.conn.write(buffer);
688	}
689
690	pub const WriteError = std.io.FixedBufferStream([]const u8).WriteError;
691	pub const Writer = std.io.Writer(*MockBufferedConnection, WriteError, write);
692
693	pub fn writer(bconn: *MockBufferedConnection) Writer {
694	return Writer{ .context = bconn };
695	}
696	};
697
698	test "HeadersParser.findHeadersEnd" {
699	var r: HeadersParser = undefined;
700	const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\nHello";
701
702	for (0..36) \|i\| {
703	r = HeadersParser.initDynamic(0);
704	try std.testing.expectEqual(@intCast(u32, i), r.findHeadersEnd(data[0..i]));
705	try std.testing.expectEqual(@intCast(u32, 35 - i), r.findHeadersEnd(data[i..]));
706	}
707	}
708
709	test "HeadersParser.findChunkedLen" {
710	var r: HeadersParser = undefined;
711	const data = "Ff\r\nf0f000 ; ext\n0\r\nffffffffffffffffffffffffffffffffffffffff\r\n";
712
713	r = HeadersParser.initDynamic(0);
714	r.state = .chunk_head_size;
715	r.next_chunk_length = 0;
716
717	const first = r.findChunkedLen(data[0..]);
718	try testing.expectEqual(@as(u32, 4), first);
719	try testing.expectEqual(@as(u64, 0xff), r.next_chunk_length);
720	try testing.expectEqual(State.chunk_data, r.state);
721	r.state = .chunk_head_size;
722	r.next_chunk_length = 0;
723
724	const second = r.findChunkedLen(data[first..]);
725	try testing.expectEqual(@as(u32, 13), second);
726	try testing.expectEqual(@as(u64, 0xf0f000), r.next_chunk_length);
727	try testing.expectEqual(State.chunk_data, r.state);
728	r.state = .chunk_head_size;
729	r.next_chunk_length = 0;
730
731	const third = r.findChunkedLen(data[first + second ..]);
732	try testing.expectEqual(@as(u32, 3), third);
733	try testing.expectEqual(@as(u64, 0), r.next_chunk_length);
734	try testing.expectEqual(State.chunk_data, r.state);
735	r.state = .chunk_head_size;
736	r.next_chunk_length = 0;
737
738	const fourth = r.findChunkedLen(data[first + second + third ..]);
739	try testing.expectEqual(@as(u32, 16), fourth);
740	try testing.expectEqual(@as(u64, 0xffffffffffffffff), r.next_chunk_length);
741	try testing.expectEqual(State.invalid, r.state);
742	}
743
744	test "HeadersParser.read length" {
745	// mock BufferedConnection for read
746
747	var r = HeadersParser.initDynamic(256);
748	defer r.header_bytes.deinit(std.testing.allocator);
749	const data = "GET / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\n\r\nHello";
750	var fbs = std.io.fixedBufferStream(data);
751
752	var bconn = MockBufferedConnection{
753	.conn = fbs,
754	};
755
756	while (true) { // read headers
757	try bconn.fill();
758
759	const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());
760	bconn.clear(@intCast(u16, nchecked));
761
762	if (r.state.isContent()) break;
763	}
764
765	var buf: [8]u8 = undefined;
766
767	r.next_chunk_length = 5;
768	const len = try r.read(&bconn, &buf, false);
769	try std.testing.expectEqual(@as(usize, 5), len);
770	try std.testing.expectEqualStrings("Hello", buf[0..len]);
771
772	try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\n\r\n", r.header_bytes.items);
773	}
774
775	test "HeadersParser.read chunked" {
776	// mock BufferedConnection for read
777
778	var r = HeadersParser.initDynamic(256);
779	defer r.header_bytes.deinit(std.testing.allocator);
780	const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n2\r\nHe\r\n2\r\nll\r\n1\r\no\r\n0\r\n\r\n";
781	var fbs = std.io.fixedBufferStream(data);
782
783	var bconn = MockBufferedConnection{
784	.conn = fbs,
785	};
786
787	while (true) { // read headers
788	try bconn.fill();
789
790	const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());
791	bconn.clear(@intCast(u16, nchecked));
792
793	if (r.state.isContent()) break;
794	}
795	var buf: [8]u8 = undefined;
796
797	r.state = .chunk_head_size;
798	const len = try r.read(&bconn, &buf, false);
799	try std.testing.expectEqual(@as(usize, 5), len);
800	try std.testing.expectEqualStrings("Hello", buf[0..len]);
801
802	try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n", r.header_bytes.items);
803	}
804
805	test "HeadersParser.read chunked trailer" {
806	// mock BufferedConnection for read
807
808	var r = HeadersParser.initDynamic(256);
809	defer r.header_bytes.deinit(std.testing.allocator);
810	const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n2\r\nHe\r\n2\r\nll\r\n1\r\no\r\n0\r\nContent-Type: text/plain\r\n\r\n";
811	var fbs = std.io.fixedBufferStream(data);
812
813	var bconn = MockBufferedConnection{
814	.conn = fbs,
815	};
816
817	while (true) { // read headers
818	try bconn.fill();
819
820	const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());
821	bconn.clear(@intCast(u16, nchecked));
822
823	if (r.state.isContent()) break;
824	}
825	var buf: [8]u8 = undefined;
826
827	r.state = .chunk_head_size;
828	const len = try r.read(&bconn, &buf, false);
829	try std.testing.expectEqual(@as(usize, 5), len);
830	try std.testing.expectEqualStrings("Hello", buf[0..len]);
831
832	while (true) { // read headers
833	try bconn.fill();
834
835	const nchecked = try r.checkCompleteHead(std.testing.allocator, bconn.peek());
836	bconn.clear(@intCast(u16, nchecked));
837
838	if (r.state.isContent()) break;
839	}
840
841	try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\n\r\nContent-Type: text/plain\r\n\r\n", r.header_bytes.items);
842	}

src/Package.zig+2

...	@@ -482,6 +482,8 @@ fn fetchAndUnpack(	...	@@ -482,6 +482,8 @@ fn fetchAndUnpack(
482	var req = try http_client.request(uri, .{}, .{});	482	var req = try http_client.request(uri, .{}, .{});
483	defer req.deinit();	483	defer req.deinit();
484		484
		485	try req.do();
		486
485	if (mem.endsWith(u8, uri.path, ".tar.gz")) {	487	if (mem.endsWith(u8, uri.path, ".tar.gz")) {
486	// I observed the gzip stream to read 1 byte at a time, so I am using a	488	// I observed the gzip stream to read 1 byte at a time, so I am using a
487	// buffered reader on the front of it.	489	// buffered reader on the front of it.