authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-15 21:52:18-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-16 00:16:15-07:00
loga0f9a5e78d1cf419e88c493f6f57e486df236e35
treeb91fc5370cf921491e50da075e0d3bc609da6a73
parent07b753f22ba87357dca4aec3e93fbca912ac2197

std: more reliable HTTP and TLS networking

* std.Io.Reader: fix confused semantics of rebase. Before it was ambiguous whether it was supposed to be based on end or seek. Now it is clearly based on seek, with an added assertion for clarity. * std.crypto.tls.Client: fix panic due to not enough buffer size available. Also, avoid unnecessary rebasing. * std.http.Reader: introduce max_head_len to limit HTTP header length. This prevents crash in underlying reader which may require a minimum buffer length. * std.http.Client: choose better buffer sizes for streams and TLS client. Crucially, the buffer shared by HTTP reader and TLS client needs to be big enough for all http headers *and* the max TLS record size. Bump HTTP header size default from 4K to 8K. fixes #24872 I have noticed however that there are still fetch problems

6 files changed, 49 insertions(+), 46 deletions(-)

lib/std/Io.zig+6-1
......@@ -717,7 +717,12 @@ pub fn Poller(comptime StreamEnum: type) type {
717717 const unused = r.buffer[r.end..];
718718 if (unused.len >= min_len) return unused;
719719 }
720 if (r.seek > 0) r.rebase(r.buffer.len) catch unreachable;
720 if (r.seek > 0) {
721 const data = r.buffer[r.seek..r.end];
722 @memmove(r.buffer[0..data.len], data);
723 r.seek = 0;
724 r.end = data.len;
725 }
721726 {
722727 var list: std.ArrayListUnmanaged(u8) = .{
723728 .items = r.buffer[0..r.end],
lib/std/Io/Reader.zig+7-24
......@@ -86,12 +86,12 @@ pub const VTable = struct {
8686 /// `Reader.buffer`, whichever is bigger.
8787 readVec: *const fn (r: *Reader, data: [][]u8) Error!usize = defaultReadVec,
8888
89 /// Ensures `capacity` more data can be buffered without rebasing.
89 /// Ensures `capacity` data can be buffered without rebasing.
9090 ///
9191 /// Asserts `capacity` is within buffer capacity, or that the stream ends
9292 /// within `capacity` bytes.
9393 ///
94 /// Only called when `capacity` cannot fit into the unused capacity of
94 /// Only called when `capacity` cannot be satisfied by unused capacity of
9595 /// `buffer`.
9696 ///
9797 /// The default implementation moves buffered data to the start of
......@@ -1037,7 +1037,7 @@ fn fillUnbuffered(r: *Reader, n: usize) Error!void {
10371037///
10381038/// Asserts buffer capacity is at least 1.
10391039pub fn fillMore(r: *Reader) Error!void {
1040 try rebase(r, 1);
1040 try rebase(r, r.end - r.seek + 1);
10411041 var bufs: [1][]u8 = .{""};
10421042 _ = try r.vtable.readVec(r, &bufs);
10431043}
......@@ -1205,24 +1205,6 @@ pub fn takeLeb128(r: *Reader, comptime Result: type) TakeLeb128Error!Result {
12051205 } }))) orelse error.Overflow;
12061206}
12071207
1208pub fn expandTotalCapacity(r: *Reader, allocator: Allocator, n: usize) Allocator.Error!void {
1209 if (n <= r.buffer.len) return;
1210 if (r.seek > 0) rebase(r, r.buffer.len);
1211 var list: ArrayList(u8) = .{
1212 .items = r.buffer[0..r.end],
1213 .capacity = r.buffer.len,
1214 };
1215 defer r.buffer = list.allocatedSlice();
1216 try list.ensureTotalCapacity(allocator, n);
1217}
1218
1219pub const FillAllocError = Error || Allocator.Error;
1220
1221pub fn fillAlloc(r: *Reader, allocator: Allocator, n: usize) FillAllocError!void {
1222 try expandTotalCapacity(r, allocator, n);
1223 return fill(r, n);
1224}
1225
12261208fn takeMultipleOf7Leb128(r: *Reader, comptime Result: type) TakeLeb128Error!Result {
12271209 const result_info = @typeInfo(Result).int;
12281210 comptime assert(result_info.bits % 7 == 0);
......@@ -1253,9 +1235,9 @@ fn takeMultipleOf7Leb128(r: *Reader, comptime Result: type) TakeLeb128Error!Resu
12531235 }
12541236}
12551237
1256/// Ensures `capacity` more data can be buffered without rebasing.
1238/// Ensures `capacity` data can be buffered without rebasing.
12571239pub fn rebase(r: *Reader, capacity: usize) RebaseError!void {
1258 if (r.end + capacity <= r.buffer.len) {
1240 if (r.buffer.len - r.seek >= capacity) {
12591241 @branchHint(.likely);
12601242 return;
12611243 }
......@@ -1263,11 +1245,12 @@ pub fn rebase(r: *Reader, capacity: usize) RebaseError!void {
12631245}
12641246
12651247pub fn defaultRebase(r: *Reader, capacity: usize) RebaseError!void {
1266 if (r.end <= r.buffer.len - capacity) return;
1248 assert(r.buffer.len - r.seek < capacity);
12671249 const data = r.buffer[r.seek..r.end];
12681250 @memmove(r.buffer[0..data.len], data);
12691251 r.seek = 0;
12701252 r.end = data.len;
1253 assert(r.buffer.len - r.seek >= capacity);
12711254}
12721255
12731256test fixed {
lib/std/crypto/tls/Client.zig+14-10
......@@ -183,7 +183,6 @@ const InitError = error{
183183/// `input` is asserted to have buffer capacity at least `min_buffer_len`.
184184pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client {
185185 assert(input.buffer.len >= min_buffer_len);
186 assert(output.buffer.len >= min_buffer_len);
187186 const host = switch (options.host) {
188187 .no_verification => "",
189188 .explicit => |host| host,
......@@ -1124,12 +1123,6 @@ fn readIndirect(c: *Client) Reader.Error!usize {
11241123 if (record_end > input.buffered().len) return 0;
11251124 }
11261125
1127 if (r.seek == r.end) {
1128 r.seek = 0;
1129 r.end = 0;
1130 }
1131 const cleartext_buffer = r.buffer[r.end..];
1132
11331126 const cleartext_len, const inner_ct: tls.ContentType = cleartext: switch (c.application_cipher) {
11341127 inline else => |*p| switch (c.tls_version) {
11351128 .tls_1_3 => {
......@@ -1145,7 +1138,8 @@ fn readIndirect(c: *Client) Reader.Error!usize {
11451138 const operand: V = pad ++ mem.toBytes(big(c.read_seq));
11461139 break :nonce @as(V, pv.server_iv) ^ operand;
11471140 };
1148 const cleartext = cleartext_buffer[0..ciphertext.len];
1141 rebase(r, ciphertext.len);
1142 const cleartext = r.buffer[r.end..][0..ciphertext.len];
11491143 P.AEAD.decrypt(cleartext, ciphertext, auth_tag, ad, nonce, pv.server_key) catch
11501144 return failRead(c, error.TlsBadRecordMac);
11511145 // TODO use scalar, non-slice version
......@@ -1171,7 +1165,8 @@ fn readIndirect(c: *Client) Reader.Error!usize {
11711165 };
11721166 const ciphertext = input.take(message_len) catch unreachable; // already peeked
11731167 const auth_tag = (input.takeArray(P.mac_length) catch unreachable).*; // already peeked
1174 const cleartext = cleartext_buffer[0..ciphertext.len];
1168 rebase(r, ciphertext.len);
1169 const cleartext = r.buffer[r.end..][0..ciphertext.len];
11751170 P.AEAD.decrypt(cleartext, ciphertext, auth_tag, ad, nonce, pv.server_write_key) catch
11761171 return failRead(c, error.TlsBadRecordMac);
11771172 break :cleartext .{ cleartext.len, ct };
......@@ -1179,7 +1174,7 @@ fn readIndirect(c: *Client) Reader.Error!usize {
11791174 else => unreachable,
11801175 },
11811176 };
1182 const cleartext = cleartext_buffer[0..cleartext_len];
1177 const cleartext = r.buffer[r.end..][0..cleartext_len];
11831178 c.read_seq = std.math.add(u64, c.read_seq, 1) catch return failRead(c, error.TlsSequenceOverflow);
11841179 switch (inner_ct) {
11851180 .alert => {
......@@ -1275,6 +1270,15 @@ fn readIndirect(c: *Client) Reader.Error!usize {
12751270 }
12761271}
12771272
1273fn rebase(r: *Reader, capacity: usize) void {
1274 if (r.buffer.len - r.end >= capacity) return;
1275 const data = r.buffer[r.seek..r.end];
1276 @memmove(r.buffer[0..data.len], data);
1277 r.seek = 0;
1278 r.end = data.len;
1279 assert(r.buffer.len - r.end >= capacity);
1280}
1281
12781282fn failRead(c: *Client, err: ReadError) error{ReadFailed} {
12791283 c.read_err = err;
12801284 return error.ReadFailed;
lib/std/http.zig+3-1
......@@ -329,6 +329,7 @@ pub const Reader = struct {
329329 /// read from `in`.
330330 trailers: []const u8 = &.{},
331331 body_err: ?BodyError = null,
332 max_head_len: usize,
332333
333334 pub const RemainingChunkLen = enum(u64) {
334335 head = 0,
......@@ -387,10 +388,11 @@ pub const Reader = struct {
387388 pub fn receiveHead(reader: *Reader) HeadError![]const u8 {
388389 reader.trailers = &.{};
389390 const in = reader.in;
391 const max_head_len = reader.max_head_len;
390392 var hp: HeadParser = .{};
391393 var head_len: usize = 0;
392394 while (true) {
393 if (in.buffer.len - head_len == 0) return error.HttpHeadersOversize;
395 if (head_len >= max_head_len) return error.HttpHeadersOversize;
394396 const remaining = in.buffered()[head_len..];
395397 if (remaining.len == 0) {
396398 in.fillMore() catch |err| switch (err) {
lib/std/http/Client.zig+17-10
......@@ -42,7 +42,7 @@ connection_pool: ConnectionPool = .{},
4242///
4343/// If the entire HTTP header cannot fit in this amount of bytes,
4444/// `error.HttpHeadersOversize` will be returned from `Request.wait`.
45read_buffer_size: usize = 4096 + if (disable_tls) 0 else std.crypto.tls.Client.min_buffer_len,
45read_buffer_size: usize = 8192,
4646/// Each `Connection` allocates this amount for the writer buffer.
4747write_buffer_size: usize = 1024,
4848
......@@ -302,18 +302,22 @@ pub const Connection = struct {
302302 const base = try gpa.alignedAlloc(u8, .of(Tls), alloc_len);
303303 errdefer gpa.free(base);
304304 const host_buffer = base[@sizeOf(Tls)..][0..remote_host.len];
305 const tls_read_buffer = host_buffer.ptr[host_buffer.len..][0..client.tls_buffer_size];
305 // The TLS client wants enough buffer for the max encrypted frame
306 // size, and the HTTP body reader wants enough buffer for the
307 // entire HTTP header. This means we need a combined upper bound.
308 const tls_read_buffer_len = client.tls_buffer_size + client.read_buffer_size;
309 const tls_read_buffer = host_buffer.ptr[host_buffer.len..][0..tls_read_buffer_len];
306310 const tls_write_buffer = tls_read_buffer.ptr[tls_read_buffer.len..][0..client.tls_buffer_size];
307 const write_buffer = tls_write_buffer.ptr[tls_write_buffer.len..][0..client.write_buffer_size];
308 const read_buffer = write_buffer.ptr[write_buffer.len..][0..client.read_buffer_size];
309 assert(base.ptr + alloc_len == read_buffer.ptr + read_buffer.len);
311 const socket_write_buffer = tls_write_buffer.ptr[tls_write_buffer.len..][0..client.write_buffer_size];
312 const socket_read_buffer = socket_write_buffer.ptr[socket_write_buffer.len..][0..client.tls_buffer_size];
313 assert(base.ptr + alloc_len == socket_read_buffer.ptr + socket_read_buffer.len);
310314 @memcpy(host_buffer, remote_host);
311315 const tls: *Tls = @ptrCast(base);
312316 tls.* = .{
313317 .connection = .{
314318 .client = client,
315319 .stream_writer = stream.writer(tls_write_buffer),
316 .stream_reader = stream.reader(tls_read_buffer),
320 .stream_reader = stream.reader(socket_read_buffer),
317321 .pool_node = .{},
318322 .port = port,
319323 .host_len = @intCast(remote_host.len),
......@@ -329,8 +333,8 @@ pub const Connection = struct {
329333 .host = .{ .explicit = remote_host },
330334 .ca = .{ .bundle = client.ca_bundle },
331335 .ssl_key_log = client.ssl_key_log,
332 .read_buffer = read_buffer,
333 .write_buffer = write_buffer,
336 .read_buffer = tls_read_buffer,
337 .write_buffer = socket_write_buffer,
334338 // This is appropriate for HTTPS because the HTTP headers contain
335339 // the content length which is used to detect truncation attacks.
336340 .allow_truncation_attacks = true,
......@@ -348,8 +352,9 @@ pub const Connection = struct {
348352 }
349353
350354 fn allocLen(client: *Client, host_len: usize) usize {
351 return @sizeOf(Tls) + host_len + client.tls_buffer_size + client.tls_buffer_size +
352 client.write_buffer_size + client.read_buffer_size;
355 const tls_read_buffer_len = client.tls_buffer_size + client.read_buffer_size;
356 return @sizeOf(Tls) + host_len + tls_read_buffer_len + client.tls_buffer_size +
357 client.write_buffer_size + client.tls_buffer_size;
353358 }
354359
355360 fn host(tls: *Tls) []u8 {
......@@ -1214,6 +1219,7 @@ pub const Request = struct {
12141219 .state = .ready,
12151220 // Populated when `http.Reader.bodyReader` is called.
12161221 .interface = undefined,
1222 .max_head_len = r.client.read_buffer_size,
12171223 };
12181224 r.redirect_behavior.subtractOne();
12191225 }
......@@ -1679,6 +1685,7 @@ pub fn request(
16791685 .state = .ready,
16801686 // Populated when `http.Reader.bodyReader` is called.
16811687 .interface = undefined,
1688 .max_head_len = client.read_buffer_size,
16821689 },
16831690 .keep_alive = options.keep_alive,
16841691 .method = method,
lib/std/http/Server.zig+2
......@@ -29,6 +29,7 @@ pub fn init(in: *Reader, out: *Writer) Server {
2929 .state = .ready,
3030 // Populated when `http.Reader.bodyReader` is called.
3131 .interface = undefined,
32 .max_head_len = in.buffer.len,
3233 },
3334 .out = out,
3435 };
......@@ -251,6 +252,7 @@ pub const Request = struct {
251252 .in = undefined,
252253 .state = .received_head,
253254 .interface = undefined,
255 .max_head_len = 4096,
254256 },
255257 .out = undefined,
256258 };