authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-16 14:47:52-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-08-16 14:47:52-07:00
log399bace2f20e64e4c10c014dc3b8e202a891c6e4
treea24ab1bbc42ca4d201bb43acd91f8e9815011fc4
parentef14c732455dc089b56aab7392584c4fa8bc2c2d
parenta0f9a5e78d1cf419e88c493f6f57e486df236e35
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #24874 from ziglang/tls-client

std: more reliable HTTP and TLS networking

7 files changed, 52 insertions(+), 48 deletions(-)

lib/std/Io.zig+6-1
......@@ -717,7 +717,12 @@ pub fn Poller(comptime StreamEnum: type) type {
717717 const unused = r.buffer[r.end..];
718718 if (unused.len >= min_len) return unused;
719719 }
720 if (r.seek > 0) r.rebase(r.buffer.len) catch unreachable;
720 if (r.seek > 0) {
721 const data = r.buffer[r.seek..r.end];
722 @memmove(r.buffer[0..data.len], data);
723 r.seek = 0;
724 r.end = data.len;
725 }
721726 {
722727 var list: std.ArrayListUnmanaged(u8) = .{
723728 .items = r.buffer[0..r.end],
lib/std/Io/Reader.zig+7-24
......@@ -86,12 +86,12 @@ pub const VTable = struct {
8686 /// `Reader.buffer`, whichever is bigger.
8787 readVec: *const fn (r: *Reader, data: [][]u8) Error!usize = defaultReadVec,
8888
89 /// Ensures `capacity` more data can be buffered without rebasing.
89 /// Ensures `capacity` data can be buffered without rebasing.
9090 ///
9191 /// Asserts `capacity` is within buffer capacity, or that the stream ends
9292 /// within `capacity` bytes.
9393 ///
94 /// Only called when `capacity` cannot fit into the unused capacity of
94 /// Only called when `capacity` cannot be satisfied by unused capacity of
9595 /// `buffer`.
9696 ///
9797 /// The default implementation moves buffered data to the start of
......@@ -1035,7 +1035,7 @@ fn fillUnbuffered(r: *Reader, n: usize) Error!void {
10351035///
10361036/// Asserts buffer capacity is at least 1.
10371037pub fn fillMore(r: *Reader) Error!void {
1038 try rebase(r, 1);
1038 try rebase(r, r.end - r.seek + 1);
10391039 var bufs: [1][]u8 = .{""};
10401040 _ = try r.vtable.readVec(r, &bufs);
10411041}
......@@ -1203,24 +1203,6 @@ pub fn takeLeb128(r: *Reader, comptime Result: type) TakeLeb128Error!Result {
12031203 } }))) orelse error.Overflow;
12041204}
12051205
1206pub fn expandTotalCapacity(r: *Reader, allocator: Allocator, n: usize) Allocator.Error!void {
1207 if (n <= r.buffer.len) return;
1208 if (r.seek > 0) rebase(r, r.buffer.len);
1209 var list: ArrayList(u8) = .{
1210 .items = r.buffer[0..r.end],
1211 .capacity = r.buffer.len,
1212 };
1213 defer r.buffer = list.allocatedSlice();
1214 try list.ensureTotalCapacity(allocator, n);
1215}
1216
1217pub const FillAllocError = Error || Allocator.Error;
1218
1219pub fn fillAlloc(r: *Reader, allocator: Allocator, n: usize) FillAllocError!void {
1220 try expandTotalCapacity(r, allocator, n);
1221 return fill(r, n);
1222}
1223
12241206fn takeMultipleOf7Leb128(r: *Reader, comptime Result: type) TakeLeb128Error!Result {
12251207 const result_info = @typeInfo(Result).int;
12261208 comptime assert(result_info.bits % 7 == 0);
......@@ -1251,9 +1233,9 @@ fn takeMultipleOf7Leb128(r: *Reader, comptime Result: type) TakeLeb128Error!Resu
12511233 }
12521234}
12531235
1254/// Ensures `capacity` more data can be buffered without rebasing.
1236/// Ensures `capacity` data can be buffered without rebasing.
12551237pub fn rebase(r: *Reader, capacity: usize) RebaseError!void {
1256 if (r.end + capacity <= r.buffer.len) {
1238 if (r.buffer.len - r.seek >= capacity) {
12571239 @branchHint(.likely);
12581240 return;
12591241 }
......@@ -1261,11 +1243,12 @@ pub fn rebase(r: *Reader, capacity: usize) RebaseError!void {
12611243}
12621244
12631245pub fn defaultRebase(r: *Reader, capacity: usize) RebaseError!void {
1264 if (r.end <= r.buffer.len - capacity) return;
1246 assert(r.buffer.len - r.seek < capacity);
12651247 const data = r.buffer[r.seek..r.end];
12661248 @memmove(r.buffer[0..data.len], data);
12671249 r.seek = 0;
12681250 r.end = data.len;
1251 assert(r.buffer.len - r.seek >= capacity);
12691252}
12701253
12711254test fixed {
lib/std/crypto/tls/Client.zig+14-10
......@@ -183,7 +183,6 @@ const InitError = error{
183183/// `input` is asserted to have buffer capacity at least `min_buffer_len`.
184184pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client {
185185 assert(input.buffer.len >= min_buffer_len);
186 assert(output.buffer.len >= min_buffer_len);
187186 const host = switch (options.host) {
188187 .no_verification => "",
189188 .explicit => |host| host,
......@@ -1124,12 +1123,6 @@ fn readIndirect(c: *Client) Reader.Error!usize {
11241123 if (record_end > input.buffered().len) return 0;
11251124 }
11261125
1127 if (r.seek == r.end) {
1128 r.seek = 0;
1129 r.end = 0;
1130 }
1131 const cleartext_buffer = r.buffer[r.end..];
1132
11331126 const cleartext_len, const inner_ct: tls.ContentType = cleartext: switch (c.application_cipher) {
11341127 inline else => |*p| switch (c.tls_version) {
11351128 .tls_1_3 => {
......@@ -1145,7 +1138,8 @@ fn readIndirect(c: *Client) Reader.Error!usize {
11451138 const operand: V = pad ++ mem.toBytes(big(c.read_seq));
11461139 break :nonce @as(V, pv.server_iv) ^ operand;
11471140 };
1148 const cleartext = cleartext_buffer[0..ciphertext.len];
1141 rebase(r, ciphertext.len);
1142 const cleartext = r.buffer[r.end..][0..ciphertext.len];
11491143 P.AEAD.decrypt(cleartext, ciphertext, auth_tag, ad, nonce, pv.server_key) catch
11501144 return failRead(c, error.TlsBadRecordMac);
11511145 // TODO use scalar, non-slice version
......@@ -1171,7 +1165,8 @@ fn readIndirect(c: *Client) Reader.Error!usize {
11711165 };
11721166 const ciphertext = input.take(message_len) catch unreachable; // already peeked
11731167 const auth_tag = (input.takeArray(P.mac_length) catch unreachable).*; // already peeked
1174 const cleartext = cleartext_buffer[0..ciphertext.len];
1168 rebase(r, ciphertext.len);
1169 const cleartext = r.buffer[r.end..][0..ciphertext.len];
11751170 P.AEAD.decrypt(cleartext, ciphertext, auth_tag, ad, nonce, pv.server_write_key) catch
11761171 return failRead(c, error.TlsBadRecordMac);
11771172 break :cleartext .{ cleartext.len, ct };
......@@ -1179,7 +1174,7 @@ fn readIndirect(c: *Client) Reader.Error!usize {
11791174 else => unreachable,
11801175 },
11811176 };
1182 const cleartext = cleartext_buffer[0..cleartext_len];
1177 const cleartext = r.buffer[r.end..][0..cleartext_len];
11831178 c.read_seq = std.math.add(u64, c.read_seq, 1) catch return failRead(c, error.TlsSequenceOverflow);
11841179 switch (inner_ct) {
11851180 .alert => {
......@@ -1275,6 +1270,15 @@ fn readIndirect(c: *Client) Reader.Error!usize {
12751270 }
12761271}
12771272
1273fn rebase(r: *Reader, capacity: usize) void {
1274 if (r.buffer.len - r.end >= capacity) return;
1275 const data = r.buffer[r.seek..r.end];
1276 @memmove(r.buffer[0..data.len], data);
1277 r.seek = 0;
1278 r.end = data.len;
1279 assert(r.buffer.len - r.end >= capacity);
1280}
1281
12781282fn failRead(c: *Client, err: ReadError) error{ReadFailed} {
12791283 c.read_err = err;
12801284 return error.ReadFailed;
lib/std/http.zig+3-1
......@@ -329,6 +329,7 @@ pub const Reader = struct {
329329 /// read from `in`.
330330 trailers: []const u8 = &.{},
331331 body_err: ?BodyError = null,
332 max_head_len: usize,
332333
333334 pub const RemainingChunkLen = enum(u64) {
334335 head = 0,
......@@ -387,10 +388,11 @@ pub const Reader = struct {
387388 pub fn receiveHead(reader: *Reader) HeadError![]const u8 {
388389 reader.trailers = &.{};
389390 const in = reader.in;
391 const max_head_len = reader.max_head_len;
390392 var hp: HeadParser = .{};
391393 var head_len: usize = 0;
392394 while (true) {
393 if (in.buffer.len - head_len == 0) return error.HttpHeadersOversize;
395 if (head_len >= max_head_len) return error.HttpHeadersOversize;
394396 const remaining = in.buffered()[head_len..];
395397 if (remaining.len == 0) {
396398 in.fillMore() catch |err| switch (err) {
lib/std/http/Client.zig+17-10
......@@ -42,7 +42,7 @@ connection_pool: ConnectionPool = .{},
4242///
4343/// If the entire HTTP header cannot fit in this amount of bytes,
4444/// `error.HttpHeadersOversize` will be returned from `Request.wait`.
45read_buffer_size: usize = 4096 + if (disable_tls) 0 else std.crypto.tls.Client.min_buffer_len,
45read_buffer_size: usize = 8192,
4646/// Each `Connection` allocates this amount for the writer buffer.
4747write_buffer_size: usize = 1024,
4848
......@@ -302,18 +302,22 @@ pub const Connection = struct {
302302 const base = try gpa.alignedAlloc(u8, .of(Tls), alloc_len);
303303 errdefer gpa.free(base);
304304 const host_buffer = base[@sizeOf(Tls)..][0..remote_host.len];
305 const tls_read_buffer = host_buffer.ptr[host_buffer.len..][0..client.tls_buffer_size];
305 // The TLS client wants enough buffer for the max encrypted frame
306 // size, and the HTTP body reader wants enough buffer for the
307 // entire HTTP header. This means we need a combined upper bound.
308 const tls_read_buffer_len = client.tls_buffer_size + client.read_buffer_size;
309 const tls_read_buffer = host_buffer.ptr[host_buffer.len..][0..tls_read_buffer_len];
306310 const tls_write_buffer = tls_read_buffer.ptr[tls_read_buffer.len..][0..client.tls_buffer_size];
307 const write_buffer = tls_write_buffer.ptr[tls_write_buffer.len..][0..client.write_buffer_size];
308 const read_buffer = write_buffer.ptr[write_buffer.len..][0..client.read_buffer_size];
309 assert(base.ptr + alloc_len == read_buffer.ptr + read_buffer.len);
311 const socket_write_buffer = tls_write_buffer.ptr[tls_write_buffer.len..][0..client.write_buffer_size];
312 const socket_read_buffer = socket_write_buffer.ptr[socket_write_buffer.len..][0..client.tls_buffer_size];
313 assert(base.ptr + alloc_len == socket_read_buffer.ptr + socket_read_buffer.len);
310314 @memcpy(host_buffer, remote_host);
311315 const tls: *Tls = @ptrCast(base);
312316 tls.* = .{
313317 .connection = .{
314318 .client = client,
315319 .stream_writer = stream.writer(tls_write_buffer),
316 .stream_reader = stream.reader(tls_read_buffer),
320 .stream_reader = stream.reader(socket_read_buffer),
317321 .pool_node = .{},
318322 .port = port,
319323 .host_len = @intCast(remote_host.len),
......@@ -329,8 +333,8 @@ pub const Connection = struct {
329333 .host = .{ .explicit = remote_host },
330334 .ca = .{ .bundle = client.ca_bundle },
331335 .ssl_key_log = client.ssl_key_log,
332 .read_buffer = read_buffer,
333 .write_buffer = write_buffer,
336 .read_buffer = tls_read_buffer,
337 .write_buffer = socket_write_buffer,
334338 // This is appropriate for HTTPS because the HTTP headers contain
335339 // the content length which is used to detect truncation attacks.
336340 .allow_truncation_attacks = true,
......@@ -348,8 +352,9 @@ pub const Connection = struct {
348352 }
349353
350354 fn allocLen(client: *Client, host_len: usize) usize {
351 return @sizeOf(Tls) + host_len + client.tls_buffer_size + client.tls_buffer_size +
352 client.write_buffer_size + client.read_buffer_size;
355 const tls_read_buffer_len = client.tls_buffer_size + client.read_buffer_size;
356 return @sizeOf(Tls) + host_len + tls_read_buffer_len + client.tls_buffer_size +
357 client.write_buffer_size + client.tls_buffer_size;
353358 }
354359
355360 fn host(tls: *Tls) []u8 {
......@@ -1214,6 +1219,7 @@ pub const Request = struct {
12141219 .state = .ready,
12151220 // Populated when `http.Reader.bodyReader` is called.
12161221 .interface = undefined,
1222 .max_head_len = r.client.read_buffer_size,
12171223 };
12181224 r.redirect_behavior.subtractOne();
12191225 }
......@@ -1679,6 +1685,7 @@ pub fn request(
16791685 .state = .ready,
16801686 // Populated when `http.Reader.bodyReader` is called.
16811687 .interface = undefined,
1688 .max_head_len = client.read_buffer_size,
16821689 },
16831690 .keep_alive = options.keep_alive,
16841691 .method = method,
lib/std/http/Server.zig+2
......@@ -29,6 +29,7 @@ pub fn init(in: *Reader, out: *Writer) Server {
2929 .state = .ready,
3030 // Populated when `http.Reader.bodyReader` is called.
3131 .interface = undefined,
32 .max_head_len = in.buffer.len,
3233 },
3334 .out = out,
3435 };
......@@ -251,6 +252,7 @@ pub const Request = struct {
251252 .in = undefined,
252253 .state = .received_head,
253254 .interface = undefined,
255 .max_head_len = 4096,
254256 },
255257 .out = undefined,
256258 };
src/Package/Fetch.zig+3-2
......@@ -1212,10 +1212,11 @@ fn unpackResource(
12121212 return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
12131213 },
12141214 .@"tar.zst" => {
1215 const window_size = std.compress.zstd.default_window_len;
1216 const window_buffer = try f.arena.allocator().create([window_size]u8);
1215 const window_len = std.compress.zstd.default_window_len;
1216 const window_buffer = try f.arena.allocator().alloc(u8, window_len + std.compress.zstd.block_size_max);
12171217 var decompress: std.compress.zstd.Decompress = .init(resource.reader(), window_buffer, .{
12181218 .verify_checksum = false,
1219 .window_len = window_len,
12191220 });
12201221 return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
12211222 },