authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-11-05 01:37:12-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-11-07 20:25:26-05:00
logde53e6e4f2dc7a41dc50b309fee87e06475e4838
tree49441fde1d68d548e15639b68fc7a5be7cc2663d
parentd86a8aedd5674819ec4af1bfc8a81b3fef91fd85

std.crypto.tls: improve debuggability of encrypted connections

By default, programs built in debug mode that open a https connection will append secrets to the file specified in the SSLKEYLOGFILE environment variable to allow protocol debugging by external programs.

3 files changed, 174 insertions(+), 31 deletions(-)

lib/std/crypto/tls/Client.zig+147-24
......@@ -33,7 +33,7 @@ received_close_notify: bool,
3333/// This makes the application vulnerable to truncation attacks unless the
3434/// application layer itself verifies that the amount of data received equals
3535/// the amount of data expected, such as HTTP with the Content-Length header.
36allow_truncation_attacks: bool = false,
36allow_truncation_attacks: bool,
3737application_cipher: tls.ApplicationCipher,
3838/// The size is enough to contain exactly one TLSCiphertext record.
3939/// This buffer is segmented into four parts:
......@@ -44,6 +44,24 @@ application_cipher: tls.ApplicationCipher,
4444/// The fields `partial_cleartext_idx`, `partial_ciphertext_idx`, and
4545/// `partial_ciphertext_end` describe the span of the segments.
4646partially_read_buffer: [tls.max_ciphertext_record_len]u8,
47/// If non-null, ssl secrets are logged to a file. Creating such a log file allows other
48/// programs with access to that file to decrypt all traffic over this connection.
49ssl_key_log: ?struct {
50 client_key_seq: u64,
51 server_key_seq: u64,
52 client_random: [32]u8,
53 file: std.fs.File,
54
55 fn clientCounter(key_log: *@This()) u64 {
56 defer key_log.client_key_seq += 1;
57 return key_log.client_key_seq;
58 }
59
60 fn serverCounter(key_log: *@This()) u64 {
61 defer key_log.server_key_seq += 1;
62 return key_log.server_key_seq;
63 }
64},
4765
4866/// This is an example of the type that is needed by the read and write
4967/// functions. It can have any fields but it must at least have these
......@@ -88,6 +106,32 @@ pub const StreamInterface = struct {
88106 }
89107};
90108
109pub const Options = struct {
110 /// How to perform host verification of server certificates.
111 host: union(enum) {
112 /// No host verification is performed, which prevents a trusted connection from
113 /// being established.
114 no_verification,
115 /// Verify that the server certificate was issues for a given host.
116 explicit: []const u8,
117 },
118 /// How to verify the authenticity of server certificates.
119 ca: union(enum) {
120 /// No ca verification is performed, which prevents a trusted connection from
121 /// being established.
122 no_verification,
123 /// Verify that the server certificate is a valid self-signed certificate.
124 /// This provides no authorization guarantees, as anyone can create a
125 /// self-signed certificate.
126 self_signed,
127 /// Verify that the server certificate is authorized by a given ca bundle.
128 bundle: Certificate.Bundle,
129 },
130 /// If non-null, ssl secrets are logged to this file. Creating such a log file allows
131 /// other programs with access to that file to decrypt all traffic over this connection.
132 ssl_key_log_file: ?std.fs.File = null,
133};
134
91135pub fn InitError(comptime Stream: type) type {
92136 return std.mem.Allocator.Error || Stream.WriteError || Stream.ReadError || tls.AlertDescription.Error || error{
93137 InsufficientEntropy,
......@@ -140,12 +184,17 @@ pub fn InitError(comptime Stream: type) type {
140184/// must conform to `StreamInterface`.
141185///
142186/// `host` is only borrowed during this function call.
143pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) InitError(@TypeOf(stream))!Client {
187pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client {
188 const host = switch (options.host) {
189 .no_verification => "",
190 .explicit => |host| host,
191 };
144192 const host_len: u16 = @intCast(host.len);
145193
146194 var random_buffer: [128]u8 = undefined;
147195 crypto.random.bytes(&random_buffer);
148196 const client_hello_rand = random_buffer[0..32].*;
197 var key_seq: u64 = 0;
149198 var server_hello_rand: [32]u8 = undefined;
150199 const legacy_session_id = random_buffer[32..64].*;
151200
......@@ -179,15 +228,21 @@ pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) In
179228 array(u16, u8, key_share.secp256r1_kp.public_key.toUncompressedSec1()) ++
180229 int(u16, @intFromEnum(tls.NamedGroup.x25519)) ++
181230 array(u16, u8, key_share.x25519_kp.public_key),
182 )) ++ int(u16, @intFromEnum(tls.ExtensionType.server_name)) ++
231 ));
232 const server_name_extension = int(u16, @intFromEnum(tls.ExtensionType.server_name)) ++
183233 int(u16, 2 + 1 + 2 + host_len) ++ // byte length of this extension payload
184234 int(u16, 1 + 2 + host_len) ++ // server_name_list byte count
185235 .{0x00} ++ // name_type
186236 int(u16, host_len);
237 const server_name_extension_len = switch (options.host) {
238 .no_verification => 0,
239 .explicit => server_name_extension.len + host_len,
240 };
187241
188242 const extensions_header =
189 int(u16, @intCast(extensions_payload.len + host_len)) ++
190 extensions_payload;
243 int(u16, @intCast(extensions_payload.len + server_name_extension_len)) ++
244 extensions_payload ++
245 server_name_extension;
191246
192247 const client_hello =
193248 int(u16, @intFromEnum(tls.ProtocolVersion.tls_1_2)) ++
......@@ -198,20 +253,24 @@ pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) In
198253 extensions_header;
199254
200255 const out_handshake = .{@intFromEnum(tls.HandshakeType.client_hello)} ++
201 int(u24, @intCast(client_hello.len + host_len)) ++
256 int(u24, @intCast(client_hello.len - server_name_extension.len + server_name_extension_len)) ++
202257 client_hello;
203258
204 const cleartext_header = .{@intFromEnum(tls.ContentType.handshake)} ++
259 const cleartext_header_buf = .{@intFromEnum(tls.ContentType.handshake)} ++
205260 int(u16, @intFromEnum(tls.ProtocolVersion.tls_1_0)) ++
206 int(u16, @intCast(out_handshake.len + host_len)) ++
261 int(u16, @intCast(out_handshake.len - server_name_extension.len + server_name_extension_len)) ++
207262 out_handshake;
263 const cleartext_header = switch (options.host) {
264 .no_verification => cleartext_header_buf[0 .. cleartext_header_buf.len - server_name_extension.len],
265 .explicit => &cleartext_header_buf,
266 };
208267
209268 {
210269 var iovecs = [_]std.posix.iovec_const{
211 .{ .base = &cleartext_header, .len = cleartext_header.len },
270 .{ .base = cleartext_header.ptr, .len = cleartext_header.len },
212271 .{ .base = host.ptr, .len = host.len },
213272 };
214 try stream.writevAll(&iovecs);
273 try stream.writevAll(iovecs[0..if (host.len == 0) 1 else 2]);
215274 }
216275
217276 var tls_version: tls.ProtocolVersion = undefined;
......@@ -472,6 +531,12 @@ pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) In
472531 pv.master_secret = P.Hkdf.extract(&ap_derived_secret, &zeroes);
473532 const client_secret = hkdfExpandLabel(P.Hkdf, pv.handshake_secret, "c hs traffic", &hello_hash, P.Hash.digest_length);
474533 const server_secret = hkdfExpandLabel(P.Hkdf, pv.handshake_secret, "s hs traffic", &hello_hash, P.Hash.digest_length);
534 if (options.ssl_key_log_file) |key_log_file| logSecrets(key_log_file, .{
535 .client_random = &client_hello_rand,
536 }, .{
537 .SERVER_HANDSHAKE_TRAFFIC_SECRET = &server_secret,
538 .CLIENT_HANDSHAKE_TRAFFIC_SECRET = &client_secret,
539 });
475540 pv.client_finished_key = hkdfExpandLabel(P.Hkdf, client_secret, "finished", "", P.Hmac.key_length);
476541 pv.server_finished_key = hkdfExpandLabel(P.Hkdf, server_secret, "finished", "", P.Hmac.key_length);
477542 pv.client_handshake_key = hkdfExpandLabel(P.Hkdf, client_secret, "key", "", P.AEAD.key_length);
......@@ -544,6 +609,13 @@ pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) In
544609 const cert_size = certs_decoder.decode(u24);
545610 const certd = try certs_decoder.sub(cert_size);
546611
612 if (tls_version == .tls_1_3) {
613 try certs_decoder.ensure(2);
614 const total_ext_size = certs_decoder.decode(u16);
615 const all_extd = try certs_decoder.sub(total_ext_size);
616 _ = all_extd;
617 }
618
547619 const subject_cert: Certificate = .{
548620 .buffer = certd.buf,
549621 .index = @intCast(certd.idx),
......@@ -551,7 +623,10 @@ pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) In
551623 const subject = try subject_cert.parse();
552624 if (cert_index == 0) {
553625 // Verify the host on the first certificate.
554 try subject.verifyHostName(host);
626 switch (options.host) {
627 .no_verification => {},
628 .explicit => try subject.verifyHostName(host),
629 }
555630
556631 // Keep track of the public key for the
557632 // certificate_verify message later.
......@@ -560,23 +635,27 @@ pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) In
560635 try prev_cert.verify(subject, now_sec);
561636 }
562637
563 if (ca_bundle.verify(subject, now_sec)) |_| {
564 handshake_state = .trust_chain_established;
565 break :cert;
566 } else |err| switch (err) {
567 error.CertificateIssuerNotFound => {},
568 else => |e| return e,
638 switch (options.ca) {
639 .no_verification => {
640 handshake_state = .trust_chain_established;
641 break :cert;
642 },
643 .self_signed => {
644 try subject.verify(subject, now_sec);
645 handshake_state = .trust_chain_established;
646 break :cert;
647 },
648 .bundle => |ca_bundle| if (ca_bundle.verify(subject, now_sec)) |_| {
649 handshake_state = .trust_chain_established;
650 break :cert;
651 } else |err| switch (err) {
652 error.CertificateIssuerNotFound => {},
653 else => |e| return e,
654 },
569655 }
570656
571657 prev_cert = subject;
572658 cert_index += 1;
573
574 if (tls_version == .tls_1_3) {
575 try certs_decoder.ensure(2);
576 const total_ext_size = certs_decoder.decode(u16);
577 const all_extd = try certs_decoder.sub(total_ext_size);
578 _ = all_extd;
579 }
580659 }
581660 },
582661 .server_key_exchange => {
......@@ -625,6 +704,11 @@ pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) In
625704 &client_hello_rand,
626705 &server_hello_rand,
627706 }, 48);
707 if (options.ssl_key_log_file) |key_log_file| logSecrets(key_log_file, .{
708 .client_random = &client_hello_rand,
709 }, .{
710 .CLIENT_RANDOM = &master_secret,
711 });
628712 const key_block = hmacExpandLabel(
629713 P.Hmac,
630714 &master_secret,
......@@ -748,6 +832,14 @@ pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) In
748832
749833 const client_secret = hkdfExpandLabel(P.Hkdf, pv.master_secret, "c ap traffic", &handshake_hash, P.Hash.digest_length);
750834 const server_secret = hkdfExpandLabel(P.Hkdf, pv.master_secret, "s ap traffic", &handshake_hash, P.Hash.digest_length);
835 if (options.ssl_key_log_file) |key_log_file| logSecrets(key_log_file, .{
836 .counter = key_seq,
837 .client_random = &client_hello_rand,
838 }, .{
839 .SERVER_TRAFFIC_SECRET = &server_secret,
840 .CLIENT_TRAFFIC_SECRET = &client_secret,
841 });
842 key_seq += 1;
751843 break :app_cipher @unionInit(tls.ApplicationCipher, @tagName(tag), .{ .tls_1_3 = .{
752844 .client_secret = client_secret,
753845 .server_secret = server_secret,
......@@ -784,8 +876,15 @@ pub fn init(stream: anytype, ca_bundle: Certificate.Bundle, host: []const u8) In
784876 .partial_ciphertext_idx = 0,
785877 .partial_ciphertext_end = @intCast(leftover.len),
786878 .received_close_notify = false,
879 .allow_truncation_attacks = false,
787880 .application_cipher = app_cipher,
788881 .partially_read_buffer = undefined,
882 .ssl_key_log = if (options.ssl_key_log_file) |key_log_file| .{
883 .client_key_seq = key_seq,
884 .server_key_seq = key_seq,
885 .client_random = client_hello_rand,
886 .file = key_log_file,
887 } else null,
789888 };
790889 @memcpy(client.partially_read_buffer[0..leftover.len], leftover);
791890 return client;
......@@ -1358,6 +1457,12 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove
13581457 const pv = &p.tls_1_3;
13591458 const P = @TypeOf(p.*);
13601459 const server_secret = hkdfExpandLabel(P.Hkdf, pv.server_secret, "traffic upd", "", P.Hash.digest_length);
1460 if (c.ssl_key_log) |*key_log| logSecrets(key_log.file, .{
1461 .counter = key_log.serverCounter(),
1462 .client_random = &key_log.client_random,
1463 }, .{
1464 .SERVER_TRAFFIC_SECRET = &server_secret,
1465 });
13611466 pv.server_secret = server_secret;
13621467 pv.server_key = hkdfExpandLabel(P.Hkdf, server_secret, "key", "", P.AEAD.key_length);
13631468 pv.server_iv = hkdfExpandLabel(P.Hkdf, server_secret, "iv", "", P.AEAD.nonce_length);
......@@ -1372,6 +1477,12 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove
13721477 const pv = &p.tls_1_3;
13731478 const P = @TypeOf(p.*);
13741479 const client_secret = hkdfExpandLabel(P.Hkdf, pv.client_secret, "traffic upd", "", P.Hash.digest_length);
1480 if (c.ssl_key_log) |*key_log| logSecrets(key_log.file, .{
1481 .counter = key_log.clientCounter(),
1482 .client_random = &key_log.client_random,
1483 }, .{
1484 .CLIENT_TRAFFIC_SECRET = &client_secret,
1485 });
13751486 pv.client_secret = client_secret;
13761487 pv.client_key = hkdfExpandLabel(P.Hkdf, client_secret, "key", "", P.AEAD.key_length);
13771488 pv.client_iv = hkdfExpandLabel(P.Hkdf, client_secret, "iv", "", P.AEAD.nonce_length);
......@@ -1426,6 +1537,18 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove
14261537 }
14271538}
14281539
1540fn logSecrets(key_log_file: std.fs.File, context: anytype, secrets: anytype) void {
1541 const locked = if (key_log_file.lock(.exclusive)) |_| true else |_| false;
1542 defer if (locked) key_log_file.unlock();
1543 key_log_file.seekFromEnd(0) catch {};
1544 inline for (@typeInfo(@TypeOf(secrets)).@"struct".fields) |field| key_log_file.writer().print("{s}" ++
1545 (if (@hasField(@TypeOf(context), "counter")) "_{d}" else "") ++ " {} {}\n", .{field.name} ++
1546 (if (@hasField(@TypeOf(context), "counter")) .{context.counter} else .{}) ++ .{
1547 std.fmt.fmtSliceHexLower(context.client_random),
1548 std.fmt.fmtSliceHexLower(@field(secrets, field.name)),
1549 }) catch {};
1550}
1551
14291552fn finishRead(c: *Client, frag: []const u8, in: usize, out: usize) usize {
14301553 const saved_buf = frag[in..];
14311554 if (c.partial_ciphertext_idx > c.partial_cleartext_idx) {
lib/std/http/Client.zig+22-7
......@@ -388,6 +388,7 @@ pub const Connection = struct {
388388
389389 // try to cleanly close the TLS connection, for any server that cares.
390390 _ = conn.tls_client.writeEnd(conn.stream, "", true) catch {};
391 if (conn.tls_client.ssl_key_log) |key_log| key_log.file.close();
391392 allocator.destroy(conn.tls_client);
392393 }
393394
......@@ -566,7 +567,7 @@ pub const Response = struct {
566567 .reason = undefined,
567568 .version = undefined,
568569 .keep_alive = false,
569 .parser = proto.HeadersParser.init(&header_buffer),
570 .parser = .init(&header_buffer),
570571 };
571572
572573 @memcpy(header_buffer[0..response_bytes.len], response_bytes);
......@@ -610,7 +611,7 @@ pub const Response = struct {
610611 }
611612
612613 pub fn iterateHeaders(r: Response) http.HeaderIterator {
613 return http.HeaderIterator.init(r.parser.get());
614 return .init(r.parser.get());
614615 }
615616
616617 test iterateHeaders {
......@@ -628,7 +629,7 @@ pub const Response = struct {
628629 .reason = undefined,
629630 .version = undefined,
630631 .keep_alive = false,
631 .parser = proto.HeadersParser.init(&header_buffer),
632 .parser = .init(&header_buffer),
632633 };
633634
634635 @memcpy(header_buffer[0..response_bytes.len], response_bytes);
......@@ -771,7 +772,7 @@ pub const Request = struct {
771772 req.client.connection_pool.release(req.client.allocator, req.connection.?);
772773 req.connection = null;
773774
774 var server_header = std.heap.FixedBufferAllocator.init(req.response.parser.header_bytes_buffer);
775 var server_header: std.heap.FixedBufferAllocator = .init(req.response.parser.header_bytes_buffer);
775776 defer req.response.parser.header_bytes_buffer = server_header.buffer[server_header.end_index..];
776777 const protocol, const valid_uri = try validateUri(uri, server_header.allocator());
777778
......@@ -1354,7 +1355,21 @@ pub fn connectTcp(client: *Client, host: []const u8, port: u16, protocol: Connec
13541355 conn.data.tls_client = try client.allocator.create(std.crypto.tls.Client);
13551356 errdefer client.allocator.destroy(conn.data.tls_client);
13561357
1357 conn.data.tls_client.* = std.crypto.tls.Client.init(stream, client.ca_bundle, host) catch return error.TlsInitializationFailed;
1358 const ssl_key_log_file: ?std.fs.File = if (std.options.http_enable_ssl_key_log_file) ssl_key_log_file: {
1359 const ssl_key_log_path = std.process.getEnvVarOwned(client.allocator, "SSLKEYLOGFILE") catch |err| switch (err) {
1360 error.EnvironmentVariableNotFound, error.InvalidWtf8 => break :ssl_key_log_file null,
1361 error.OutOfMemory => return error.OutOfMemory,
1362 };
1363 defer client.allocator.free(ssl_key_log_path);
1364 break :ssl_key_log_file std.fs.cwd().createFile(ssl_key_log_path, .{ .truncate = false }) catch null;
1365 } else null;
1366 errdefer if (ssl_key_log_file) |key_log_file| key_log_file.close();
1367
1368 conn.data.tls_client.* = std.crypto.tls.Client.init(stream, .{
1369 .host = .{ .explicit = host },
1370 .ca = .{ .bundle = client.ca_bundle },
1371 .ssl_key_log_file = ssl_key_log_file,
1372 }) catch return error.TlsInitializationFailed;
13581373 // This is appropriate for HTTPS because the HTTP headers contain
13591374 // the content length which is used to detect truncation attacks.
13601375 conn.data.tls_client.allow_truncation_attacks = true;
......@@ -1620,7 +1635,7 @@ pub fn open(
16201635 }
16211636 }
16221637
1623 var server_header = std.heap.FixedBufferAllocator.init(options.server_header_buffer);
1638 var server_header: std.heap.FixedBufferAllocator = .init(options.server_header_buffer);
16241639 const protocol, const valid_uri = try validateUri(uri, server_header.allocator());
16251640
16261641 if (protocol == .tls and @atomicLoad(bool, &client.next_https_rescan_certs, .acquire)) {
......@@ -1654,7 +1669,7 @@ pub fn open(
16541669 .status = undefined,
16551670 .reason = undefined,
16561671 .keep_alive = undefined,
1657 .parser = proto.HeadersParser.init(server_header.buffer[server_header.end_index..]),
1672 .parser = .init(server_header.buffer[server_header.end_index..]),
16581673 },
16591674 .headers = options.headers,
16601675 .extra_headers = options.extra_headers,
lib/std/std.zig+5
......@@ -146,6 +146,11 @@ pub const Options = struct {
146146 /// make a HTTPS connection.
147147 http_disable_tls: bool = false,
148148
149 /// This enables `std.http.Client` to log ssl secrets to the file specified by the SSLKEYLOGFILE
150 /// env var. Creating such a log file allows other programs with access to that file to decrypt
151 /// all `std.http.Client` traffic made by this program.
152 http_enable_ssl_key_log_file: bool = @import("builtin").mode == .Debug,
153
149154 side_channels_mitigations: crypto.SideChannelsMitigations = crypto.default_side_channels_mitigations,
150155};
151156