authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-12-28 16:37:22-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-02 16:57:16-07:00
log940d368e7ea95d2bb8185e71af3d1ec0328917dc
treea2338ec0d47f74c0badfe0a22d2e77e5044e3082
parent21ab99174eabc9ae8efa2b19890d9cab51773b35

std.crypto.tls.Client: fix the read function

The read function has been renamed to readAdvanced since it has slightly different semantics than typical read functions, specifically regarding the end-of-file. A higher level read function is implemented on top. Now, API users may pass small buffers to the read function and everything will work fine. This is done by re-decrypting the same ciphertext record with each call to read() until the record is finished being transmitted. If the buffer supplied to read() is large enough, then any given ciphertext record will only be decrypted once, since it decrypts directly to the read() buffer and therefore does not need any memcpy. On the other hand, if the buffer supplied to read() is small, then the ciphertext is decrypted into a stack buffer, a subset is copied to the read() buffer, and then the entire ciphertext record is saved for the next call to read().

3 files changed, 136 insertions(+), 46 deletions(-)

lib/std/crypto/tls/Client.zig+129-34
...@@ -18,14 +18,20 @@ const array = tls.array;...@@ -18,14 +18,20 @@ const array = tls.array;
18const enum_array = tls.enum_array;18const enum_array = tls.enum_array;
19const Certificate = crypto.Certificate;19const Certificate = crypto.Certificate;
2020
21application_cipher: ApplicationCipher,
22read_seq: u64,21read_seq: u64,
23write_seq: u64,22write_seq: u64,
24/// The size is enough to contain exactly one TLSCiphertext record.
25partially_read_buffer: [tls.max_ciphertext_record_len]u8,
26/// The number of partially read bytes inside `partially_read_buffer`.23/// The number of partially read bytes inside `partially_read_buffer`.
27partially_read_len: u15,24partially_read_len: u15,
25/// The number of cleartext bytes from decoding `partially_read_buffer` which
26/// have already been transferred via read() calls. This implementation will
27/// re-decrypt bytes from `partially_read_buffer` when the buffer supplied by
28/// the read() API user is not large enough.
29partial_cleartext_index: u15,
30application_cipher: ApplicationCipher,
28eof: bool,31eof: bool,
32/// The size is enough to contain exactly one TLSCiphertext record.
33/// Contains encrypted bytes.
34partially_read_buffer: [tls.max_ciphertext_record_len]u8,
2935
30/// `host` is only borrowed during this function call.36/// `host` is only borrowed during this function call.
31pub fn init(stream: net.Stream, ca_bundle: Certificate.Bundle, host: []const u8) !Client {37pub fn init(stream: net.Stream, ca_bundle: Certificate.Bundle, host: []const u8) !Client {
...@@ -596,6 +602,7 @@ pub fn init(stream: net.Stream, ca_bundle: Certificate.Bundle, host: []const u8)...@@ -596,6 +602,7 @@ pub fn init(stream: net.Stream, ca_bundle: Certificate.Bundle, host: []const u8)
596 .application_cipher = app_cipher,602 .application_cipher = app_cipher,
597 .read_seq = 0,603 .read_seq = 0,
598 .write_seq = 0,604 .write_seq = 0,
605 .partial_cleartext_index = 0,
599 .partially_read_buffer = undefined,606 .partially_read_buffer = undefined,
600 .partially_read_len = @intCast(u15, len - end),607 .partially_read_len = @intCast(u15, len - end),
601 .eof = false,608 .eof = false,
...@@ -722,27 +729,85 @@ pub fn writeAll(c: *Client, stream: net.Stream, bytes: []const u8) !void {...@@ -722,27 +729,85 @@ pub fn writeAll(c: *Client, stream: net.Stream, bytes: []const u8) !void {
722 }729 }
723}730}
724731
725/// Returns number of bytes that have been read, which are now populated inside732/// Returns the number of bytes read, calling the underlying read function the
726/// `buffer`. A return value of zero bytes does not necessarily mean end of733/// minimal number of times until the buffer has at least `len` bytes filled.
727/// stream. Instead, the `eof` flag is set upon end of stream. The `eof` flag734/// If the number read is less than `len` it means the stream reached the end.
728/// may be set after any call to `read`, including when greater than zero bytes735/// Reaching the end of the stream is not an error condition.
729/// are returned, and this function asserts that `eof` is `false`.736pub fn readAtLeast(c: *Client, stream: anytype, buffer: []u8, len: usize) !usize {
730pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {737 assert(len <= buffer.len);
738 if (c.eof) return 0;
739 var index: usize = 0;
740 while (index < len) {
741 index += try c.readAdvanced(stream, buffer[index..]);
742 if (c.eof) break;
743 }
744 return index;
745}
746
747pub fn read(c: *Client, stream: anytype, buffer: []u8) !usize {
748 return readAtLeast(c, stream, buffer, 1);
749}
750
751/// Returns the number of bytes read. If the number read is smaller than
752/// `buffer.len`, it means the stream reached the end. Reaching the end of the
753/// stream is not an error condition.
754pub fn readAll(c: *Client, stream: anytype, buffer: []u8) !usize {
755 return readAtLeast(c, stream, buffer, buffer.len);
756}
757
758/// Returns number of bytes that have been read, populated inside `buffer`. A
759/// return value of zero bytes does not mean end of stream. Instead, the `eof`
760/// flag is set upon end of stream. The `eof` flag may be set after any call to
761/// `read`, including when greater than zero bytes are returned, and this
762/// function asserts that `eof` is `false`.
763/// See `read` for a higher level function that has the same, familiar API
764/// as other read functions, such as `std.fs.File.read`.
765/// It is recommended to use a buffer size with length at least
766/// `tls.max_ciphertext_len` bytes to avoid redundantly decrypting the same
767/// encoded data.
768pub fn readAdvanced(c: *Client, stream: net.Stream, buffer: []u8) !usize {
731 assert(!c.eof);769 assert(!c.eof);
732 const prev_len = c.partially_read_len;770 const prev_len = c.partially_read_len;
733 var in_buf: [max_ciphertext_len * 4]u8 = undefined;771 // Ideally, this buffer would never be used. It is needed when `buffer` is too small
734 mem.copy(u8, &in_buf, c.partially_read_buffer[0..prev_len]);772 // to fit the cleartext, which may be as large as `max_ciphertext_len`.
773 var cleartext_stack_buffer: [max_ciphertext_len]u8 = undefined;
774 // This buffer is typically used, except, as an optimization when a very large
775 // `buffer` is provided, we use half of it for buffering ciphertext and the
776 // other half for outputting cleartext.
777 var in_stack_buffer: [max_ciphertext_len * 4]u8 = undefined;
778 const half_buffer_len = buffer.len / 2;
779 const out_in: struct { []u8, []u8 } = if (half_buffer_len >= in_stack_buffer.len) .{
780 buffer[0..half_buffer_len],
781 buffer[half_buffer_len..],
782 } else .{
783 buffer,
784 &in_stack_buffer,
785 };
786 const out_buf = out_in[0];
787 const in_buf = out_in[1];
788 mem.copy(u8, in_buf, c.partially_read_buffer[0..prev_len]);
735789
736 // Capacity of output buffer, in records, rounded up.790 // Capacity of output buffer, in records, rounded up.
737 const buf_cap = (buffer.len +| (max_ciphertext_len - 1)) / max_ciphertext_len;791 const buf_cap = (out_buf.len +| (max_ciphertext_len - 1)) / max_ciphertext_len;
738 const wanted_read_len = buf_cap * (max_ciphertext_len + tls.ciphertext_record_header_len);792 const wanted_read_len = buf_cap * (max_ciphertext_len + tls.ciphertext_record_header_len);
739 const ask_slice = in_buf[prev_len..@min(wanted_read_len, in_buf.len)];793 const ask_len = @max(wanted_read_len, cleartext_stack_buffer.len);
740 const actual_read_len = try stream.read(ask_slice);794 const ask_slice = in_buf[prev_len..][0..@min(ask_len, in_buf.len - prev_len)];
741 const frag = in_buf[0 .. prev_len + actual_read_len];795 assert(ask_slice.len > 0);
742 if (frag.len == 0) {796 const frag = frag: {
743 // This is either a truncation attack, or a bug in the server.797 if (prev_len >= 5) {
744 return error.TlsConnectionTruncated;798 const record_size = mem.readIntBig(u16, in_buf[3..][0..2]);
745 }799 if (prev_len >= 5 + record_size) {
800 // We can use our buffered data without calling read().
801 break :frag in_buf[0..prev_len];
802 }
803 }
804 const actual_read_len = try stream.read(ask_slice);
805 if (actual_read_len == 0) {
806 // This is either a truncation attack, or a bug in the server.
807 return error.TlsConnectionTruncated;
808 }
809 break :frag in_buf[0 .. prev_len + actual_read_len];
810 };
746 var in: usize = 0;811 var in: usize = 0;
747 var out: usize = 0;812 var out: usize = 0;
748813
...@@ -750,6 +815,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {...@@ -750,6 +815,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
750 if (in + tls.ciphertext_record_header_len > frag.len) {815 if (in + tls.ciphertext_record_header_len > frag.len) {
751 return finishRead(c, frag, in, out);816 return finishRead(c, frag, in, out);
752 }817 }
818 const record_start = in;
753 const ct = @intToEnum(ContentType, frag[in]);819 const ct = @intToEnum(ContentType, frag[in]);
754 in += 1;820 in += 1;
755 const legacy_version = mem.readIntBig(u16, frag[in..][0..2]);821 const legacy_version = mem.readIntBig(u16, frag[in..][0..2]);
...@@ -767,7 +833,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {...@@ -767,7 +833,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
767 @panic("TODO handle an alert here");833 @panic("TODO handle an alert here");
768 },834 },
769 .application_data => {835 .application_data => {
770 const cleartext_len = switch (c.application_cipher) {836 const cleartext = switch (c.application_cipher) {
771 inline else => |*p| c: {837 inline else => |*p| c: {
772 const P = @TypeOf(p.*);838 const P = @TypeOf(p.*);
773 const V = @Vector(P.AEAD.nonce_length, u8);839 const V = @Vector(P.AEAD.nonce_length, u8);
...@@ -776,29 +842,29 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {...@@ -776,29 +842,29 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
776 const ciphertext = frag[in..][0..ciphertext_len];842 const ciphertext = frag[in..][0..ciphertext_len];
777 in += ciphertext_len;843 in += ciphertext_len;
778 const auth_tag = frag[in..][0..P.AEAD.tag_length].*;844 const auth_tag = frag[in..][0..P.AEAD.tag_length].*;
779 const cleartext = buffer[out..][0..ciphertext_len];
780 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);845 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
846 // Here we use read_seq and then intentionally don't
847 // increment it until later when it is certain the same
848 // ciphertext does not need to be decrypted again.
781 const operand: V = pad ++ @bitCast([8]u8, big(c.read_seq));849 const operand: V = pad ++ @bitCast([8]u8, big(c.read_seq));
782 c.read_seq += 1;
783 const nonce: [P.AEAD.nonce_length]u8 = @as(V, p.server_iv) ^ operand;850 const nonce: [P.AEAD.nonce_length]u8 = @as(V, p.server_iv) ^ operand;
784 //std.debug.print("seq: {d} nonce: {} server_key: {} server_iv: {}\n", .{851 const cleartext_buf = if (c.partial_cleartext_index == 0 and out + ciphertext.len <= out_buf.len)
785 // c.read_seq - 1,852 out_buf[out..]
786 // std.fmt.fmtSliceHexLower(&nonce),853 else
787 // std.fmt.fmtSliceHexLower(&p.server_key),854 &cleartext_stack_buffer;
788 // std.fmt.fmtSliceHexLower(&p.server_iv),855 const cleartext = cleartext_buf[0..ciphertext.len];
789 //});
790 P.AEAD.decrypt(cleartext, ciphertext, auth_tag, ad, nonce, p.server_key) catch856 P.AEAD.decrypt(cleartext, ciphertext, auth_tag, ad, nonce, p.server_key) catch
791 return error.TlsBadRecordMac;857 return error.TlsBadRecordMac;
792 break :c cleartext.len;858 break :c cleartext;
793 },859 },
794 };860 };
795861
796 const cleartext = buffer[out..][0..cleartext_len];
797 const inner_ct = @intToEnum(ContentType, cleartext[cleartext.len - 1]);862 const inner_ct = @intToEnum(ContentType, cleartext[cleartext.len - 1]);
798 switch (inner_ct) {863 switch (inner_ct) {
799 .alert => {864 .alert => {
800 const level = @intToEnum(tls.AlertLevel, buffer[out]);865 c.read_seq += 1;
801 const desc = @intToEnum(tls.AlertDescription, buffer[out + 1]);866 const level = @intToEnum(tls.AlertLevel, out_buf[out]);
867 const desc = @intToEnum(tls.AlertDescription, out_buf[out + 1]);
802 if (desc == .close_notify) {868 if (desc == .close_notify) {
803 c.eof = true;869 c.eof = true;
804 return out;870 return out;
...@@ -807,6 +873,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {...@@ -807,6 +873,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
807 return error.TlsAlert;873 return error.TlsAlert;
808 },874 },
809 .handshake => {875 .handshake => {
876 c.read_seq += 1;
810 var ct_i: usize = 0;877 var ct_i: usize = 0;
811 while (true) {878 while (true) {
812 const handshake_type = @intToEnum(tls.HandshakeType, cleartext[ct_i]);879 const handshake_type = @intToEnum(tls.HandshakeType, cleartext[ct_i]);
...@@ -819,7 +886,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {...@@ -819,7 +886,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
819 const handshake = cleartext[ct_i..next_handshake_i];886 const handshake = cleartext[ct_i..next_handshake_i];
820 switch (handshake_type) {887 switch (handshake_type) {
821 .new_session_ticket => {888 .new_session_ticket => {
822 std.debug.print("server sent a new session ticket\n", .{});889 // This client implementation ignores new session tickets.
823 },890 },
824 .key_update => {891 .key_update => {
825 switch (c.application_cipher) {892 switch (c.application_cipher) {
...@@ -859,7 +926,35 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {...@@ -859,7 +926,35 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
859 }926 }
860 },927 },
861 .application_data => {928 .application_data => {
862 out += cleartext_len - 1;929 // Determine whether the output buffer or a stack
930 // buffer was used for storing the cleartext.
931 if (c.partial_cleartext_index == 0 and
932 out + cleartext.len <= out_buf.len)
933 {
934 // Output buffer was used directly which means no
935 // memory copying needs to occur, and we can move
936 // on to the next ciphertext record.
937 out += cleartext.len - 1;
938 c.read_seq += 1;
939 } else {
940 // Stack buffer was used, so we must copy to the output buffer.
941 const dest = out_buf[out..];
942 const rest = cleartext[c.partial_cleartext_index..];
943 const src = rest[0..@min(rest.len, dest.len)];
944 mem.copy(u8, dest, src);
945 out += src.len;
946 c.partial_cleartext_index = @intCast(
947 @TypeOf(c.partial_cleartext_index),
948 c.partial_cleartext_index + src.len,
949 );
950 if (c.partial_cleartext_index >= cleartext.len) {
951 c.partial_cleartext_index = 0;
952 c.read_seq += 1;
953 } else {
954 in = record_start;
955 return finishRead(c, frag, in, out);
956 }
957 }
863 },958 },
864 else => {959 else => {
865 std.debug.print("inner content type: {d}\n", .{inner_ct});960 std.debug.print("inner content type: {d}\n", .{inner_ct});
lib/std/http/Client.zig+3-9
...@@ -63,16 +63,10 @@ pub const Request = struct {...@@ -63,16 +63,10 @@ pub const Request = struct {
63 }63 }
6464
65 pub fn readAtLeast(req: *Request, buffer: []u8, len: usize) !usize {65 pub fn readAtLeast(req: *Request, buffer: []u8, len: usize) !usize {
66 var index: usize = 0;66 switch (req.protocol) {
67 while (index < len) {67 .http => return req.stream.readAtLeast(buffer, len),
68 const amt = try req.read(buffer[index..]);68 .https => return req.tls_client.readAtLeast(req.stream, buffer, len),
69 index += amt;
70 switch (req.protocol) {
71 .http => if (amt == 0) break,
72 .https => if (req.tls_client.eof) break,
73 }
74 }69 }
75 return index;
76 }70 }
77};71};
7872
lib/std/net.zig+4-3
...@@ -1680,11 +1680,12 @@ pub const Stream = struct {...@@ -1680,11 +1680,12 @@ pub const Stream = struct {
1680 }1680 }
16811681
1682 /// Returns the number of bytes read, calling the underlying read function1682 /// Returns the number of bytes read, calling the underlying read function
1683 /// the minimal number of times until at least the buffer has at least1683 /// the minimal number of times until the buffer has at least `len` bytes
1684 /// `len` bytes filled. If the number read is less than `len` it means the1684 /// filled. If the number read is less than `len` it means the stream
1685 /// stream reached the end. Reaching the end of the stream is not an error1685 /// reached the end. Reaching the end of the stream is not an error
1686 /// condition.1686 /// condition.
1687 pub fn readAtLeast(s: Stream, buffer: []u8, len: usize) ReadError!usize {1687 pub fn readAtLeast(s: Stream, buffer: []u8, len: usize) ReadError!usize {
1688 assert(len <= buffer.len);
1688 var index: usize = 0;1689 var index: usize = 0;
1689 while (index < len) {1690 while (index < len) {
1690 const amt = try s.read(buffer[index..]);1691 const amt = try s.read(buffer[index..]);