authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-02 20:49:11+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-20 09:09:06+11:00
log89f9c5cb373c81af5cb052cd9e68e44a123d0b04
tree1c4d428ad6138ff50d62c0d054acb5b23e31926d
parent7e2755646f5c9cab9973708a79c8aaa369d148e7

std.compress.zstandard: improve doc comments

2 files changed, 137 insertions(+), 67 deletions(-)

lib/std/compress/zstandard/decode/block.zig+60-36
......@@ -23,7 +23,6 @@ pub const Error = error{
2323 ReservedBlock,
2424 MalformedRleBlock,
2525 MalformedCompressedBlock,
26 EndOfStream,
2726};
2827
2928pub const DecodeState = struct {
......@@ -92,11 +91,17 @@ pub const DecodeState = struct {
9291 /// stream and Huffman tree from `literals` and reads the FSE tables from
9392 /// `source`.
9493 ///
95 /// Errors:
96 /// - returns `error.BitStreamHasNoStartBit` if the (reversed) literal bitstream's
97 /// first byte does not have any bits set.
98 /// - returns `error.TreelessLiteralsFirst` `literals` is a treeless literals section
99 /// and the decode state does not have a Huffman tree from a previous block.
94 /// Errors returned:
95 /// - `error.BitStreamHasNoStartBit` if the (reversed) literal bitstream's
96 /// first byte does not have any bits set
97 /// - `error.TreelessLiteralsFirst` `literals` is a treeless literals
98 /// section and the decode state does not have a Huffman tree from a
99 /// previous block
100 /// - `error.RepeatModeFirst` on the first call if one of the sequence FSE
101 /// tables is set to repeat mode
102 /// - `error.MalformedAccuracyLog` if an FSE table has an invalid accuracy
103 /// - `error.MalformedFseTable` if there are errors decoding an FSE table
104 /// - `error.EndOfStream` if `source` ends before all FSE tables are read
100105 pub fn prepare(
101106 self: *DecodeState,
102107 source: anytype,
......@@ -132,8 +137,10 @@ pub const DecodeState = struct {
132137 }
133138 }
134139
135 /// Read initial FSE states for sequence decoding. Returns `error.EndOfStream`
136 /// if `bit_reader` does not contain enough bits.
140 /// Read initial FSE states for sequence decoding.
141 ///
142 /// Errors returned:
143 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits.
137144 pub fn readInitialFseState(self: *DecodeState, bit_reader: *readers.ReverseBitReader) error{EndOfStream}!void {
138145 self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
139146 self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
......@@ -308,13 +315,19 @@ pub const DecodeState = struct {
308315 } || DecodeLiteralsError;
309316
310317 /// Decode one sequence from `bit_reader` into `dest`, written starting at
311 /// `write_pos` and update FSE states if `last_sequence` is `false`. Returns
312 /// `error.MalformedSequence` error if the decompressed sequence would be longer
313 /// than `sequence_size_limit` or the sequence's offset is too large; returns
314 /// `error.EndOfStream` if `bit_reader` does not contain enough bits; returns
315 /// `error.UnexpectedEndOfLiteralStream` if the decoder state's literal streams
316 /// do not contain enough literals for the sequence (this may mean the literal
317 /// stream or the sequence is malformed).
318 /// `write_pos` and update FSE states if `last_sequence` is `false`.
319 /// `prepare()` must be called for the block before attempting to decode
320 /// sequences.
321 ///
322 /// Errors returned:
323 /// - `error.MalformedSequence` if the decompressed sequence would be
324 /// longer than `sequence_size_limit` or the sequence's offset is too
325 /// large
326 /// - `error.UnexpectedEndOfLiteralStream` if the decoder state's literal
327 /// streams do not contain enough literals for the sequence (this may
328 /// mean the literal stream or the sequence is malformed).
329 /// - `error.OffsetCodeTooLarge` if an invalid offset code is found
330 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits
318331 pub fn decodeSequenceSlice(
319332 self: *DecodeState,
320333 dest: []u8,
......@@ -336,7 +349,8 @@ pub const DecodeState = struct {
336349 return sequence_length;
337350 }
338351
339 /// Decode one sequence from `bit_reader` into `dest`; see `decodeSequenceSlice`.
352 /// Decode one sequence from `bit_reader` into `dest`; see
353 /// `decodeSequenceSlice`.
340354 pub fn decodeSequenceRingBuffer(
341355 self: *DecodeState,
342356 dest: *RingBuffer,
......@@ -364,7 +378,7 @@ pub const DecodeState = struct {
364378 try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
365379 }
366380
367 pub fn initLiteralStream(self: *DecodeState, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
381 fn initLiteralStream(self: *DecodeState, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
368382 try self.literal_stream_reader.init(bytes);
369383 }
370384
......@@ -393,12 +407,14 @@ pub const DecodeState = struct {
393407 PrefixNotFound,
394408 } || LiteralBitsError;
395409
396 /// Decode `len` bytes of literals into `dest`. `literals` should be the
397 /// `LiteralsSection` that was passed to `prepare()`. Returns
398 /// `error.MalformedLiteralsLength` if the number of literal bytes decoded by
399 /// `self` plus `len` is greater than the regenerated size of `literals`.
400 /// Returns `error.UnexpectedEndOfLiteralStream` and `error.PrefixNotFound` if
401 /// there are problems decoding Huffman compressed literals.
410 /// Decode `len` bytes of literals into `dest`.
411 ///
412 /// Errors returned:
413 /// - `error.MalformedLiteralsLength` if the number of literal bytes
414 /// decoded by `self` plus `len` is greater than the regenerated size of
415 /// `literals`
416 /// - `error.UnexpectedEndOfLiteralStream` and `error.PrefixNotFound` if
417 /// there are problems decoding Huffman compressed literals
402418 pub fn decodeLiteralsSlice(
403419 self: *DecodeState,
404420 dest: []u8,
......@@ -561,7 +577,6 @@ pub const DecodeState = struct {
561577/// - `error.MalformedRleBlock` if the block is an RLE block and `src.len < 1`
562578/// - `error.MalformedCompressedBlock` if there are errors decoding a
563579/// compressed block
564/// - `error.EndOfStream` if the sequence bit stream ends unexpectedly
565580pub fn decodeBlock(
566581 dest: []u8,
567582 src: []const u8,
......@@ -738,7 +753,8 @@ pub fn decodeBlockRingBuffer(
738753/// `error.SequenceBufferTooSmall` are returned (the maximum block size is an
739754/// upper bound for the size of both buffers). See `decodeBlock`
740755/// and `decodeBlockRingBuffer` for function that can decode a block without
741/// these extra copies.
756/// these extra copies. `error.EndOfStream` is returned if `source` does not
757/// contain enough bytes.
742758pub fn decodeBlockReader(
743759 dest: *RingBuffer,
744760 source: anytype,
......@@ -820,6 +836,10 @@ pub fn decodeBlockHeader(src: *const [3]u8) frame.ZStandard.Block.Header {
820836 };
821837}
822838
839/// Decode the header of a block.
840///
841/// Errors returned:
842/// - `error.EndOfStream` if `src.len < 3`
823843pub fn decodeBlockHeaderSlice(src: []const u8) error{EndOfStream}!frame.ZStandard.Block.Header {
824844 if (src.len < 3) return error.EndOfStream;
825845 return decodeBlockHeader(src[0..3]);
......@@ -828,9 +848,14 @@ pub fn decodeBlockHeaderSlice(src: []const u8) error{EndOfStream}!frame.ZStandar
828848/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
829849/// number of bytes the section uses.
830850///
831/// Errors:
832/// - returns `error.MalformedLiteralsHeader` if the header is invalid
833/// - returns `error.MalformedLiteralsSection` if there are errors decoding
851/// Errors returned:
852/// - `error.MalformedLiteralsHeader` if the header is invalid
853/// - `error.MalformedLiteralsSection` if there are decoding errors
854/// - `error.MalformedAccuracyLog` if compressed literals have invalid
855/// accuracy
856/// - `error.MalformedFseTable` if compressed literals have invalid FSE table
857/// - `error.MalformedHuffmanTree` if there are errors decoding a Huffamn tree
858/// - `error.EndOfStream` if there are not enough bytes in `src`
834859pub fn decodeLiteralsSectionSlice(
835860 src: []const u8,
836861 consumed_count: *usize,
......@@ -886,11 +911,7 @@ pub fn decodeLiteralsSectionSlice(
886911}
887912
888913/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
889/// number of bytes the section uses.
890///
891/// Errors:
892/// - returns `error.MalformedLiteralsHeader` if the header is invalid
893/// - returns `error.MalformedLiteralsSection` if there are errors decoding
914/// number of bytes the section uses. See `decodeLiterasSectionSlice()`.
894915pub fn decodeLiteralsSection(
895916 source: anytype,
896917 buffer: []u8,
......@@ -961,6 +982,9 @@ fn decodeStreams(size_format: u2, stream_data: []const u8) !LiteralsSection.Stre
961982}
962983
963984/// Decode a literals section header.
985///
986/// Errors returned:
987/// - `error.EndOfStream` if there are not enough bytes in `source`
964988pub fn decodeLiteralsHeader(source: anytype) !LiteralsSection.Header {
965989 const byte0 = try source.readByte();
966990 const block_type = @intToEnum(LiteralsSection.BlockType, byte0 & 0b11);
......@@ -1011,9 +1035,9 @@ pub fn decodeLiteralsHeader(source: anytype) !LiteralsSection.Header {
10111035
10121036/// Decode a sequences section header.
10131037///
1014/// Errors:
1015/// - returns `error.ReservedBitSet` is the reserved bit is set
1016/// - returns `error.MalformedSequencesHeader` if the header is invalid
1038/// Errors returned:
1039/// - `error.ReservedBitSet` if the reserved bit is set
1040/// - `error.EndOfStream` if there are not enough bytes in `source`
10171041pub fn decodeSequencesHeader(
10181042 source: anytype,
10191043) !SequencesSection.Header {
lib/std/compress/zstandard/decompress.zig+77-31
......@@ -25,11 +25,12 @@ pub fn isSkippableMagic(magic: u32) bool {
2525
2626/// Returns the kind of frame at the beginning of `src`.
2727///
28/// Errors:
29/// - returns `error.BadMagic` if `source` begins with bytes not equal to the
28/// Errors returned:
29/// - `error.BadMagic` if `source` begins with bytes not equal to the
3030/// Zstandard frame magic number, or outside the range of magic numbers for
3131/// skippable frames.
32pub fn decodeFrameType(source: anytype) !frame.Kind {
32/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
33pub fn decodeFrameType(source: anytype) error{ BadMagic, EndOfStream }!frame.Kind {
3334 const magic = try source.readIntLittle(u32);
3435 return if (magic == frame.ZStandard.magic_number)
3536 .zstandard
......@@ -45,12 +46,23 @@ const ReadWriteCount = struct {
4546};
4647
4748/// Decodes the frame at the start of `src` into `dest`. Returns the number of
48/// bytes read from `src` and written to `dest`.
49/// bytes read from `src` and written to `dest`. This function can only decode
50/// frames that declare the decompressed content size.
4951///
50/// Errors:
51/// - returns `error.UnknownContentSizeUnsupported`
52/// - returns `error.ContentTooLarge`
53/// - returns `error.BadMagic`
52/// Errors returned:
53/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
54/// uncompressed content size
55/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
56/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
57/// number for a Zstandard or Skippable frame
58/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
59/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
60/// contains a checksum that does not match the checksum of the decompressed
61/// data
62/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
63/// - `error.UnusedBitSet` if the unused bit of the frame header is set
64/// - `error.EndOfStream` if `src` does not contain a complete frame
65/// - an error in `block.Error` if there are errors decoding a block
5466pub fn decodeFrame(
5567 dest: []u8,
5668 src: []const u8,
......@@ -66,6 +78,7 @@ pub fn decodeFrame(
6678 };
6779}
6880
81/// Returns the frame checksum corresponding to the data fed into `hasher`
6982pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 {
7083 const hash = hasher.final();
7184 return @intCast(u32, hash & 0xFFFFFFFF);
......@@ -74,20 +87,31 @@ pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 {
7487const FrameError = error{
7588 DictionaryIdFlagUnsupported,
7689 ChecksumFailure,
90 EndOfStream,
7791} || InvalidBit || block.Error;
7892
7993/// Decode a Zstandard frame from `src` into `dest`, returning the number of
80/// bytes read from `src` and written to `dest`; if the frame does not declare
81/// its decompressed content size `error.UnknownContentSizeUnsupported` is
82/// returned. Returns `error.DictionaryIdFlagUnsupported` if the frame uses a
83/// dictionary, and `error.ChecksumFailure` if `verify_checksum` is `true` and
84/// the frame contains a checksum that does not match the checksum computed from
85/// the decompressed frame.
94/// bytes read from `src` and written to `dest`. The first four bytes of `src`
95/// must be the magic number for a Zstandard frame.
96///
97/// Error returned:
98/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
99/// uncompressed content size
100/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
101/// number for a Zstandard or Skippable frame
102/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
103/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
104/// contains a checksum that does not match the checksum of the decompressed
105/// data
106/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
107/// - `error.UnusedBitSet` if the unused bit of the frame header is set
108/// - `error.EndOfStream` if `src` does not contain a complete frame
109/// - an error in `block.Error` if there are errors decoding a block
86110pub fn decodeZStandardFrame(
87111 dest: []u8,
88112 src: []const u8,
89113 verify_checksum: bool,
90) (error{ UnknownContentSizeUnsupported, ContentTooLarge, EndOfStream } || FrameError)!ReadWriteCount {
114) (error{ UnknownContentSizeUnsupported, ContentTooLarge } || FrameError)!ReadWriteCount {
91115 assert(readInt(u32, src[0..4]) == frame.ZStandard.magic_number);
92116 var consumed_count: usize = 4;
93117
......@@ -127,7 +151,18 @@ pub const FrameContext = struct {
127151 has_checksum: bool,
128152 block_size_max: usize,
129153
130 pub fn init(frame_header: frame.ZStandard.Header, window_size_max: usize, verify_checksum: bool) !FrameContext {
154 const Error = error{ DictionaryIdFlagUnsupported, WindowSizeUnknown, WindowTooLarge };
155 /// Validates `frame_header` and returns the associated `FrameContext`.
156 ///
157 /// Errors returned:
158 /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
159 /// - `error.WindowSizeUnknown` if the frame does not have a valid window size
160 /// - `error.WindowTooLarge` if the window size is larger than
161 pub fn init(
162 frame_header: frame.ZStandard.Header,
163 window_size_max: usize,
164 verify_checksum: bool,
165 ) Error!FrameContext {
131166 if (frame_header.descriptor.dictionary_id_flag != 0) return error.DictionaryIdFlagUnsupported;
132167
133168 const window_size_raw = frameWindowSize(frame_header) orelse return error.WindowSizeUnknown;
......@@ -147,19 +182,29 @@ pub const FrameContext = struct {
147182};
148183
149184/// Decode a Zstandard from from `src` and return the decompressed bytes; see
150/// `decodeZStandardFrame()`. Returns `error.WindowSizeUnknown` if the frame
151/// does not declare its content size or a window descriptor (this indicates a
152/// malformed frame).
185/// `decodeZStandardFrame()`. `allocator` is used to allocate both the returned
186/// slice and internal buffers used during decoding. The first four bytes of
187/// `src` must be the magic number for a Zstandard frame.
153188///
154/// Errors:
155/// - returns `error.WindowTooLarge`
156/// - returns `error.WindowSizeUnknown`
189/// Errors returned:
190/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
191/// - `error.WindowTooLarge` if the window size is larger than
192/// `window_size_max`
193/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
194/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
195/// contains a checksum that does not match the checksum of the decompressed
196/// data
197/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
198/// - `error.UnusedBitSet` if the unused bit of the frame header is set
199/// - `error.EndOfStream` if `src` does not contain a complete frame
200/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
201/// - an error in `block.Error` if there are errors decoding a block
157202pub fn decodeZStandardFrameAlloc(
158203 allocator: std.mem.Allocator,
159204 src: []const u8,
160205 verify_checksum: bool,
161206 window_size_max: usize,
162) (error{ WindowSizeUnknown, WindowTooLarge, OutOfMemory, EndOfStream } || FrameError)![]u8 {
207) (error{OutOfMemory} || FrameContext.Error || FrameError)![]u8 {
163208 var result = std.ArrayList(u8).init(allocator);
164209 assert(readInt(u32, src[0..4]) == frame.ZStandard.magic_number);
165210 var consumed_count: usize = 4;
......@@ -222,7 +267,7 @@ fn decodeFrameBlocks(
222267 src: []const u8,
223268 consumed_count: *usize,
224269 hash: ?*std.hash.XxHash64,
225) block.Error!usize {
270) (error{EndOfStream} || block.Error)!usize {
226271 // These tables take 7680 bytes
227272 var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
228273 var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;
......@@ -252,7 +297,8 @@ fn decodeFrameBlocks(
252297 return written_count;
253298}
254299
255/// Decode the header of a skippable frame.
300/// Decode the header of a skippable frame. The first four bytes of `src` must
301/// be a valid magic number for a Skippable frame.
256302pub fn decodeSkippableHeader(src: *const [8]u8) frame.Skippable.Header {
257303 const magic = readInt(u32, src[0..4]);
258304 assert(isSkippableMagic(magic));
......@@ -263,8 +309,8 @@ pub fn decodeSkippableHeader(src: *const [8]u8) frame.Skippable.Header {
263309 };
264310}
265311
266/// Returns the window size required to decompress a frame, or `null` if it cannot be
267/// determined, which indicates a malformed frame header.
312/// Returns the window size required to decompress a frame, or `null` if it
313/// cannot be determined (which indicates a malformed frame header).
268314pub fn frameWindowSize(header: frame.ZStandard.Header) ?u64 {
269315 if (header.window_descriptor) |descriptor| {
270316 const exponent = (descriptor & 0b11111000) >> 3;
......@@ -279,10 +325,10 @@ pub fn frameWindowSize(header: frame.ZStandard.Header) ?u64 {
279325const InvalidBit = error{ UnusedBitSet, ReservedBitSet };
280326/// Decode the header of a Zstandard frame.
281327///
282/// Errors:
283/// - returns `error.UnusedBitSet` if the unused bits of the header are set
284/// - returns `error.ReservedBitSet` if the reserved bits of the header are
285/// set
328/// Errors returned:
329/// - `error.UnusedBitSet` if the unused bits of the header are set
330/// - `error.ReservedBitSet` if the reserved bits of the header are set
331/// - `error.EndOfStream` if `source` does not contain a complete header
286332pub fn decodeZStandardHeader(source: anytype) (error{EndOfStream} || InvalidBit)!frame.ZStandard.Header {
287333 const descriptor = @bitCast(frame.ZStandard.Header.Descriptor, try source.readByte());
288334