authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-01-25 01:30:17+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-20 09:09:06+11:00
log7558bf64513ec2be59b95aecc5e0ac50ad88b1f5
treef72e51f091a278859bebd41630113bcd2e539d6d
parentab18adf5c38e2640411039600f8de7de817ac200

std.compress.zstandard: minor cleanup and add doc comments

1 files changed, 68 insertions(+), 4 deletions(-)

lib/std/compress/zstandard/decompress.zig+68-4
......@@ -18,6 +18,10 @@ fn isSkippableMagic(magic: u32) bool {
1818 return frame.Skippable.magic_number_min <= magic and magic <= frame.Skippable.magic_number_max;
1919}
2020
21/// Returns the decompressed size of the frame at the start of `src`. Returns 0
22/// if the the frame is skippable, `null` for Zstanndard frames that do not
23/// declare their content size. Returns `UnusedBitSet` and `ReservedBitSet`
24/// errors if the respective bits of the the frame descriptor are set.
2125pub fn getFrameDecompressedSize(src: []const u8) !?usize {
2226 switch (try frameType(src)) {
2327 .zstandard => {
......@@ -28,7 +32,10 @@ pub fn getFrameDecompressedSize(src: []const u8) !?usize {
2832 }
2933}
3034
31pub fn frameType(src: []const u8) !frame.Kind {
35/// Returns the kind of frame at the beginning of `src`. Returns `BadMagic` if
36/// `src` begin with bytes not equal to the Zstandard frame magic number, or
37/// outside the range of magic numbers for skippable frames.
38pub fn frameType(src: []const u8) error{BadMagic}!frame.Kind {
3239 const magic = readInt(u32, src[0..4]);
3340 return if (magic == frame.ZStandard.magic_number)
3441 .zstandard
......@@ -43,11 +50,13 @@ const ReadWriteCount = struct {
4350 write_count: usize,
4451};
4552
53/// Decodes the frame at the start of `src` into `dest`. Returns the number of
54/// bytes read from `src` and written to `dest`.
4655pub fn decodeFrame(dest: []u8, src: []const u8, verify_checksum: bool) !ReadWriteCount {
4756 return switch (try frameType(src)) {
4857 .zstandard => decodeZStandardFrame(dest, src, verify_checksum),
4958 .skippable => ReadWriteCount{
50 .read_count = try skippableFrameSize(src[0..8]) + 8,
59 .read_count = skippableFrameSize(src[0..8]) + 8,
5160 .write_count = 0,
5261 },
5362 };
......@@ -82,6 +91,10 @@ pub const DecodeState = struct {
8291 };
8392 }
8493
94 /// Prepare the decoder to decode a compressed block. Loads the literals
95 /// stream and Huffman tree from `literals` and reads the FSE tables from `src`.
96 /// Returns `error.BitStreamHasNoStartBit` if the (reversed) literal bitstream's
97 /// first byte does not have any bits set.
8598 pub fn prepare(
8699 self: *DecodeState,
87100 src: []const u8,
......@@ -130,6 +143,8 @@ pub const DecodeState = struct {
130143 return 0;
131144 }
132145
146 /// Read initial FSE states for sequence decoding. Returns `error.EndOfStream`
147 /// if `bit_reader` does not contain enough bits.
133148 pub fn readInitialFseState(self: *DecodeState, bit_reader: anytype) !void {
134149 self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
135150 self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
......@@ -283,6 +298,14 @@ pub const DecodeState = struct {
283298 for (copy_slice.second) |b| dest.writeAssumeCapacity(b);
284299 }
285300
301 /// Decode one sequence from `bit_reader` into `dest`, written starting at
302 /// `write_pos` and update FSE states if `last_sequence` is `false`. Returns
303 /// `error.MalformedSequence` error if the decompressed sequence would be longer
304 /// than `sequence_size_limit` or the sequence's offset is too large; returns
305 /// `error.EndOfStream` if `bit_reader` does not contain enough bits; returns
306 /// `error.UnexpectedEndOfLiteralStream` if the decoder state's literal streams
307 /// do not contain enough literals for the sequence (this may mean the literal
308 /// stream or the sequence is malformed).
286309 pub fn decodeSequenceSlice(
287310 self: *DecodeState,
288311 dest: []u8,
......@@ -305,6 +328,7 @@ pub const DecodeState = struct {
305328 return sequence_length;
306329 }
307330
331 /// Decode one sequence from `bit_reader` into `dest`; see `decodeSequenceSlice`.
308332 pub fn decodeSequenceRingBuffer(
309333 self: *DecodeState,
310334 dest: *RingBuffer,
......@@ -335,6 +359,12 @@ pub const DecodeState = struct {
335359 try self.literal_stream_reader.init(bytes);
336360 }
337361
362 /// Decode `len` bytes of literals into `dest`. `literals` should be the
363 /// `LiteralsSection` that was passed to `prepare()`. Returns
364 /// `error.MalformedLiteralsLength` if the number of literal bytes decoded by
365 /// `self` plus `len` is greater than the regenerated size of `literals`.
366 /// Returns `error.UnexpectedEndOfLiteralStream` and `error.PrefixNotFound` if
367 /// there are problems decoding Huffman compressed literals.
338368 pub fn decodeLiteralsSlice(self: *DecodeState, dest: []u8, literals: LiteralsSection, len: usize) !void {
339369 if (self.literal_written_count + len > literals.header.regenerated_size) return error.MalformedLiteralsLength;
340370 switch (literals.header.block_type) {
......@@ -403,6 +433,7 @@ pub const DecodeState = struct {
403433 }
404434 }
405435
436 /// Decode literals into `dest`; see `decodeLiteralsSlice()`.
406437 pub fn decodeLiteralsRingBuffer(self: *DecodeState, dest: *RingBuffer, literals: LiteralsSection, len: usize) !void {
407438 if (self.literal_written_count + len > literals.header.regenerated_size) return error.MalformedLiteralsLength;
408439 switch (literals.header.block_type) {
......@@ -483,6 +514,13 @@ const literal_table_size_max = 1 << types.compressed_block.table_accuracy_log_ma
483514const match_table_size_max = 1 << types.compressed_block.table_accuracy_log_max.match;
484515const offset_table_size_max = 1 << types.compressed_block.table_accuracy_log_max.match;
485516
517/// Decode a Zstandard frame from `src` into `dest`, returning the number of
518/// bytes read from `src` and written to `dest`; if the frame does not declare
519/// its decompressed content size `error.UnknownContentSizeUnsupported` is
520/// returned. Returns `error.DictionaryIdFlagUnsupported` if the frame uses a
521/// dictionary, and `error.ChecksumFailure` if `verify_checksum` is `true` and
522/// the frame contains a checksum that does not match the checksum computed from
523/// the decompressed frame.
486524pub fn decodeZStandardFrame(dest: []u8, src: []const u8, verify_checksum: bool) !ReadWriteCount {
487525 assert(readInt(u32, src[0..4]) == frame.ZStandard.magic_number);
488526 var consumed_count: usize = 4;
......@@ -520,6 +558,10 @@ pub fn decodeZStandardFrame(dest: []u8, src: []const u8, verify_checksum: bool)
520558 return ReadWriteCount{ .read_count = consumed_count, .write_count = written_count };
521559}
522560
561/// Decode a Zstandard from from `src` and return the decompressed bytes; see
562/// `decodeZStandardFrame()`. Returns `error.WindowSizeUnknown` if the frame
563/// does not declare its content size or a window descriptor (this indicates a
564/// malformed frame).
523565pub fn decodeZStandardFrameAlloc(allocator: std.mem.Allocator, src: []const u8, verify_checksum: bool) ![]u8 {
524566 var result = std.ArrayList(u8).init(allocator);
525567 assert(readInt(u32, src[0..4]) == frame.ZStandard.magic_number);
......@@ -599,6 +641,7 @@ pub fn decodeZStandardFrameAlloc(allocator: std.mem.Allocator, src: []const u8,
599641 return result.toOwnedSlice();
600642}
601643
644/// Convenience wrapper for decoding all blocks in a frame; see `decodeBlock()`.
602645pub fn decodeFrameBlocks(dest: []u8, src: []const u8, consumed_count: *usize, hash: ?*std.hash.XxHash64) !usize {
603646 // These tables take 7680 bytes
604647 var literal_fse_data: [literal_table_size_max]Table.Fse = undefined;
......@@ -686,6 +729,10 @@ fn decodeRleBlockRingBuffer(dest: *RingBuffer, src: []const u8, block_size: u21,
686729 return block_size;
687730}
688731
732/// Decode a single block from `src` into `dest`. The beginning of `src` should
733/// be the start of the block content (i.e. directly after the block header).
734/// Increments `consumed_count` by the number of bytes read from `src` to decode
735/// the block and returns the decompressed size of the block.
689736pub fn decodeBlock(
690737 dest: []u8,
691738 src: []const u8,
......@@ -750,6 +797,9 @@ pub fn decodeBlock(
750797 }
751798}
752799
800/// Decode a single block from `src` into `dest`; see `decodeBlock()`. Returns
801/// the size of the decompressed block, which can be used with `dest.sliceLast()`
802/// to get the decompressed bytes.
753803pub fn decodeBlockRingBuffer(
754804 dest: *RingBuffer,
755805 src: []const u8,
......@@ -811,6 +861,7 @@ pub fn decodeBlockRingBuffer(
811861 }
812862}
813863
864/// Decode the header of a skippable frame.
814865pub fn decodeSkippableHeader(src: *const [8]u8) frame.Skippable.Header {
815866 const magic = readInt(u32, src[0..4]);
816867 assert(isSkippableMagic(magic));
......@@ -821,12 +872,15 @@ pub fn decodeSkippableHeader(src: *const [8]u8) frame.Skippable.Header {
821872 };
822873}
823874
824pub fn skippableFrameSize(src: *const [8]u8) !usize {
875/// Returns the content size of a skippable frame.
876pub fn skippableFrameSize(src: *const [8]u8) usize {
825877 assert(isSkippableMagic(readInt(u32, src[0..4])));
826878 const frame_size = readInt(u32, src[4..8]);
827879 return frame_size;
828880}
829881
882/// Returns the window size required to decompress a frame, or `null` if it cannot be
883/// determined, which indicates a malformed frame header.
830884pub fn frameWindowSize(header: frame.ZStandard.Header) ?u64 {
831885 if (header.window_descriptor) |descriptor| {
832886 const exponent = (descriptor & 0b11111000) >> 3;
......@@ -838,6 +892,8 @@ pub fn frameWindowSize(header: frame.ZStandard.Header) ?u64 {
838892 } else return header.content_size;
839893}
840894
895/// Decode the header of a Zstandard frame. Returns `error.UnusedBitSet` or
896/// `error.ReservedBitSet` if the corresponding bits are sets.
841897pub fn decodeZStandardHeader(src: []const u8, consumed_count: ?*usize) !frame.ZStandard.Header {
842898 const descriptor = @bitCast(frame.ZStandard.Header.Descriptor, src[0]);
843899
......@@ -879,6 +935,7 @@ pub fn decodeZStandardHeader(src: []const u8, consumed_count: ?*usize) !frame.ZS
879935 return header;
880936}
881937
938/// Decode the header of a block.
882939pub fn decodeBlockHeader(src: *const [3]u8) frame.ZStandard.Block.Header {
883940 const last_block = src[0] & 1 == 1;
884941 const block_type = @intToEnum(frame.ZStandard.Block.Type, (src[0] & 0b110) >> 1);
......@@ -890,6 +947,8 @@ pub fn decodeBlockHeader(src: *const [3]u8) frame.ZStandard.Block.Header {
890947 };
891948}
892949
950/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
951/// number of bytes the section uses.
893952pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !LiteralsSection {
894953 var bytes_read: usize = 0;
895954 const header = try decodeLiteralsHeader(src, &bytes_read);
......@@ -1107,6 +1166,7 @@ fn lessThanByWeight(
11071166 return weights[lhs.symbol] < weights[rhs.symbol];
11081167}
11091168
1169/// Decode a literals section header.
11101170pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) !LiteralsSection.Header {
11111171 if (src.len == 0) return error.MalformedLiteralsSection;
11121172 const byte0 = src[0];
......@@ -1172,6 +1232,7 @@ pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) !LiteralsSe
11721232 };
11731233}
11741234
1235/// Decode a sequences section header.
11751236pub fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !SequencesSection.Header {
11761237 if (src.len == 0) return error.MalformedSequencesSection;
11771238 var sequence_count: u24 = undefined;
......@@ -1241,7 +1302,8 @@ fn buildFseTable(values: []const u16, entries: []Table.Fse) !void {
12411302 if (value == 0 or value == 1) continue;
12421303 const probability = value - 1;
12431304
1244 const state_share_dividend = try std.math.ceilPowerOfTwo(u16, probability);
1305 const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
1306 return error.MalformedFseTable;
12451307 const share_size = @divExact(total_probability, state_share_dividend);
12461308 const double_state_count = state_share_dividend - probability;
12471309 const single_state_count = probability - double_state_count;
......@@ -1363,6 +1425,8 @@ const ReversedByteReader = struct {
13631425 }
13641426};
13651427
1428/// A bit reader for reading the reversed bit streams used to encode
1429/// FSE compressed data.
13661430pub const ReverseBitReader = struct {
13671431 byte_reader: ReversedByteReader,
13681432 bit_reader: std.io.BitReader(.Big, ReversedByteReader.Reader),