authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-05-01 18:23:21-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:28-07:00
log0d29c78af5d3f9c38c64399eec5509e11e10047a
treefe1ad3ec4326f67468b4a5f42ed943dcf46410a5
parent990031444dde4da223adcadfd2f2824f5baa0812

std.compress.zstd: tests passing

2 files changed, 43 insertions(+), 86 deletions(-)

lib/std/compress/zstd.zig+6-5
......@@ -81,12 +81,11 @@ pub const table_size_max = struct {
8181fn testDecompress(gpa: std.mem.Allocator, compressed: []const u8) ![]u8 {
8282 var out: std.ArrayListUnmanaged(u8) = .empty;
8383 defer out.deinit(gpa);
84 try out.ensureUnusedCapacity(gpa, default_window_len);
8584
8685 var in: std.io.BufferedReader = undefined;
8786 in.initFixed(@constCast(compressed));
8887 var zstd_stream: Decompress = .init(&in, .{});
89 try zstd_stream.reader().readRemainingArrayList(gpa, null, &out, .unlimited);
88 try zstd_stream.reader().readRemainingArrayList(gpa, null, &out, .unlimited, default_window_len);
9089
9190 return out.toOwnedSlice(gpa);
9291}
......@@ -103,16 +102,18 @@ fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void {
103102
104103 var out: std.ArrayListUnmanaged(u8) = .empty;
105104 defer out.deinit(gpa);
106 try out.ensureUnusedCapacity(gpa, default_window_len);
107105
108106 var in: std.io.BufferedReader = undefined;
109107 in.initFixed(@constCast(compressed));
110108 var zstd_stream: Decompress = .init(&in, .{});
111 try std.testing.expectError(error.ReadFailed, zstd_stream.reader().readRemainingArrayList(gpa, null, &out, .unlimited));
109 try std.testing.expectError(
110 error.ReadFailed,
111 zstd_stream.reader().readRemainingArrayList(gpa, null, &out, .unlimited, default_window_len),
112 );
112113 try std.testing.expectError(err, zstd_stream.err orelse {});
113114}
114115
115test "decompression" {
116test Decompress {
116117 const uncompressed = @embedFile("testdata/rfc8478.txt");
117118 const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
118119 const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
lib/std/compress/zstd/Decompress.zig+37-81
......@@ -149,7 +149,8 @@ fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state:
149149 const header_bytes = try in.takeArray(3);
150150 const block_header: Frame.Zstandard.Block.Header = @bitCast(header_bytes.*);
151151 const block_size = block_header.size;
152 if (state.frame.block_size_max < block_size) return error.BlockOversize;
152 const frame_block_size_max = state.frame.block_size_max;
153 if (frame_block_size_max < block_size) return error.BlockOversize;
153154 if (@intFromEnum(limit) < block_size) return error.OutputBufferUndersize;
154155 var bytes_written: usize = 0;
155156 switch (block_header.type) {
......@@ -185,17 +186,18 @@ fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state:
185186 if (sequences_header.sequence_count > 0) {
186187 try decode.readInitialFseState(&bit_stream);
187188
188 var sequence_size_limit = state.frame.block_size_max;
189 for (0..sequences_header.sequence_count) |i| {
190 const decompressed_size = try decode.decodeSequence(
191 bw,
192 &bit_stream,
193 sequence_size_limit,
194 i == sequences_header.sequence_count - 1,
195 );
196 sequence_size_limit -= decompressed_size;
197 bytes_written += decompressed_size;
189 // Ensures the following calls to `decodeSequence` will not flush.
190 if (frame_block_size_max > bw.buffer.len) return error.OutputBufferUndersize;
191 const dest = (try bw.writableSliceGreedy(frame_block_size_max))[0..frame_block_size_max];
192 for (0..sequences_header.sequence_count - 1) |_| {
193 bytes_written += try decode.decodeSequence(dest, bytes_written, &bit_stream);
194 try decode.updateState(.literal, &bit_stream);
195 try decode.updateState(.match, &bit_stream);
196 try decode.updateState(.offset, &bit_stream);
198197 }
198 bytes_written += try decode.decodeSequence(dest, bytes_written, &bit_stream);
199 if (bytes_written > dest.len) return error.MalformedSequence;
200 bw.advance(bytes_written);
199201 }
200202
201203 if (!bit_stream.isEmpty()) {
......@@ -206,6 +208,7 @@ fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state:
206208 if (decode.literal_written_count < literals.header.regenerated_size) {
207209 const len = literals.header.regenerated_size - decode.literal_written_count;
208210 try decode.decodeLiterals(bw, len);
211 decode.literal_written_count += len;
209212 bytes_written += len;
210213 }
211214
......@@ -216,8 +219,7 @@ fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state:
216219 .raw, .rle => {},
217220 }
218221
219 if (bytes_written > state.frame.block_size_max) return error.BlockOversize;
220 if (remaining.nonzero()) return error.MalformedCompressedBlock;
222 if (bytes_written > frame_block_size_max) return error.BlockOversize;
221223
222224 state.decompressed_size += bytes_written;
223225 if (state.frame.content_size) |size| {
......@@ -649,63 +651,35 @@ pub const Frame = struct {
649651 };
650652 }
651653
652 const DecodeSequenceError = error{
653 /// The decompressed sequence would be longer than
654 /// `sequence_size_limit` or the sequence's offset is too large
655 MalformedSequence,
656 /// The decoder state's literal streams do not contain enough
657 /// literals for the sequence (this may mean the literal stream or the
658 /// sequence is malformed).
659 UnexpectedEndOfLiteralStream,
660 /// The FSE sequence bitstream is malformed
661 InvalidBitStream,
662 /// `bit_reader` does not contain enough bits
663 EndOfStream,
664 /// The `BufferedWriter` storage capacity is not large enough to
665 /// accept this stream.
666 OutputBufferUndersize,
667 WriteFailed,
668 MalformedLiteralsLength,
669 MalformedFseBits,
670 MissingStartBit,
671 HuffmanTreeIncomplete,
672 };
673
674654 /// Decode one sequence from `bit_reader` into `dest`. Updates FSE states
675655 /// if `last_sequence` is `false`. Assumes `prepare` called for the block
676656 /// before attempting to decode sequences.
677657 fn decodeSequence(
678 self: *Decode,
679 dest: *BufferedWriter,
658 decode: *Decode,
659 dest: []u8,
660 write_pos: usize,
680661 bit_reader: *ReverseBitReader,
681 sequence_size_limit: usize,
682 last_sequence: bool,
683 ) DecodeSequenceError!usize {
684 const sequence = try self.nextSequence(bit_reader);
662 ) !usize {
663 const sequence = try decode.nextSequence(bit_reader);
685664 const literal_length: usize = sequence.literal_length;
686665 const match_length: usize = sequence.match_length;
687666 const sequence_length = literal_length + match_length;
688 if (sequence_length > sequence_size_limit) return error.MalformedSequence;
689 if (sequence_length > dest.buffer.len) return error.OutputBufferUndersize;
690
691 if (sequence.offset > literal_length) return error.MalformedSequence;
692 // Ensures the following call to `decodeLiterals` will not cause a
693 // flush and therefore be at the beginning of `out`.
694 const out = try dest.writableSlice(sequence_length);
695 const expected_end = dest.end + out.len;
696 try decodeLiterals(self, dest, literal_length);
697 @memmove(
698 out[literal_length..][0..match_length],
699 out[literal_length - sequence.offset ..][0..match_length],
700 );
701 dest.advance(match_length);
702 assert(dest.end == expected_end);
703
704 if (!last_sequence) {
705 try self.updateState(.literal, bit_reader);
706 try self.updateState(.match, bit_reader);
707 try self.updateState(.offset, bit_reader);
708 }
667
668 const copy_start = std.math.sub(usize, write_pos + sequence.literal_length, sequence.offset) catch
669 return error.MalformedSequence;
670
671 if (decode.literal_written_count + literal_length > decode.literal_header.regenerated_size)
672 return error.MalformedLiteralsLength;
673 var sub_bw: BufferedWriter = undefined;
674 sub_bw.initFixed(dest[write_pos..]);
675 try decodeLiterals(decode, &sub_bw, literal_length);
676 decode.literal_written_count += literal_length;
677 // This is not a @memmove; it intentionally repeats patterns
678 // caused by iterating one byte at a time.
679 for (
680 dest[write_pos + literal_length ..][0..match_length],
681 dest[copy_start..][0..match_length],
682 ) |*d, s| d.* = s;
709683 return sequence_length;
710684 }
711685
......@@ -744,31 +718,14 @@ pub const Frame = struct {
744718 };
745719 }
746720
747 const DecodeLiteralsError = error{
748 /// The number of literal bytes decoded by `self` plus `len` is greater
749 /// than the regenerated size of `literals`
750 MalformedLiteralsLength,
751 /// Problems decoding Huffman compressed literals
752 UnexpectedEndOfLiteralStream,
753 OutputBufferUndersize,
754 WriteFailed,
755 MissingStartBit,
756 HuffmanTreeIncomplete,
757 };
758
759721 /// Decode `len` bytes of literals into `dest`.
760 pub fn decodeLiterals(self: *Decode, dest: *BufferedWriter, len: usize) DecodeLiteralsError!void {
761 if (self.literal_written_count + len > self.literal_header.regenerated_size)
762 return error.MalformedLiteralsLength;
763
722 fn decodeLiterals(self: *Decode, dest: *BufferedWriter, len: usize) !void {
764723 switch (self.literal_header.block_type) {
765724 .raw => {
766725 try dest.writeAll(self.literal_streams.one[self.literal_written_count..][0..len]);
767 self.literal_written_count += len;
768726 },
769727 .rle => {
770728 try dest.splatByteAll(self.literal_streams.one[0], len);
771 self.literal_written_count += len;
772729 },
773730 .compressed, .treeless => {
774731 if (len > dest.buffer.len) return error.OutputBufferUndersize;
......@@ -810,7 +767,6 @@ pub const Frame = struct {
810767 }
811768 }
812769 }
813 self.literal_written_count += len;
814770 },
815771 }
816772 }