authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-10 01:33:38+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-20 09:09:06+11:00
logee5af3c74c27ebf366ef51486119487650f80468
treef86094baf16f8c00df2bd19855df6af962cb07d5
parent31cc4605aba68edc44a31f8383eaa39a906d6ec8

std.compress.zstandard: cleanup high-level api docs and error sets


5 files changed, 173 insertions(+), 69 deletions(-)

lib/std/compress/zstandard.zig+28-12
...@@ -7,7 +7,11 @@ const RingBuffer = @import("zstandard/RingBuffer.zig");...@@ -7,7 +7,11 @@ const RingBuffer = @import("zstandard/RingBuffer.zig");
7pub const decompress = @import("zstandard/decompress.zig");7pub const decompress = @import("zstandard/decompress.zig");
8pub usingnamespace @import("zstandard/types.zig");8pub usingnamespace @import("zstandard/types.zig");
99
10pub fn ZstandardStream(comptime ReaderType: type, comptime verify_checksum: bool, comptime window_size_max: usize) type {10pub fn ZstandardStream(
11 comptime ReaderType: type,
12 comptime verify_checksum: bool,
13 comptime window_size_max: usize,
14) type {
11 return struct {15 return struct {
12 const Self = @This();16 const Self = @This();
1317
...@@ -24,11 +28,16 @@ pub fn ZstandardStream(comptime ReaderType: type, comptime verify_checksum: bool...@@ -24,11 +28,16 @@ pub fn ZstandardStream(comptime ReaderType: type, comptime verify_checksum: bool
24 sequence_buffer: []u8,28 sequence_buffer: []u8,
25 checksum: if (verify_checksum) ?u32 else void,29 checksum: if (verify_checksum) ?u32 else void,
2630
27 pub const Error = ReaderType.Error || error{ ChecksumFailure, MalformedBlock, MalformedFrame, OutOfMemory };31 pub const Error = ReaderType.Error || error{
32 ChecksumFailure,
33 MalformedBlock,
34 MalformedFrame,
35 OutOfMemory,
36 };
2837
29 pub const Reader = std.io.Reader(*Self, Error, read);38 pub const Reader = std.io.Reader(*Self, Error, read);
3039
31 pub fn init(allocator: Allocator, source: ReaderType) !Self {40 pub fn init(allocator: Allocator, source: ReaderType) Self {
32 return Self{41 return Self{
33 .allocator = allocator,42 .allocator = allocator,
34 .source = std.io.countingReader(source),43 .source = std.io.countingReader(source),
...@@ -146,7 +155,8 @@ pub fn ZstandardStream(comptime ReaderType: type, comptime verify_checksum: bool...@@ -146,7 +155,8 @@ pub fn ZstandardStream(comptime ReaderType: type, comptime verify_checksum: bool
146155
147 const source_reader = self.source.reader();156 const source_reader = self.source.reader();
148 while (self.buffer.isEmpty() and self.state != .LastBlock) {157 while (self.buffer.isEmpty() and self.state != .LastBlock) {
149 const header_bytes = source_reader.readBytesNoEof(3) catch return error.MalformedFrame;158 const header_bytes = source_reader.readBytesNoEof(3) catch
159 return error.MalformedFrame;
150 const block_header = decompress.block.decodeBlockHeader(&header_bytes);160 const block_header = decompress.block.decodeBlockHeader(&header_bytes);
151161
152 decompress.block.decodeBlockReader(162 decompress.block.decodeBlockReader(
...@@ -171,10 +181,12 @@ pub fn ZstandardStream(comptime ReaderType: type, comptime verify_checksum: bool...@@ -171,10 +181,12 @@ pub fn ZstandardStream(comptime ReaderType: type, comptime verify_checksum: bool
171 if (block_header.last_block) {181 if (block_header.last_block) {
172 self.state = .LastBlock;182 self.state = .LastBlock;
173 if (self.frame_context.has_checksum) {183 if (self.frame_context.has_checksum) {
174 const checksum = source_reader.readIntLittle(u32) catch return error.MalformedFrame;184 const checksum = source_reader.readIntLittle(u32) catch
185 return error.MalformedFrame;
175 if (comptime verify_checksum) {186 if (comptime verify_checksum) {
176 if (self.frame_context.hasher_opt) |*hasher| {187 if (self.frame_context.hasher_opt) |*hasher| {
177 if (checksum != decompress.computeChecksum(hasher)) return error.ChecksumFailure;188 if (checksum != decompress.computeChecksum(hasher))
189 return error.ChecksumFailure;
178 }190 }
179 }191 }
180 }192 }
...@@ -182,9 +194,9 @@ pub fn ZstandardStream(comptime ReaderType: type, comptime verify_checksum: bool...@@ -182,9 +194,9 @@ pub fn ZstandardStream(comptime ReaderType: type, comptime verify_checksum: bool
182 }194 }
183195
184 const decoded_data_len = self.buffer.len();196 const decoded_data_len = self.buffer.len();
185 var written_count: usize = 0;197 var count: usize = 0;
186 while (written_count < decoded_data_len and written_count < buffer.len) : (written_count += 1) {198 while (count < decoded_data_len and count < buffer.len) : (count += 1) {
187 buffer[written_count] = self.buffer.read().?;199 buffer[count] = self.buffer.read().?;
188 }200 }
189 if (self.state == .LastBlock and self.buffer.len() == 0) {201 if (self.state == .LastBlock and self.buffer.len() == 0) {
190 self.state = .NewFrame;202 self.state = .NewFrame;
...@@ -195,18 +207,22 @@ pub fn ZstandardStream(comptime ReaderType: type, comptime verify_checksum: bool...@@ -195,18 +207,22 @@ pub fn ZstandardStream(comptime ReaderType: type, comptime verify_checksum: bool
195 self.allocator.free(self.sequence_buffer);207 self.allocator.free(self.sequence_buffer);
196 self.buffer.deinit(self.allocator);208 self.buffer.deinit(self.allocator);
197 }209 }
198 return written_count;210 return count;
199 }211 }
200 };212 };
201}213}
202214
203pub fn zstandardStream(allocator: Allocator, reader: anytype) !ZstandardStream(@TypeOf(reader), true, 8 * (1 << 20)) {215pub fn zstandardStream(
216 allocator: Allocator,
217 reader: anytype,
218 comptime window_size_max: usize,
219) ZstandardStream(@TypeOf(reader), true, window_size_max) {
204 return ZstandardStream(@TypeOf(reader), true, 8 * (1 << 20)).init(allocator, reader);220 return ZstandardStream(@TypeOf(reader), true, 8 * (1 << 20)).init(allocator, reader);
205}221}
206222
207fn testDecompress(data: []const u8) ![]u8 {223fn testDecompress(data: []const u8) ![]u8 {
208 var in_stream = std.io.fixedBufferStream(data);224 var in_stream = std.io.fixedBufferStream(data);
209 var stream = try zstandardStream(std.testing.allocator, in_stream.reader());225 var stream = zstandardStream(std.testing.allocator, in_stream.reader(), 1 << 23);
210 defer stream.deinit();226 defer stream.deinit();
211 const result = stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));227 const result = stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
212 return result;228 return result;
lib/std/compress/zstandard/RingBuffer.zig+6-3
...@@ -13,6 +13,8 @@ data: []u8,...@@ -13,6 +13,8 @@ data: []u8,
13read_index: usize,13read_index: usize,
14write_index: usize,14write_index: usize,
1515
16pub const Error = error{Full};
17
16/// Allocate a new `RingBuffer`18/// Allocate a new `RingBuffer`
17pub fn init(allocator: Allocator, capacity: usize) Allocator.Error!RingBuffer {19pub fn init(allocator: Allocator, capacity: usize) Allocator.Error!RingBuffer {
18 const bytes = try allocator.alloc(u8, capacity);20 const bytes = try allocator.alloc(u8, capacity);
...@@ -41,7 +43,7 @@ pub fn mask2(self: RingBuffer, index: usize) usize {...@@ -41,7 +43,7 @@ pub fn mask2(self: RingBuffer, index: usize) usize {
4143
42/// Write `byte` into the ring buffer. Returns `error.Full` if the ring44/// Write `byte` into the ring buffer. Returns `error.Full` if the ring
43/// buffer is full.45/// buffer is full.
44pub fn write(self: *RingBuffer, byte: u8) !void {46pub fn write(self: *RingBuffer, byte: u8) Error!void {
45 if (self.isFull()) return error.Full;47 if (self.isFull()) return error.Full;
46 self.writeAssumeCapacity(byte);48 self.writeAssumeCapacity(byte);
47}49}
...@@ -55,7 +57,7 @@ pub fn writeAssumeCapacity(self: *RingBuffer, byte: u8) void {...@@ -55,7 +57,7 @@ pub fn writeAssumeCapacity(self: *RingBuffer, byte: u8) void {
5557
56/// Write `bytes` into the ring bufffer. Returns `error.Full` if the ring58/// Write `bytes` into the ring bufffer. Returns `error.Full` if the ring
57/// buffer does not have enough space, without writing any data.59/// buffer does not have enough space, without writing any data.
58pub fn writeSlice(self: *RingBuffer, bytes: []const u8) !void {60pub fn writeSlice(self: *RingBuffer, bytes: []const u8) Error!void {
59 if (self.len() + bytes.len > self.data.len) return error.Full;61 if (self.len() + bytes.len > self.data.len) return error.Full;
60 self.writeSliceAssumeCapacity(bytes);62 self.writeSliceAssumeCapacity(bytes);
61}63}
...@@ -87,7 +89,8 @@ pub fn isFull(self: RingBuffer) bool {...@@ -87,7 +89,8 @@ pub fn isFull(self: RingBuffer) bool {
8789
88/// Returns the length90/// Returns the length
89pub fn len(self: RingBuffer) usize {91pub fn len(self: RingBuffer) usize {
90 const adjusted_write_index = self.write_index + @as(usize, @boolToInt(self.write_index < self.read_index)) * 2 * self.data.len;92 const wrap_offset = 2 * self.data.len * @boolToInt(self.write_index < self.read_index);
93 const adjusted_write_index = self.write_index + wrap_offset;
91 return adjusted_write_index - self.read_index;94 return adjusted_write_index - self.read_index;
92}95}
9396
lib/std/compress/zstandard/decode/block.zig+3-3
...@@ -413,7 +413,7 @@ pub const DecodeState = struct {...@@ -413,7 +413,7 @@ pub const DecodeState = struct {
413413
414 const DecodeLiteralsError = error{414 const DecodeLiteralsError = error{
415 MalformedLiteralsLength,415 MalformedLiteralsLength,
416 PrefixNotFound,416 NotFound,
417 } || LiteralBitsError;417 } || LiteralBitsError;
418418
419 /// Decode `len` bytes of literals into `dest`.419 /// Decode `len` bytes of literals into `dest`.
...@@ -422,8 +422,8 @@ pub const DecodeState = struct {...@@ -422,8 +422,8 @@ pub const DecodeState = struct {
422 /// - `error.MalformedLiteralsLength` if the number of literal bytes422 /// - `error.MalformedLiteralsLength` if the number of literal bytes
423 /// decoded by `self` plus `len` is greater than the regenerated size of423 /// decoded by `self` plus `len` is greater than the regenerated size of
424 /// `literals`424 /// `literals`
425 /// - `error.UnexpectedEndOfLiteralStream` and `error.PrefixNotFound` if425 /// - `error.UnexpectedEndOfLiteralStream` and `error.NotFound` if there
426 /// there are problems decoding Huffman compressed literals426 /// are problems decoding Huffman compressed literals
427 pub fn decodeLiteralsSlice(427 pub fn decodeLiteralsSlice(
428 self: *DecodeState,428 self: *DecodeState,
429 dest: []u8,429 dest: []u8,
lib/std/compress/zstandard/decompress.zig+126-43
...@@ -6,6 +6,8 @@ const types = @import("types.zig");...@@ -6,6 +6,8 @@ const types = @import("types.zig");
6const frame = types.frame;6const frame = types.frame;
7const LiteralsSection = types.compressed_block.LiteralsSection;7const LiteralsSection = types.compressed_block.LiteralsSection;
8const SequencesSection = types.compressed_block.SequencesSection;8const SequencesSection = types.compressed_block.SequencesSection;
9const SkippableHeader = types.frame.Skippable.Header;
10const ZstandardHeader = types.frame.Zstandard.Header;
9const Table = types.compressed_block.Table;11const Table = types.compressed_block.Table;
1012
11pub const block = @import("decode/block.zig");13pub const block = @import("decode/block.zig");
...@@ -16,15 +18,13 @@ const readers = @import("readers.zig");...@@ -16,15 +18,13 @@ const readers = @import("readers.zig");
1618
17const readInt = std.mem.readIntLittle;19const readInt = std.mem.readIntLittle;
18const readIntSlice = std.mem.readIntSliceLittle;20const readIntSlice = std.mem.readIntSliceLittle;
19fn readVarInt(comptime T: type, bytes: []const u8) T {
20 return std.mem.readVarInt(T, bytes, .Little);
21}
2221
22/// Returns `true` is `magic` is a valid magic number for a skippable frame
23pub fn isSkippableMagic(magic: u32) bool {23pub fn isSkippableMagic(magic: u32) bool {
24 return frame.Skippable.magic_number_min <= magic and magic <= frame.Skippable.magic_number_max;24 return frame.Skippable.magic_number_min <= magic and magic <= frame.Skippable.magic_number_max;
25}25}
2626
27/// Returns the kind of frame at the beginning of `src`.27/// Returns the kind of frame at the beginning of `source`.
28///28///
29/// Errors returned:29/// Errors returned:
30/// - `error.BadMagic` if `source` begins with bytes not equal to the30/// - `error.BadMagic` if `source` begins with bytes not equal to the
...@@ -50,11 +50,22 @@ pub fn frameType(magic: u32) error{BadMagic}!frame.Kind {...@@ -50,11 +50,22 @@ pub fn frameType(magic: u32) error{BadMagic}!frame.Kind {
50}50}
5151
52pub const FrameHeader = union(enum) {52pub const FrameHeader = union(enum) {
53 zstandard: types.frame.Zstandard.Header,53 zstandard: ZstandardHeader,
54 skippable: types.frame.Skippable.Header,54 skippable: SkippableHeader,
55};55};
5656
57pub fn decodeFrameHeader(source: anytype) error{ BadMagic, EndOfStream, ReservedBitSet }!FrameHeader {57pub const HeaderError = error{ BadMagic, EndOfStream, ReservedBitSet };
58
59/// Returns the header of the frame at the beginning of `source`.
60///
61/// Errors returned:
62/// - `error.BadMagic` if `source` begins with bytes not equal to the
63/// Zstandard frame magic number, or outside the range of magic numbers for
64/// skippable frames.
65/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
66/// - `error.ReservedBitSet` if the frame is a Zstandard frame and any of the
67/// reserved bits are set
68pub fn decodeFrameHeader(source: anytype) HeaderError!FrameHeader {
58 const magic = try source.readIntLittle(u32);69 const magic = try source.readIntLittle(u32);
59 const frame_type = try frameType(magic);70 const frame_type = try frameType(magic);
60 switch (frame_type) {71 switch (frame_type) {
...@@ -68,41 +79,74 @@ pub fn decodeFrameHeader(source: anytype) error{ BadMagic, EndOfStream, Reserved...@@ -68,41 +79,74 @@ pub fn decodeFrameHeader(source: anytype) error{ BadMagic, EndOfStream, Reserved
68 }79 }
69}80}
7081
71const ReadWriteCount = struct {82pub const ReadWriteCount = struct {
72 read_count: usize,83 read_count: usize,
73 write_count: usize,84 write_count: usize,
74};85};
7586
76/// Decodes frames from `src` into `dest`; see `decodeFrame()`.87/// Decodes frames from `src` into `dest`; returns the length of the result.
77pub fn decode(dest: []u8, src: []const u8, verify_checksum: bool) !usize {88/// The stream should not have extra trailing bytes - either all bytes in `src`
89/// will be decoded, or an error will be returned. An error will be returned if
90/// a Zstandard frame in `src` does not declare its content size.
91///
92/// Errors returned:
93/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
94/// uses a dictionary
95/// - `error.MalformedFrame` if a frame in `src` is invalid
96/// - `error.UnknownContentSizeUnsupported` if a frame in `src` does not
97/// declare its content size
98pub fn decode(dest: []u8, src: []const u8, verify_checksum: bool) error{
99 MalformedFrame,
100 UnknownContentSizeUnsupported,
101 DictionaryIdFlagUnsupported,
102}!usize {
78 var write_count: usize = 0;103 var write_count: usize = 0;
79 var read_count: usize = 0;104 var read_count: usize = 0;
80 while (read_count < src.len) {105 while (read_count < src.len) {
81 const counts = try decodeFrame(dest, src[read_count..], verify_checksum);106 const counts = decodeFrame(dest, src[read_count..], verify_checksum) catch |err| {
107 switch (err) {
108 error.UnknownContentSizeUnsupported => return error.UnknownContentSizeUnsupported,
109 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
110 else => return error.MalformedFrame,
111 }
112 };
82 read_count += counts.read_count;113 read_count += counts.read_count;
83 write_count += counts.write_count;114 write_count += counts.write_count;
84 }115 }
85 return write_count;116 return write_count;
86}117}
87118
119/// Decodes a stream of frames from `src`; returns the decoded bytes. The stream
120/// should not have extra trailing bytes - either all bytes in `src` will be
121/// decoded, or an error will be returned.
122///
123/// Errors returned:
124/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
125/// uses a dictionary
126/// - `error.MalformedFrame` if a frame in `src` is invalid
127/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
88pub fn decodeAlloc(128pub fn decodeAlloc(
89 allocator: Allocator,129 allocator: Allocator,
90 src: []const u8,130 src: []const u8,
91 verify_checksum: bool,131 verify_checksum: bool,
92 window_size_max: usize,132 window_size_max: usize,
93) ![]u8 {133) error{ DictionaryIdFlagUnsupported, MalformedFrame, OutOfMemory }![]u8 {
94 var result = std.ArrayList(u8).init(allocator);134 var result = std.ArrayList(u8).init(allocator);
95 errdefer result.deinit();135 errdefer result.deinit();
96136
97 var read_count: usize = 0;137 var read_count: usize = 0;
98 while (read_count < src.len) {138 while (read_count < src.len) {
99 read_count += try decodeFrameArrayList(139 read_count += decodeFrameArrayList(
100 allocator,140 allocator,
101 &result,141 &result,
102 src[read_count..],142 src[read_count..],
103 verify_checksum,143 verify_checksum,
104 window_size_max,144 window_size_max,
105 );145 ) catch |err| switch (err) {
146 error.OutOfMemory => return error.OutOfMemory,
147 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
148 else => return error.MalformedFrame,
149 };
106 }150 }
107 return result.toOwnedSlice();151 return result.toOwnedSlice();
108}152}
...@@ -112,18 +156,24 @@ pub fn decodeAlloc(...@@ -112,18 +156,24 @@ pub fn decodeAlloc(
112/// frames that declare the decompressed content size.156/// frames that declare the decompressed content size.
113///157///
114/// Errors returned:158/// Errors returned:
159/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
160/// number for a Zstandard or skippable frame
115/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the161/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
116/// uncompressed content size162/// uncompressed content size
163/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
117/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data164/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
118/// size declared by the frame header165/// size declared by the frame header
119/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic166/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
120/// number for a Zstandard or Skippable frame167/// that is larger than `std.math.maxInt(usize)`
121/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary168/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
122/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame169/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
123/// contains a checksum that does not match the checksum of the decompressed170/// contains a checksum that does not match the checksum of the decompressed
124/// data171/// data
125/// - `error.ReservedBitSet` if the reserved bit of the frame header is set172/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
173/// are set
126/// - `error.EndOfStream` if `src` does not contain a complete frame174/// - `error.EndOfStream` if `src` does not contain a complete frame
175/// - `error.BadContentSize` if the content size declared by the frame does
176/// not equal the actual size of decompressed data
127/// - an error in `block.Error` if there are errors decoding a block177/// - an error in `block.Error` if there are errors decoding a block
128/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a178/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
129/// size greater than `src.len`179/// size greater than `src.len`
...@@ -131,7 +181,15 @@ pub fn decodeFrame(...@@ -131,7 +181,15 @@ pub fn decodeFrame(
131 dest: []u8,181 dest: []u8,
132 src: []const u8,182 src: []const u8,
133 verify_checksum: bool,183 verify_checksum: bool,
134) !ReadWriteCount {184) (error{
185 BadMagic,
186 UnknownContentSizeUnsupported,
187 ContentTooLarge,
188 ContentSizeTooLarge,
189 WindowSizeUnknown,
190 DictionaryIdFlagUnsupported,
191 SkippableSizeTooLarge,
192} || FrameError)!ReadWriteCount {
135 var fbs = std.io.fixedBufferStream(src);193 var fbs = std.io.fixedBufferStream(src);
136 switch (try decodeFrameType(fbs.reader())) {194 switch (try decodeFrameType(fbs.reader())) {
137 .zstandard => return decodeZstandardFrame(dest, src, verify_checksum),195 .zstandard => return decodeZstandardFrame(dest, src, verify_checksum),
...@@ -153,16 +211,21 @@ pub fn decodeFrame(...@@ -153,16 +211,21 @@ pub fn decodeFrame(
153///211///
154/// Errors returned:212/// Errors returned:
155/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic213/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
156/// number for a Zstandard or Skippable frame214/// number for a Zstandard or skippable frame
157/// - `error.WindowSizeUnknown` if the frame does not have a valid window size215/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
158/// - `error.WindowTooLarge` if the window size is larger than216/// - `error.WindowTooLarge` if the window size is larger than
159/// `window_size_max`217/// `window_size_max`
218/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
219/// that is larger than `std.math.maxInt(usize)`
160/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary220/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
161/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame221/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
162/// contains a checksum that does not match the checksum of the decompressed222/// contains a checksum that does not match the checksum of the decompressed
163/// data223/// data
164/// - `error.ReservedBitSet` if the reserved bit of the frame header is set224/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
225/// are set
165/// - `error.EndOfStream` if `src` does not contain a complete frame226/// - `error.EndOfStream` if `src` does not contain a complete frame
227/// - `error.BadContentSize` if the content size declared by the frame does
228/// not equal the actual size of decompressed data
166/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory229/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
167/// - an error in `block.Error` if there are errors decoding a block230/// - an error in `block.Error` if there are errors decoding a block
168/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a231/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
...@@ -173,12 +236,18 @@ pub fn decodeFrameArrayList(...@@ -173,12 +236,18 @@ pub fn decodeFrameArrayList(
173 src: []const u8,236 src: []const u8,
174 verify_checksum: bool,237 verify_checksum: bool,
175 window_size_max: usize,238 window_size_max: usize,
176) !usize {239) (error{ BadMagic, OutOfMemory, SkippableSizeTooLarge } || FrameContext.Error || FrameError)!usize {
177 var fbs = std.io.fixedBufferStream(src);240 var fbs = std.io.fixedBufferStream(src);
178 const reader = fbs.reader();241 const reader = fbs.reader();
179 const magic = try reader.readIntLittle(u32);242 const magic = try reader.readIntLittle(u32);
180 switch (try frameType(magic)) {243 switch (try frameType(magic)) {
181 .zstandard => return decodeZstandardFrameArrayList(allocator, dest, src, verify_checksum, window_size_max),244 .zstandard => return decodeZstandardFrameArrayList(
245 allocator,
246 dest,
247 src,
248 verify_checksum,
249 window_size_max,
250 ),
182 .skippable => {251 .skippable => {
183 const content_size = try fbs.reader().readIntLittle(u32);252 const content_size = try fbs.reader().readIntLittle(u32);
184 if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;253 if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;
...@@ -211,7 +280,10 @@ const FrameError = error{...@@ -211,7 +280,10 @@ const FrameError = error{
211/// uncompressed content size280/// uncompressed content size
212/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data281/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
213/// size declared by the frame header282/// size declared by the frame header
283/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
214/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary284/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
285/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
286/// that is larger than `std.math.maxInt(usize)`
215/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame287/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
216/// contains a checksum that does not match the checksum of the decompressed288/// contains a checksum that does not match the checksum of the decompressed
217/// data289/// data
...@@ -239,7 +311,11 @@ pub fn decodeZstandardFrame(...@@ -239,7 +311,11 @@ pub fn decodeZstandardFrame(
239 var source = fbs.reader();311 var source = fbs.reader();
240 const frame_header = try decodeZstandardHeader(source);312 const frame_header = try decodeZstandardHeader(source);
241 consumed_count += fbs.pos;313 consumed_count += fbs.pos;
242 break :context FrameContext.init(frame_header, std.math.maxInt(usize), verify_checksum) catch |err| switch (err) {314 break :context FrameContext.init(
315 frame_header,
316 std.math.maxInt(usize),
317 verify_checksum,
318 ) catch |err| switch (err) {
243 error.WindowTooLarge => unreachable,319 error.WindowTooLarge => unreachable,
244 inline else => |e| return e,320 inline else => |e| return e,
245 };321 };
...@@ -260,7 +336,8 @@ pub fn decodeZStandardFrameBlocks(...@@ -260,7 +336,8 @@ pub fn decodeZStandardFrameBlocks(
260 src: []const u8,336 src: []const u8,
261 frame_context: *FrameContext,337 frame_context: *FrameContext,
262) (error{ ContentTooLarge, UnknownContentSizeUnsupported } || FrameError)!ReadWriteCount {338) (error{ ContentTooLarge, UnknownContentSizeUnsupported } || FrameError)!ReadWriteCount {
263 const content_size = frame_context.content_size orelse return error.UnknownContentSizeUnsupported;339 const content_size = frame_context.content_size orelse
340 return error.UnknownContentSizeUnsupported;
264 if (dest.len < content_size) return error.ContentTooLarge;341 if (dest.len < content_size) return error.ContentTooLarge;
265342
266 var consumed_count: usize = 0;343 var consumed_count: usize = 0;
...@@ -304,14 +381,19 @@ pub const FrameContext = struct {...@@ -304,14 +381,19 @@ pub const FrameContext = struct {
304 ///381 ///
305 /// Errors returned:382 /// Errors returned:
306 /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary383 /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
307 /// - `error.WindowSizeUnknown` if the frame does not have a valid window size384 /// - `error.WindowSizeUnknown` if the frame does not have a valid window
308 /// - `error.WindowTooLarge` if the window size is larger than `window_size_max`385 /// size
386 /// - `error.WindowTooLarge` if the window size is larger than
387 /// `window_size_max`
388 /// - `error.ContentSizeTooLarge` if the frame header indicates a content
389 /// size larger than `std.math.maxInt(usize)`
309 pub fn init(390 pub fn init(
310 frame_header: frame.Zstandard.Header,391 frame_header: ZstandardHeader,
311 window_size_max: usize,392 window_size_max: usize,
312 verify_checksum: bool,393 verify_checksum: bool,
313 ) Error!FrameContext {394 ) Error!FrameContext {
314 if (frame_header.descriptor.dictionary_id_flag != 0) return error.DictionaryIdFlagUnsupported;395 if (frame_header.descriptor.dictionary_id_flag != 0)
396 return error.DictionaryIdFlagUnsupported;
315397
316 const window_size_raw = frameWindowSize(frame_header) orelse return error.WindowSizeUnknown;398 const window_size_raw = frameWindowSize(frame_header) orelse return error.WindowSizeUnknown;
317 const window_size = if (window_size_raw > window_size_max)399 const window_size = if (window_size_raw > window_size_max)
...@@ -319,7 +401,8 @@ pub const FrameContext = struct {...@@ -319,7 +401,8 @@ pub const FrameContext = struct {
319 else401 else
320 @intCast(usize, window_size_raw);402 @intCast(usize, window_size_raw);
321403
322 const should_compute_checksum = frame_header.descriptor.content_checksum_flag and verify_checksum;404 const should_compute_checksum =
405 frame_header.descriptor.content_checksum_flag and verify_checksum;
323406
324 const content_size = if (frame_header.content_size) |size|407 const content_size = if (frame_header.content_size) |size|
325 std.math.cast(usize, size) orelse return error.ContentSizeTooLarge408 std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
...@@ -345,6 +428,8 @@ pub const FrameContext = struct {...@@ -345,6 +428,8 @@ pub const FrameContext = struct {
345/// - `error.WindowTooLarge` if the window size is larger than428/// - `error.WindowTooLarge` if the window size is larger than
346/// `window_size_max`429/// `window_size_max`
347/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary430/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
431/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
432/// that is larger than `std.math.maxInt(usize)`
348/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame433/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
349/// contains a checksum that does not match the checksum of the decompressed434/// contains a checksum that does not match the checksum of the decompressed
350/// data435/// data
...@@ -441,8 +526,6 @@ pub fn decodeZstandardFrameBlocksArrayList(...@@ -441,8 +526,6 @@ pub fn decodeZstandardFrameBlocksArrayList(
441 return consumed_count;526 return consumed_count;
442}527}
443528
444/// Convenience wrapper for decoding all blocks in a frame; see
445/// `decodeZStandardFrameBlocks()`.
446fn decodeFrameBlocksInner(529fn decodeFrameBlocksInner(
447 dest: []u8,530 dest: []u8,
448 src: []const u8,531 src: []const u8,
...@@ -459,7 +542,7 @@ fn decodeFrameBlocksInner(...@@ -459,7 +542,7 @@ fn decodeFrameBlocksInner(
459 var bytes_read: usize = 3;542 var bytes_read: usize = 3;
460 defer consumed_count.* += bytes_read;543 defer consumed_count.* += bytes_read;
461 var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);544 var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);
462 var written_count: usize = 0;545 var count: usize = 0;
463 while (true) : ({546 while (true) : ({
464 block_header = try block.decodeBlockHeaderSlice(src[bytes_read..]);547 block_header = try block.decodeBlockHeaderSlice(src[bytes_read..]);
465 bytes_read += 3;548 bytes_read += 3;
...@@ -471,18 +554,18 @@ fn decodeFrameBlocksInner(...@@ -471,18 +554,18 @@ fn decodeFrameBlocksInner(
471 &decode_state,554 &decode_state,
472 &bytes_read,555 &bytes_read,
473 block_size_max,556 block_size_max,
474 written_count,557 count,
475 );558 );
476 if (hash) |hash_state| hash_state.update(dest[written_count .. written_count + written_size]);559 if (hash) |hash_state| hash_state.update(dest[count .. count + written_size]);
477 written_count += written_size;560 count += written_size;
478 if (block_header.last_block) break;561 if (block_header.last_block) break;
479 }562 }
480 return written_count;563 return count;
481}564}
482565
483/// Decode the header of a skippable frame. The first four bytes of `src` must566/// Decode the header of a skippable frame. The first four bytes of `src` must
484/// be a valid magic number for a Skippable frame.567/// be a valid magic number for a skippable frame.
485pub fn decodeSkippableHeader(src: *const [8]u8) frame.Skippable.Header {568pub fn decodeSkippableHeader(src: *const [8]u8) SkippableHeader {
486 const magic = readInt(u32, src[0..4]);569 const magic = readInt(u32, src[0..4]);
487 assert(isSkippableMagic(magic));570 assert(isSkippableMagic(magic));
488 const frame_size = readInt(u32, src[4..8]);571 const frame_size = readInt(u32, src[4..8]);
...@@ -494,7 +577,7 @@ pub fn decodeSkippableHeader(src: *const [8]u8) frame.Skippable.Header {...@@ -494,7 +577,7 @@ pub fn decodeSkippableHeader(src: *const [8]u8) frame.Skippable.Header {
494577
495/// Returns the window size required to decompress a frame, or `null` if it578/// Returns the window size required to decompress a frame, or `null` if it
496/// cannot be determined (which indicates a malformed frame header).579/// cannot be determined (which indicates a malformed frame header).
497pub fn frameWindowSize(header: frame.Zstandard.Header) ?u64 {580pub fn frameWindowSize(header: ZstandardHeader) ?u64 {
498 if (header.window_descriptor) |descriptor| {581 if (header.window_descriptor) |descriptor| {
499 const exponent = (descriptor & 0b11111000) >> 3;582 const exponent = (descriptor & 0b11111000) >> 3;
500 const mantissa = descriptor & 0b00000111;583 const mantissa = descriptor & 0b00000111;
...@@ -508,10 +591,10 @@ pub fn frameWindowSize(header: frame.Zstandard.Header) ?u64 {...@@ -508,10 +591,10 @@ pub fn frameWindowSize(header: frame.Zstandard.Header) ?u64 {
508/// Decode the header of a Zstandard frame.591/// Decode the header of a Zstandard frame.
509///592///
510/// Errors returned:593/// Errors returned:
511/// - `error.ReservedBitSet` if the reserved bits of the header are set594/// - `error.ReservedBitSet` if any of the reserved bits of the header are set
512/// - `error.EndOfStream` if `source` does not contain a complete header595/// - `error.EndOfStream` if `source` does not contain a complete header
513pub fn decodeZstandardHeader(source: anytype) error{ EndOfStream, ReservedBitSet }!frame.Zstandard.Header {596pub fn decodeZstandardHeader(source: anytype) error{ EndOfStream, ReservedBitSet }!ZstandardHeader {
514 const descriptor = @bitCast(frame.Zstandard.Header.Descriptor, try source.readByte());597 const descriptor = @bitCast(ZstandardHeader.Descriptor, try source.readByte());
515598
516 if (descriptor.reserved) return error.ReservedBitSet;599 if (descriptor.reserved) return error.ReservedBitSet;
517600
...@@ -534,7 +617,7 @@ pub fn decodeZstandardHeader(source: anytype) error{ EndOfStream, ReservedBitSet...@@ -534,7 +617,7 @@ pub fn decodeZstandardHeader(source: anytype) error{ EndOfStream, ReservedBitSet
534 if (field_size == 2) content_size.? += 256;617 if (field_size == 2) content_size.? += 256;
535 }618 }
536619
537 const header = frame.Zstandard.Header{620 const header = ZstandardHeader{
538 .descriptor = descriptor,621 .descriptor = descriptor,
539 .window_descriptor = window_descriptor,622 .window_descriptor = window_descriptor,
540 .dictionary_id = dictionary_id,623 .dictionary_id = dictionary_id,
lib/std/compress/zstandard/types.zig+10-8
...@@ -92,13 +92,13 @@ pub const compressed_block = struct {...@@ -92,13 +92,13 @@ pub const compressed_block = struct {
92 index: usize,92 index: usize,
93 };93 };
9494
95 pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{PrefixNotFound}!Result {95 pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{NotFound}!Result {
96 var node = self.nodes[index];96 var node = self.nodes[index];
97 const weight = node.weight;97 const weight = node.weight;
98 var i: usize = index;98 var i: usize = index;
99 while (node.weight == weight) {99 while (node.weight == weight) {
100 if (node.prefix == prefix) return Result{ .symbol = node.symbol };100 if (node.prefix == prefix) return Result{ .symbol = node.symbol };
101 if (i == 0) return error.PrefixNotFound;101 if (i == 0) return error.NotFound;
102 i -= 1;102 i -= 1;
103 node = self.nodes[i];103 node = self.nodes[i];
104 }104 }
...@@ -164,12 +164,14 @@ pub const compressed_block = struct {...@@ -164,12 +164,14 @@ pub const compressed_block = struct {
164 };164 };
165165
166 pub const match_length_code_table = [53]struct { u32, u5 }{166 pub const match_length_code_table = [53]struct { u32, u5 }{
167 .{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 },167 .{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
168 .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 },168 .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
169 .{ 19, 0 }, .{ 20, 0 }, .{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },169 .{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
170 .{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 }, .{ 33, 0 }, .{ 34, 0 },170 .{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
171 .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 }, .{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 },171 .{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
172 .{ 67, 4 }, .{ 83, 4 }, .{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },172 .{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
173 .{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
174 .{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
173 .{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },175 .{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
174 };176 };
175177