authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-21 13:59:14-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-02-21 13:59:14-05:00
logb52be973dfb7d1408218b8e75800a2da3dc69108
tree4fd6baa7704ddba74b00315f65f9daf5be9c40a7
parent98dd041d536aad1d4936353b4f5e4a2e0aab0fe1
parent765a6d34139771cb94c73e0af71b02db2f1e0f98
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14394 from dweiller/zstandard

Zstandard decompressor

16 files changed, 6383 insertions(+), 0 deletions(-)

build.zig+3
...@@ -113,8 +113,11 @@ pub fn build(b: *std.Build) !void {...@@ -113,8 +113,11 @@ pub fn build(b: *std.Build) !void {
113 ".gz",113 ".gz",
114 ".z.0",114 ".z.0",
115 ".z.9",115 ".z.9",
116 ".zstd.3",
117 ".zstd.19",
116 "rfc1951.txt",118 "rfc1951.txt",
117 "rfc1952.txt",119 "rfc1952.txt",
120 "rfc8478.txt",
118 // exclude files from lib/std/compress/deflate/testdata121 // exclude files from lib/std/compress/deflate/testdata
119 ".expect",122 ".expect",
120 ".expect-noinput",123 ".expect-noinput",
lib/std/RingBuffer.zig created+136
...@@ -0,0 +1,136 @@
1//! This ring buffer stores read and write indices while being able to utilise
2//! the full backing slice by incrementing the indices modulo twice the slice's
3//! length and reducing indices modulo the slice's length on slice access. This
4//! means that whether the ring buffer if full or empty can be distinguished by
5//! looking at the difference between the read and write indices without adding
6//! an extra boolean flag or having to reserve a slot in the buffer.
7//!
8//! This ring buffer has not been implemented with thread safety in mind, and
9//! therefore should not be assumed to be suitable for use cases involving
10//! separate reader and writer threads.
11
12const Allocator = @import("std").mem.Allocator;
13const assert = @import("std").debug.assert;
14
15const RingBuffer = @This();
16
17data: []u8,
18read_index: usize,
19write_index: usize,
20
21pub const Error = error{Full};
22
23/// Allocate a new `RingBuffer`; `deinit()` should be called to free the buffer.
24pub fn init(allocator: Allocator, capacity: usize) Allocator.Error!RingBuffer {
25 const bytes = try allocator.alloc(u8, capacity);
26 return RingBuffer{
27 .data = bytes,
28 .write_index = 0,
29 .read_index = 0,
30 };
31}
32
33/// Free the data backing a `RingBuffer`; must be passed the same `Allocator` as
34/// `init()`.
35pub fn deinit(self: *RingBuffer, allocator: Allocator) void {
36 allocator.free(self.data);
37 self.* = undefined;
38}
39
40/// Returns `index` modulo the length of the backing slice.
41pub fn mask(self: RingBuffer, index: usize) usize {
42 return index % self.data.len;
43}
44
45/// Returns `index` modulo twice the length of the backing slice.
46pub fn mask2(self: RingBuffer, index: usize) usize {
47 return index % (2 * self.data.len);
48}
49
50/// Write `byte` into the ring buffer. Returns `error.Full` if the ring
51/// buffer is full.
52pub fn write(self: *RingBuffer, byte: u8) Error!void {
53 if (self.isFull()) return error.Full;
54 self.writeAssumeCapacity(byte);
55}
56
57/// Write `byte` into the ring buffer. If the ring buffer is full, the
58/// oldest byte is overwritten.
59pub fn writeAssumeCapacity(self: *RingBuffer, byte: u8) void {
60 self.data[self.mask(self.write_index)] = byte;
61 self.write_index = self.mask2(self.write_index + 1);
62}
63
64/// Write `bytes` into the ring buffer. Returns `error.Full` if the ring
65/// buffer does not have enough space, without writing any data.
66pub fn writeSlice(self: *RingBuffer, bytes: []const u8) Error!void {
67 if (self.len() + bytes.len > self.data.len) return error.Full;
68 self.writeSliceAssumeCapacity(bytes);
69}
70
71/// Write `bytes` into the ring buffer. If there is not enough space, older
72/// bytes will be overwritten.
73pub fn writeSliceAssumeCapacity(self: *RingBuffer, bytes: []const u8) void {
74 for (bytes) |b| self.writeAssumeCapacity(b);
75}
76
77/// Consume a byte from the ring buffer and return it. Returns `null` if the
78/// ring buffer is empty.
79pub fn read(self: *RingBuffer) ?u8 {
80 if (self.isEmpty()) return null;
81 return self.readAssumeLength();
82}
83
84/// Consume a byte from the ring buffer and return it; asserts that the buffer
85/// is not empty.
86pub fn readAssumeLength(self: *RingBuffer) u8 {
87 assert(!self.isEmpty());
88 const byte = self.data[self.mask(self.read_index)];
89 self.read_index = self.mask2(self.read_index + 1);
90 return byte;
91}
92
93/// Returns `true` if the ring buffer is empty and `false` otherwise.
94pub fn isEmpty(self: RingBuffer) bool {
95 return self.write_index == self.read_index;
96}
97
98/// Returns `true` if the ring buffer is full and `false` otherwise.
99pub fn isFull(self: RingBuffer) bool {
100 return self.mask2(self.write_index + self.data.len) == self.read_index;
101}
102
103/// Returns the length
104pub fn len(self: RingBuffer) usize {
105 const wrap_offset = 2 * self.data.len * @boolToInt(self.write_index < self.read_index);
106 const adjusted_write_index = self.write_index + wrap_offset;
107 return adjusted_write_index - self.read_index;
108}
109
110/// A `Slice` represents a region of a ring buffer. The region is split into two
111/// sections as the ring buffer data will not be contiguous if the desired
112/// region wraps to the start of the backing slice.
113pub const Slice = struct {
114 first: []u8,
115 second: []u8,
116};
117
118/// Returns a `Slice` for the region of the ring buffer starting at
119/// `self.mask(start_unmasked)` with the specified length.
120pub fn sliceAt(self: RingBuffer, start_unmasked: usize, length: usize) Slice {
121 assert(length <= self.data.len);
122 const slice1_start = self.mask(start_unmasked);
123 const slice1_end = @min(self.data.len, slice1_start + length);
124 const slice1 = self.data[slice1_start..slice1_end];
125 const slice2 = self.data[0 .. length - slice1.len];
126 return Slice{
127 .first = slice1,
128 .second = slice2,
129 };
130}
131
132/// Returns a `Slice` for the last `length` bytes written to the ring buffer.
133/// Does not check that any bytes have been written into the region.
134pub fn sliceLast(self: RingBuffer, length: usize) Slice {
135 return self.sliceAt(self.write_index + self.data.len - length, length);
136}
lib/std/compress.zig+2
...@@ -6,6 +6,7 @@ pub const lzma = @import("compress/lzma.zig");...@@ -6,6 +6,7 @@ pub const lzma = @import("compress/lzma.zig");
6pub const lzma2 = @import("compress/lzma2.zig");6pub const lzma2 = @import("compress/lzma2.zig");
7pub const xz = @import("compress/xz.zig");7pub const xz = @import("compress/xz.zig");
8pub const zlib = @import("compress/zlib.zig");8pub const zlib = @import("compress/zlib.zig");
9pub const zstd = @import("compress/zstandard.zig");
910
10pub fn HashedReader(11pub fn HashedReader(
11 comptime ReaderType: anytype,12 comptime ReaderType: anytype,
...@@ -44,4 +45,5 @@ test {...@@ -44,4 +45,5 @@ test {
44 _ = lzma2;45 _ = lzma2;
45 _ = xz;46 _ = xz;
46 _ = zlib;47 _ = zlib;
48 _ = zstd;
47}49}
lib/std/compress/testdata/rfc8478.txt created+3027
...@@ -0,0 +1,3027 @@
1
2
3
4
5
6
7Internet Engineering Task Force (IETF) Y. Collet
8Request for Comments: 8478 M. Kucherawy, Ed.
9Category: Informational Facebook
10ISSN: 2070-1721 October 2018
11
12
13 Zstandard Compression and the application/zstd Media Type
14
15Abstract
16
17 Zstandard, or "zstd" (pronounced "zee standard"), is a data
18 compression mechanism. This document describes the mechanism and
19 registers a media type and content encoding to be used when
20 transporting zstd-compressed content via Multipurpose Internet Mail
21 Extensions (MIME).
22
23 Despite use of the word "standard" as part of its name, readers are
24 advised that this document is not an Internet Standards Track
25 specification; it is being published for informational purposes only.
26
27Status of This Memo
28
29 This document is not an Internet Standards Track specification; it is
30 published for informational purposes.
31
32 This document is a product of the Internet Engineering Task Force
33 (IETF). It represents the consensus of the IETF community. It has
34 received public review and has been approved for publication by the
35 Internet Engineering Steering Group (IESG). Not all documents
36 approved by the IESG are candidates for any level of Internet
37 Standard; see Section 2 of RFC 7841.
38
39 Information about the current status of this document, any errata,
40 and how to provide feedback on it may be obtained at
41 https://www.rfc-editor.org/info/rfc8478.
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58Collet & Kucherawy Informational [Page 1]
59
60RFC 8478 application/zstd October 2018
61
62
63Copyright Notice
64
65 Copyright (c) 2018 IETF Trust and the persons identified as the
66 document authors. All rights reserved.
67
68 This document is subject to BCP 78 and the IETF Trust's Legal
69 Provisions Relating to IETF Documents
70 (https://trustee.ietf.org/license-info) in effect on the date of
71 publication of this document. Please review these documents
72 carefully, as they describe your rights and restrictions with respect
73 to this document. Code Components extracted from this document must
74 include Simplified BSD License text as described in Section 4.e of
75 the Trust Legal Provisions and are provided without warranty as
76 described in the Simplified BSD License.
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114Collet & Kucherawy Informational [Page 2]
115
116RFC 8478 application/zstd October 2018
117
118
119Table of Contents
120
121 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 4
122 2. Definitions . . . . . . . . . . . . . . . . . . . . . . . . . 4
123 3. Compression Algorithm . . . . . . . . . . . . . . . . . . . . 5
124 3.1. Frames . . . . . . . . . . . . . . . . . . . . . . . . . 6
125 3.1.1. Zstandard Frames . . . . . . . . . . . . . . . . . . 6
126 3.1.1.1. Frame Header . . . . . . . . . . . . . . . . . . 7
127 3.1.1.2. Blocks . . . . . . . . . . . . . . . . . . . . . 12
128 3.1.1.3. Compressed Blocks . . . . . . . . . . . . . . . . 14
129 3.1.1.4. Sequence Execution . . . . . . . . . . . . . . . 28
130 3.1.1.5. Repeat Offsets . . . . . . . . . . . . . . . . . 29
131 3.1.2. Skippable Frames . . . . . . . . . . . . . . . . . . 30
132 4. Entropy Encoding . . . . . . . . . . . . . . . . . . . . . . 30
133 4.1. FSE . . . . . . . . . . . . . . . . . . . . . . . . . . . 31
134 4.1.1. FSE Table Description . . . . . . . . . . . . . . . . 31
135 4.2. Huffman Coding . . . . . . . . . . . . . . . . . . . . . 34
136 4.2.1. Huffman Tree Description . . . . . . . . . . . . . . 35
137 4.2.1.1. Huffman Tree Header . . . . . . . . . . . . . . . 36
138 4.2.1.2. FSE Compression of Huffman Weights . . . . . . . 37
139 4.2.1.3. Conversion from Weights to Huffman Prefix Codes . 38
140 4.2.2. Huffman-Coded Streams . . . . . . . . . . . . . . . . 39
141 5. Dictionary Format . . . . . . . . . . . . . . . . . . . . . . 40
142 6. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 42
143 6.1. The 'application/zstd' Media Type . . . . . . . . . . . . 42
144 6.2. Content Encoding . . . . . . . . . . . . . . . . . . . . 43
145 6.3. Dictionaries . . . . . . . . . . . . . . . . . . . . . . 43
146 7. Security Considerations . . . . . . . . . . . . . . . . . . . 43
147 8. Implementation Status . . . . . . . . . . . . . . . . . . . . 44
148 9. References . . . . . . . . . . . . . . . . . . . . . . . . . 45
149 9.1. Normative References . . . . . . . . . . . . . . . . . . 45
150 9.2. Informative References . . . . . . . . . . . . . . . . . 45
151 Appendix A. Decoding Tables for Predefined Codes . . . . . . . . 46
152 A.1. Literal Length Code Table . . . . . . . . . . . . . . . . 46
153 A.2. Match Length Code Table . . . . . . . . . . . . . . . . . 49
154 A.3. Offset Code Table . . . . . . . . . . . . . . . . . . . . 52
155 Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . . . 53
156 Authors' Addresses . . . . . . . . . . . . . . . . . . . . . . . 54
157
158
159
160
161
162
163
164
165
166
167
168
169
170Collet & Kucherawy Informational [Page 3]
171
172RFC 8478 application/zstd October 2018
173
174
1751. Introduction
176
177 Zstandard, or "zstd" (pronounced "zee standard"), is a data
178 compression mechanism, akin to gzip [RFC1952].
179
180 Despite use of the word "standard" as part of its name, readers are
181 advised that this document is not an Internet Standards Track
182 specification; it is being published for informational purposes only.
183
184 This document describes the Zstandard format. Also, to enable the
185 transport of a data object compressed with Zstandard, this document
186 registers a media type that can be used to identify such content when
187 it is used in a payload encoded using Multipurpose Internet Mail
188 Extensions (MIME).
189
1902. Definitions
191
192 Some terms used elsewhere in this document are defined here for
193 clarity.
194
195 uncompressed: Describes an arbitrary set of bytes in their original
196 form, prior to being subjected to compression.
197
198 compress, compression: The act of processing a set of bytes via the
199 compression mechanism described here.
200
201 compressed: Describes the result of passing a set of bytes through
202 this mechanism. The original input has thus been compressed.
203
204 decompress, decompression: The act of processing a set of bytes
205 through the inverse of the compression mechanism described here,
206 in an attempt to recover the original set of bytes prior to
207 compression.
208
209 decompressed: Describes the result of passing a set of bytes through
210 the reverse of this mechanism. When this is successful, the
211 decompressed payload and the uncompressed payload are
212 indistinguishable.
213
214 encode: The process of translating data from one form to another;
215 this may include compression or it may refer to other translations
216 done as part of this specification.
217
218 decode: The reverse of "encode"; describes a process of reversing a
219 prior encoding to recover the original content.
220
221
222
223
224
225
226Collet & Kucherawy Informational [Page 4]
227
228RFC 8478 application/zstd October 2018
229
230
231 frame: Content compressed by Zstandard is transformed into a
232 Zstandard frame. Multiple frames can be appended into a single
233 file or stream. A frame is completely independent, has a defined
234 beginning and end, and has a set of parameters that tells the
235 decoder how to decompress it.
236
237 block: A frame encapsulates one or multiple blocks. Each block
238 contains arbitrary content, which is described by its header, and
239 has a guaranteed maximum content size that depends upon frame
240 parameters. Unlike frames, each block depends on previous blocks
241 for proper decoding. However, each block can be decompressed
242 without waiting for its successor, allowing streaming operations.
243
244 natural order: A sequence or ordering of objects or values that is
245 typical of that type of object or value. A set of unique
246 integers, for example, is in "natural order" if when progressing
247 from one element in the set or sequence to the next, there is
248 never a decrease in value.
249
250 The naming convention for identifiers within the specification is
251 Mixed_Case_With_Underscores. Identifiers inside square brackets
252 indicate that the identifier is optional in the presented context.
253
2543. Compression Algorithm
255
256 This section describes the Zstandard algorithm.
257
258 The purpose of this document is to define a lossless compressed data
259 format that is a) independent of the CPU type, operating system, file
260 system, and character set and b) is suitable for file compression and
261 pipe and streaming compression, using the Zstandard algorithm. The
262 text of the specification assumes a basic background in programming
263 at the level of bits and other primitive data representations.
264
265 The data can be produced or consumed, even for an arbitrarily long
266 sequentially presented input data stream, using only an a priori
267 bounded amount of intermediate storage, and hence can be used in data
268 communications. The format uses the Zstandard compression method,
269 and an optional xxHash-64 checksum method [XXHASH], for detection of
270 data corruption.
271
272 The data format defined by this specification does not attempt to
273 allow random access to compressed data.
274
275 Unless otherwise indicated below, a compliant compressor must produce
276 data sets that conform to the specifications presented here.
277 However, it does not need to support all options.
278
279
280
281
282Collet & Kucherawy Informational [Page 5]
283
284RFC 8478 application/zstd October 2018
285
286
287 A compliant decompressor must be able to decompress at least one
288 working set of parameters that conforms to the specifications
289 presented here. It may also ignore informative fields, such as the
290 checksum. Whenever it does not support a parameter defined in the
291 compressed stream, it must produce a non-ambiguous error code and
292 associated error message explaining which parameter is unsupported.
293
294 This specification is intended for use by implementers of software to
295 compress data into Zstandard format and/or decompress data from
296 Zstandard format. The Zstandard format is supported by an open
297 source reference implementation, written in portable C, and available
298 at [ZSTD].
299
3003.1. Frames
301
302 Zstandard compressed data is made up of one or more frames. Each
303 frame is independent and can be decompressed independently of other
304 frames. The decompressed content of multiple concatenated frames is
305 the concatenation of each frame's decompressed content.
306
307 There are two frame formats defined for Zstandard: Zstandard frames
308 and skippable frames. Zstandard frames contain compressed data,
309 while skippable frames contain custom user metadata.
310
3113.1.1. Zstandard Frames
312
313 The structure of a single Zstandard frame is as follows:
314
315 +--------------------+------------+
316 | Magic_Number | 4 bytes |
317 +--------------------+------------+
318 | Frame_Header | 2-14 bytes |
319 +--------------------+------------+
320 | Data_Block | n bytes |
321 +--------------------+------------+
322 | [More Data_Blocks] | |
323 +--------------------+------------+
324 | [Content_Checksum] | 0-4 bytes |
325 +--------------------+------------+
326
327 Magic_Number: 4 bytes, little-endian format. Value: 0xFD2FB528.
328
329 Frame_Header: 2 to 14 bytes, detailed in Section 3.1.1.1.
330
331 Data_Block: Detailed in Section 3.1.1.2. This is where data
332 appears.
333
334
335
336
337
338Collet & Kucherawy Informational [Page 6]
339
340RFC 8478 application/zstd October 2018
341
342
343 Content_Checksum: An optional 32-bit checksum, only present if
344 Content_Checksum_Flag is set. The content checksum is the result
345 of the XXH64() hash function [XXHASH] digesting the original
346 (decoded) data as input, and a seed of zero. The low 4 bytes of
347 the checksum are stored in little-endian format.
348
349 The magic number was selected to be less probable to find at the
350 beginning of an arbitrary file. It avoids trivial patterns (0x00,
351 0xFF, repeated bytes, increasing bytes, etc.), contains byte values
352 outside of ASCII range, and doesn't map into UTF-8 space, all of
353 which reduce the likelihood of its appearance at the top of a text
354 file.
355
3563.1.1.1. Frame Header
357
358 The frame header has a variable size, with a minimum of 2 bytes and
359 up to 14 bytes depending on optional parameters. The structure of
360 Frame_Header is as follows:
361
362 +-------------------------+-----------+
363 | Frame_Header_Descriptor | 1 byte |
364 +-------------------------+-----------+
365 | [Window_Descriptor] | 0-1 byte |
366 +-------------------------+-----------+
367 | [Dictionary_ID] | 0-4 bytes |
368 +-------------------------+-----------+
369 | [Frame_Content_Size] | 0-8 bytes |
370 +-------------------------+-----------+
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394Collet & Kucherawy Informational [Page 7]
395
396RFC 8478 application/zstd October 2018
397
398
3993.1.1.1.1. Frame_Header_Descriptor
400
401 The first header's byte is called the Frame_Header_Descriptor. It
402 describes which other fields are present. Decoding this byte is
403 enough to tell the size of Frame_Header.
404
405 +------------+-------------------------+
406 | Bit Number | Field Name |
407 +------------+-------------------------+
408 | 7-6 | Frame_Content_Size_Flag |
409 +------------+-------------------------+
410 | 5 | Single_Segment_Flag |
411 +------------+-------------------------+
412 | 4 | (unused) |
413 +------------+-------------------------+
414 | 3 | (reserved) |
415 +------------+-------------------------+
416 | 2 | Content_Checksum_Flag |
417 +------------+-------------------------+
418 | 1-0 | Dictionary_ID_Flag |
419 +------------+-------------------------+
420
421 In this table, bit 7 is the highest bit, while bit 0 is the lowest
422 one.
423
4243.1.1.1.1.1. Frame_Content_Size_Flag
425
426 This is a 2-bit flag (equivalent to Frame_Header_Descriptor right-
427 shifted 6 bits) specifying whether Frame_Content_Size (the
428 decompressed data size) is provided within the header. Flag_Value
429 provides FCS_Field_Size, which is the number of bytes used by
430 Frame_Content_Size according to the following table:
431
432 +----------------+--------+---+---+---+
433 | Flag_Value | 0 | 1 | 2 | 3 |
434 +----------------+--------+---+---+---+
435 | FCS_Field_Size | 0 or 1 | 2 | 4 | 8 |
436 +----------------+--------+---+---+---+
437
438 When Flag_Value is 0, FCS_Field_Size depends on Single_Segment_Flag:
439 If Single_Segment_Flag is set, FCS_Field_Size is 1. Otherwise,
440 FCS_Field_Size is 0; Frame_Content_Size is not provided.
441
442
443
444
445
446
447
448
449
450Collet & Kucherawy Informational [Page 8]
451
452RFC 8478 application/zstd October 2018
453
454
4553.1.1.1.1.2. Single_Segment_Flag
456
457 If this flag is set, data must be regenerated within a single
458 continuous memory segment.
459
460 In this case, Window_Descriptor byte is skipped, but
461 Frame_Content_Size is necessarily present. As a consequence, the
462 decoder must allocate a memory segment of size equal or larger than
463 Frame_Content_Size.
464
465 In order to protect the decoder from unreasonable memory
466 requirements, a decoder is allowed to reject a compressed frame that
467 requests a memory size beyond the decoder's authorized range.
468
469 For broader compatibility, decoders are recommended to support memory
470 sizes of at least 8 MB. This is only a recommendation; each decoder
471 is free to support higher or lower limits, depending on local
472 limitations.
473
4743.1.1.1.1.3. Unused Bit
475
476 A decoder compliant with this specification version shall not
477 interpret this bit. It might be used in a future version, to signal
478 a property that is not mandatory to properly decode the frame. An
479 encoder compliant with this specification must set this bit to zero.
480
4813.1.1.1.1.4. Reserved Bit
482
483 This bit is reserved for some future feature. Its value must be
484 zero. A decoder compliant with this specification version must
485 ensure it is not set. This bit may be used in a future revision, to
486 signal a feature that must be interpreted to decode the frame
487 correctly.
488
4893.1.1.1.1.5. Content_Checksum_Flag
490
491 If this flag is set, a 32-bit Content_Checksum will be present at the
492 frame's end. See the description of Content_Checksum above.
493
494
495
496
497
498
499
500
501
502
503
504
505
506Collet & Kucherawy Informational [Page 9]
507
508RFC 8478 application/zstd October 2018
509
510
5113.1.1.1.1.6. Dictionary_ID_Flag
512
513 This is a 2-bit flag (= Frame_Header_Descriptor & 0x3) indicating
514 whether a dictionary ID is provided within the header. It also
515 specifies the size of this field as DID_Field_Size:
516
517 +----------------+---+---+---+---+
518 | Flag_Value | 0 | 1 | 2 | 3 |
519 +----------------+---+---+---+---+
520 | DID_Field_Size | 0 | 1 | 2 | 4 |
521 +----------------+---+---+---+---+
522
5233.1.1.1.2. Window Descriptor
524
525 This provides guarantees about the minimum memory buffer required to
526 decompress a frame. This information is important for decoders to
527 allocate enough memory.
528
529 The Window_Descriptor byte is optional. When Single_Segment_Flag is
530 set, Window_Descriptor is not present. In this case, Window_Size is
531 Frame_Content_Size, which can be any value from 0 to 2^64-1 bytes (16
532 ExaBytes).
533
534 +------------+----------+----------+
535 | Bit Number | 7-3 | 2-0 |
536 +------------+----------+----------+
537 | Field Name | Exponent | Mantissa |
538 +------------+----------+----------+
539
540 The minimum memory buffer size is called Window_Size. It is
541 described by the following formulae:
542
543 windowLog = 10 + Exponent;
544 windowBase = 1 << windowLog;
545 windowAdd = (windowBase / 8) * Mantissa;
546 Window_Size = windowBase + windowAdd;
547
548 The minimum Window_Size is 1 KB. The maximum Window_Size is (1<<41)
549 + 7*(1<<38) bytes, which is 3.75 TB.
550
551 In general, larger Window_Size values tend to improve the compression
552 ratio, but at the cost of increased memory usage.
553
554 To properly decode compressed data, a decoder will need to allocate a
555 buffer of at least Window_Size bytes.
556
557
558
559
560
561
562Collet & Kucherawy Informational [Page 10]
563
564RFC 8478 application/zstd October 2018
565
566
567 In order to protect decoders from unreasonable memory requirements, a
568 decoder is allowed to reject a compressed frame that requests a
569 memory size beyond decoder's authorized range.
570
571 For improved interoperability, it's recommended for decoders to
572 support values of Window_Size up to 8 MB and for encoders not to
573 generate frames requiring a Window_Size larger than 8 MB. It's
574 merely a recommendation though, and decoders are free to support
575 larger or lower limits, depending on local limitations.
576
5773.1.1.1.3. Dictionary_ID
578
579 This is a variable size field, which contains the ID of the
580 dictionary required to properly decode the frame. This field is
581 optional. When it's not present, it's up to the decoder to know
582 which dictionary to use.
583
584 Dictionary_ID field size is provided by DID_Field_Size.
585 DID_Field_Size is directly derived from the value of
586 Dictionary_ID_Flag. One byte can represent an ID 0-255; 2 bytes can
587 represent an ID 0-65535; 4 bytes can represent an ID 0-4294967295.
588 Format is little-endian.
589
590 It is permitted to represent a small ID (for example, 13) with a
591 large 4-byte dictionary ID, even if it is less efficient.
592
593 Within private environments, any dictionary ID can be used. However,
594 for frames and dictionaries distributed in public space,
595 Dictionary_ID must be attributed carefully. The following ranges are
596 reserved for use only with dictionaries that have been registered
597 with IANA (see Section 6.3):
598
599 low range: <= 32767
600 high range: >= (1 << 31)
601
602 Any other value for Dictionary_ID can be used by private arrangement
603 between participants.
604
605 Any payload presented for decompression that references an
606 unregistered reserved dictionary ID results in an error.
607
608
609
610
611
612
613
614
615
616
617
618Collet & Kucherawy Informational [Page 11]
619
620RFC 8478 application/zstd October 2018
621
622
6233.1.1.1.4. Frame Content Size
624
625 This is the original (uncompressed) size. This information is
626 optional. Frame_Content_Size uses a variable number of bytes,
627 provided by FCS_Field_Size. FCS_Field_Size is provided by the value
628 of Frame_Content_Size_Flag. FCS_Field_Size can be equal to 0 (not
629 present), 1, 2, 4, or 8 bytes.
630
631 +----------------+--------------+
632 | FCS Field Size | Range |
633 +----------------+--------------+
634 | 0 | unknown |
635 +----------------+--------------+
636 | 1 | 0 - 255 |
637 +----------------+--------------+
638 | 2 | 256 - 65791 |
639 +----------------+--------------+
640 | 4 | 0 - 2^32 - 1 |
641 +----------------+--------------+
642 | 8 | 0 - 2^64 - 1 |
643 +----------------+--------------+
644
645 Frame_Content_Size format is little-endian. When FCS_Field_Size is
646 1, 4, or 8 bytes, the value is read directly. When FCS_Field_Size is
647 2, the offset of 256 is added. It's allowed to represent a small
648 size (for example 18) using any compatible variant.
649
6503.1.1.2. Blocks
651
652 After Magic_Number and Frame_Header, there are some number of blocks.
653 Each frame must have at least 1 block, but there is no upper limit on
654 the number of blocks per frame.
655
656 The structure of a block is as follows:
657
658 +--------------+---------------+
659 | Block_Header | Block_Content |
660 +--------------+---------------+
661 | 3 bytes | n bytes |
662 +--------------+---------------+
663
664
665
666
667
668
669
670
671
672
673
674Collet & Kucherawy Informational [Page 12]
675
676RFC 8478 application/zstd October 2018
677
678
679 Block_Header uses 3 bytes, written using little-endian convention.
680 It contains three fields:
681
682 +------------+------------+------------+
683 | Last_Block | Block_Type | Block_Size |
684 +------------+------------+------------+
685 | bit 0 | bits 1-2 | bits 3-23 |
686 +------------+------------+------------+
687
6883.1.1.2.1. Last_Block
689
690 The lowest bit (Last_Block) signals whether this block is the last
691 one. The frame will end after this last block. It may be followed
692 by an optional Content_Checksum (see Section 3.1.1).
693
6943.1.1.2.2. Block_Type
695
696 The next 2 bits represent the Block_Type. There are four block
697 types:
698
699 +-----------+------------------+
700 | Value | Block_Type |
701 +-----------+------------------+
702 | 0 | Raw_Block |
703 +-----------+------------------+
704 | 1 | RLE_Block |
705 +-----------+------------------+
706 | 2 | Compressed_Block |
707 +-----------+------------------+
708 | 3 | Reserved |
709 +-----------+------------------+
710
711 Raw_Block: This is an uncompressed block. Block_Content contains
712 Block_Size bytes.
713
714 RLE_Block: This is a single byte, repeated Block_Size times.
715 Block_Content consists of a single byte. On the decompression
716 side, this byte must be repeated Block_Size times.
717
718 Compressed_Block: This is a compressed block as described in
719 Section 3.1.1.3. Block_Size is the length of Block_Content,
720 namely the compressed data. The decompressed size is not known,
721 but its maximum possible value is guaranteed (see below).
722
723 Reserved: This is not a block. This value cannot be used with the
724 current specification. If such a value is present, it is
725 considered to be corrupt data.
726
727
728
729
730Collet & Kucherawy Informational [Page 13]
731
732RFC 8478 application/zstd October 2018
733
734
7353.1.1.2.3. Block_Size
736
737 The upper 21 bits of Block_Header represent the Block_Size.
738 Block_Size is the size of the block excluding the header. A block
739 can contain any number of bytes (even zero), up to
740 Block_Maximum_Decompressed_Size, which is the smallest of:
741
742 o Window_Size
743
744 o 128 KB
745
746 A Compressed_Block has the extra restriction that Block_Size is
747 always strictly less than the decompressed size. If this condition
748 cannot be respected, the block must be sent uncompressed instead
749 (i.e., treated as a Raw_Block).
750
7513.1.1.3. Compressed Blocks
752
753 To decompress a compressed block, the compressed size must be
754 provided from the Block_Size field within Block_Header.
755
756 A compressed block consists of two sections: a Literals
757 Section (Section 3.1.1.3.1) and a
758 Sequences_Section (Section 3.1.1.3.2). The results of the two
759 sections are then combined to produce the decompressed data in
760 Sequence Execution (Section 3.1.1.4).
761
762 To decode a compressed block, the following elements are necessary:
763
764 o Previous decoded data, up to a distance of Window_Size, or the
765 beginning of the Frame, whichever is smaller. Single_Segment_Flag
766 will be set in the latter case.
767
768 o List of "recent offsets" from the previous Compressed_Block.
769
770 o The previous Huffman tree, required by Treeless_Literals_Block
771 type.
772
773 o Previous Finite State Entropy (FSE) decoding tables, required by
774 Repeat_Mode, for each symbol type (literals lengths, match
775 lengths, offsets).
776
777 Note that decoding tables are not always from the previous
778 Compressed_Block:
779
780 o Every decoding table can come from a dictionary.
781
782
783
784
785
786Collet & Kucherawy Informational [Page 14]
787
788RFC 8478 application/zstd October 2018
789
790
791 o The Huffman tree comes from the previous
792 Compressed_Literals_Block.
793
7943.1.1.3.1. Literals_Section_Header
795
796 All literals are regrouped in the first part of the block. They can
797 be decoded first and then copied during Sequence Execution (see
798 Section 3.1.1.4), or they can be decoded on the flow during Sequence
799 Execution.
800
801 Literals can be stored uncompressed or compressed using Huffman
802 prefix codes. When compressed, an optional tree description can be
803 present, followed by 1 or 4 streams.
804
805 +----------------------------+
806 | Literals_Section_Header |
807 +----------------------------+
808 | [Huffman_Tree_Description] |
809 +----------------------------+
810 | [Jump_Table] |
811 +----------------------------+
812 | Stream_1 |
813 +----------------------------+
814 | [Stream_2] |
815 +----------------------------+
816 | [Stream_3] |
817 +----------------------------+
818 | [Stream_4] |
819 +----------------------------+
820
8213.1.1.3.1.1. Literals_Section_Header
822
823 This field describes how literals are packed. It's a byte-aligned
824 variable-size bit field, ranging from 1 to 5 bytes, using little-
825 endian convention.
826
827 +---------------------+-----------+
828 | Literals_Block_Type | 2 bits |
829 +---------------------+-----------+
830 | Size_Format | 1-2 bits |
831 +---------------------+-----------+
832 | Regenerated_Size | 5-20 bits |
833 +---------------------+-----------+
834 | [Compressed_Size] | 0-18 bits |
835 +---------------------+-----------+
836
837 In this representation, bits at the top are the lowest bits.
838
839
840
841
842Collet & Kucherawy Informational [Page 15]
843
844RFC 8478 application/zstd October 2018
845
846
847 The Literals_Block_Type field uses the two lowest bits of the first
848 byte, describing four different block types:
849
850 +---------------------------+-------+
851 | Literals_Block_Type | Value |
852 +---------------------------+-------+
853 | Raw_Literals_Block | 0 |
854 +---------------------------+-------+
855 | RLE_Literals_Block | 1 |
856 +---------------------------+-------+
857 | Compressed_Literals_Block | 2 |
858 +---------------------------+-------+
859 | Treeless_Literals_Block | 3 |
860 +---------------------------+-------+
861
862 Raw_Literals_Block: Literals are stored uncompressed.
863 Literals_Section_Content is Regenerated_Size.
864
865 RLE_Literals_Block: Literals consist of a single-byte value repeated
866 Regenerated_Size times. Literals_Section_Content is 1.
867
868 Compressed_Literals_Block: This is a standard Huffman-compressed
869 block, starting with a Huffman tree description. See details
870 below. Literals_Section_Content is Compressed_Size.
871
872 Treeless_Literals_Block: This is a Huffman-compressed block, using
873 the Huffman tree from the previous Compressed_Literals_Block, or a
874 dictionary if there is no previous Huffman-compressed literals
875 block. Huffman_Tree_Description will be skipped. Note that if
876 this mode is triggered without any previous Huffman-table in the
877 frame (or dictionary, per Section 5), it should be treated as data
878 corruption. Literals_Section_Content is Compressed_Size.
879
880 The Size_Format is divided into two families:
881
882 o For Raw_Literals_Block and RLE_Literals_Block, it's only necessary
883 to decode Regenerated_Size. There is no Compressed_Size field.
884
885 o For Compressed_Block and Treeless_Literals_Block, it's required to
886 decode both Compressed_Size and Regenerated_Size (the decompressed
887 size). It's also necessary to decode the number of streams (1 or
888 4).
889
890 For values spanning several bytes, the convention is little endian.
891
892 Size_Format for Raw_Literals_Block and RLE_Literals_Block uses 1 or 2
893 bits. Its value is (Literals_Section_Header[0]>>2) & 0x3.
894
895
896
897
898Collet & Kucherawy Informational [Page 16]
899
900RFC 8478 application/zstd October 2018
901
902
903 Size_Format == 00 or 10: Size_Format uses 1 bit. Regenerated_Size
904 uses 5 bits (value 0-31). Literals_Section_Header uses 1 byte.
905 Regenerated_Size = Literal_Section_Header[0]>>3.
906
907 Size_Format == 01: Size_Format uses 2 bits. Regenerated_Size uses
908 12 bits (values 0-4095). Literals_Section_Header uses 2 bytes.
909 Regenerated_Size = (Literals_Section_Header[0]>>4) +
910 (Literals_Section_Header[1]<<4).
911
912 Size_Format == 11: Size_Format uses 2 bits. Regenerated_Size uses
913 20 bits (values 0-1048575). Literals_Section_Header uses 3 bytes.
914 Regenerated_Size = (Literals_Section_Header[0]>>4) +
915 (Literals_Section_Header[1]<<4) + (Literals_Section_Header[2]<<12)
916
917 Only Stream_1 is present for these cases. Note that it is permitted
918 to represent a short value (for example, 13) using a long format,
919 even if it's less efficient.
920
921 Size_Format for Compressed_Literals_Block and Treeless_Literals_Block
922 always uses 2 bits.
923
924 Size_Format == 00: A single stream. Both Regenerated_Size and
925 Compressed_Size use 10 bits (values 0-1023).
926 Literals_Section_Header uses 3 bytes.
927
928 Size_Format == 01: 4 streams. Both Regenerated_Size and
929 Compressed_Size use 10 bits (values 0-1023).
930 Literals_Section_Header uses 3 bytes.
931
932 Size_Format == 10: 4 streams. Both Regenerated_Size and
933 Compressed_Size use 14 bits (values 0-16383).
934 Literals_Section_Header uses 4 bytes.
935
936 Size_Format == 11: 4 streams. Both Regenerated_Size and
937 Compressed_Size use 18 bits (values 0-262143).
938 Literals_Section_Header uses 5 bytes.
939
940 Both the Compressed_Size and Regenerated_Size fields follow little-
941 endian convention. Note that Compressed_Size includes the size of
942 the Huffman_Tree_Description when it is present.
943
9443.1.1.3.1.2. Raw_Literals_Block
945
946 The data in Stream_1 is Regenerated_Size bytes long. It contains the
947 raw literals data to be used during Sequence Execution
948 (Section 3.1.1.3.2).
949
950
951
952
953
954Collet & Kucherawy Informational [Page 17]
955
956RFC 8478 application/zstd October 2018
957
958
9593.1.1.3.1.3. RLE_Literals_Block
960
961 Stream_1 consists of a single byte that should be repeated
962 Regenerated_Size times to generate the decoded literals.
963
9643.1.1.3.1.4. Compressed_Literals_Block and Treeless_Literals_Block
965
966 Both of these modes contain Huffman-encoded data. For
967 Treeless_Literals_Block, the Huffman table comes from the previously
968 compressed literals block, or from a dictionary; see Section 5.
969
9703.1.1.3.1.5. Huffman_Tree_Description
971
972 This section is only present when the Literals_Block_Type type is
973 Compressed_Literals_Block (2). The format of
974 Huffman_Tree_Description can be found in Section 4.2.1. The size of
975 Huffman_Tree_Description is determined during the decoding process.
976 It must be used to determine where streams begin.
977
978 Total_Streams_Size = Compressed_Size
979 - Huffman_Tree_Description_Size
980
9813.1.1.3.1.6. Jump_Table
982
983 The Jump_Table is only present when there are 4 Huffman-coded
984 streams.
985
986 (Reminder: Huffman-compressed data consists of either 1 or 4 Huffman-
987 coded streams.)
988
989 If only 1 stream is present, it is a single bitstream occupying the
990 entire remaining portion of the literals block, encoded as described
991 within Section 4.2.2.
992
993 If there are 4 streams, Literals_Section_Header only provides enough
994 information to know the decompressed and compressed sizes of all 4
995 streams combined. The decompressed size of each stream is equal to
996 (Regenerated_Size+3)/4, except for the last stream, which may be up
997 to 3 bytes smaller, to reach a total decompressed size as specified
998 in Regenerated_Size.
999
1000 The compressed size of each stream is provided explicitly in the
1001 Jump_Table. The Jump_Table is 6 bytes long and consists of three
1002 2-byte little-endian fields, describing the compressed sizes of the
1003 first 3 streams. Stream4_Size is computed from Total_Streams_Size
1004 minus sizes of other streams.
1005
1006
1007
1008
1009
1010Collet & Kucherawy Informational [Page 18]
1011
1012RFC 8478 application/zstd October 2018
1013
1014
1015 Stream4_Size = Total_Streams_Size - 6
1016 - Stream1_Size - Stream2_Size
1017 - Stream3_Size
1018
1019 Note that if Stream1_Size + Stream2_Size + Stream3_Size exceeds
1020 Total_Streams_Size, the data are considered corrupted.
1021
1022 Each of these 4 bitstreams is then decoded independently as a
1023 Huffman-Coded stream, as described in Section 4.2.2.
1024
10253.1.1.3.2. Sequences_Section
1026
1027 A compressed block is a succession of sequences. A sequence is a
1028 literal copy command, followed by a match copy command. A literal
1029 copy command specifies a length. It is the number of bytes to be
1030 copied (or extracted) from the Literals Section. A match copy
1031 command specifies an offset and a length.
1032
1033 When all sequences are decoded, if there are literals left in the
1034 literals section, these bytes are added at the end of the block.
1035
1036 This is described in more detail in Section 3.1.1.4.
1037
1038 The Sequences_Section regroups all symbols required to decode
1039 commands. There are three symbol types: literals lengths, offsets,
1040 and match lengths. They are encoded together, interleaved, in a
1041 single "bitstream".
1042
1043 The Sequences_Section starts by a header, followed by optional
1044 probability tables for each symbol type, followed by the bitstream.
1045
1046 Sequences_Section_Header
1047 [Literals_Length_Table]
1048 [Offset_Table]
1049 [Match_Length_Table]
1050 bitStream
1051
1052 To decode the Sequences_Section, it's necessary to know its size.
1053 This size is deduced from the size of the Literals_Section:
1054 Sequences_Section_Size = Block_Size - Literals_Section_Header -
1055 Literals_Section_Content
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066Collet & Kucherawy Informational [Page 19]
1067
1068RFC 8478 application/zstd October 2018
1069
1070
10713.1.1.3.2.1. Sequences_Section_Header
1072
1073 This header consists of two items:
1074
1075 o Number_of_Sequences
1076
1077 o Symbol_Compression_Modes
1078
1079 Number_of_Sequences is a variable size field using between 1 and 3
1080 bytes. If the first byte is "byte0":
1081
1082 o if (byte0 == 0): there are no sequences. The sequence section
1083 stops here. Decompressed content is defined entirely as Literals
1084 Section content. The FSE tables used in Repeat_Mode are not
1085 updated.
1086
1087 o if (byte0 < 128): Number_of_Sequences = byte0. Uses 1 byte.
1088
1089 o if (byte0 < 255): Number_of_Sequences = ((byte0 - 128) << 8) +
1090 byte1. Uses 2 bytes.
1091
1092 o if (byte0 == 255): Number_of_Sequences = byte1 + (byte2 << 8) +
1093 0x7F00. Uses 3 bytes.
1094
1095 Symbol_Compression_Modes is a single byte, defining the compression
1096 mode of each symbol type.
1097
1098 +-------------+----------------------+
1099 | Bit Number | Field Name |
1100 +-------------+----------------------+
1101 | 7-6 | Literal_Lengths_Mode |
1102 +-------------+----------------------+
1103 | 5-4 | Offsets_Mode |
1104 +-------------+----------------------+
1105 | 3-2 | Match_Lengths_Mode |
1106 +-------------+----------------------+
1107 | 1-0 | Reserved |
1108 +-------------+----------------------+
1109
1110 The last field, Reserved, must be all zeroes.
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122Collet & Kucherawy Informational [Page 20]
1123
1124RFC 8478 application/zstd October 2018
1125
1126
1127 Literals_Lengths_Mode, Offsets_Mode, and Match_Lengths_Mode define
1128 the Compression_Mode of literals lengths, offsets, and match lengths
1129 symbols, respectively. They follow the same enumeration:
1130
1131 +-------+---------------------+
1132 | Value | Compression_Mode |
1133 +-------+---------------------+
1134 | 0 | Predefined_Mode |
1135 +-------+---------------------+
1136 | 1 | RLE_Mode |
1137 +-------+---------------------+
1138 | 2 | FSE_Compressed_Mode |
1139 +-------+---------------------+
1140 | 3 | Repeat_Mode |
1141 +-------+---------------------+
1142
1143 Predefined_Mode: A predefined FSE (see Section 4.1) distribution
1144 table is used, as defined in Section 3.1.1.3.2.2. No distribution
1145 table will be present.
1146
1147 RLE_Mode: The table description consists of a single byte, which
1148 contains the symbol's value. This symbol will be used for all
1149 sequences.
1150
1151 FSE_Compressed_Mode: Standard FSE compression. A distribution table
1152 will be present. The format of this distribution table is
1153 described in Section 4.1.1. Note that the maximum allowed
1154 accuracy log for literals length and match length tables is 9, and
1155 the maximum accuracy log for the offsets table is 8. This mode
1156 must not be used when only one symbol is present; RLE_Mode should
1157 be used instead (although any other mode will work).
1158
1159 Repeat_Mode: The table used in the previous Compressed_Block with
1160 Number_Of_Sequences > 0 will be used again, or if this is the
1161 first block, the table in the dictionary will be used. Note that
1162 this includes RLE_Mode, so if Repeat_Mode follows RLE_Mode, the
1163 same symbol will be repeated. It also includes Predefined_Mode,
1164 in which case Repeat_Mode will have the same outcome as
1165 Predefined_Mode. No distribution table will be present. If this
1166 mode is used without any previous sequence table in the frame (or
1167 dictionary; see Section 5) to repeat, this should be treated as
1168 corruption.
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178Collet & Kucherawy Informational [Page 21]
1179
1180RFC 8478 application/zstd October 2018
1181
1182
11833.1.1.3.2.1.1. Sequence Codes for Lengths and Offsets
1184
1185 Each symbol is a code in its own context, which specifies Baseline
1186 and Number_of_Bits to add. Codes are FSE compressed and interleaved
1187 with raw additional bits in the same bitstream.
1188
1189 Literals length codes are values ranging from 0 to 35 inclusive.
1190 They define lengths from 0 to 131071 bytes. The literals length is
1191 equal to the decoded Baseline plus the result of reading
1192 Number_of_Bits bits from the bitstream, as a little-endian value.
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234Collet & Kucherawy Informational [Page 22]
1235
1236RFC 8478 application/zstd October 2018
1237
1238
1239 +----------------------+----------+----------------+
1240 | Literals_Length_Code | Baseline | Number_of_Bits |
1241 +----------------------+----------+----------------+
1242 | 0-15 | length | 0 |
1243 +----------------------+----------+----------------+
1244 | 16 | 16 | 1 |
1245 +----------------------+----------+----------------+
1246 | 17 | 18 | 1 |
1247 +----------------------+----------+----------------+
1248 | 18 | 20 | 1 |
1249 +----------------------+----------+----------------+
1250 | 19 | 22 | 1 |
1251 +----------------------+----------+----------------+
1252 | 20 | 24 | 2 |
1253 +----------------------+----------+----------------+
1254 | 21 | 28 | 2 |
1255 +----------------------+----------+----------------+
1256 | 22 | 32 | 3 |
1257 +----------------------+----------+----------------+
1258 | 23 | 40 | 3 |
1259 +----------------------+----------+----------------+
1260 | 24 | 48 | 4 |
1261 +----------------------+----------+----------------+
1262 | 25 | 64 | 6 |
1263 +----------------------+----------+----------------+
1264 | 26 | 128 | 7 |
1265 +----------------------+----------+----------------+
1266 | 27 | 256 | 8 |
1267 +----------------------+----------+----------------+
1268 | 28 | 512 | 9 |
1269 +----------------------+----------+----------------+
1270 | 29 | 1024 | 10 |
1271 +----------------------+----------+----------------+
1272 | 30 | 2048 | 11 |
1273 +----------------------+----------+----------------+
1274 | 31 | 4096 | 12 |
1275 +----------------------+----------+----------------+
1276 | 32 | 8192 | 13 |
1277 +----------------------+----------+----------------+
1278 | 33 | 16384 | 14 |
1279 +----------------------+----------+----------------+
1280 | 34 | 32768 | 15 |
1281 +----------------------+----------+----------------+
1282 | 35 | 65536 | 16 |
1283 +----------------------+----------+----------------+
1284
1285
1286
1287
1288
1289
1290Collet & Kucherawy Informational [Page 23]
1291
1292RFC 8478 application/zstd October 2018
1293
1294
1295 Match length codes are values ranging from 0 to 52 inclusive. They
1296 define lengths from 3 to 131074 bytes. The match length is equal to
1297 the decoded Baseline plus the result of reading Number_of_Bits bits
1298 from the bitstream, as a little-endian value.
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346Collet & Kucherawy Informational [Page 24]
1347
1348RFC 8478 application/zstd October 2018
1349
1350
1351 +-------------------+-----------------------+----------------+
1352 | Match_Length_Code | Baseline | Number_of_Bits |
1353 +-------------------+-----------------------+----------------+
1354 | 0-31 | Match_Length_Code + 3 | 0 |
1355 +-------------------+-----------------------+----------------+
1356 | 32 | 35 | 1 |
1357 +-------------------+-----------------------+----------------+
1358 | 33 | 37 | 1 |
1359 +-------------------+-----------------------+----------------+
1360 | 34 | 39 | 1 |
1361 +-------------------+-----------------------+----------------+
1362 | 35 | 41 | 1 |
1363 +-------------------+-----------------------+----------------+
1364 | 36 | 43 | 2 |
1365 +-------------------+-----------------------+----------------+
1366 | 37 | 47 | 2 |
1367 +-------------------+-----------------------+----------------+
1368 | 38 | 51 | 3 |
1369 +-------------------+-----------------------+----------------+
1370 | 39 | 59 | 3 |
1371 +-------------------+-----------------------+----------------+
1372 | 40 | 67 | 4 |
1373 +-------------------+-----------------------+----------------+
1374 | 41 | 83 | 4 |
1375 +-------------------+-----------------------+----------------+
1376 | 42 | 99 | 5 |
1377 +-------------------+-----------------------+----------------+
1378 | 43 | 131 | 7 |
1379 +-------------------+-----------------------+----------------+
1380 | 44 | 259 | 8 |
1381 +-------------------+-----------------------+----------------+
1382 | 45 | 515 | 9 |
1383 +-------------------+-----------------------+----------------+
1384 | 46 | 1027 | 10 |
1385 +-------------------+-----------------------+----------------+
1386 | 47 | 2051 | 11 |
1387 +-------------------+-----------------------+----------------+
1388 | 48 | 4099 | 12 |
1389 +-------------------+-----------------------+----------------+
1390 | 49 | 8195 | 13 |
1391 +-------------------+-----------------------+----------------+
1392 | 50 | 16387 | 14 |
1393 +-------------------+-----------------------+----------------+
1394 | 51 | 32771 | 15 |
1395 +-------------------+-----------------------+----------------+
1396 | 52 | 65539 | 16 |
1397 +-------------------+-----------------------+----------------+
1398
1399
1400
1401
1402Collet & Kucherawy Informational [Page 25]
1403
1404RFC 8478 application/zstd October 2018
1405
1406
1407 Offset codes are values ranging from 0 to N.
1408
1409 A decoder is free to limit its maximum supported value for N.
1410 Support for values of at least 22 is recommended. At the time of
1411 this writing, the reference decoder supports a maximum N value of 31.
1412
1413 An offset code is also the number of additional bits to read in
1414 little-endian fashion and can be translated into an Offset_Value
1415 using the following formulas:
1416
1417 Offset_Value = (1 << offsetCode) + readNBits(offsetCode);
1418 if (Offset_Value > 3) Offset = Offset_Value - 3;
1419
1420 This means that maximum Offset_Value is (2^(N+1))-1, supporting back-
1421 reference distance up to (2^(N+1))-4, but it is limited by the
1422 maximum back-reference distance (see Section 3.1.1.1.2).
1423
1424 Offset_Value from 1 to 3 are special: they define "repeat codes".
1425 This is described in more detail in Section 3.1.1.5.
1426
14273.1.1.3.2.1.2. Decoding Sequences
1428
1429 FSE bitstreams are read in reverse of the direction they are written.
1430 In zstd, the compressor writes bits forward into a block, and the
1431 decompressor must read the bitstream backwards.
1432
1433 To find the start of the bitstream, it is therefore necessary to know
1434 the offset of the last byte of the block, which can be found by
1435 counting Block_Size bytes after the block header.
1436
1437 After writing the last bit containing information, the compressor
1438 writes a single 1 bit and then fills the byte with 0-7 zero bits of
1439 padding. The last byte of the compressed bitstream cannot be zero
1440 for that reason.
1441
1442 When decompressing, the last byte containing the padding is the first
1443 byte to read. The decompressor needs to skip 0-7 initial zero bits
1444 until the first 1 bit occurs. Afterwards, the useful part of the
1445 bitstream begins.
1446
1447 FSE decoding requires a 'state' to be carried from symbol to symbol.
1448 For more explanation on FSE decoding, see Section 4.1.
1449
1450 For sequence decoding, a separate state keeps track of each literal
1451 lengths, offsets, and match lengths symbols. Some FSE primitives are
1452 also used. For more details on the operation of these primitives,
1453 see Section 4.1.
1454
1455
1456
1457
1458Collet & Kucherawy Informational [Page 26]
1459
1460RFC 8478 application/zstd October 2018
1461
1462
1463 The bitstream starts with initial FSE state values, each using the
1464 required number of bits in their respective accuracy, decoded
1465 previously from their normalized distribution. It starts with
1466 Literals_Length_State, followed by Offset_State, and finally
1467 Match_Length_State.
1468
1469 Note that all values are read backward, so the 'start' of the
1470 bitstream is at the highest position in memory, immediately before
1471 the last 1 bit for padding.
1472
1473 After decoding the starting states, a single sequence is decoded
1474 Number_Of_Sequences times. These sequences are decoded in order from
1475 first to last. Since the compressor writes the bitstream in the
1476 forward direction, this means the compressor must encode the
1477 sequences starting with the last one and ending with the first.
1478
1479 For each of the symbol types, the FSE state can be used to determine
1480 the appropriate code. The code then defines the Baseline and
1481 Number_of_Bits to read for each type. The description of the codes
1482 for how to determine these values can be found in
1483 Section 3.1.1.3.2.1.
1484
1485 Decoding starts by reading the Number_of_Bits required to decode
1486 offset. It does the same for Match_Length and then for
1487 Literals_Length. This sequence is then used for Sequence Execution
1488 (see Section 3.1.1.4).
1489
1490 If it is not the last sequence in the block, the next operation is to
1491 update states. Using the rules pre-calculated in the decoding
1492 tables, Literals_Length_State is updated, followed by
1493 Match_Length_State, and then Offset_State. See Section 4.1 for
1494 details on how to update states from the bitstream.
1495
1496 This operation will be repeated Number_of_Sequences times. At the
1497 end, the bitstream shall be entirely consumed; otherwise, the
1498 bitstream is considered corrupted.
1499
15003.1.1.3.2.2. Default Distributions
1501
1502 If Predefined_Mode is selected for a symbol type, its FSE decoding
1503 table is generated from a predefined distribution table defined here.
1504 For details on how to convert this distribution into a decoding
1505 table, see Section 4.1.
1506
1507
1508
1509
1510
1511
1512
1513
1514Collet & Kucherawy Informational [Page 27]
1515
1516RFC 8478 application/zstd October 2018
1517
1518
15193.1.1.3.2.2.1. Literals Length
1520
1521 The decoding table uses an accuracy log of 6 bits (64 states).
1522
1523 short literalsLength_defaultDistribution[36] =
1524 { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
1525 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
1526 -1,-1,-1,-1
1527 };
1528
15293.1.1.3.2.2.2. Match Length
1530
1531 The decoding table uses an accuracy log of 6 bits (64 states).
1532
1533 short matchLengths_defaultDistribution[53] =
1534 { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
1535 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1536 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
1537 -1,-1,-1,-1,-1
1538 };
1539
15403.1.1.3.2.2.3. Offset Codes
1541
1542 The decoding table uses an accuracy log of 5 bits (32 states), and
1543 supports a maximum N value of 28, allowing offset values up to
1544 536,870,908.
1545
1546 If any sequence in the compressed block requires a larger offset than
1547 this, it's not possible to use the default distribution to represent
1548 it.
1549
1550 short offsetCodes_defaultDistribution[29] =
1551 { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
1552 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1
1553 };
1554
15553.1.1.4. Sequence Execution
1556
1557 Once literals and sequences have been decoded, they are combined to
1558 produce the decoded content of a block.
1559
1560 Each sequence consists of a tuple of (literals_length, offset_value,
1561 match_length), decoded as described in the
1562 Sequences_Section (Section 3.1.1.3.2). To execute a sequence, first
1563 copy literals_length bytes from the decoded literals to the output.
1564
1565
1566
1567
1568
1569
1570Collet & Kucherawy Informational [Page 28]
1571
1572RFC 8478 application/zstd October 2018
1573
1574
1575 Then, match_length bytes are copied from previous decoded data. The
1576 offset to copy from is determined by offset_value:
1577
1578 o if Offset_Value > 3, then the offset is Offset_Value - 3;
1579
1580 o if Offset_Value is from 1-3, the offset is a special repeat offset
1581 value. See Section 3.1.1.5 for how the offset is determined in
1582 this case.
1583
1584 The offset is defined as from the current position (after copying the
1585 literals), so an offset of 6 and a match length of 3 means that 3
1586 bytes should be copied from 6 bytes back. Note that all offsets
1587 leading to previously decoded data must be smaller than Window_Size
1588 defined in Frame_Header_Descriptor (Section 3.1.1.1.1).
1589
15903.1.1.5. Repeat Offsets
1591
1592 As seen above, the first three values define a repeated offset; we
1593 will call them Repeated_Offset1, Repeated_Offset2, and
1594 Repeated_Offset3. They are sorted in recency order, with
1595 Repeated_Offset1 meaning "most recent one".
1596
1597 If offset_value is 1, then the offset used is Repeated_Offset1, etc.
1598
1599 There is one exception: When the current sequence's literals_length
1600 is 0, repeated offsets are shifted by 1, so an offset_value of 1
1601 means Repeated_Offset2, an offset_value of 2 means Repeated_Offset3,
1602 and an offset_value of 3 means Repeated_Offset1 - 1_byte.
1603
1604 For the first block, the starting offset history is populated with
1605 the following values: Repeated_Offset1 (1), Repeated_Offset2 (4), and
1606 Repeated_Offset3 (8), unless a dictionary is used, in which case they
1607 come from the dictionary.
1608
1609 Then each block gets its starting offset history from the ending
1610 values of the most recent Compressed_Block. Note that blocks that
1611 are not Compressed_Block are skipped; they do not contribute to
1612 offset history.
1613
1614 The newest offset takes the lead in offset history, shifting others
1615 back (up to its previous place if it was already present). This
1616 means that when Repeated_Offset1 (most recent) is used, history is
1617 unmodified. When Repeated_Offset2 is used, it is swapped with
1618 Repeated_Offset1. If any other offset is used, it becomes
1619 Repeated_Offset1, and the rest are shifted back by 1.
1620
1621
1622
1623
1624
1625
1626Collet & Kucherawy Informational [Page 29]
1627
1628RFC 8478 application/zstd October 2018
1629
1630
16313.1.2. Skippable Frames
1632
1633 +--------------+------------+-----------+
1634 | Magic_Number | Frame_Size | User_Data |
1635 +--------------+------------+-----------+
1636 | 4 bytes | 4 bytes | n bytes |
1637 +--------------+------------+-----------+
1638
1639 Skippable frames allow the insertion of user-defined metadata into a
1640 flow of concatenated frames.
1641
1642 Skippable frames defined in this specification are compatible with
1643 skippable frames in [LZ4].
1644
1645 From a compliant decoder perspective, skippable frames simply need to
1646 be skipped, and their content ignored, resuming decoding after the
1647 skippable frame.
1648
1649 It should be noted that a skippable frame can be used to watermark a
1650 stream of concatenated frames embedding any kind of tracking
1651 information (even just a Universally Unique Identifier (UUID)).
1652 Users wary of such possibility should scan the stream of concatenated
1653 frames in an attempt to detect such frames for analysis or removal.
1654
1655 The fields are:
1656
1657 Magic_Number: 4 bytes, little-endian format. Value: 0x184D2A5?,
1658 which means any value from 0x184D2A50 to 0x184D2A5F. All 16
1659 values are valid to identify a skippable frame. This
1660 specification does not detail any specific tagging methods for
1661 skippable frames.
1662
1663 Frame_Size: This is the size, in bytes, of the following User_Data
1664 (without including the magic number nor the size field itself).
1665 This field is represented using 4 bytes, little-endian format,
1666 unsigned 32 bits. This means User_Data can't be bigger than
1667 (2^32-1) bytes.
1668
1669 User_Data: This field can be anything. Data will just be skipped by
1670 the decoder.
1671
16724. Entropy Encoding
1673
1674 Two types of entropy encoding are used by the Zstandard format: FSE
1675 and Huffman coding. Huffman is used to compress literals, while FSE
1676 is used for all other symbols (Literals_Length_Code,
1677 Match_Length_Code, and offset codes) and to compress Huffman headers.
1678
1679
1680
1681
1682Collet & Kucherawy Informational [Page 30]
1683
1684RFC 8478 application/zstd October 2018
1685
1686
16874.1. FSE
1688
1689 FSE, short for Finite State Entropy, is an entropy codec based on
1690 [ANS]. FSE encoding/decoding involves a state that is carried over
1691 between symbols, so decoding must be done in the opposite direction
1692 as encoding. Therefore, all FSE bitstreams are read from end to
1693 beginning. Note that the order of the bits in the stream is not
1694 reversed; they are simply read in the reverse order from which they
1695 were written.
1696
1697 For additional details on FSE, see Finite State Entropy [FSE].
1698
1699 FSE decoding involves a decoding table that has a power of 2 size and
1700 contains three elements: Symbol, Num_Bits, and Baseline. The base 2
1701 logarithm of the table size is its Accuracy_Log. An FSE state value
1702 represents an index in this table.
1703
1704 To obtain the initial state value, consume Accuracy_Log bits from the
1705 stream as a little-endian value. The next symbol in the stream is
1706 the Symbol indicated in the table for that state. To obtain the next
1707 state value, the decoder should consume Num_Bits bits from the stream
1708 as a little-endian value and add it to Baseline.
1709
17104.1.1. FSE Table Description
1711
1712 To decode FSE streams, it is necessary to construct the decoding
1713 table. The Zstandard format encodes FSE table descriptions as
1714 described here.
1715
1716 An FSE distribution table describes the probabilities of all symbols
1717 from 0 to the last present one (included) on a normalized scale of
1718 (1 << Accuracy_Log). Note that there must be two or more symbols
1719 with non-zero probability.
1720
1721 A bitstream is read forward, in little-endian fashion. It is not
1722 necessary to know its exact size, since the size will be discovered
1723 and reported by the decoding process. The bitstream starts by
1724 reporting on which scale it operates. If low4bits designates the
1725 lowest 4 bits of the first byte, then Accuracy_Log = low4bits + 5.
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738Collet & Kucherawy Informational [Page 31]
1739
1740RFC 8478 application/zstd October 2018
1741
1742
1743 This is followed by each symbol value, from 0 to the last present
1744 one. The number of bits used by each field is variable and depends
1745 on:
1746
1747 Remaining probabilities + 1: For example, presuming an Accuracy_Log
1748 of 8, and presuming 100 probabilities points have already been
1749 distributed, the decoder may read any value from 0 to
1750 (256 - 100 + 1) == 157, inclusive. Therefore, it must read
1751 log2sup(157) == 8 bits.
1752
1753 Value decoded: Small values use 1 fewer bit. For example, presuming
1754 values from 0 to 157 (inclusive) are possible, 255 - 157 = 98
1755 values are remaining in an 8-bit field. The first 98 values
1756 (hence from 0 to 97) use only 7 bits, and values from 98 to 157
1757 use 8 bits. This is achieved through this scheme:
1758
1759 +------------+---------------+-----------+
1760 | Value Read | Value Decoded | Bits Used |
1761 +------------+---------------+-----------+
1762 | 0 - 97 | 0 - 97 | 7 |
1763 +------------+---------------+-----------+
1764 | 98 - 127 | 98 - 127 | 8 |
1765 +------------+---------------+-----------+
1766 | 128 - 225 | 0 - 97 | 7 |
1767 +------------+---------------+-----------+
1768 | 226 - 255 | 128 - 157 | 8 |
1769 +------------+---------------+-----------+
1770
1771 Symbol probabilities are read one by one, in order. The probability
1772 is obtained from Value decoded using the formula P = Value - 1. This
1773 means the value 0 becomes the negative probability -1. This is a
1774 special probability that means "less than 1". Its effect on the
1775 distribution table is described below. For the purpose of
1776 calculating total allocated probability points, it counts as 1.
1777
1778 When a symbol has a probability of zero, it is followed by a 2-bit
1779 repeat flag. This repeat flag tells how many probabilities of zeroes
1780 follow the current one. It provides a number ranging from 0 to 3.
1781 If it is a 3, another 2-bit repeat flag follows, and so on.
1782
1783 When the last symbol reaches a cumulated total of
1784 (1 << Accuracy_Log), decoding is complete. If the last symbol makes
1785 the cumulated total go above (1 << Accuracy_Log), distribution is
1786 considered corrupted.
1787
1788
1789
1790
1791
1792
1793
1794Collet & Kucherawy Informational [Page 32]
1795
1796RFC 8478 application/zstd October 2018
1797
1798
1799 Finally, the decoder can tell how many bytes were used in this
1800 process and how many symbols are present. The bitstream consumes a
1801 round number of bytes. Any remaining bit within the last byte is
1802 simply unused.
1803
1804 The distribution of normalized probabilities is enough to create a
1805 unique decoding table. The table has a size of (1 << Accuracy_Log).
1806 Each cell describes the symbol decoded and instructions to get the
1807 next state.
1808
1809 Symbols are scanned in their natural order for "less than 1"
1810 probabilities as described above. Symbols with this probability are
1811 being attributed a single cell, starting from the end of the table
1812 and retreating. These symbols define a full state reset, reading
1813 Accuracy_Log bits.
1814
1815 All remaining symbols are allocated in their natural order. Starting
1816 from symbol 0 and table position 0, each symbol gets allocated as
1817 many cells as its probability. Cell allocation is spread, not
1818 linear; each successor position follows this rule:
1819
1820 position += (tableSize >> 1) + (tableSize >> 3) + 3;
1821 position &= tableSize - 1;
1822
1823 A position is skipped if it is already occupied by a "less than 1"
1824 probability symbol. Position does not reset between symbols; it
1825 simply iterates through each position in the table, switching to the
1826 next symbol when enough states have been allocated to the current
1827 one.
1828
1829 The result is a list of state values. Each state will decode the
1830 current symbol.
1831
1832 To get the Number_of_Bits and Baseline required for the next state,
1833 it is first necessary to sort all states in their natural order. The
1834 lower states will need 1 more bit than higher ones. The process is
1835 repeated for each symbol.
1836
1837 For example, presuming a symbol has a probability of 5, it receives
1838 five state values. States are sorted in natural order. The next
1839 power of 2 is 8. The space of probabilities is divided into 8 equal
1840 parts. Presuming the Accuracy_Log is 7, this defines 128 states, and
1841 each share (divided by 8) is 16 in size. In order to reach 8, 8 - 5
1842 = 3 lowest states will count "double", doubling the number of shares
1843 (32 in width), requiring 1 more bit in the process.
1844
1845
1846
1847
1848
1849
1850Collet & Kucherawy Informational [Page 33]
1851
1852RFC 8478 application/zstd October 2018
1853
1854
1855 Baseline is assigned starting from the higher states using fewer
1856 bits, and proceeding naturally, then resuming at the first state,
1857 each taking its allocated width from Baseline.
1858
1859 +----------------+-------+-------+--------+------+-------+
1860 | state order | 0 | 1 | 2 | 3 | 4 |
1861 +----------------+-------+-------+--------+------+-------+
1862 | width | 32 | 32 | 32 | 16 | 16 |
1863 +----------------+-------+-------+--------+------+-------+
1864 | Number_of_Bits | 5 | 5 | 5 | 4 | 4 |
1865 +----------------+-------+-------+--------+------+-------+
1866 | range number | 2 | 4 | 6 | 0 | 1 |
1867 +----------------+-------+-------+--------+------+-------+
1868 | Baseline | 32 | 64 | 96 | 0 | 16 |
1869 +----------------+-------+-------+--------+------+-------+
1870 | range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
1871 +----------------+-------+-------+--------+------+-------+
1872
1873 The next state is determined from the current state by reading the
1874 required Number_of_Bits and adding the specified Baseline.
1875
1876 See Appendix A for the results of this process that are applied to
1877 the default distributions.
1878
18794.2. Huffman Coding
1880
1881 Zstandard Huffman-coded streams are read backwards, similar to the
1882 FSE bitstreams. Therefore, to find the start of the bitstream, it is
1883 necessary to know the offset of the last byte of the Huffman-coded
1884 stream.
1885
1886 After writing the last bit containing information, the compressor
1887 writes a single 1 bit and then fills the byte with 0-7 0 bits of
1888 padding. The last byte of the compressed bitstream cannot be 0 for
1889 that reason.
1890
1891 When decompressing, the last byte containing the padding is the first
1892 byte to read. The decompressor needs to skip 0-7 initial 0 bits and
1893 the first 1 bit that occurs. Afterwards, the useful part of the
1894 bitstream begins.
1895
1896 The bitstream contains Huffman-coded symbols in little-endian order,
1897 with the codes defined by the method below.
1898
1899
1900
1901
1902
1903
1904
1905
1906Collet & Kucherawy Informational [Page 34]
1907
1908RFC 8478 application/zstd October 2018
1909
1910
19114.2.1. Huffman Tree Description
1912
1913 Prefix coding represents symbols from an a priori known alphabet by
1914 bit sequences (codewords), one codeword for each symbol, in a manner
1915 such that different symbols may be represented by bit sequences of
1916 different lengths, but a parser can always parse an encoded string
1917 unambiguously symbol by symbol.
1918
1919 Given an alphabet with known symbol frequencies, the Huffman
1920 algorithm allows the construction of an optimal prefix code using the
1921 fewest bits of any possible prefix codes for that alphabet.
1922
1923 The prefix code must not exceed a maximum code length. More bits
1924 improve accuracy but yield a larger header size and require more
1925 memory or more complex decoding operations. This specification
1926 limits the maximum code length to 11 bits.
1927
1928 All literal values from zero (included) to the last present one
1929 (excluded) are represented by Weight with values from 0 to
1930 Max_Number_of_Bits. Transformation from Weight to Number_of_Bits
1931 follows this pseudocode:
1932
1933 if Weight == 0
1934 Number_of_Bits = 0
1935 else
1936 Number_of_Bits = Max_Number_of_Bits + 1 - Weight
1937
1938 The last symbol's Weight is deduced from previously decoded ones, by
1939 completing to the nearest power of 2. This power of 2 gives
1940 Max_Number_of_Bits the depth of the current tree.
1941
1942 For example, presume the following Huffman tree must be described:
1943
1944 +---------------+----------------+
1945 | Literal Value | Number_of_Bits |
1946 +---------------+----------------+
1947 | 0 | 1 |
1948 +---------------+----------------+
1949 | 1 | 2 |
1950 +---------------+----------------+
1951 | 2 | 3 |
1952 +---------------+----------------+
1953 | 3 | 0 |
1954 +---------------+----------------+
1955 | 4 | 4 |
1956 +---------------+----------------+
1957 | 5 | 4 |
1958 +---------------+----------------+
1959
1960
1961
1962Collet & Kucherawy Informational [Page 35]
1963
1964RFC 8478 application/zstd October 2018
1965
1966
1967 The tree depth is 4, since its longest element uses 4 bits. (The
1968 longest elements are those with the smallest frequencies.) Value 5
1969 will not be listed as it can be determined from the values for 0-4,
1970 nor will values above 5 as they are all 0. Values from 0 to 4 will
1971 be listed using Weight instead of Number_of_Bits. The pseudocode to
1972 determine Weight is:
1973
1974 if Number_of_Bits == 0
1975 Weight = 0
1976 else
1977 Weight = Max_Number_of_Bits + 1 - Number_of_Bits
1978
1979 It gives the following series of weights:
1980
1981 +---------------+--------+
1982 | Literal Value | Weight |
1983 +---------------+--------+
1984 | 0 | 4 |
1985 +---------------+--------+
1986 | 1 | 3 |
1987 +---------------+--------+
1988 | 2 | 2 |
1989 +---------------+--------+
1990 | 3 | 0 |
1991 +---------------+--------+
1992 | 4 | 1 |
1993 +---------------+--------+
1994
1995 The decoder will do the inverse operation: having collected weights
1996 of literals from 0 to 4, it knows the last literal, 5, is present
1997 with a non-zero Weight. The Weight of 5 can be determined by
1998 advancing to the next power of 2. The sum of 2^(Weight-1) (excluding
1999 0's) is 15. The nearest power of 2 is 16. Therefore,
2000 Max_Number_of_Bits = 4 and Weight[5] = 16 - 15 = 1.
2001
20024.2.1.1. Huffman Tree Header
2003
2004 This is a single byte value (0-255), which describes how the series
2005 of weights is encoded.
2006
2007 headerByte < 128: The series of weights is compressed using FSE (see
2008 below). The length of the FSE-compressed series is equal to
2009 headerByte (0-127).
2010
2011
2012
2013
2014
2015
2016
2017
2018Collet & Kucherawy Informational [Page 36]
2019
2020RFC 8478 application/zstd October 2018
2021
2022
2023 headerByte >= 128: This is a direct representation, where each
2024 Weight is written directly as a 4-bit field (0-15). They are
2025 encoded forward, 2 weights to a byte with the first weight taking
2026 the top 4 bits and the second taking the bottom 4; for example,
2027 the following operations could be used to read the weights:
2028
2029 Weight[0] = (Byte[0] >> 4)
2030 Weight[1] = (Byte[0] & 0xf),
2031 etc.
2032
2033 The full representation occupies ceiling(Number_of_Symbols/2)
2034 bytes, meaning it uses only full bytes even if Number_of_Symbols
2035 is odd. Number_of_Symbols = headerByte - 127. Note that maximum
2036 Number_of_Symbols is 255 - 127 = 128. If any literal has a value
2037 over 128, raw header mode is not possible, and it is necessary to
2038 use FSE compression.
2039
20404.2.1.2. FSE Compression of Huffman Weights
2041
2042 In this case, the series of Huffman weights is compressed using FSE
2043 compression. It is a single bitstream with two interleaved states,
2044 sharing a single distribution table.
2045
2046 To decode an FSE bitstream, it is necessary to know its compressed
2047 size. Compressed size is provided by headerByte. It's also
2048 necessary to know its maximum possible decompressed size, which is
2049 255, since literal values span from 0 to 255, and the last symbol's
2050 Weight is not represented.
2051
2052 An FSE bitstream starts by a header, describing probabilities
2053 distribution. It will create a decoding table. For a list of
2054 Huffman weights, the maximum accuracy log is 6 bits. For more
2055 details, see Section 4.1.1.
2056
2057 The Huffman header compression uses two states, which share the same
2058 FSE distribution table. The first state (State1) encodes the even-
2059 numbered index symbols, and the second (State2) encodes the odd-
2060 numbered index symbols. State1 is initialized first, and then
2061 State2, and they take turns decoding a single symbol and updating
2062 their state. For more details on these FSE operations, see
2063 Section 4.1.
2064
2065 The number of symbols to be decoded is determined by tracking the
2066 bitStream overflow condition: If updating state after decoding a
2067 symbol would require more bits than remain in the stream, it is
2068 assumed that extra bits are zero. Then, symbols for each of the
2069 final states are decoded and the process is complete.
2070
2071
2072
2073
2074Collet & Kucherawy Informational [Page 37]
2075
2076RFC 8478 application/zstd October 2018
2077
2078
20794.2.1.3. Conversion from Weights to Huffman Prefix Codes
2080
2081 All present symbols will now have a Weight value. It is possible to
2082 transform weights into Number_of_Bits, using this formula:
2083
2084 if Weight > 0
2085 Number_of_Bits = Max_Number_of_Bits + 1 - Weight
2086 else
2087 Number_of_Bits = 0
2088
2089 Symbols are sorted by Weight. Within the same Weight, symbols keep
2090 natural sequential order. Symbols with a Weight of zero are removed.
2091 Then, starting from the lowest Weight, prefix codes are distributed
2092 in sequential order.
2093
2094 For example, assume the following list of weights has been decoded:
2095
2096 +---------+--------+
2097 | Literal | Weight |
2098 +---------+--------+
2099 | 0 | 4 |
2100 +---------+--------+
2101 | 1 | 3 |
2102 +---------+--------+
2103 | 2 | 2 |
2104 +---------+--------+
2105 | 3 | 0 |
2106 +---------+--------+
2107 | 4 | 1 |
2108 +---------+--------+
2109 | 5 | 1 |
2110 +---------+--------+
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130Collet & Kucherawy Informational [Page 38]
2131
2132RFC 8478 application/zstd October 2018
2133
2134
2135 Sorting by weight and then the natural sequential order yields the
2136 following distribution:
2137
2138 +---------+--------+----------------+--------------+
2139 | Literal | Weight | Number_Of_Bits | Prefix Codes |
2140 +---------+--------+----------------|--------------+
2141 | 3 | 0 | 0 | N/A |
2142 +---------+--------+----------------|--------------+
2143 | 4 | 1 | 4 | 0000 |
2144 +---------+--------+----------------|--------------+
2145 | 5 | 1 | 4 | 0001 |
2146 +---------+--------+----------------|--------------+
2147 | 2 | 2 | 3 | 001 |
2148 +---------+--------+----------------|--------------+
2149 | 1 | 3 | 2 | 01 |
2150 +---------+--------+----------------|--------------+
2151 | 0 | 4 | 1 | 1 |
2152 +---------+--------+----------------|--------------+
2153
21544.2.2. Huffman-Coded Streams
2155
2156 Given a Huffman decoding table, it is possible to decode a Huffman-
2157 coded stream.
2158
2159 Each bitstream must be read backward, which starts from the end and
2160 goes up to the beginning. Therefore, it is necessary to know the
2161 size of each bitstream.
2162
2163 It is also necessary to know exactly which bit is the last. This is
2164 detected by a final bit flag: the highest bit of the last byte is a
2165 final-bit-flag. Consequently, a last byte of 0 is not possible. And
2166 the final-bit-flag itself is not part of the useful bitstream.
2167 Hence, the last byte contains between 0 and 7 useful bits.
2168
2169 Starting from the end, it is possible to read the bitstream in a
2170 little-endian fashion, keeping track of already used bits. Since the
2171 bitstream is encoded in reverse order, starting from the end, read
2172 symbols in forward order.
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186Collet & Kucherawy Informational [Page 39]
2187
2188RFC 8478 application/zstd October 2018
2189
2190
2191 For example, if the literal sequence "0145" was encoded using the
2192 above prefix code, it would be encoded (in reverse order) as:
2193
2194 +---------+----------+
2195 | Symbol | Encoding |
2196 +---------+----------+
2197 | 5 | 0000 |
2198 +---------+----------+
2199 | 4 | 0001 |
2200 +---------+----------+
2201 | 1 | 01 |
2202 +---------+----------+
2203 | 0 | 1 |
2204 +---------+----------+
2205 | Padding | 00001 |
2206 +---------+----------+
2207
2208 This results in the following 2-byte bitstream:
2209
2210 00010000 00001101
2211
2212 Here is an alternative representation with the symbol codes separated
2213 by underscores:
2214
2215 0001_0000 00001_1_01
2216
2217 Reading the highest Max_Number_of_Bits bits, it's possible to compare
2218 the extracted value to the decoding table, determining the symbol to
2219 decode and number of bits to discard.
2220
2221 The process continues reading up to the required number of symbols
2222 per stream. If a bitstream is not entirely and exactly consumed,
2223 hence reaching exactly its beginning position with all bits consumed,
2224 the decoding process is considered faulty.
2225
22265. Dictionary Format
2227
2228 Zstandard is compatible with "raw content" dictionaries, free of any
2229 format restriction, except that they must be at least 8 bytes. These
2230 dictionaries function as if they were just the content part of a
2231 formatted dictionary.
2232
2233 However, dictionaries created by "zstd --train" in the reference
2234 implementation follow a specific format, described here.
2235
2236 Dictionaries are not included in the compressed content but rather
2237 are provided out of band. That is, the Dictionary_ID identifies
2238 which should be used, but this specification does not describe the
2239
2240
2241
2242Collet & Kucherawy Informational [Page 40]
2243
2244RFC 8478 application/zstd October 2018
2245
2246
2247 mechanism by which the dictionary is obtained prior to use during
2248 compression or decompression.
2249
2250 A dictionary has a size, defined either by a buffer limit or a file
2251 size. The general format is:
2252
2253 +--------------+---------------+----------------+---------+
2254 | Magic_Number | Dictionary_ID | Entropy_Tables | Content |
2255 +--------------+---------------+----------------+---------+
2256
2257 Magic_Number: 4 bytes ID, value 0xEC30A437, little-endian format.
2258
2259 Dictionary_ID: 4 bytes, stored in little-endian format.
2260 Dictionary_ID can be any value, except 0 (which means no
2261 Dictionary_ID). It is used by decoders to check if they use the
2262 correct dictionary. If the frame is going to be distributed in a
2263 private environment, any Dictionary_ID can be used. However, for
2264 public distribution of compressed frames, the following ranges are
2265 reserved and shall not be used:
2266
2267 low range: <= 32767
2268 high range: >= (2^31)
2269
2270 Entropy_Tables: Follow the same format as the tables in compressed
2271 blocks. See the relevant FSE and Huffman sections for how to
2272 decode these tables. They are stored in the following order:
2273 Huffman table for literals, FSE table for offsets, FSE table for
2274 match lengths, and FSE table for literals lengths. These tables
2275 populate the Repeat Stats literals mode and Repeat distribution
2276 mode for sequence decoding. It is finally followed by 3 offset
2277 values, populating repeat offsets (instead of using {1,4,8}),
2278 stored in order, 4-bytes little-endian each, for a total of 12
2279 bytes. Each repeat offset must have a value less than the
2280 dictionary size.
2281
2282 Content: The rest of the dictionary is its content. The content
2283 acts as a "past" in front of data to be compressed or
2284 decompressed, so it can be referenced in sequence commands. As
2285 long as the amount of data decoded from this frame is less than or
2286 equal to Window_Size, sequence commands may specify offsets longer
2287 than the total length of decoded output so far to reference back
2288 to the dictionary, even parts of the dictionary with offsets
2289 larger than Window_Size. After the total output has surpassed
2290 Window_Size, however, this is no longer allowed, and the
2291 dictionary is no longer accessible.
2292
2293
2294
2295
2296
2297
2298Collet & Kucherawy Informational [Page 41]
2299
2300RFC 8478 application/zstd October 2018
2301
2302
23036. IANA Considerations
2304
2305 IANA has made two registrations, as described below.
2306
23076.1. The 'application/zstd' Media Type
2308
2309 The 'application/zstd' media type identifies a block of data that is
2310 compressed using zstd compression. The data is a stream of bytes as
2311 described in this document. IANA has added the following to the
2312 "Media Types" registry:
2313
2314 Type name: application
2315
2316 Subtype name: zstd
2317
2318 Required parameters: N/A
2319
2320 Optional parameters: N/A
2321
2322 Encoding considerations: binary
2323
2324 Security considerations: See Section 7 of RFC 8478
2325
2326 Interoperability considerations: N/A
2327
2328 Published specification: RFC 8478
2329
2330 Applications that use this media type: anywhere data size is an
2331 issue
2332
2333 Additional information:
2334
2335 Magic number(s): 4 bytes, little-endian format.
2336 Value: 0xFD2FB528
2337
2338 File extension(s): zst
2339
2340 Macintosh file type code(s): N/A
2341
2342 For further information: See [ZSTD]
2343
2344 Intended usage: common
2345
2346 Restrictions on usage: N/A
2347
2348 Author: Murray S. Kucherawy
2349
2350 Change Controller: IETF
2351
2352
2353
2354Collet & Kucherawy Informational [Page 42]
2355
2356RFC 8478 application/zstd October 2018
2357
2358
2359 Provisional registration: no
2360
23616.2. Content Encoding
2362
2363 IANA has added the following entry to the "HTTP Content Coding
2364 Registry" within the "Hypertext Transfer Protocol (HTTP) Parameters"
2365 registry:
2366
2367 Name: zstd
2368
2369 Description: A stream of bytes compressed using the Zstandard
2370 protocol
2371
2372 Pointer to specification text: RFC 8478
2373
23746.3. Dictionaries
2375
2376 Work in progress includes development of dictionaries that will
2377 optimize compression and decompression of particular types of data.
2378 Specification of such dictionaries for public use will necessitate
2379 registration of a code point from the reserved range described in
2380 Section 3.1.1.1.3 and its association with a specific dictionary.
2381
2382 However, there are at present no such dictionaries published for
2383 public use, so this document makes no immediate request of IANA to
2384 create such a registry.
2385
23867. Security Considerations
2387
2388 Any data compression method involves the reduction of redundancy in
2389 the data. Zstandard is no exception, and the usual precautions
2390 apply.
2391
2392 One should never compress a message whose content must remain secret
2393 with a message generated by a third party. Such a compression can be
2394 used to guess the content of the secret message through analysis of
2395 entropy reduction. This was demonstrated in the Compression Ratio
2396 Info-leak Made Easy (CRIME) attack [CRIME], for example.
2397
2398 A decoder has to demonstrate capabilities to detect and prevent any
2399 kind of data tampering in the compressed frame from triggering system
2400 faults, such as reading or writing beyond allowed memory ranges.
2401 This can be guaranteed by either the implementation language or
2402 careful bound checkings. Of particular note is the encoding of
2403 Number_of_Sequences values that cause the decoder to read into the
2404 block header (and beyond), as well as the indication of a
2405 Frame_Content_Size that is smaller than the actual decompressed data,
2406 in an attempt to trigger a buffer overflow. It is highly recommended
2407
2408
2409
2410Collet & Kucherawy Informational [Page 43]
2411
2412RFC 8478 application/zstd October 2018
2413
2414
2415 to fuzz-test (i.e., provide invalid, unexpected, or random input and
2416 verify safe operation of) decoder implementations to test and harden
2417 their capability to detect bad frames and deal with them without any
2418 adverse system side effect.
2419
2420 An attacker may provide correctly formed compressed frames with
2421 unreasonable memory requirements. A decoder must always control
2422 memory requirements and enforce some (system-specific) limits in
2423 order to protect memory usage from such scenarios.
2424
2425 Compression can be optimized by training a dictionary on a variety of
2426 related content payloads. This dictionary must then be available at
2427 the decoder for decompression of the payload to be possible. While
2428 this document does not specify how to acquire a dictionary for a
2429 given compressed payload, it is worth noting that third-party
2430 dictionaries may interact unexpectedly with a decoder, leading to
2431 possible memory or other resource exhaustion attacks. We expect such
2432 topics to be discussed in further detail in the Security
2433 Considerations section of a forthcoming RFC for dictionary
2434 acquisition and transmission, but highlight this issue now out of an
2435 abundance of caution.
2436
2437 As discussed in Section 3.1.2, it is possible to store arbitrary user
2438 metadata in skippable frames. While such frames are ignored during
2439 decompression of the data, they can be used as a watermark to track
2440 the path of the compressed payload.
2441
24428. Implementation Status
2443
2444 Source code for a C language implementation of a Zstandard-compliant
2445 library is available at [ZSTD-GITHUB]. This implementation is
2446 considered to be the reference implementation and is production
2447 ready; it implements the full range of the specification. It is
2448 routinely tested against security hazards and widely deployed within
2449 Facebook infrastructure.
2450
2451 The reference version is optimized for speed and is highly portable.
2452 It has been proven to run safely on multiple architectures (e.g.,
2453 x86, x64, ARM, MIPS, PowerPC, IA64) featuring 32- or 64-bit
2454 addressing schemes, a little- or big-endian storage scheme, a number
2455 of different operating systems (e.g., UNIX (including Linux, BSD,
2456 OS-X, and Solaris) and Windows), and a number of compilers (e.g.,
2457 gcc, clang, visual, and icc).
2458
2459
2460
2461
2462
2463
2464
2465
2466Collet & Kucherawy Informational [Page 44]
2467
2468RFC 8478 application/zstd October 2018
2469
2470
24719. References
2472
24739.1. Normative References
2474
2475 [ZSTD] "Zstandard", <http://www.zstd.net>.
2476
24779.2. Informative References
2478
2479 [ANS] Duda, J., "Asymmetric numeral systems: entropy coding
2480 combining speed of Huffman coding with compression rate of
2481 arithmetic coding", January 2014,
2482 <https://arxiv.org/pdf/1311.2540>.
2483
2484 [CRIME] "CRIME", June 2018, <https://en.wikipedia.org/w/
2485 index.php?title=CRIME&oldid=844538656>.
2486
2487 [FSE] "FiniteStateEntropy", commit 6efa78a, June 2018,
2488 <https://github.com/Cyan4973/FiniteStateEntropy/>.
2489
2490 [LZ4] "LZ4 Frame Format Description", commit d03224b, January
2491 2018, <https://github.com/lz4/lz4/blob/master/doc/
2492 lz4_Frame_format.md>.
2493
2494 [RFC1952] Deutsch, P., "GZIP file format specification version 4.3",
2495 RFC 1952, DOI 10.17487/RFC1952, May 1996,
2496 <https://www.rfc-editor.org/info/rfc1952>.
2497
2498 [XXHASH] "XXHASH Algorithm", <http://www.xxhash.org>.
2499
2500 [ZSTD-GITHUB]
2501 "zstd", commit 8514bd8, August 2018,
2502 <https://github.com/facebook/zstd>.
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522Collet & Kucherawy Informational [Page 45]
2523
2524RFC 8478 application/zstd October 2018
2525
2526
2527Appendix A. Decoding Tables for Predefined Codes
2528
2529 This appendix contains FSE decoding tables for the predefined literal
2530 length, match length, and offset codes. The tables have been
2531 constructed using the algorithm as given above in Section 4.1.1. The
2532 tables here can be used as examples to crosscheck that an
2533 implementation has built its decoding tables correctly.
2534
2535A.1. Literal Length Code Table
2536
2537 +-------+--------+----------------+------+
2538 | State | Symbol | Number_Of_Bits | Base |
2539 +-------+--------+----------------+------+
2540 | 0 | 0 | 0 | 0 |
2541 +-------+--------+----------------+------+
2542 | 0 | 0 | 4 | 0 |
2543 +-------+--------+----------------+------+
2544 | 1 | 0 | 4 | 16 |
2545 +-------+--------+----------------+------+
2546 | 2 | 1 | 5 | 32 |
2547 +-------+--------+----------------+------+
2548 | 3 | 3 | 5 | 0 |
2549 +-------+--------+----------------+------+
2550 | 4 | 4 | 5 | 0 |
2551 +-------+--------+----------------+------+
2552 | 5 | 6 | 5 | 0 |
2553 +-------+--------+----------------+------+
2554 | 6 | 7 | 5 | 0 |
2555 +-------+--------+----------------+------+
2556 | 7 | 9 | 5 | 0 |
2557 +-------+--------+----------------+------+
2558 | 8 | 10 | 5 | 0 |
2559 +-------+--------+----------------+------+
2560 | 9 | 12 | 5 | 0 |
2561 +-------+--------+----------------+------+
2562 | 10 | 14 | 6 | 0 |
2563 +-------+--------+----------------+------+
2564 | 11 | 16 | 5 | 0 |
2565 +-------+--------+----------------+------+
2566 | 12 | 18 | 5 | 0 |
2567 +-------+--------+----------------+------+
2568 | 13 | 19 | 5 | 0 |
2569 +-------+--------+----------------+------+
2570 | 14 | 21 | 5 | 0 |
2571 +-------+--------+----------------+------+
2572 | 15 | 22 | 5 | 0 |
2573 +-------+--------+----------------+------+
2574 | 16 | 24 | 5 | 0 |
2575
2576
2577
2578Collet & Kucherawy Informational [Page 46]
2579
2580RFC 8478 application/zstd October 2018
2581
2582
2583 +-------+--------+----------------+------+
2584 | 17 | 25 | 5 | 32 |
2585 +-------+--------+----------------+------+
2586 | 18 | 26 | 5 | 0 |
2587 +-------+--------+----------------+------+
2588 | 19 | 27 | 6 | 0 |
2589 +-------+--------+----------------+------+
2590 | 20 | 29 | 6 | 0 |
2591 +-------+--------+----------------+------+
2592 | 21 | 31 | 6 | 0 |
2593 +-------+--------+----------------+------+
2594 | 22 | 0 | 4 | 32 |
2595 +-------+--------+----------------+------+
2596 | 23 | 1 | 4 | 0 |
2597 +-------+--------+----------------+------+
2598 | 24 | 2 | 5 | 0 |
2599 +-------+--------+----------------+------+
2600 | 25 | 4 | 5 | 32 |
2601 +-------+--------+----------------+------+
2602 | 26 | 5 | 5 | 0 |
2603 +-------+--------+----------------+------+
2604 | 27 | 7 | 5 | 32 |
2605 +-------+--------+----------------+------+
2606 | 28 | 8 | 5 | 0 |
2607 +-------+--------+----------------+------+
2608 | 29 | 10 | 5 | 32 |
2609 +-------+--------+----------------+------+
2610 | 30 | 11 | 5 | 0 |
2611 +-------+--------+----------------+------+
2612 | 31 | 13 | 6 | 0 |
2613 +-------+--------+----------------+------+
2614 | 32 | 16 | 5 | 32 |
2615 +-------+--------+----------------+------+
2616 | 33 | 17 | 5 | 0 |
2617 +-------+--------+----------------+------+
2618 | 34 | 19 | 5 | 32 |
2619 +-------+--------+----------------+------+
2620 | 35 | 20 | 5 | 0 |
2621 +-------+--------+----------------+------+
2622 | 36 | 22 | 5 | 32 |
2623 +-------+--------+----------------+------+
2624 | 37 | 23 | 5 | 0 |
2625 +-------+--------+----------------+------+
2626 | 38 | 25 | 4 | 0 |
2627 +-------+--------+----------------+------+
2628 | 39 | 25 | 4 | 16 |
2629 +-------+--------+----------------+------+
2630 | 40 | 26 | 5 | 32 |
2631
2632
2633
2634Collet & Kucherawy Informational [Page 47]
2635
2636RFC 8478 application/zstd October 2018
2637
2638
2639 +-------+--------+----------------+------+
2640 | 41 | 28 | 6 | 0 |
2641 +-------+--------+----------------+------+
2642 | 42 | 30 | 6 | 0 |
2643 +-------+--------+----------------+------+
2644 | 43 | 0 | 4 | 48 |
2645 +-------+--------+----------------+------+
2646 | 44 | 1 | 4 | 16 |
2647 +-------+--------+----------------+------+
2648 | 45 | 2 | 5 | 32 |
2649 +-------+--------+----------------+------+
2650 | 46 | 3 | 5 | 32 |
2651 +-------+--------+----------------+------+
2652 | 47 | 5 | 5 | 32 |
2653 +-------+--------+----------------+------+
2654 | 48 | 6 | 5 | 32 |
2655 +-------+--------+----------------+------+
2656 | 49 | 8 | 5 | 32 |
2657 +-------+--------+----------------+------+
2658 | 50 | 9 | 5 | 32 |
2659 +-------+--------+----------------+------+
2660 | 51 | 11 | 5 | 32 |
2661 +-------+--------+----------------+------+
2662 | 52 | 12 | 5 | 32 |
2663 +-------+--------+----------------+------+
2664 | 53 | 15 | 6 | 0 |
2665 +-------+--------+----------------+------+
2666 | 54 | 17 | 5 | 32 |
2667 +-------+--------+----------------+------+
2668 | 55 | 18 | 5 | 32 |
2669 +-------+--------+----------------+------+
2670 | 56 | 20 | 5 | 32 |
2671 +-------+--------+----------------+------+
2672 | 57 | 21 | 5 | 32 |
2673 +-------+--------+----------------+------+
2674 | 58 | 23 | 5 | 32 |
2675 +-------+--------+----------------+------+
2676 | 59 | 24 | 5 | 32 |
2677 +-------+--------+----------------+------+
2678 | 60 | 35 | 6 | 0 |
2679 +-------+--------+----------------+------+
2680 | 61 | 34 | 6 | 0 |
2681 +-------+--------+----------------+------+
2682 | 62 | 33 | 6 | 0 |
2683 +-------+--------+----------------+------+
2684 | 63 | 32 | 6 | 0 |
2685 +-------+--------+----------------+------+
2686
2687
2688
2689
2690Collet & Kucherawy Informational [Page 48]
2691
2692RFC 8478 application/zstd October 2018
2693
2694
2695A.2. Match Length Code Table
2696
2697 +-------+--------+----------------+------+
2698 | State | Symbol | Number_Of_Bits | Base |
2699 +-------+--------+----------------+------+
2700 | 0 | 0 | 0 | 0 |
2701 +-------+--------+----------------+------+
2702 | 0 | 0 | 6 | 0 |
2703 +-------+--------+----------------+------+
2704 | 1 | 1 | 4 | 0 |
2705 +-------+--------+----------------+------+
2706 | 2 | 2 | 5 | 32 |
2707 +-------+--------+----------------+------+
2708 | 3 | 3 | 5 | 0 |
2709 +-------+--------+----------------+------+
2710 | 4 | 5 | 5 | 0 |
2711 +-------+--------+----------------+------+
2712 | 5 | 6 | 5 | 0 |
2713 +-------+--------+----------------+------+
2714 | 6 | 8 | 5 | 0 |
2715 +-------+--------+----------------+------+
2716 | 7 | 10 | 6 | 0 |
2717 +-------+--------+----------------+------+
2718 | 8 | 13 | 6 | 0 |
2719 +-------+--------+----------------+------+
2720 | 9 | 16 | 6 | 0 |
2721 +-------+--------+----------------+------+
2722 | 10 | 19 | 6 | 0 |
2723 +-------+--------+----------------+------+
2724 | 11 | 22 | 6 | 0 |
2725 +-------+--------+----------------+------+
2726 | 12 | 25 | 6 | 0 |
2727 +-------+--------+----------------+------+
2728 | 13 | 28 | 6 | 0 |
2729 +-------+--------+----------------+------+
2730 | 14 | 31 | 6 | 0 |
2731 +-------+--------+----------------+------+
2732 | 15 | 33 | 6 | 0 |
2733 +-------+--------+----------------+------+
2734 | 16 | 35 | 6 | 0 |
2735 +-------+--------+----------------+------+
2736 | 17 | 37 | 6 | 0 |
2737 +-------+--------+----------------+------+
2738 | 18 | 39 | 6 | 0 |
2739 +-------+--------+----------------+------+
2740 | 19 | 41 | 6 | 0 |
2741 +-------+--------+----------------+------+
2742 | 20 | 43 | 6 | 0 |
2743
2744
2745
2746Collet & Kucherawy Informational [Page 49]
2747
2748RFC 8478 application/zstd October 2018
2749
2750
2751 +-------+--------+----------------+------+
2752 | 21 | 45 | 6 | 0 |
2753 +-------+--------+----------------+------+
2754 | 22 | 1 | 4 | 16 |
2755 +-------+--------+----------------+------+
2756 | 23 | 2 | 4 | 0 |
2757 +-------+--------+----------------+------+
2758 | 24 | 3 | 5 | 32 |
2759 +-------+--------+----------------+------+
2760 | 25 | 4 | 5 | 0 |
2761 +-------+--------+----------------+------+
2762 | 26 | 6 | 5 | 32 |
2763 +-------+--------+----------------+------+
2764 | 27 | 7 | 5 | 0 |
2765 +-------+--------+----------------+------+
2766 | 28 | 9 | 6 | 0 |
2767 +-------+--------+----------------+------+
2768 | 29 | 12 | 6 | 0 |
2769 +-------+--------+----------------+------+
2770 | 30 | 15 | 6 | 0 |
2771 +-------+--------+----------------+------+
2772 | 31 | 18 | 6 | 0 |
2773 +-------+--------+----------------+------+
2774 | 32 | 21 | 6 | 0 |
2775 +-------+--------+----------------+------+
2776 | 33 | 24 | 6 | 0 |
2777 +-------+--------+----------------+------+
2778 | 34 | 27 | 6 | 0 |
2779 +-------+--------+----------------+------+
2780 | 35 | 30 | 6 | 0 |
2781 +-------+--------+----------------+------+
2782 | 36 | 32 | 6 | 0 |
2783 +-------+--------+----------------+------+
2784 | 37 | 34 | 6 | 0 |
2785 +-------+--------+----------------+------+
2786 | 38 | 36 | 6 | 0 |
2787 +-------+--------+----------------+------+
2788 | 39 | 38 | 6 | 0 |
2789 +-------+--------+----------------+------+
2790 | 40 | 40 | 6 | 0 |
2791 +-------+--------+----------------+------+
2792 | 41 | 42 | 6 | 0 |
2793 +-------+--------+----------------+------+
2794 | 42 | 44 | 6 | 0 |
2795 +-------+--------+----------------+------+
2796 | 43 | 1 | 4 | 32 |
2797 +-------+--------+----------------+------+
2798 | 44 | 1 | 4 | 48 |
2799
2800
2801
2802Collet & Kucherawy Informational [Page 50]
2803
2804RFC 8478 application/zstd October 2018
2805
2806
2807 +-------+--------+----------------+------+
2808 | 45 | 2 | 4 | 16 |
2809 +-------+--------+----------------+------+
2810 | 46 | 4 | 5 | 32 |
2811 +-------+--------+----------------+------+
2812 | 47 | 5 | 5 | 32 |
2813 +-------+--------+----------------+------+
2814 | 48 | 7 | 5 | 32 |
2815 +-------+--------+----------------+------+
2816 | 49 | 8 | 5 | 32 |
2817 +-------+--------+----------------+------+
2818 | 50 | 11 | 6 | 0 |
2819 +-------+--------+----------------+------+
2820 | 51 | 14 | 6 | 0 |
2821 +-------+--------+----------------+------+
2822 | 52 | 17 | 6 | 0 |
2823 +-------+--------+----------------+------+
2824 | 53 | 20 | 6 | 0 |
2825 +-------+--------+----------------+------+
2826 | 54 | 23 | 6 | 0 |
2827 +-------+--------+----------------+------+
2828 | 55 | 26 | 6 | 0 |
2829 +-------+--------+----------------+------+
2830 | 56 | 29 | 6 | 0 |
2831 +-------+--------+----------------+------+
2832 | 57 | 52 | 6 | 0 |
2833 +-------+--------+----------------+------+
2834 | 58 | 51 | 6 | 0 |
2835 +-------+--------+----------------+------+
2836 | 59 | 50 | 6 | 0 |
2837 +-------+--------+----------------+------+
2838 | 60 | 49 | 6 | 0 |
2839 +-------+--------+----------------+------+
2840 | 61 | 48 | 6 | 0 |
2841 +-------+--------+----------------+------+
2842 | 62 | 47 | 6 | 0 |
2843 +-------+--------+----------------+------+
2844 | 63 | 46 | 6 | 0 |
2845 +-------+--------+----------------+------+
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858Collet & Kucherawy Informational [Page 51]
2859
2860RFC 8478 application/zstd October 2018
2861
2862
2863A.3. Offset Code Table
2864
2865 +-------+--------+----------------+------+
2866 | State | Symbol | Number_Of_Bits | Base |
2867 +-------+--------+----------------+------+
2868 | 0 | 0 | 0 | 0 |
2869 +-------+--------+----------------+------+
2870 | 0 | 0 | 5 | 0 |
2871 +-------+--------+----------------+------+
2872 | 1 | 6 | 4 | 0 |
2873 +-------+--------+----------------+------+
2874 | 2 | 9 | 5 | 0 |
2875 +-------+--------+----------------+------+
2876 | 3 | 15 | 5 | 0 |
2877 +-------+--------+----------------+------+
2878 | 4 | 21 | 5 | 0 |
2879 +-------+--------+----------------+------+
2880 | 5 | 3 | 5 | 0 |
2881 +-------+--------+----------------+------+
2882 | 6 | 7 | 4 | 0 |
2883 +-------+--------+----------------+------+
2884 | 7 | 12 | 5 | 0 |
2885 +-------+--------+----------------+------+
2886 | 8 | 18 | 5 | 0 |
2887 +-------+--------+----------------+------+
2888 | 9 | 23 | 5 | 0 |
2889 +-------+--------+----------------+------+
2890 | 10 | 5 | 5 | 0 |
2891 +-------+--------+----------------+------+
2892 | 11 | 8 | 4 | 0 |
2893 +-------+--------+----------------+------+
2894 | 12 | 14 | 5 | 0 |
2895 +-------+--------+----------------+------+
2896 | 13 | 20 | 5 | 0 |
2897 +-------+--------+----------------+------+
2898 | 14 | 2 | 5 | 0 |
2899 +-------+--------+----------------+------+
2900 | 15 | 7 | 4 | 16 |
2901 +-------+--------+----------------+------+
2902 | 16 | 11 | 5 | 0 |
2903 +-------+--------+----------------+------+
2904 | 17 | 17 | 5 | 0 |
2905 +-------+--------+----------------+------+
2906 | 18 | 22 | 5 | 0 |
2907 +-------+--------+----------------+------+
2908 | 19 | 4 | 5 | 0 |
2909 +-------+--------+----------------+------+
2910 | 20 | 8 | 4 | 16 |
2911
2912
2913
2914Collet & Kucherawy Informational [Page 52]
2915
2916RFC 8478 application/zstd October 2018
2917
2918
2919 +-------+--------+----------------+------+
2920 | 21 | 13 | 5 | 0 |
2921 +-------+--------+----------------+------+
2922 | 22 | 19 | 5 | 0 |
2923 +-------+--------+----------------+------+
2924 | 23 | 1 | 5 | 0 |
2925 +-------+--------+----------------+------+
2926 | 24 | 6 | 4 | 16 |
2927 +-------+--------+----------------+------+
2928 | 25 | 10 | 5 | 0 |
2929 +-------+--------+----------------+------+
2930 | 26 | 16 | 5 | 0 |
2931 +-------+--------+----------------+------+
2932 | 27 | 28 | 5 | 0 |
2933 +-------+--------+----------------+------+
2934 | 28 | 27 | 5 | 0 |
2935 +-------+--------+----------------+------+
2936 | 29 | 26 | 5 | 0 |
2937 +-------+--------+----------------+------+
2938 | 30 | 25 | 5 | 0 |
2939 +-------+--------+----------------+------+
2940 | 31 | 24 | 5 | 0 |
2941 +-------+--------+----------------+------+
2942
2943Acknowledgments
2944
2945 zstd was developed by Yann Collet.
2946
2947 Bobo Bose-Kolanu, Felix Handte, Kyle Nekritz, Nick Terrell, and David
2948 Schleimer provided helpful feedback during the development of this
2949 document.
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970Collet & Kucherawy Informational [Page 53]
2971
2972RFC 8478 application/zstd October 2018
2973
2974
2975Authors' Addresses
2976
2977 Yann Collet
2978 Facebook
2979 1 Hacker Way
2980 Menlo Park, CA 94025
2981 United States of America
2982
2983 Email: cyan@fb.com
2984
2985
2986 Murray S. Kucherawy (editor)
2987 Facebook
2988 1 Hacker Way
2989 Menlo Park, CA 94025
2990 United States of America
2991
2992 Email: msk@fb.com
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026Collet & Kucherawy Informational [Page 54]
3027
lib/std/compress/testdata/rfc8478.txt.zst.19 created
Binary files /dev/null and b/lib/std/compress/testdata/rfc8478.txt.zst.19 differ
lib/std/compress/testdata/rfc8478.txt.zst.3 created
Binary files /dev/null and b/lib/std/compress/testdata/rfc8478.txt.zst.3 differ
lib/std/compress/zstandard.zig created+286
...@@ -0,0 +1,286 @@
1const std = @import("std");
2const Allocator = std.mem.Allocator;
3const RingBuffer = std.RingBuffer;
4
5const types = @import("zstandard/types.zig");
6pub const frame = types.frame;
7pub const compressed_block = types.compressed_block;
8
9pub const decompress = @import("zstandard/decompress.zig");
10
11pub const DecompressStreamOptions = struct {
12 verify_checksum: bool = true,
13 window_size_max: usize = 1 << 23, // 8MiB default maximum window size,
14};
15
16pub fn DecompressStream(
17 comptime ReaderType: type,
18 comptime options: DecompressStreamOptions,
19) type {
20 return struct {
21 const Self = @This();
22
23 allocator: Allocator,
24 source: std.io.CountingReader(ReaderType),
25 state: enum { NewFrame, InFrame, LastBlock },
26 decode_state: decompress.block.DecodeState,
27 frame_context: decompress.FrameContext,
28 buffer: RingBuffer,
29 literal_fse_buffer: []types.compressed_block.Table.Fse,
30 match_fse_buffer: []types.compressed_block.Table.Fse,
31 offset_fse_buffer: []types.compressed_block.Table.Fse,
32 literals_buffer: []u8,
33 sequence_buffer: []u8,
34 checksum: if (options.verify_checksum) ?u32 else void,
35 current_frame_decompressed_size: usize,
36
37 pub const Error = ReaderType.Error || error{
38 ChecksumFailure,
39 DictionaryIdFlagUnsupported,
40 MalformedBlock,
41 MalformedFrame,
42 OutOfMemory,
43 };
44
45 pub const Reader = std.io.Reader(*Self, Error, read);
46
47 pub fn init(allocator: Allocator, source: ReaderType) Self {
48 return Self{
49 .allocator = allocator,
50 .source = std.io.countingReader(source),
51 .state = .NewFrame,
52 .decode_state = undefined,
53 .frame_context = undefined,
54 .buffer = undefined,
55 .literal_fse_buffer = undefined,
56 .match_fse_buffer = undefined,
57 .offset_fse_buffer = undefined,
58 .literals_buffer = undefined,
59 .sequence_buffer = undefined,
60 .checksum = undefined,
61 .current_frame_decompressed_size = undefined,
62 };
63 }
64
65 fn frameInit(self: *Self) !void {
66 const source_reader = self.source.reader();
67 switch (try decompress.decodeFrameHeader(source_reader)) {
68 .skippable => |header| {
69 try source_reader.skipBytes(header.frame_size, .{});
70 self.state = .NewFrame;
71 },
72 .zstandard => |header| {
73 const frame_context = context: {
74 break :context try decompress.FrameContext.init(
75 header,
76 options.window_size_max,
77 options.verify_checksum,
78 );
79 };
80
81 const literal_fse_buffer = try self.allocator.alloc(
82 types.compressed_block.Table.Fse,
83 types.compressed_block.table_size_max.literal,
84 );
85 errdefer self.allocator.free(literal_fse_buffer);
86
87 const match_fse_buffer = try self.allocator.alloc(
88 types.compressed_block.Table.Fse,
89 types.compressed_block.table_size_max.match,
90 );
91 errdefer self.allocator.free(match_fse_buffer);
92
93 const offset_fse_buffer = try self.allocator.alloc(
94 types.compressed_block.Table.Fse,
95 types.compressed_block.table_size_max.offset,
96 );
97 errdefer self.allocator.free(offset_fse_buffer);
98
99 const decode_state = decompress.block.DecodeState.init(
100 literal_fse_buffer,
101 match_fse_buffer,
102 offset_fse_buffer,
103 );
104 const buffer = try RingBuffer.init(self.allocator, frame_context.window_size);
105
106 const literals_data = try self.allocator.alloc(u8, options.window_size_max);
107 errdefer self.allocator.free(literals_data);
108
109 const sequence_data = try self.allocator.alloc(u8, options.window_size_max);
110 errdefer self.allocator.free(sequence_data);
111
112 self.literal_fse_buffer = literal_fse_buffer;
113 self.match_fse_buffer = match_fse_buffer;
114 self.offset_fse_buffer = offset_fse_buffer;
115 self.literals_buffer = literals_data;
116 self.sequence_buffer = sequence_data;
117
118 self.buffer = buffer;
119
120 self.decode_state = decode_state;
121 self.frame_context = frame_context;
122
123 self.checksum = if (options.verify_checksum) null else {};
124 self.current_frame_decompressed_size = 0;
125
126 self.state = .InFrame;
127 },
128 }
129 }
130
131 pub fn deinit(self: *Self) void {
132 if (self.state == .NewFrame) return;
133 self.allocator.free(self.decode_state.literal_fse_buffer);
134 self.allocator.free(self.decode_state.match_fse_buffer);
135 self.allocator.free(self.decode_state.offset_fse_buffer);
136 self.allocator.free(self.literals_buffer);
137 self.allocator.free(self.sequence_buffer);
138 self.buffer.deinit(self.allocator);
139 }
140
141 pub fn reader(self: *Self) Reader {
142 return .{ .context = self };
143 }
144
145 pub fn read(self: *Self, buffer: []u8) Error!usize {
146 if (buffer.len == 0) return 0;
147
148 var size: usize = 0;
149 while (size == 0) {
150 while (self.state == .NewFrame) {
151 const initial_count = self.source.bytes_read;
152 self.frameInit() catch |err| switch (err) {
153 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
154 error.EndOfStream => return if (self.source.bytes_read == initial_count)
155 0
156 else
157 error.MalformedFrame,
158 error.OutOfMemory => return error.OutOfMemory,
159 else => return error.MalformedFrame,
160 };
161 }
162 size = try self.readInner(buffer);
163 }
164 return size;
165 }
166
167 fn readInner(self: *Self, buffer: []u8) Error!usize {
168 std.debug.assert(self.state != .NewFrame);
169
170 const source_reader = self.source.reader();
171 while (self.buffer.isEmpty() and self.state != .LastBlock) {
172 const header_bytes = source_reader.readBytesNoEof(3) catch
173 return error.MalformedFrame;
174 const block_header = decompress.block.decodeBlockHeader(&header_bytes);
175
176 decompress.block.decodeBlockReader(
177 &self.buffer,
178 source_reader,
179 block_header,
180 &self.decode_state,
181 self.frame_context.block_size_max,
182 self.literals_buffer,
183 self.sequence_buffer,
184 ) catch
185 return error.MalformedBlock;
186
187 if (self.frame_context.content_size) |size| {
188 if (self.current_frame_decompressed_size > size) return error.MalformedFrame;
189 }
190
191 const size = self.buffer.len();
192 self.current_frame_decompressed_size += size;
193
194 if (self.frame_context.hasher_opt) |*hasher| {
195 if (size > 0) {
196 const written_slice = self.buffer.sliceLast(size);
197 hasher.update(written_slice.first);
198 hasher.update(written_slice.second);
199 }
200 }
201 if (block_header.last_block) {
202 self.state = .LastBlock;
203 if (self.frame_context.has_checksum) {
204 const checksum = source_reader.readIntLittle(u32) catch
205 return error.MalformedFrame;
206 if (comptime options.verify_checksum) {
207 if (self.frame_context.hasher_opt) |*hasher| {
208 if (checksum != decompress.computeChecksum(hasher))
209 return error.ChecksumFailure;
210 }
211 }
212 }
213 if (self.frame_context.content_size) |content_size| {
214 if (content_size != self.current_frame_decompressed_size) {
215 return error.MalformedFrame;
216 }
217 }
218 }
219 }
220
221 const size = @min(self.buffer.len(), buffer.len);
222 for (0..size) |i| {
223 buffer[i] = self.buffer.read().?;
224 }
225 if (self.state == .LastBlock and self.buffer.len() == 0) {
226 self.state = .NewFrame;
227 self.allocator.free(self.literal_fse_buffer);
228 self.allocator.free(self.match_fse_buffer);
229 self.allocator.free(self.offset_fse_buffer);
230 self.allocator.free(self.literals_buffer);
231 self.allocator.free(self.sequence_buffer);
232 self.buffer.deinit(self.allocator);
233 }
234 return size;
235 }
236 };
237}
238
239pub fn decompressStreamOptions(
240 allocator: Allocator,
241 reader: anytype,
242 comptime options: DecompressStreamOptions,
243) DecompressStream(@TypeOf(reader, options)) {
244 return DecompressStream(@TypeOf(reader), options).init(allocator, reader);
245}
246
247pub fn decompressStream(
248 allocator: Allocator,
249 reader: anytype,
250) DecompressStream(@TypeOf(reader), .{}) {
251 return DecompressStream(@TypeOf(reader), .{}).init(allocator, reader);
252}
253
254fn testDecompress(data: []const u8) ![]u8 {
255 var in_stream = std.io.fixedBufferStream(data);
256 var zstd_stream = decompressStream(std.testing.allocator, in_stream.reader());
257 defer zstd_stream.deinit();
258 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
259 return result;
260}
261
262fn testReader(data: []const u8, comptime expected: []const u8) !void {
263 const buf = try testDecompress(data);
264 defer std.testing.allocator.free(buf);
265 try std.testing.expectEqualSlices(u8, expected, buf);
266}
267
268test "zstandard decompression" {
269 const uncompressed = @embedFile("testdata/rfc8478.txt");
270 const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
271 const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
272
273 var buffer = try std.testing.allocator.alloc(u8, uncompressed.len);
274 defer std.testing.allocator.free(buffer);
275
276 const res3 = try decompress.decode(buffer, compressed3, true);
277 try std.testing.expectEqual(uncompressed.len, res3);
278 try std.testing.expectEqualSlices(u8, uncompressed, buffer);
279
280 const res19 = try decompress.decode(buffer, compressed19, true);
281 try std.testing.expectEqual(uncompressed.len, res19);
282 try std.testing.expectEqualSlices(u8, uncompressed, buffer);
283
284 try testReader(compressed3, uncompressed);
285 try testReader(compressed19, uncompressed);
286}
lib/std/compress/zstandard/decode/block.zig created+1149
...@@ -0,0 +1,1149 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const RingBuffer = std.RingBuffer;
4
5const types = @import("../types.zig");
6const frame = types.frame;
7const Table = types.compressed_block.Table;
8const LiteralsSection = types.compressed_block.LiteralsSection;
9const SequencesSection = types.compressed_block.SequencesSection;
10
11const huffman = @import("huffman.zig");
12const readers = @import("../readers.zig");
13
14const decodeFseTable = @import("fse.zig").decodeFseTable;
15
16const readInt = std.mem.readIntLittle;
17
18pub const Error = error{
19 BlockSizeOverMaximum,
20 MalformedBlockSize,
21 ReservedBlock,
22 MalformedRleBlock,
23 MalformedCompressedBlock,
24};
25
26pub const DecodeState = struct {
27 repeat_offsets: [3]u32,
28
29 offset: StateData(8),
30 match: StateData(9),
31 literal: StateData(9),
32
33 offset_fse_buffer: []Table.Fse,
34 match_fse_buffer: []Table.Fse,
35 literal_fse_buffer: []Table.Fse,
36
37 fse_tables_undefined: bool,
38
39 literal_stream_reader: readers.ReverseBitReader,
40 literal_stream_index: usize,
41 literal_streams: LiteralsSection.Streams,
42 literal_header: LiteralsSection.Header,
43 huffman_tree: ?LiteralsSection.HuffmanTree,
44
45 literal_written_count: usize,
46 written_count: usize = 0,
47
48 fn StateData(comptime max_accuracy_log: comptime_int) type {
49 return struct {
50 state: State,
51 table: Table,
52 accuracy_log: u8,
53
54 const State = std.meta.Int(.unsigned, max_accuracy_log);
55 };
56 }
57
58 pub fn init(
59 literal_fse_buffer: []Table.Fse,
60 match_fse_buffer: []Table.Fse,
61 offset_fse_buffer: []Table.Fse,
62 ) DecodeState {
63 return DecodeState{
64 .repeat_offsets = .{
65 types.compressed_block.start_repeated_offset_1,
66 types.compressed_block.start_repeated_offset_2,
67 types.compressed_block.start_repeated_offset_3,
68 },
69
70 .offset = undefined,
71 .match = undefined,
72 .literal = undefined,
73
74 .literal_fse_buffer = literal_fse_buffer,
75 .match_fse_buffer = match_fse_buffer,
76 .offset_fse_buffer = offset_fse_buffer,
77
78 .fse_tables_undefined = true,
79
80 .literal_written_count = 0,
81 .literal_header = undefined,
82 .literal_streams = undefined,
83 .literal_stream_reader = undefined,
84 .literal_stream_index = undefined,
85 .huffman_tree = null,
86
87 .written_count = 0,
88 };
89 }
90
91 /// Prepare the decoder to decode a compressed block. Loads the literals
92 /// stream and Huffman tree from `literals` and reads the FSE tables from
93 /// `source`.
94 ///
95 /// Errors returned:
96 /// - `error.BitStreamHasNoStartBit` if the (reversed) literal bitstream's
97 /// first byte does not have any bits set
98 /// - `error.TreelessLiteralsFirst` `literals` is a treeless literals
99 /// section and the decode state does not have a Huffman tree from a
100 /// previous block
101 /// - `error.RepeatModeFirst` on the first call if one of the sequence FSE
102 /// tables is set to repeat mode
103 /// - `error.MalformedAccuracyLog` if an FSE table has an invalid accuracy
104 /// - `error.MalformedFseTable` if there are errors decoding an FSE table
105 /// - `error.EndOfStream` if `source` ends before all FSE tables are read
106 pub fn prepare(
107 self: *DecodeState,
108 source: anytype,
109 literals: LiteralsSection,
110 sequences_header: SequencesSection.Header,
111 ) !void {
112 self.literal_written_count = 0;
113 self.literal_header = literals.header;
114 self.literal_streams = literals.streams;
115
116 if (literals.huffman_tree) |tree| {
117 self.huffman_tree = tree;
118 } else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
119 return error.TreelessLiteralsFirst;
120 }
121
122 switch (literals.header.block_type) {
123 .raw, .rle => {},
124 .compressed, .treeless => {
125 self.literal_stream_index = 0;
126 switch (literals.streams) {
127 .one => |slice| try self.initLiteralStream(slice),
128 .four => |streams| try self.initLiteralStream(streams[0]),
129 }
130 },
131 }
132
133 if (sequences_header.sequence_count > 0) {
134 try self.updateFseTable(source, .literal, sequences_header.literal_lengths);
135 try self.updateFseTable(source, .offset, sequences_header.offsets);
136 try self.updateFseTable(source, .match, sequences_header.match_lengths);
137 self.fse_tables_undefined = false;
138 }
139 }
140
141 /// Read initial FSE states for sequence decoding.
142 ///
143 /// Errors returned:
144 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits.
145 pub fn readInitialFseState(self: *DecodeState, bit_reader: *readers.ReverseBitReader) error{EndOfStream}!void {
146 self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
147 self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
148 self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
149 }
150
151 fn updateRepeatOffset(self: *DecodeState, offset: u32) void {
152 self.repeat_offsets[2] = self.repeat_offsets[1];
153 self.repeat_offsets[1] = self.repeat_offsets[0];
154 self.repeat_offsets[0] = offset;
155 }
156
157 fn useRepeatOffset(self: *DecodeState, index: usize) u32 {
158 if (index == 1)
159 std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
160 else if (index == 2) {
161 std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
162 std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
163 }
164 return self.repeat_offsets[0];
165 }
166
167 const DataType = enum { offset, match, literal };
168
169 fn updateState(
170 self: *DecodeState,
171 comptime choice: DataType,
172 bit_reader: *readers.ReverseBitReader,
173 ) error{ MalformedFseBits, EndOfStream }!void {
174 switch (@field(self, @tagName(choice)).table) {
175 .rle => {},
176 .fse => |table| {
177 const data = table[@field(self, @tagName(choice)).state];
178 const T = @TypeOf(@field(self, @tagName(choice))).State;
179 const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
180 const next_state = std.math.cast(
181 @TypeOf(@field(self, @tagName(choice))).State,
182 data.baseline + bits_summand,
183 ) orelse return error.MalformedFseBits;
184 @field(self, @tagName(choice)).state = next_state;
185 },
186 }
187 }
188
189 const FseTableError = error{
190 MalformedFseTable,
191 MalformedAccuracyLog,
192 RepeatModeFirst,
193 EndOfStream,
194 };
195
196 fn updateFseTable(
197 self: *DecodeState,
198 source: anytype,
199 comptime choice: DataType,
200 mode: SequencesSection.Header.Mode,
201 ) !void {
202 const field_name = @tagName(choice);
203 switch (mode) {
204 .predefined => {
205 @field(self, field_name).accuracy_log =
206 @field(types.compressed_block.default_accuracy_log, field_name);
207
208 @field(self, field_name).table =
209 @field(types.compressed_block, "predefined_" ++ field_name ++ "_fse_table");
210 },
211 .rle => {
212 @field(self, field_name).accuracy_log = 0;
213 @field(self, field_name).table = .{ .rle = try source.readByte() };
214 },
215 .fse => {
216 var bit_reader = readers.bitReader(source);
217
218 const table_size = try decodeFseTable(
219 &bit_reader,
220 @field(types.compressed_block.table_symbol_count_max, field_name),
221 @field(types.compressed_block.table_accuracy_log_max, field_name),
222 @field(self, field_name ++ "_fse_buffer"),
223 );
224 @field(self, field_name).table = .{
225 .fse = @field(self, field_name ++ "_fse_buffer")[0..table_size],
226 };
227 @field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
228 },
229 .repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
230 }
231 }
232
233 const Sequence = struct {
234 literal_length: u32,
235 match_length: u32,
236 offset: u32,
237 };
238
239 fn nextSequence(
240 self: *DecodeState,
241 bit_reader: *readers.ReverseBitReader,
242 ) error{ InvalidBitStream, EndOfStream }!Sequence {
243 const raw_code = self.getCode(.offset);
244 const offset_code = std.math.cast(u5, raw_code) orelse {
245 return error.InvalidBitStream;
246 };
247 const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
248
249 const match_code = self.getCode(.match);
250 if (match_code >= types.compressed_block.match_length_code_table.len)
251 return error.InvalidBitStream;
252 const match = types.compressed_block.match_length_code_table[match_code];
253 const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
254
255 const literal_code = self.getCode(.literal);
256 if (literal_code >= types.compressed_block.literals_length_code_table.len)
257 return error.InvalidBitStream;
258 const literal = types.compressed_block.literals_length_code_table[literal_code];
259 const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
260
261 const offset = if (offset_value > 3) offset: {
262 const offset = offset_value - 3;
263 self.updateRepeatOffset(offset);
264 break :offset offset;
265 } else offset: {
266 if (literal_length == 0) {
267 if (offset_value == 3) {
268 const offset = self.repeat_offsets[0] - 1;
269 self.updateRepeatOffset(offset);
270 break :offset offset;
271 }
272 break :offset self.useRepeatOffset(offset_value);
273 }
274 break :offset self.useRepeatOffset(offset_value - 1);
275 };
276
277 if (offset == 0) return error.InvalidBitStream;
278
279 return .{
280 .literal_length = literal_length,
281 .match_length = match_length,
282 .offset = offset,
283 };
284 }
285
286 fn executeSequenceSlice(
287 self: *DecodeState,
288 dest: []u8,
289 write_pos: usize,
290 sequence: Sequence,
291 ) (error{MalformedSequence} || DecodeLiteralsError)!void {
292 if (sequence.offset > write_pos + sequence.literal_length) return error.MalformedSequence;
293
294 try self.decodeLiteralsSlice(dest[write_pos..], sequence.literal_length);
295 const copy_start = write_pos + sequence.literal_length - sequence.offset;
296 const copy_end = copy_start + sequence.match_length;
297 // NOTE: we ignore the usage message for std.mem.copy and copy with dest.ptr >= src.ptr
298 // to allow repeats
299 std.mem.copy(u8, dest[write_pos + sequence.literal_length ..], dest[copy_start..copy_end]);
300 self.written_count += sequence.match_length;
301 }
302
303 fn executeSequenceRingBuffer(
304 self: *DecodeState,
305 dest: *RingBuffer,
306 sequence: Sequence,
307 ) (error{MalformedSequence} || DecodeLiteralsError)!void {
308 if (sequence.offset > @min(dest.data.len, self.written_count + sequence.literal_length))
309 return error.MalformedSequence;
310
311 try self.decodeLiteralsRingBuffer(dest, sequence.literal_length);
312 const copy_start = dest.write_index + dest.data.len - sequence.offset;
313 const copy_slice = dest.sliceAt(copy_start, sequence.match_length);
314 // TODO: would std.mem.copy and figuring out dest slice be better/faster?
315 for (copy_slice.first) |b| dest.writeAssumeCapacity(b);
316 for (copy_slice.second) |b| dest.writeAssumeCapacity(b);
317 self.written_count += sequence.match_length;
318 }
319
320 const DecodeSequenceError = error{
321 InvalidBitStream,
322 EndOfStream,
323 MalformedSequence,
324 MalformedFseBits,
325 } || DecodeLiteralsError;
326
327 /// Decode one sequence from `bit_reader` into `dest`, written starting at
328 /// `write_pos` and update FSE states if `last_sequence` is `false`.
329 /// `prepare()` must be called for the block before attempting to decode
330 /// sequences.
331 ///
332 /// Errors returned:
333 /// - `error.MalformedSequence` if the decompressed sequence would be
334 /// longer than `sequence_size_limit` or the sequence's offset is too
335 /// large
336 /// - `error.UnexpectedEndOfLiteralStream` if the decoder state's literal
337 /// streams do not contain enough literals for the sequence (this may
338 /// mean the literal stream or the sequence is malformed).
339 /// - `error.InvalidBitStream` if the FSE sequence bitstream is malformed
340 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits
341 /// - `error.DestTooSmall` if `dest` is not large enough to holde the
342 /// decompressed sequence
343 pub fn decodeSequenceSlice(
344 self: *DecodeState,
345 dest: []u8,
346 write_pos: usize,
347 bit_reader: *readers.ReverseBitReader,
348 sequence_size_limit: usize,
349 last_sequence: bool,
350 ) (error{DestTooSmall} || DecodeSequenceError)!usize {
351 const sequence = try self.nextSequence(bit_reader);
352 const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
353 if (sequence_length > sequence_size_limit) return error.MalformedSequence;
354 if (sequence_length > dest[write_pos..].len) return error.DestTooSmall;
355
356 try self.executeSequenceSlice(dest, write_pos, sequence);
357 if (!last_sequence) {
358 try self.updateState(.literal, bit_reader);
359 try self.updateState(.match, bit_reader);
360 try self.updateState(.offset, bit_reader);
361 }
362 return sequence_length;
363 }
364
365 /// Decode one sequence from `bit_reader` into `dest`; see
366 /// `decodeSequenceSlice`.
367 pub fn decodeSequenceRingBuffer(
368 self: *DecodeState,
369 dest: *RingBuffer,
370 bit_reader: anytype,
371 sequence_size_limit: usize,
372 last_sequence: bool,
373 ) DecodeSequenceError!usize {
374 const sequence = try self.nextSequence(bit_reader);
375 const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
376 if (sequence_length > sequence_size_limit) return error.MalformedSequence;
377
378 try self.executeSequenceRingBuffer(dest, sequence);
379 if (!last_sequence) {
380 try self.updateState(.literal, bit_reader);
381 try self.updateState(.match, bit_reader);
382 try self.updateState(.offset, bit_reader);
383 }
384 return sequence_length;
385 }
386
387 fn nextLiteralMultiStream(
388 self: *DecodeState,
389 ) error{BitStreamHasNoStartBit}!void {
390 self.literal_stream_index += 1;
391 try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
392 }
393
394 fn initLiteralStream(self: *DecodeState, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
395 try self.literal_stream_reader.init(bytes);
396 }
397
398 fn isLiteralStreamEmpty(self: *DecodeState) bool {
399 switch (self.literal_streams) {
400 .one => return self.literal_stream_reader.isEmpty(),
401 .four => return self.literal_stream_index == 3 and self.literal_stream_reader.isEmpty(),
402 }
403 }
404
405 const LiteralBitsError = error{
406 BitStreamHasNoStartBit,
407 UnexpectedEndOfLiteralStream,
408 };
409 fn readLiteralsBits(
410 self: *DecodeState,
411 bit_count_to_read: usize,
412 ) LiteralBitsError!u16 {
413 return self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch bits: {
414 if (self.literal_streams == .four and self.literal_stream_index < 3) {
415 try self.nextLiteralMultiStream();
416 break :bits self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch
417 return error.UnexpectedEndOfLiteralStream;
418 } else {
419 return error.UnexpectedEndOfLiteralStream;
420 }
421 };
422 }
423
424 const DecodeLiteralsError = error{
425 MalformedLiteralsLength,
426 NotFound,
427 } || LiteralBitsError;
428
429 /// Decode `len` bytes of literals into `dest`.
430 ///
431 /// Errors returned:
432 /// - `error.MalformedLiteralsLength` if the number of literal bytes
433 /// decoded by `self` plus `len` is greater than the regenerated size of
434 /// `literals`
435 /// - `error.UnexpectedEndOfLiteralStream` and `error.NotFound` if there
436 /// are problems decoding Huffman compressed literals
437 pub fn decodeLiteralsSlice(
438 self: *DecodeState,
439 dest: []u8,
440 len: usize,
441 ) DecodeLiteralsError!void {
442 if (self.literal_written_count + len > self.literal_header.regenerated_size)
443 return error.MalformedLiteralsLength;
444
445 switch (self.literal_header.block_type) {
446 .raw => {
447 const literals_end = self.literal_written_count + len;
448 const literal_data = self.literal_streams.one[self.literal_written_count..literals_end];
449 std.mem.copy(u8, dest, literal_data);
450 self.literal_written_count += len;
451 self.written_count += len;
452 },
453 .rle => {
454 for (0..len) |i| {
455 dest[i] = self.literal_streams.one[0];
456 }
457 self.literal_written_count += len;
458 self.written_count += len;
459 },
460 .compressed, .treeless => {
461 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
462 const huffman_tree = self.huffman_tree orelse unreachable;
463 const max_bit_count = huffman_tree.max_bit_count;
464 const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
465 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
466 max_bit_count,
467 );
468 var bits_read: u4 = 0;
469 var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
470 var bit_count_to_read: u4 = starting_bit_count;
471 for (0..len) |i| {
472 var prefix: u16 = 0;
473 while (true) {
474 const new_bits = self.readLiteralsBits(bit_count_to_read) catch |err| {
475 return err;
476 };
477 prefix <<= bit_count_to_read;
478 prefix |= new_bits;
479 bits_read += bit_count_to_read;
480 const result = huffman_tree.query(huffman_tree_index, prefix) catch |err| {
481 return err;
482 };
483
484 switch (result) {
485 .symbol => |sym| {
486 dest[i] = sym;
487 bit_count_to_read = starting_bit_count;
488 bits_read = 0;
489 huffman_tree_index = huffman_tree.symbol_count_minus_one;
490 break;
491 },
492 .index => |index| {
493 huffman_tree_index = index;
494 const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
495 huffman_tree.nodes[index].weight,
496 max_bit_count,
497 );
498 bit_count_to_read = bit_count - bits_read;
499 },
500 }
501 }
502 }
503 self.literal_written_count += len;
504 self.written_count += len;
505 },
506 }
507 }
508
509 /// Decode literals into `dest`; see `decodeLiteralsSlice()`.
510 pub fn decodeLiteralsRingBuffer(
511 self: *DecodeState,
512 dest: *RingBuffer,
513 len: usize,
514 ) DecodeLiteralsError!void {
515 if (self.literal_written_count + len > self.literal_header.regenerated_size)
516 return error.MalformedLiteralsLength;
517
518 switch (self.literal_header.block_type) {
519 .raw => {
520 const literals_end = self.literal_written_count + len;
521 const literal_data = self.literal_streams.one[self.literal_written_count..literals_end];
522 dest.writeSliceAssumeCapacity(literal_data);
523 self.literal_written_count += len;
524 self.written_count += len;
525 },
526 .rle => {
527 for (0..len) |_| {
528 dest.writeAssumeCapacity(self.literal_streams.one[0]);
529 }
530 self.literal_written_count += len;
531 self.written_count += len;
532 },
533 .compressed, .treeless => {
534 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
535 const huffman_tree = self.huffman_tree orelse unreachable;
536 const max_bit_count = huffman_tree.max_bit_count;
537 const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
538 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
539 max_bit_count,
540 );
541 var bits_read: u4 = 0;
542 var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
543 var bit_count_to_read: u4 = starting_bit_count;
544 for (0..len) |_| {
545 var prefix: u16 = 0;
546 while (true) {
547 const new_bits = try self.readLiteralsBits(bit_count_to_read);
548 prefix <<= bit_count_to_read;
549 prefix |= new_bits;
550 bits_read += bit_count_to_read;
551 const result = try huffman_tree.query(huffman_tree_index, prefix);
552
553 switch (result) {
554 .symbol => |sym| {
555 dest.writeAssumeCapacity(sym);
556 bit_count_to_read = starting_bit_count;
557 bits_read = 0;
558 huffman_tree_index = huffman_tree.symbol_count_minus_one;
559 break;
560 },
561 .index => |index| {
562 huffman_tree_index = index;
563 const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
564 huffman_tree.nodes[index].weight,
565 max_bit_count,
566 );
567 bit_count_to_read = bit_count - bits_read;
568 },
569 }
570 }
571 }
572 self.literal_written_count += len;
573 self.written_count += len;
574 },
575 }
576 }
577
578 fn getCode(self: *DecodeState, comptime choice: DataType) u32 {
579 return switch (@field(self, @tagName(choice)).table) {
580 .rle => |value| value,
581 .fse => |table| table[@field(self, @tagName(choice)).state].symbol,
582 };
583 }
584};
585
586/// Decode a single block from `src` into `dest`. The beginning of `src` must be
587/// the start of the block content (i.e. directly after the block header).
588/// Increments `consumed_count` by the number of bytes read from `src` to decode
589/// the block and returns the decompressed size of the block.
590///
591/// Errors returned:
592///
593/// - `error.BlockSizeOverMaximum` if block's size is larger than 1 << 17 or
594/// `dest[written_count..].len`
595/// - `error.MalformedBlockSize` if `src.len` is smaller than the block size
596/// and the block is a raw or compressed block
597/// - `error.ReservedBlock` if the block is a reserved block
598/// - `error.MalformedRleBlock` if the block is an RLE block and `src.len < 1`
599/// - `error.MalformedCompressedBlock` if there are errors decoding a
600/// compressed block
601/// - `error.DestTooSmall` is `dest` is not large enough to hold the
602/// decompressed block
603pub fn decodeBlock(
604 dest: []u8,
605 src: []const u8,
606 block_header: frame.Zstandard.Block.Header,
607 decode_state: *DecodeState,
608 consumed_count: *usize,
609 block_size_max: usize,
610 written_count: usize,
611) (error{DestTooSmall} || Error)!usize {
612 const block_size = block_header.block_size;
613 if (block_size_max < block_size) return error.BlockSizeOverMaximum;
614 switch (block_header.block_type) {
615 .raw => {
616 if (src.len < block_size) return error.MalformedBlockSize;
617 if (dest[written_count..].len < block_size) return error.DestTooSmall;
618 const data = src[0..block_size];
619 std.mem.copy(u8, dest[written_count..], data);
620 consumed_count.* += block_size;
621 decode_state.written_count += block_size;
622 return block_size;
623 },
624 .rle => {
625 if (src.len < 1) return error.MalformedRleBlock;
626 if (dest[written_count..].len < block_size) return error.DestTooSmall;
627 for (written_count..block_size + written_count) |write_pos| {
628 dest[write_pos] = src[0];
629 }
630 consumed_count.* += 1;
631 decode_state.written_count += block_size;
632 return block_size;
633 },
634 .compressed => {
635 if (src.len < block_size) return error.MalformedBlockSize;
636 var bytes_read: usize = 0;
637 const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
638 return error.MalformedCompressedBlock;
639 var fbs = std.io.fixedBufferStream(src[bytes_read..block_size]);
640 const fbs_reader = fbs.reader();
641 const sequences_header = decodeSequencesHeader(fbs_reader) catch
642 return error.MalformedCompressedBlock;
643
644 decode_state.prepare(fbs_reader, literals, sequences_header) catch
645 return error.MalformedCompressedBlock;
646
647 bytes_read += fbs.pos;
648
649 var bytes_written: usize = 0;
650 {
651 const bit_stream_bytes = src[bytes_read..block_size];
652 var bit_stream: readers.ReverseBitReader = undefined;
653 bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
654
655 if (sequences_header.sequence_count > 0) {
656 decode_state.readInitialFseState(&bit_stream) catch
657 return error.MalformedCompressedBlock;
658
659 var sequence_size_limit = block_size_max;
660 for (0..sequences_header.sequence_count) |i| {
661 const write_pos = written_count + bytes_written;
662 const decompressed_size = decode_state.decodeSequenceSlice(
663 dest,
664 write_pos,
665 &bit_stream,
666 sequence_size_limit,
667 i == sequences_header.sequence_count - 1,
668 ) catch |err| switch (err) {
669 error.DestTooSmall => return error.DestTooSmall,
670 else => return error.MalformedCompressedBlock,
671 };
672 bytes_written += decompressed_size;
673 sequence_size_limit -= decompressed_size;
674 }
675 }
676
677 if (!bit_stream.isEmpty()) {
678 return error.MalformedCompressedBlock;
679 }
680 }
681
682 if (decode_state.literal_written_count < literals.header.regenerated_size) {
683 const len = literals.header.regenerated_size - decode_state.literal_written_count;
684 if (len > dest[written_count + bytes_written ..].len) return error.DestTooSmall;
685 decode_state.decodeLiteralsSlice(dest[written_count + bytes_written ..], len) catch
686 return error.MalformedCompressedBlock;
687 bytes_written += len;
688 }
689
690 switch (decode_state.literal_header.block_type) {
691 .treeless, .compressed => {
692 if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
693 },
694 .raw, .rle => {},
695 }
696
697 consumed_count.* += block_size;
698 return bytes_written;
699 },
700 .reserved => return error.ReservedBlock,
701 }
702}
703
704/// Decode a single block from `src` into `dest`; see `decodeBlock()`. Returns
705/// the size of the decompressed block, which can be used with `dest.sliceLast()`
706/// to get the decompressed bytes. `error.BlockSizeOverMaximum` is returned if
707/// the block's compressed or decompressed size is larger than `block_size_max`.
708pub fn decodeBlockRingBuffer(
709 dest: *RingBuffer,
710 src: []const u8,
711 block_header: frame.Zstandard.Block.Header,
712 decode_state: *DecodeState,
713 consumed_count: *usize,
714 block_size_max: usize,
715) Error!usize {
716 const block_size = block_header.block_size;
717 if (block_size_max < block_size) return error.BlockSizeOverMaximum;
718 switch (block_header.block_type) {
719 .raw => {
720 if (src.len < block_size) return error.MalformedBlockSize;
721 const data = src[0..block_size];
722 dest.writeSliceAssumeCapacity(data);
723 consumed_count.* += block_size;
724 decode_state.written_count += block_size;
725 return block_size;
726 },
727 .rle => {
728 if (src.len < 1) return error.MalformedRleBlock;
729 for (0..block_size) |_| {
730 dest.writeAssumeCapacity(src[0]);
731 }
732 consumed_count.* += 1;
733 decode_state.written_count += block_size;
734 return block_size;
735 },
736 .compressed => {
737 if (src.len < block_size) return error.MalformedBlockSize;
738 var bytes_read: usize = 0;
739 const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
740 return error.MalformedCompressedBlock;
741 var fbs = std.io.fixedBufferStream(src[bytes_read..block_size]);
742 const fbs_reader = fbs.reader();
743 const sequences_header = decodeSequencesHeader(fbs_reader) catch
744 return error.MalformedCompressedBlock;
745
746 decode_state.prepare(fbs_reader, literals, sequences_header) catch
747 return error.MalformedCompressedBlock;
748
749 bytes_read += fbs.pos;
750
751 var bytes_written: usize = 0;
752 {
753 const bit_stream_bytes = src[bytes_read..block_size];
754 var bit_stream: readers.ReverseBitReader = undefined;
755 bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
756
757 if (sequences_header.sequence_count > 0) {
758 decode_state.readInitialFseState(&bit_stream) catch
759 return error.MalformedCompressedBlock;
760
761 var sequence_size_limit = block_size_max;
762 for (0..sequences_header.sequence_count) |i| {
763 const decompressed_size = decode_state.decodeSequenceRingBuffer(
764 dest,
765 &bit_stream,
766 sequence_size_limit,
767 i == sequences_header.sequence_count - 1,
768 ) catch return error.MalformedCompressedBlock;
769 bytes_written += decompressed_size;
770 sequence_size_limit -= decompressed_size;
771 }
772 }
773
774 if (!bit_stream.isEmpty()) {
775 return error.MalformedCompressedBlock;
776 }
777 }
778
779 if (decode_state.literal_written_count < literals.header.regenerated_size) {
780 const len = literals.header.regenerated_size - decode_state.literal_written_count;
781 decode_state.decodeLiteralsRingBuffer(dest, len) catch
782 return error.MalformedCompressedBlock;
783 bytes_written += len;
784 }
785
786 switch (decode_state.literal_header.block_type) {
787 .treeless, .compressed => {
788 if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
789 },
790 .raw, .rle => {},
791 }
792
793 consumed_count.* += block_size;
794 if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
795 return bytes_written;
796 },
797 .reserved => return error.ReservedBlock,
798 }
799}
800
801/// Decode a single block from `source` into `dest`. Literal and sequence data
802/// from the block is copied into `literals_buffer` and `sequence_buffer`, which
803/// must be large enough or `error.LiteralsBufferTooSmall` and
804/// `error.SequenceBufferTooSmall` are returned (the maximum block size is an
805/// upper bound for the size of both buffers). See `decodeBlock`
806/// and `decodeBlockRingBuffer` for function that can decode a block without
807/// these extra copies. `error.EndOfStream` is returned if `source` does not
808/// contain enough bytes.
809pub fn decodeBlockReader(
810 dest: *RingBuffer,
811 source: anytype,
812 block_header: frame.Zstandard.Block.Header,
813 decode_state: *DecodeState,
814 block_size_max: usize,
815 literals_buffer: []u8,
816 sequence_buffer: []u8,
817) !void {
818 const block_size = block_header.block_size;
819 var block_reader_limited = std.io.limitedReader(source, block_size);
820 const block_reader = block_reader_limited.reader();
821 if (block_size_max < block_size) return error.BlockSizeOverMaximum;
822 switch (block_header.block_type) {
823 .raw => {
824 if (block_size == 0) return;
825 const slice = dest.sliceAt(dest.write_index, block_size);
826 try source.readNoEof(slice.first);
827 try source.readNoEof(slice.second);
828 dest.write_index = dest.mask2(dest.write_index + block_size);
829 decode_state.written_count += block_size;
830 },
831 .rle => {
832 const byte = try source.readByte();
833 for (0..block_size) |_| {
834 dest.writeAssumeCapacity(byte);
835 }
836 decode_state.written_count += block_size;
837 },
838 .compressed => {
839 const literals = try decodeLiteralsSection(block_reader, literals_buffer);
840 const sequences_header = try decodeSequencesHeader(block_reader);
841
842 try decode_state.prepare(block_reader, literals, sequences_header);
843
844 var bytes_written: usize = 0;
845 {
846 const size = try block_reader.readAll(sequence_buffer);
847 var bit_stream: readers.ReverseBitReader = undefined;
848 try bit_stream.init(sequence_buffer[0..size]);
849
850 if (sequences_header.sequence_count > 0) {
851 if (sequence_buffer.len < block_reader_limited.bytes_left)
852 return error.SequenceBufferTooSmall;
853
854 decode_state.readInitialFseState(&bit_stream) catch
855 return error.MalformedCompressedBlock;
856
857 var sequence_size_limit = block_size_max;
858 for (0..sequences_header.sequence_count) |i| {
859 const decompressed_size = decode_state.decodeSequenceRingBuffer(
860 dest,
861 &bit_stream,
862 sequence_size_limit,
863 i == sequences_header.sequence_count - 1,
864 ) catch return error.MalformedCompressedBlock;
865 sequence_size_limit -= decompressed_size;
866 bytes_written += decompressed_size;
867 }
868 }
869
870 if (!bit_stream.isEmpty()) {
871 return error.MalformedCompressedBlock;
872 }
873 }
874
875 if (decode_state.literal_written_count < literals.header.regenerated_size) {
876 const len = literals.header.regenerated_size - decode_state.literal_written_count;
877 decode_state.decodeLiteralsRingBuffer(dest, len) catch
878 return error.MalformedCompressedBlock;
879 bytes_written += len;
880 }
881
882 switch (decode_state.literal_header.block_type) {
883 .treeless, .compressed => {
884 if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
885 },
886 .raw, .rle => {},
887 }
888
889 if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
890 if (block_reader_limited.bytes_left != 0) return error.MalformedCompressedBlock;
891 decode_state.literal_written_count = 0;
892 },
893 .reserved => return error.ReservedBlock,
894 }
895}
896
897/// Decode the header of a block.
898pub fn decodeBlockHeader(src: *const [3]u8) frame.Zstandard.Block.Header {
899 const last_block = src[0] & 1 == 1;
900 const block_type = @intToEnum(frame.Zstandard.Block.Type, (src[0] & 0b110) >> 1);
901 const block_size = ((src[0] & 0b11111000) >> 3) + (@as(u21, src[1]) << 5) + (@as(u21, src[2]) << 13);
902 return .{
903 .last_block = last_block,
904 .block_type = block_type,
905 .block_size = block_size,
906 };
907}
908
909/// Decode the header of a block.
910///
911/// Errors returned:
912/// - `error.EndOfStream` if `src.len < 3`
913pub fn decodeBlockHeaderSlice(src: []const u8) error{EndOfStream}!frame.Zstandard.Block.Header {
914 if (src.len < 3) return error.EndOfStream;
915 return decodeBlockHeader(src[0..3]);
916}
917
918/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
919/// number of bytes the section uses.
920///
921/// Errors returned:
922/// - `error.MalformedLiteralsHeader` if the header is invalid
923/// - `error.MalformedLiteralsSection` if there are decoding errors
924/// - `error.MalformedAccuracyLog` if compressed literals have invalid
925/// accuracy
926/// - `error.MalformedFseTable` if compressed literals have invalid FSE table
927/// - `error.MalformedHuffmanTree` if there are errors decoding a Huffamn tree
928/// - `error.EndOfStream` if there are not enough bytes in `src`
929pub fn decodeLiteralsSectionSlice(
930 src: []const u8,
931 consumed_count: *usize,
932) (error{ MalformedLiteralsHeader, MalformedLiteralsSection, EndOfStream } || huffman.Error)!LiteralsSection {
933 var bytes_read: usize = 0;
934 const header = header: {
935 var fbs = std.io.fixedBufferStream(src);
936 defer bytes_read = fbs.pos;
937 break :header decodeLiteralsHeader(fbs.reader()) catch return error.MalformedLiteralsHeader;
938 };
939 switch (header.block_type) {
940 .raw => {
941 if (src.len < bytes_read + header.regenerated_size) return error.MalformedLiteralsSection;
942 const stream = src[bytes_read .. bytes_read + header.regenerated_size];
943 consumed_count.* += header.regenerated_size + bytes_read;
944 return LiteralsSection{
945 .header = header,
946 .huffman_tree = null,
947 .streams = .{ .one = stream },
948 };
949 },
950 .rle => {
951 if (src.len < bytes_read + 1) return error.MalformedLiteralsSection;
952 const stream = src[bytes_read .. bytes_read + 1];
953 consumed_count.* += 1 + bytes_read;
954 return LiteralsSection{
955 .header = header,
956 .huffman_tree = null,
957 .streams = .{ .one = stream },
958 };
959 },
960 .compressed, .treeless => {
961 const huffman_tree_start = bytes_read;
962 const huffman_tree = if (header.block_type == .compressed)
963 try huffman.decodeHuffmanTreeSlice(src[bytes_read..], &bytes_read)
964 else
965 null;
966 const huffman_tree_size = bytes_read - huffman_tree_start;
967 const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
968 return error.MalformedLiteralsSection;
969
970 if (src.len < bytes_read + total_streams_size) return error.MalformedLiteralsSection;
971 const stream_data = src[bytes_read .. bytes_read + total_streams_size];
972
973 const streams = try decodeStreams(header.size_format, stream_data);
974 consumed_count.* += bytes_read + total_streams_size;
975 return LiteralsSection{
976 .header = header,
977 .huffman_tree = huffman_tree,
978 .streams = streams,
979 };
980 },
981 }
982}
983
984/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
985/// number of bytes the section uses. See `decodeLiterasSectionSlice()`.
986pub fn decodeLiteralsSection(
987 source: anytype,
988 buffer: []u8,
989) !LiteralsSection {
990 const header = try decodeLiteralsHeader(source);
991 switch (header.block_type) {
992 .raw => {
993 try source.readNoEof(buffer[0..header.regenerated_size]);
994 return LiteralsSection{
995 .header = header,
996 .huffman_tree = null,
997 .streams = .{ .one = buffer },
998 };
999 },
1000 .rle => {
1001 buffer[0] = try source.readByte();
1002 return LiteralsSection{
1003 .header = header,
1004 .huffman_tree = null,
1005 .streams = .{ .one = buffer[0..1] },
1006 };
1007 },
1008 .compressed, .treeless => {
1009 var counting_reader = std.io.countingReader(source);
1010 const huffman_tree = if (header.block_type == .compressed)
1011 try huffman.decodeHuffmanTree(counting_reader.reader(), buffer)
1012 else
1013 null;
1014 const huffman_tree_size = @intCast(usize, counting_reader.bytes_read);
1015 const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
1016 return error.MalformedLiteralsSection;
1017
1018 if (total_streams_size > buffer.len) return error.LiteralsBufferTooSmall;
1019 try source.readNoEof(buffer[0..total_streams_size]);
1020 const stream_data = buffer[0..total_streams_size];
1021
1022 const streams = try decodeStreams(header.size_format, stream_data);
1023 return LiteralsSection{
1024 .header = header,
1025 .huffman_tree = huffman_tree,
1026 .streams = streams,
1027 };
1028 },
1029 }
1030}
1031
1032fn decodeStreams(size_format: u2, stream_data: []const u8) !LiteralsSection.Streams {
1033 if (size_format == 0) {
1034 return .{ .one = stream_data };
1035 }
1036
1037 if (stream_data.len < 6) return error.MalformedLiteralsSection;
1038
1039 const stream_1_length = @as(usize, readInt(u16, stream_data[0..2]));
1040 const stream_2_length = @as(usize, readInt(u16, stream_data[2..4]));
1041 const stream_3_length = @as(usize, readInt(u16, stream_data[4..6]));
1042
1043 const stream_1_start = 6;
1044 const stream_2_start = stream_1_start + stream_1_length;
1045 const stream_3_start = stream_2_start + stream_2_length;
1046 const stream_4_start = stream_3_start + stream_3_length;
1047
1048 if (stream_data.len < stream_4_start) return error.MalformedLiteralsSection;
1049
1050 return .{ .four = .{
1051 stream_data[stream_1_start .. stream_1_start + stream_1_length],
1052 stream_data[stream_2_start .. stream_2_start + stream_2_length],
1053 stream_data[stream_3_start .. stream_3_start + stream_3_length],
1054 stream_data[stream_4_start..],
1055 } };
1056}
1057
1058/// Decode a literals section header.
1059///
1060/// Errors returned:
1061/// - `error.EndOfStream` if there are not enough bytes in `source`
1062pub fn decodeLiteralsHeader(source: anytype) !LiteralsSection.Header {
1063 const byte0 = try source.readByte();
1064 const block_type = @intToEnum(LiteralsSection.BlockType, byte0 & 0b11);
1065 const size_format = @intCast(u2, (byte0 & 0b1100) >> 2);
1066 var regenerated_size: u20 = undefined;
1067 var compressed_size: ?u18 = null;
1068 switch (block_type) {
1069 .raw, .rle => {
1070 switch (size_format) {
1071 0, 2 => {
1072 regenerated_size = byte0 >> 3;
1073 },
1074 1 => regenerated_size = (byte0 >> 4) + (@as(u20, try source.readByte()) << 4),
1075 3 => regenerated_size = (byte0 >> 4) +
1076 (@as(u20, try source.readByte()) << 4) +
1077 (@as(u20, try source.readByte()) << 12),
1078 }
1079 },
1080 .compressed, .treeless => {
1081 const byte1 = try source.readByte();
1082 const byte2 = try source.readByte();
1083 switch (size_format) {
1084 0, 1 => {
1085 regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
1086 compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
1087 },
1088 2 => {
1089 const byte3 = try source.readByte();
1090 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
1091 compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
1092 },
1093 3 => {
1094 const byte3 = try source.readByte();
1095 const byte4 = try source.readByte();
1096 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
1097 compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
1098 },
1099 }
1100 },
1101 }
1102 return LiteralsSection.Header{
1103 .block_type = block_type,
1104 .size_format = size_format,
1105 .regenerated_size = regenerated_size,
1106 .compressed_size = compressed_size,
1107 };
1108}
1109
1110/// Decode a sequences section header.
1111///
1112/// Errors returned:
1113/// - `error.ReservedBitSet` if the reserved bit is set
1114/// - `error.EndOfStream` if there are not enough bytes in `source`
1115pub fn decodeSequencesHeader(
1116 source: anytype,
1117) !SequencesSection.Header {
1118 var sequence_count: u24 = undefined;
1119
1120 const byte0 = try source.readByte();
1121 if (byte0 == 0) {
1122 return SequencesSection.Header{
1123 .sequence_count = 0,
1124 .offsets = undefined,
1125 .match_lengths = undefined,
1126 .literal_lengths = undefined,
1127 };
1128 } else if (byte0 < 128) {
1129 sequence_count = byte0;
1130 } else if (byte0 < 255) {
1131 sequence_count = (@as(u24, (byte0 - 128)) << 8) + try source.readByte();
1132 } else {
1133 sequence_count = (try source.readByte()) + (@as(u24, try source.readByte()) << 8) + 0x7F00;
1134 }
1135
1136 const compression_modes = try source.readByte();
1137
1138 const matches_mode = @intToEnum(SequencesSection.Header.Mode, (compression_modes & 0b00001100) >> 2);
1139 const offsets_mode = @intToEnum(SequencesSection.Header.Mode, (compression_modes & 0b00110000) >> 4);
1140 const literal_mode = @intToEnum(SequencesSection.Header.Mode, (compression_modes & 0b11000000) >> 6);
1141 if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
1142
1143 return SequencesSection.Header{
1144 .sequence_count = sequence_count,
1145 .offsets = offsets_mode,
1146 .match_lengths = matches_mode,
1147 .literal_lengths = literal_mode,
1148 };
1149}
lib/std/compress/zstandard/decode/fse.zig created+153
...@@ -0,0 +1,153 @@
1const std = @import("std");
2const assert = std.debug.assert;
3
4const types = @import("../types.zig");
5const Table = types.compressed_block.Table;
6
7pub fn decodeFseTable(
8 bit_reader: anytype,
9 expected_symbol_count: usize,
10 max_accuracy_log: u4,
11 entries: []Table.Fse,
12) !usize {
13 const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
14 if (accuracy_log_biased > max_accuracy_log -| 5) return error.MalformedAccuracyLog;
15 const accuracy_log = accuracy_log_biased + 5;
16
17 var values: [256]u16 = undefined;
18 var value_count: usize = 0;
19
20 const total_probability = @as(u16, 1) << accuracy_log;
21 var accumulated_probability: u16 = 0;
22
23 while (accumulated_probability < total_probability) {
24 // WARNING: The RFC in poorly worded, and would suggest std.math.log2_int_ceil is correct here,
25 // but power of two (remaining probabilities + 1) need max bits set to 1 more.
26 const max_bits = std.math.log2_int(u16, total_probability - accumulated_probability + 1) + 1;
27 const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
28
29 const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
30
31 const value = if (small < cutoff)
32 small
33 else value: {
34 const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
35 break :value if (value_read < @as(u16, 1) << (max_bits - 1))
36 value_read
37 else
38 value_read - cutoff;
39 };
40
41 accumulated_probability += if (value != 0) value - 1 else 1;
42
43 values[value_count] = value;
44 value_count += 1;
45
46 if (value == 1) {
47 while (true) {
48 const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
49 if (repeat_flag + value_count > 256) return error.MalformedFseTable;
50 for (0..repeat_flag) |_| {
51 values[value_count] = 1;
52 value_count += 1;
53 }
54 if (repeat_flag < 3) break;
55 }
56 }
57 if (value_count == 256) break;
58 }
59 bit_reader.alignToByte();
60
61 if (value_count < 2) return error.MalformedFseTable;
62 if (accumulated_probability != total_probability) return error.MalformedFseTable;
63 if (value_count > expected_symbol_count) return error.MalformedFseTable;
64
65 const table_size = total_probability;
66
67 try buildFseTable(values[0..value_count], entries[0..table_size]);
68 return table_size;
69}
70
71fn buildFseTable(values: []const u16, entries: []Table.Fse) !void {
72 const total_probability = @intCast(u16, entries.len);
73 const accuracy_log = std.math.log2_int(u16, total_probability);
74 assert(total_probability <= 1 << 9);
75
76 var less_than_one_count: usize = 0;
77 for (values, 0..) |value, i| {
78 if (value == 0) {
79 entries[entries.len - 1 - less_than_one_count] = Table.Fse{
80 .symbol = @intCast(u8, i),
81 .baseline = 0,
82 .bits = accuracy_log,
83 };
84 less_than_one_count += 1;
85 }
86 }
87
88 var position: usize = 0;
89 var temp_states: [1 << 9]u16 = undefined;
90 for (values, 0..) |value, symbol| {
91 if (value == 0 or value == 1) continue;
92 const probability = value - 1;
93
94 const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
95 return error.MalformedFseTable;
96 const share_size = @divExact(total_probability, state_share_dividend);
97 const double_state_count = state_share_dividend - probability;
98 const single_state_count = probability - double_state_count;
99 const share_size_log = std.math.log2_int(u16, share_size);
100
101 for (0..probability) |i| {
102 temp_states[i] = @intCast(u16, position);
103 position += (entries.len >> 1) + (entries.len >> 3) + 3;
104 position &= entries.len - 1;
105 while (position >= entries.len - less_than_one_count) {
106 position += (entries.len >> 1) + (entries.len >> 3) + 3;
107 position &= entries.len - 1;
108 }
109 }
110 std.sort.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
111 for (0..probability) |i| {
112 entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
113 .symbol = @intCast(u8, symbol),
114 .bits = share_size_log + 1,
115 .baseline = single_state_count * share_size + @intCast(u16, i) * 2 * share_size,
116 } else Table.Fse{
117 .symbol = @intCast(u8, symbol),
118 .bits = share_size_log,
119 .baseline = (@intCast(u16, i) - double_state_count) * share_size,
120 };
121 }
122 }
123}
124
125test buildFseTable {
126 const literals_length_default_values = [36]u16{
127 5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
128 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
129 0, 0, 0, 0,
130 };
131
132 const match_lengths_default_values = [53]u16{
133 2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
134 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
135 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
136 0, 0, 0, 0, 0,
137 };
138
139 const offset_codes_default_values = [29]u16{
140 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
141 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
142 };
143
144 var entries: [64]Table.Fse = undefined;
145 try buildFseTable(&literals_length_default_values, &entries);
146 try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_literal_fse_table.fse, &entries);
147
148 try buildFseTable(&match_lengths_default_values, &entries);
149 try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_match_fse_table.fse, &entries);
150
151 try buildFseTable(&offset_codes_default_values, entries[0..32]);
152 try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_offset_fse_table.fse, entries[0..32]);
153}
lib/std/compress/zstandard/decode/huffman.zig created+234
...@@ -0,0 +1,234 @@
1const std = @import("std");
2
3const types = @import("../types.zig");
4const LiteralsSection = types.compressed_block.LiteralsSection;
5const Table = types.compressed_block.Table;
6
7const readers = @import("../readers.zig");
8
9const decodeFseTable = @import("fse.zig").decodeFseTable;
10
11pub const Error = error{
12 MalformedHuffmanTree,
13 MalformedFseTable,
14 MalformedAccuracyLog,
15 EndOfStream,
16};
17
18fn decodeFseHuffmanTree(
19 source: anytype,
20 compressed_size: usize,
21 buffer: []u8,
22 weights: *[256]u4,
23) !usize {
24 var stream = std.io.limitedReader(source, compressed_size);
25 var bit_reader = readers.bitReader(stream.reader());
26
27 var entries: [1 << 6]Table.Fse = undefined;
28 const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch |err| switch (err) {
29 error.MalformedAccuracyLog, error.MalformedFseTable => |e| return e,
30 error.EndOfStream => return error.MalformedFseTable,
31 else => |e| return e,
32 };
33 const accuracy_log = std.math.log2_int_ceil(usize, table_size);
34
35 const amount = try stream.reader().readAll(buffer);
36 var huff_bits: readers.ReverseBitReader = undefined;
37 huff_bits.init(buffer[0..amount]) catch return error.MalformedHuffmanTree;
38
39 return assignWeights(&huff_bits, accuracy_log, &entries, weights);
40}
41
42fn decodeFseHuffmanTreeSlice(src: []const u8, compressed_size: usize, weights: *[256]u4) !usize {
43 if (src.len < compressed_size) return error.MalformedHuffmanTree;
44 var stream = std.io.fixedBufferStream(src[0..compressed_size]);
45 var counting_reader = std.io.countingReader(stream.reader());
46 var bit_reader = readers.bitReader(counting_reader.reader());
47
48 var entries: [1 << 6]Table.Fse = undefined;
49 const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch |err| switch (err) {
50 error.MalformedAccuracyLog, error.MalformedFseTable => |e| return e,
51 error.EndOfStream => return error.MalformedFseTable,
52 };
53 const accuracy_log = std.math.log2_int_ceil(usize, table_size);
54
55 const start_index = std.math.cast(usize, counting_reader.bytes_read) orelse
56 return error.MalformedHuffmanTree;
57 var huff_data = src[start_index..compressed_size];
58 var huff_bits: readers.ReverseBitReader = undefined;
59 huff_bits.init(huff_data) catch return error.MalformedHuffmanTree;
60
61 return assignWeights(&huff_bits, accuracy_log, &entries, weights);
62}
63
64fn assignWeights(
65 huff_bits: *readers.ReverseBitReader,
66 accuracy_log: usize,
67 entries: *[1 << 6]Table.Fse,
68 weights: *[256]u4,
69) !usize {
70 var i: usize = 0;
71 var even_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
72 var odd_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
73
74 while (i < 254) {
75 const even_data = entries[even_state];
76 var read_bits: usize = 0;
77 const even_bits = huff_bits.readBits(u32, even_data.bits, &read_bits) catch unreachable;
78 weights[i] = std.math.cast(u4, even_data.symbol) orelse return error.MalformedHuffmanTree;
79 i += 1;
80 if (read_bits < even_data.bits) {
81 weights[i] = std.math.cast(u4, entries[odd_state].symbol) orelse return error.MalformedHuffmanTree;
82 i += 1;
83 break;
84 }
85 even_state = even_data.baseline + even_bits;
86
87 read_bits = 0;
88 const odd_data = entries[odd_state];
89 const odd_bits = huff_bits.readBits(u32, odd_data.bits, &read_bits) catch unreachable;
90 weights[i] = std.math.cast(u4, odd_data.symbol) orelse return error.MalformedHuffmanTree;
91 i += 1;
92 if (read_bits < odd_data.bits) {
93 if (i == 255) return error.MalformedHuffmanTree;
94 weights[i] = std.math.cast(u4, entries[even_state].symbol) orelse return error.MalformedHuffmanTree;
95 i += 1;
96 break;
97 }
98 odd_state = odd_data.baseline + odd_bits;
99 } else return error.MalformedHuffmanTree;
100
101 if (!huff_bits.isEmpty()) {
102 return error.MalformedHuffmanTree;
103 }
104
105 return i + 1; // stream contains all but the last symbol
106}
107
108fn decodeDirectHuffmanTree(source: anytype, encoded_symbol_count: usize, weights: *[256]u4) !usize {
109 const weights_byte_count = (encoded_symbol_count + 1) / 2;
110 for (0..weights_byte_count) |i| {
111 const byte = try source.readByte();
112 weights[2 * i] = @intCast(u4, byte >> 4);
113 weights[2 * i + 1] = @intCast(u4, byte & 0xF);
114 }
115 return encoded_symbol_count + 1;
116}
117
118fn assignSymbols(weight_sorted_prefixed_symbols: []LiteralsSection.HuffmanTree.PrefixedSymbol, weights: [256]u4) usize {
119 for (0..weight_sorted_prefixed_symbols.len) |i| {
120 weight_sorted_prefixed_symbols[i] = .{
121 .symbol = @intCast(u8, i),
122 .weight = undefined,
123 .prefix = undefined,
124 };
125 }
126
127 std.sort.sort(
128 LiteralsSection.HuffmanTree.PrefixedSymbol,
129 weight_sorted_prefixed_symbols,
130 weights,
131 lessThanByWeight,
132 );
133
134 var prefix: u16 = 0;
135 var prefixed_symbol_count: usize = 0;
136 var sorted_index: usize = 0;
137 const symbol_count = weight_sorted_prefixed_symbols.len;
138 while (sorted_index < symbol_count) {
139 var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
140 const weight = weights[symbol];
141 if (weight == 0) {
142 sorted_index += 1;
143 continue;
144 }
145
146 while (sorted_index < symbol_count) : ({
147 sorted_index += 1;
148 prefixed_symbol_count += 1;
149 prefix += 1;
150 }) {
151 symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
152 if (weights[symbol] != weight) {
153 prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
154 break;
155 }
156 weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
157 weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
158 weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
159 }
160 }
161 return prefixed_symbol_count;
162}
163
164fn buildHuffmanTree(weights: *[256]u4, symbol_count: usize) error{MalformedHuffmanTree}!LiteralsSection.HuffmanTree {
165 var weight_power_sum_big: u32 = 0;
166 for (weights[0 .. symbol_count - 1]) |value| {
167 weight_power_sum_big += (@as(u16, 1) << value) >> 1;
168 }
169 if (weight_power_sum_big >= 1 << 11) return error.MalformedHuffmanTree;
170 const weight_power_sum = @intCast(u16, weight_power_sum_big);
171
172 // advance to next power of two (even if weight_power_sum is a power of 2)
173 // TODO: is it valid to have weight_power_sum == 0?
174 const max_number_of_bits = if (weight_power_sum == 0) 1 else std.math.log2_int(u16, weight_power_sum) + 1;
175 const next_power_of_two = @as(u16, 1) << max_number_of_bits;
176 weights[symbol_count - 1] = std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1;
177
178 var weight_sorted_prefixed_symbols: [256]LiteralsSection.HuffmanTree.PrefixedSymbol = undefined;
179 const prefixed_symbol_count = assignSymbols(weight_sorted_prefixed_symbols[0..symbol_count], weights.*);
180 const tree = LiteralsSection.HuffmanTree{
181 .max_bit_count = max_number_of_bits,
182 .symbol_count_minus_one = @intCast(u8, prefixed_symbol_count - 1),
183 .nodes = weight_sorted_prefixed_symbols,
184 };
185 return tree;
186}
187
188pub fn decodeHuffmanTree(
189 source: anytype,
190 buffer: []u8,
191) (@TypeOf(source).Error || Error)!LiteralsSection.HuffmanTree {
192 const header = try source.readByte();
193 var weights: [256]u4 = undefined;
194 const symbol_count = if (header < 128)
195 // FSE compressed weights
196 try decodeFseHuffmanTree(source, header, buffer, &weights)
197 else
198 try decodeDirectHuffmanTree(source, header - 127, &weights);
199
200 return buildHuffmanTree(&weights, symbol_count);
201}
202
203pub fn decodeHuffmanTreeSlice(
204 src: []const u8,
205 consumed_count: *usize,
206) Error!LiteralsSection.HuffmanTree {
207 if (src.len == 0) return error.MalformedHuffmanTree;
208 const header = src[0];
209 var bytes_read: usize = 1;
210 var weights: [256]u4 = undefined;
211 const symbol_count = if (header < 128) count: {
212 // FSE compressed weights
213 bytes_read += header;
214 break :count try decodeFseHuffmanTreeSlice(src[1..], header, &weights);
215 } else count: {
216 var fbs = std.io.fixedBufferStream(src[1..]);
217 defer bytes_read += fbs.pos;
218 break :count try decodeDirectHuffmanTree(fbs.reader(), header - 127, &weights);
219 };
220
221 consumed_count.* += bytes_read;
222 return buildHuffmanTree(&weights, symbol_count);
223}
224
225fn lessThanByWeight(
226 weights: [256]u4,
227 lhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
228 rhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
229) bool {
230 // NOTE: this function relies on the use of a stable sorting algorithm,
231 // otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
232 // should be added
233 return weights[lhs.symbol] < weights[rhs.symbol];
234}
lib/std/compress/zstandard/decompress.zig created+636
...@@ -0,0 +1,636 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const Allocator = std.mem.Allocator;
4const RingBuffer = std.RingBuffer;
5
6const types = @import("types.zig");
7const frame = types.frame;
8const LiteralsSection = types.compressed_block.LiteralsSection;
9const SequencesSection = types.compressed_block.SequencesSection;
10const SkippableHeader = types.frame.Skippable.Header;
11const ZstandardHeader = types.frame.Zstandard.Header;
12const Table = types.compressed_block.Table;
13
14pub const block = @import("decode/block.zig");
15
16const readers = @import("readers.zig");
17
18const readInt = std.mem.readIntLittle;
19const readIntSlice = std.mem.readIntSliceLittle;
20
21/// Returns `true` is `magic` is a valid magic number for a skippable frame
22pub fn isSkippableMagic(magic: u32) bool {
23 return frame.Skippable.magic_number_min <= magic and magic <= frame.Skippable.magic_number_max;
24}
25
26/// Returns the kind of frame at the beginning of `source`.
27///
28/// Errors returned:
29/// - `error.BadMagic` if `source` begins with bytes not equal to the
30/// Zstandard frame magic number, or outside the range of magic numbers for
31/// skippable frames.
32/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
33pub fn decodeFrameType(source: anytype) error{ BadMagic, EndOfStream }!frame.Kind {
34 const magic = try source.readIntLittle(u32);
35 return frameType(magic);
36}
37
38/// Returns the kind of frame associated to `magic`.
39///
40/// Errors returned:
41/// - `error.BadMagic` if `magic` is not a valid magic number.
42pub fn frameType(magic: u32) error{BadMagic}!frame.Kind {
43 return if (magic == frame.Zstandard.magic_number)
44 .zstandard
45 else if (isSkippableMagic(magic))
46 .skippable
47 else
48 error.BadMagic;
49}
50
51pub const FrameHeader = union(enum) {
52 zstandard: ZstandardHeader,
53 skippable: SkippableHeader,
54};
55
56pub const HeaderError = error{ BadMagic, EndOfStream, ReservedBitSet };
57
58/// Returns the header of the frame at the beginning of `source`.
59///
60/// Errors returned:
61/// - `error.BadMagic` if `source` begins with bytes not equal to the
62/// Zstandard frame magic number, or outside the range of magic numbers for
63/// skippable frames.
64/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
65/// - `error.ReservedBitSet` if the frame is a Zstandard frame and any of the
66/// reserved bits are set
67pub fn decodeFrameHeader(source: anytype) (@TypeOf(source).Error || HeaderError)!FrameHeader {
68 const magic = try source.readIntLittle(u32);
69 const frame_type = try frameType(magic);
70 switch (frame_type) {
71 .zstandard => return FrameHeader{ .zstandard = try decodeZstandardHeader(source) },
72 .skippable => return FrameHeader{
73 .skippable = .{
74 .magic_number = magic,
75 .frame_size = try source.readIntLittle(u32),
76 },
77 },
78 }
79}
80
81pub const ReadWriteCount = struct {
82 read_count: usize,
83 write_count: usize,
84};
85
86/// Decodes frames from `src` into `dest`; returns the length of the result.
87/// The stream should not have extra trailing bytes - either all bytes in `src`
88/// will be decoded, or an error will be returned. An error will be returned if
89/// a Zstandard frame in `src` does not declare its content size.
90///
91/// Errors returned:
92/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
93/// uses a dictionary
94/// - `error.MalformedFrame` if a frame in `src` is invalid
95/// - `error.UnknownContentSizeUnsupported` if a frame in `src` does not
96/// declare its content size
97pub fn decode(dest: []u8, src: []const u8, verify_checksum: bool) error{
98 MalformedFrame,
99 UnknownContentSizeUnsupported,
100 DictionaryIdFlagUnsupported,
101}!usize {
102 var write_count: usize = 0;
103 var read_count: usize = 0;
104 while (read_count < src.len) {
105 const counts = decodeFrame(dest, src[read_count..], verify_checksum) catch |err| {
106 switch (err) {
107 error.UnknownContentSizeUnsupported => return error.UnknownContentSizeUnsupported,
108 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
109 else => return error.MalformedFrame,
110 }
111 };
112 read_count += counts.read_count;
113 write_count += counts.write_count;
114 }
115 return write_count;
116}
117
118/// Decodes a stream of frames from `src`; returns the decoded bytes. The stream
119/// should not have extra trailing bytes - either all bytes in `src` will be
120/// decoded, or an error will be returned.
121///
122/// Errors returned:
123/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
124/// uses a dictionary
125/// - `error.MalformedFrame` if a frame in `src` is invalid
126/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
127pub fn decodeAlloc(
128 allocator: Allocator,
129 src: []const u8,
130 verify_checksum: bool,
131 window_size_max: usize,
132) error{ DictionaryIdFlagUnsupported, MalformedFrame, OutOfMemory }![]u8 {
133 var result = std.ArrayList(u8).init(allocator);
134 errdefer result.deinit();
135
136 var read_count: usize = 0;
137 while (read_count < src.len) {
138 read_count += decodeFrameArrayList(
139 allocator,
140 &result,
141 src[read_count..],
142 verify_checksum,
143 window_size_max,
144 ) catch |err| switch (err) {
145 error.OutOfMemory => return error.OutOfMemory,
146 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
147 else => return error.MalformedFrame,
148 };
149 }
150 return result.toOwnedSlice();
151}
152
153/// Decodes the frame at the start of `src` into `dest`. Returns the number of
154/// bytes read from `src` and written to `dest`. This function can only decode
155/// frames that declare the decompressed content size.
156///
157/// Errors returned:
158/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
159/// number for a Zstandard or skippable frame
160/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
161/// uncompressed content size
162/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
163/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
164/// size declared by the frame header
165/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
166/// that is larger than `std.math.maxInt(usize)`
167/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
168/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
169/// contains a checksum that does not match the checksum of the decompressed
170/// data
171/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
172/// are set
173/// - `error.EndOfStream` if `src` does not contain a complete frame
174/// - `error.BadContentSize` if the content size declared by the frame does
175/// not equal the actual size of decompressed data
176/// - an error in `block.Error` if there are errors decoding a block
177/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
178/// size greater than `src.len`
179pub fn decodeFrame(
180 dest: []u8,
181 src: []const u8,
182 verify_checksum: bool,
183) (error{
184 BadMagic,
185 UnknownContentSizeUnsupported,
186 ContentTooLarge,
187 ContentSizeTooLarge,
188 WindowSizeUnknown,
189 DictionaryIdFlagUnsupported,
190 SkippableSizeTooLarge,
191} || FrameError)!ReadWriteCount {
192 var fbs = std.io.fixedBufferStream(src);
193 switch (try decodeFrameType(fbs.reader())) {
194 .zstandard => return decodeZstandardFrame(dest, src, verify_checksum),
195 .skippable => {
196 const content_size = try fbs.reader().readIntLittle(u32);
197 if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;
198 const read_count = @as(usize, content_size) + 8;
199 if (read_count > src.len) return error.SkippableSizeTooLarge;
200 return ReadWriteCount{
201 .read_count = read_count,
202 .write_count = 0,
203 };
204 },
205 }
206}
207
208/// Decodes the frame at the start of `src` into `dest`. Returns the number of
209/// bytes read from `src`.
210///
211/// Errors returned:
212/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
213/// number for a Zstandard or skippable frame
214/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
215/// - `error.WindowTooLarge` if the window size is larger than
216/// `window_size_max`
217/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
218/// that is larger than `std.math.maxInt(usize)`
219/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
220/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
221/// contains a checksum that does not match the checksum of the decompressed
222/// data
223/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
224/// are set
225/// - `error.EndOfStream` if `src` does not contain a complete frame
226/// - `error.BadContentSize` if the content size declared by the frame does
227/// not equal the actual size of decompressed data
228/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
229/// - an error in `block.Error` if there are errors decoding a block
230/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
231/// size greater than `src.len`
232pub fn decodeFrameArrayList(
233 allocator: Allocator,
234 dest: *std.ArrayList(u8),
235 src: []const u8,
236 verify_checksum: bool,
237 window_size_max: usize,
238) (error{ BadMagic, OutOfMemory, SkippableSizeTooLarge } || FrameContext.Error || FrameError)!usize {
239 var fbs = std.io.fixedBufferStream(src);
240 const reader = fbs.reader();
241 const magic = try reader.readIntLittle(u32);
242 switch (try frameType(magic)) {
243 .zstandard => return decodeZstandardFrameArrayList(
244 allocator,
245 dest,
246 src,
247 verify_checksum,
248 window_size_max,
249 ),
250 .skippable => {
251 const content_size = try fbs.reader().readIntLittle(u32);
252 if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;
253 const read_count = @as(usize, content_size) + 8;
254 if (read_count > src.len) return error.SkippableSizeTooLarge;
255 return read_count;
256 },
257 }
258}
259
260/// Returns the frame checksum corresponding to the data fed into `hasher`
261pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 {
262 const hash = hasher.final();
263 return @intCast(u32, hash & 0xFFFFFFFF);
264}
265
266const FrameError = error{
267 ChecksumFailure,
268 BadContentSize,
269 EndOfStream,
270 ReservedBitSet,
271} || block.Error;
272
273/// Decode a Zstandard frame from `src` into `dest`, returning the number of
274/// bytes read from `src` and written to `dest`. The first four bytes of `src`
275/// must be the magic number for a Zstandard frame.
276///
277/// Error returned:
278/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
279/// uncompressed content size
280/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
281/// size declared by the frame header
282/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
283/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
284/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
285/// that is larger than `std.math.maxInt(usize)`
286/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
287/// contains a checksum that does not match the checksum of the decompressed
288/// data
289/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
290/// - `error.EndOfStream` if `src` does not contain a complete frame
291/// - an error in `block.Error` if there are errors decoding a block
292/// - `error.BadContentSize` if the content size declared by the frame does
293/// not equal the actual size of decompressed data
294pub fn decodeZstandardFrame(
295 dest: []u8,
296 src: []const u8,
297 verify_checksum: bool,
298) (error{
299 UnknownContentSizeUnsupported,
300 ContentTooLarge,
301 ContentSizeTooLarge,
302 WindowSizeUnknown,
303 DictionaryIdFlagUnsupported,
304} || FrameError)!ReadWriteCount {
305 assert(readInt(u32, src[0..4]) == frame.Zstandard.magic_number);
306 var consumed_count: usize = 4;
307
308 var frame_context = context: {
309 var fbs = std.io.fixedBufferStream(src[consumed_count..]);
310 var source = fbs.reader();
311 const frame_header = try decodeZstandardHeader(source);
312 consumed_count += fbs.pos;
313 break :context FrameContext.init(
314 frame_header,
315 std.math.maxInt(usize),
316 verify_checksum,
317 ) catch |err| switch (err) {
318 error.WindowTooLarge => unreachable,
319 inline else => |e| return e,
320 };
321 };
322 const counts = try decodeZStandardFrameBlocks(
323 dest,
324 src[consumed_count..],
325 &frame_context,
326 );
327 return ReadWriteCount{
328 .read_count = counts.read_count + consumed_count,
329 .write_count = counts.write_count,
330 };
331}
332
333pub fn decodeZStandardFrameBlocks(
334 dest: []u8,
335 src: []const u8,
336 frame_context: *FrameContext,
337) (error{ ContentTooLarge, UnknownContentSizeUnsupported } || FrameError)!ReadWriteCount {
338 const content_size = frame_context.content_size orelse
339 return error.UnknownContentSizeUnsupported;
340 if (dest.len < content_size) return error.ContentTooLarge;
341
342 var consumed_count: usize = 0;
343 const written_count = decodeFrameBlocksInner(
344 dest[0..content_size],
345 src[consumed_count..],
346 &consumed_count,
347 if (frame_context.hasher_opt) |*hasher| hasher else null,
348 frame_context.block_size_max,
349 ) catch |err| switch (err) {
350 error.DestTooSmall => return error.BadContentSize,
351 inline else => |e| return e,
352 };
353
354 if (written_count != content_size) return error.BadContentSize;
355 if (frame_context.has_checksum) {
356 if (src.len < consumed_count + 4) return error.EndOfStream;
357 const checksum = readIntSlice(u32, src[consumed_count .. consumed_count + 4]);
358 consumed_count += 4;
359 if (frame_context.hasher_opt) |*hasher| {
360 if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
361 }
362 }
363 return ReadWriteCount{ .read_count = consumed_count, .write_count = written_count };
364}
365
366pub const FrameContext = struct {
367 hasher_opt: ?std.hash.XxHash64,
368 window_size: usize,
369 has_checksum: bool,
370 block_size_max: usize,
371 content_size: ?usize,
372
373 const Error = error{
374 DictionaryIdFlagUnsupported,
375 WindowSizeUnknown,
376 WindowTooLarge,
377 ContentSizeTooLarge,
378 };
379 /// Validates `frame_header` and returns the associated `FrameContext`.
380 ///
381 /// Errors returned:
382 /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
383 /// - `error.WindowSizeUnknown` if the frame does not have a valid window
384 /// size
385 /// - `error.WindowTooLarge` if the window size is larger than
386 /// `window_size_max`
387 /// - `error.ContentSizeTooLarge` if the frame header indicates a content
388 /// size larger than `std.math.maxInt(usize)`
389 pub fn init(
390 frame_header: ZstandardHeader,
391 window_size_max: usize,
392 verify_checksum: bool,
393 ) Error!FrameContext {
394 if (frame_header.descriptor.dictionary_id_flag != 0)
395 return error.DictionaryIdFlagUnsupported;
396
397 const window_size_raw = frameWindowSize(frame_header) orelse return error.WindowSizeUnknown;
398 const window_size = if (window_size_raw > window_size_max)
399 return error.WindowTooLarge
400 else
401 @intCast(usize, window_size_raw);
402
403 const should_compute_checksum =
404 frame_header.descriptor.content_checksum_flag and verify_checksum;
405
406 const content_size = if (frame_header.content_size) |size|
407 std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
408 else
409 null;
410
411 return .{
412 .hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
413 .window_size = window_size,
414 .has_checksum = frame_header.descriptor.content_checksum_flag,
415 .block_size_max = @min(1 << 17, window_size),
416 .content_size = content_size,
417 };
418 }
419};
420
421/// Decode a Zstandard from from `src` and return number of bytes read; see
422/// `decodeZstandardFrame()`. The first four bytes of `src` must be the magic
423/// number for a Zstandard frame.
424///
425/// Errors returned:
426/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
427/// - `error.WindowTooLarge` if the window size is larger than
428/// `window_size_max`
429/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
430/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
431/// that is larger than `std.math.maxInt(usize)`
432/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
433/// contains a checksum that does not match the checksum of the decompressed
434/// data
435/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
436/// - `error.EndOfStream` if `src` does not contain a complete frame
437/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
438/// - an error in `block.Error` if there are errors decoding a block
439/// - `error.BadContentSize` if the content size declared by the frame does
440/// not equal the size of decompressed data
441pub fn decodeZstandardFrameArrayList(
442 allocator: Allocator,
443 dest: *std.ArrayList(u8),
444 src: []const u8,
445 verify_checksum: bool,
446 window_size_max: usize,
447) (error{OutOfMemory} || FrameContext.Error || FrameError)!usize {
448 assert(readInt(u32, src[0..4]) == frame.Zstandard.magic_number);
449 var consumed_count: usize = 4;
450
451 var frame_context = context: {
452 var fbs = std.io.fixedBufferStream(src[consumed_count..]);
453 var source = fbs.reader();
454 const frame_header = try decodeZstandardHeader(source);
455 consumed_count += fbs.pos;
456 break :context try FrameContext.init(frame_header, window_size_max, verify_checksum);
457 };
458
459 consumed_count += try decodeZstandardFrameBlocksArrayList(
460 allocator,
461 dest,
462 src[consumed_count..],
463 &frame_context,
464 );
465 return consumed_count;
466}
467
468pub fn decodeZstandardFrameBlocksArrayList(
469 allocator: Allocator,
470 dest: *std.ArrayList(u8),
471 src: []const u8,
472 frame_context: *FrameContext,
473) (error{OutOfMemory} || FrameError)!usize {
474 const initial_len = dest.items.len;
475
476 var ring_buffer = try RingBuffer.init(allocator, frame_context.window_size);
477 defer ring_buffer.deinit(allocator);
478
479 // These tables take 7680 bytes
480 var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
481 var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;
482 var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined;
483
484 var block_header = try block.decodeBlockHeaderSlice(src);
485 var consumed_count: usize = 3;
486 var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);
487 while (true) : ({
488 block_header = try block.decodeBlockHeaderSlice(src[consumed_count..]);
489 consumed_count += 3;
490 }) {
491 const written_size = try block.decodeBlockRingBuffer(
492 &ring_buffer,
493 src[consumed_count..],
494 block_header,
495 &decode_state,
496 &consumed_count,
497 frame_context.block_size_max,
498 );
499 if (frame_context.content_size) |size| {
500 if (dest.items.len - initial_len > size) {
501 return error.BadContentSize;
502 }
503 }
504 if (written_size > 0) {
505 const written_slice = ring_buffer.sliceLast(written_size);
506 try dest.appendSlice(written_slice.first);
507 try dest.appendSlice(written_slice.second);
508 if (frame_context.hasher_opt) |*hasher| {
509 hasher.update(written_slice.first);
510 hasher.update(written_slice.second);
511 }
512 }
513 if (block_header.last_block) break;
514 }
515 if (frame_context.content_size) |size| {
516 if (dest.items.len - initial_len != size) {
517 return error.BadContentSize;
518 }
519 }
520
521 if (frame_context.has_checksum) {
522 if (src.len < consumed_count + 4) return error.EndOfStream;
523 const checksum = readIntSlice(u32, src[consumed_count .. consumed_count + 4]);
524 consumed_count += 4;
525 if (frame_context.hasher_opt) |*hasher| {
526 if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
527 }
528 }
529 return consumed_count;
530}
531
532fn decodeFrameBlocksInner(
533 dest: []u8,
534 src: []const u8,
535 consumed_count: *usize,
536 hash: ?*std.hash.XxHash64,
537 block_size_max: usize,
538) (error{ EndOfStream, DestTooSmall } || block.Error)!usize {
539 // These tables take 7680 bytes
540 var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
541 var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;
542 var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined;
543
544 var block_header = try block.decodeBlockHeaderSlice(src);
545 var bytes_read: usize = 3;
546 defer consumed_count.* += bytes_read;
547 var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);
548 var count: usize = 0;
549 while (true) : ({
550 block_header = try block.decodeBlockHeaderSlice(src[bytes_read..]);
551 bytes_read += 3;
552 }) {
553 const written_size = try block.decodeBlock(
554 dest,
555 src[bytes_read..],
556 block_header,
557 &decode_state,
558 &bytes_read,
559 block_size_max,
560 count,
561 );
562 if (hash) |hash_state| hash_state.update(dest[count .. count + written_size]);
563 count += written_size;
564 if (block_header.last_block) break;
565 }
566 return count;
567}
568
569/// Decode the header of a skippable frame. The first four bytes of `src` must
570/// be a valid magic number for a skippable frame.
571pub fn decodeSkippableHeader(src: *const [8]u8) SkippableHeader {
572 const magic = readInt(u32, src[0..4]);
573 assert(isSkippableMagic(magic));
574 const frame_size = readInt(u32, src[4..8]);
575 return .{
576 .magic_number = magic,
577 .frame_size = frame_size,
578 };
579}
580
581/// Returns the window size required to decompress a frame, or `null` if it
582/// cannot be determined (which indicates a malformed frame header).
583pub fn frameWindowSize(header: ZstandardHeader) ?u64 {
584 if (header.window_descriptor) |descriptor| {
585 const exponent = (descriptor & 0b11111000) >> 3;
586 const mantissa = descriptor & 0b00000111;
587 const window_log = 10 + exponent;
588 const window_base = @as(u64, 1) << @intCast(u6, window_log);
589 const window_add = (window_base / 8) * mantissa;
590 return window_base + window_add;
591 } else return header.content_size;
592}
593
594/// Decode the header of a Zstandard frame.
595///
596/// Errors returned:
597/// - `error.ReservedBitSet` if any of the reserved bits of the header are set
598/// - `error.EndOfStream` if `source` does not contain a complete header
599pub fn decodeZstandardHeader(
600 source: anytype,
601) (@TypeOf(source).Error || error{ EndOfStream, ReservedBitSet })!ZstandardHeader {
602 const descriptor = @bitCast(ZstandardHeader.Descriptor, try source.readByte());
603
604 if (descriptor.reserved) return error.ReservedBitSet;
605
606 var window_descriptor: ?u8 = null;
607 if (!descriptor.single_segment_flag) {
608 window_descriptor = try source.readByte();
609 }
610
611 var dictionary_id: ?u32 = null;
612 if (descriptor.dictionary_id_flag > 0) {
613 // if flag is 3 then field_size = 4, else field_size = flag
614 const field_size = (@as(u4, 1) << descriptor.dictionary_id_flag) >> 1;
615 dictionary_id = try source.readVarInt(u32, .Little, field_size);
616 }
617
618 var content_size: ?u64 = null;
619 if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) {
620 const field_size = @as(u4, 1) << descriptor.content_size_flag;
621 content_size = try source.readVarInt(u64, .Little, field_size);
622 if (field_size == 2) content_size.? += 256;
623 }
624
625 const header = ZstandardHeader{
626 .descriptor = descriptor,
627 .window_descriptor = window_descriptor,
628 .dictionary_id = dictionary_id,
629 .content_size = content_size,
630 };
631 return header;
632}
633
634test {
635 std.testing.refAllDecls(@This());
636}
lib/std/compress/zstandard/readers.zig created+82
...@@ -0,0 +1,82 @@
1const std = @import("std");
2
3pub const ReversedByteReader = struct {
4 remaining_bytes: usize,
5 bytes: []const u8,
6
7 const Reader = std.io.Reader(*ReversedByteReader, error{}, readFn);
8
9 pub fn init(bytes: []const u8) ReversedByteReader {
10 return .{
11 .bytes = bytes,
12 .remaining_bytes = bytes.len,
13 };
14 }
15
16 pub fn reader(self: *ReversedByteReader) Reader {
17 return .{ .context = self };
18 }
19
20 fn readFn(ctx: *ReversedByteReader, buffer: []u8) !usize {
21 if (ctx.remaining_bytes == 0) return 0;
22 const byte_index = ctx.remaining_bytes - 1;
23 buffer[0] = ctx.bytes[byte_index];
24 // buffer[0] = @bitReverse(ctx.bytes[byte_index]);
25 ctx.remaining_bytes = byte_index;
26 return 1;
27 }
28};
29
30/// A bit reader for reading the reversed bit streams used to encode
31/// FSE compressed data.
32pub const ReverseBitReader = struct {
33 byte_reader: ReversedByteReader,
34 bit_reader: std.io.BitReader(.Big, ReversedByteReader.Reader),
35
36 pub fn init(self: *ReverseBitReader, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
37 self.byte_reader = ReversedByteReader.init(bytes);
38 self.bit_reader = std.io.bitReader(.Big, self.byte_reader.reader());
39 if (bytes.len == 0) return;
40 var i: usize = 0;
41 while (i < 8 and 0 == self.readBitsNoEof(u1, 1) catch unreachable) : (i += 1) {}
42 if (i == 8) return error.BitStreamHasNoStartBit;
43 }
44
45 pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: usize) error{EndOfStream}!U {
46 return self.bit_reader.readBitsNoEof(U, num_bits);
47 }
48
49 pub fn readBits(self: *@This(), comptime U: type, num_bits: usize, out_bits: *usize) error{}!U {
50 return try self.bit_reader.readBits(U, num_bits, out_bits);
51 }
52
53 pub fn alignToByte(self: *@This()) void {
54 self.bit_reader.alignToByte();
55 }
56
57 pub fn isEmpty(self: ReverseBitReader) bool {
58 return self.byte_reader.remaining_bytes == 0 and self.bit_reader.bit_count == 0;
59 }
60};
61
62pub fn BitReader(comptime Reader: type) type {
63 return struct {
64 underlying: std.io.BitReader(.Little, Reader),
65
66 pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: usize) !U {
67 return self.underlying.readBitsNoEof(U, num_bits);
68 }
69
70 pub fn readBits(self: *@This(), comptime U: type, num_bits: usize, out_bits: *usize) !U {
71 return self.underlying.readBits(U, num_bits, out_bits);
72 }
73
74 pub fn alignToByte(self: *@This()) void {
75 self.underlying.alignToByte();
76 }
77 };
78}
79
80pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {
81 return .{ .underlying = std.io.bitReader(.Little, reader) };
82}
lib/std/compress/zstandard/types.zig created+401
...@@ -0,0 +1,401 @@
1pub const frame = struct {
2 pub const Kind = enum { zstandard, skippable };
3
4 pub const Zstandard = struct {
5 pub const magic_number = 0xFD2FB528;
6
7 header: Header,
8 data_blocks: []Block,
9 checksum: ?u32,
10
11 pub const Header = struct {
12 descriptor: Descriptor,
13 window_descriptor: ?u8,
14 dictionary_id: ?u32,
15 content_size: ?u64,
16
17 pub const Descriptor = packed struct {
18 dictionary_id_flag: u2,
19 content_checksum_flag: bool,
20 reserved: bool,
21 unused: bool,
22 single_segment_flag: bool,
23 content_size_flag: u2,
24 };
25 };
26
27 pub const Block = struct {
28 pub const Header = struct {
29 last_block: bool,
30 block_type: Block.Type,
31 block_size: u21,
32 };
33
34 pub const Type = enum(u2) {
35 raw,
36 rle,
37 compressed,
38 reserved,
39 };
40 };
41 };
42
43 pub const Skippable = struct {
44 pub const magic_number_min = 0x184D2A50;
45 pub const magic_number_max = 0x184D2A5F;
46
47 pub const Header = struct {
48 magic_number: u32,
49 frame_size: u32,
50 };
51 };
52};
53
54pub const compressed_block = struct {
55 pub const LiteralsSection = struct {
56 header: Header,
57 huffman_tree: ?HuffmanTree,
58 streams: Streams,
59
60 pub const Streams = union(enum) {
61 one: []const u8,
62 four: [4][]const u8,
63 };
64
65 pub const Header = struct {
66 block_type: BlockType,
67 size_format: u2,
68 regenerated_size: u20,
69 compressed_size: ?u18,
70 };
71
72 pub const BlockType = enum(u2) {
73 raw,
74 rle,
75 compressed,
76 treeless,
77 };
78
79 pub const HuffmanTree = struct {
80 max_bit_count: u4,
81 symbol_count_minus_one: u8,
82 nodes: [256]PrefixedSymbol,
83
84 pub const PrefixedSymbol = struct {
85 symbol: u8,
86 prefix: u16,
87 weight: u4,
88 };
89
90 pub const Result = union(enum) {
91 symbol: u8,
92 index: usize,
93 };
94
95 pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{NotFound}!Result {
96 var node = self.nodes[index];
97 const weight = node.weight;
98 var i: usize = index;
99 while (node.weight == weight) {
100 if (node.prefix == prefix) return Result{ .symbol = node.symbol };
101 if (i == 0) return error.NotFound;
102 i -= 1;
103 node = self.nodes[i];
104 }
105 return Result{ .index = i };
106 }
107
108 pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
109 return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
110 }
111 };
112
113 pub const StreamCount = enum { one, four };
114 pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
115 return switch (block_type) {
116 .raw, .rle => .one,
117 .compressed, .treeless => if (size_format == 0) .one else .four,
118 };
119 }
120 };
121
122 pub const SequencesSection = struct {
123 header: SequencesSection.Header,
124 literals_length_table: Table,
125 offset_table: Table,
126 match_length_table: Table,
127
128 pub const Header = struct {
129 sequence_count: u24,
130 match_lengths: Mode,
131 offsets: Mode,
132 literal_lengths: Mode,
133
134 pub const Mode = enum(u2) {
135 predefined,
136 rle,
137 fse,
138 repeat,
139 };
140 };
141 };
142
143 pub const Table = union(enum) {
144 fse: []const Fse,
145 rle: u8,
146
147 pub const Fse = struct {
148 symbol: u8,
149 baseline: u16,
150 bits: u8,
151 };
152 };
153
154 pub const literals_length_code_table = [36]struct { u32, u5 }{
155 .{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
156 .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
157 .{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
158 .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
159 .{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
160 .{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
161 .{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
162 .{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
163 .{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
164 };
165
166 pub const match_length_code_table = [53]struct { u32, u5 }{
167 .{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
168 .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
169 .{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
170 .{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
171 .{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
172 .{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
173 .{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
174 .{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
175 .{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
176 };
177
178 pub const literals_length_default_distribution = [36]i16{
179 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
180 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
181 -1, -1, -1, -1,
182 };
183
184 pub const match_lengths_default_distribution = [53]i16{
185 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
186 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
187 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
188 -1, -1, -1, -1, -1,
189 };
190
191 pub const offset_codes_default_distribution = [29]i16{
192 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
193 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
194 };
195
196 pub const predefined_literal_fse_table = Table{
197 .fse = &[64]Table.Fse{
198 .{ .symbol = 0, .bits = 4, .baseline = 0 },
199 .{ .symbol = 0, .bits = 4, .baseline = 16 },
200 .{ .symbol = 1, .bits = 5, .baseline = 32 },
201 .{ .symbol = 3, .bits = 5, .baseline = 0 },
202 .{ .symbol = 4, .bits = 5, .baseline = 0 },
203 .{ .symbol = 6, .bits = 5, .baseline = 0 },
204 .{ .symbol = 7, .bits = 5, .baseline = 0 },
205 .{ .symbol = 9, .bits = 5, .baseline = 0 },
206 .{ .symbol = 10, .bits = 5, .baseline = 0 },
207 .{ .symbol = 12, .bits = 5, .baseline = 0 },
208 .{ .symbol = 14, .bits = 6, .baseline = 0 },
209 .{ .symbol = 16, .bits = 5, .baseline = 0 },
210 .{ .symbol = 18, .bits = 5, .baseline = 0 },
211 .{ .symbol = 19, .bits = 5, .baseline = 0 },
212 .{ .symbol = 21, .bits = 5, .baseline = 0 },
213 .{ .symbol = 22, .bits = 5, .baseline = 0 },
214 .{ .symbol = 24, .bits = 5, .baseline = 0 },
215 .{ .symbol = 25, .bits = 5, .baseline = 32 },
216 .{ .symbol = 26, .bits = 5, .baseline = 0 },
217 .{ .symbol = 27, .bits = 6, .baseline = 0 },
218 .{ .symbol = 29, .bits = 6, .baseline = 0 },
219 .{ .symbol = 31, .bits = 6, .baseline = 0 },
220 .{ .symbol = 0, .bits = 4, .baseline = 32 },
221 .{ .symbol = 1, .bits = 4, .baseline = 0 },
222 .{ .symbol = 2, .bits = 5, .baseline = 0 },
223 .{ .symbol = 4, .bits = 5, .baseline = 32 },
224 .{ .symbol = 5, .bits = 5, .baseline = 0 },
225 .{ .symbol = 7, .bits = 5, .baseline = 32 },
226 .{ .symbol = 8, .bits = 5, .baseline = 0 },
227 .{ .symbol = 10, .bits = 5, .baseline = 32 },
228 .{ .symbol = 11, .bits = 5, .baseline = 0 },
229 .{ .symbol = 13, .bits = 6, .baseline = 0 },
230 .{ .symbol = 16, .bits = 5, .baseline = 32 },
231 .{ .symbol = 17, .bits = 5, .baseline = 0 },
232 .{ .symbol = 19, .bits = 5, .baseline = 32 },
233 .{ .symbol = 20, .bits = 5, .baseline = 0 },
234 .{ .symbol = 22, .bits = 5, .baseline = 32 },
235 .{ .symbol = 23, .bits = 5, .baseline = 0 },
236 .{ .symbol = 25, .bits = 4, .baseline = 0 },
237 .{ .symbol = 25, .bits = 4, .baseline = 16 },
238 .{ .symbol = 26, .bits = 5, .baseline = 32 },
239 .{ .symbol = 28, .bits = 6, .baseline = 0 },
240 .{ .symbol = 30, .bits = 6, .baseline = 0 },
241 .{ .symbol = 0, .bits = 4, .baseline = 48 },
242 .{ .symbol = 1, .bits = 4, .baseline = 16 },
243 .{ .symbol = 2, .bits = 5, .baseline = 32 },
244 .{ .symbol = 3, .bits = 5, .baseline = 32 },
245 .{ .symbol = 5, .bits = 5, .baseline = 32 },
246 .{ .symbol = 6, .bits = 5, .baseline = 32 },
247 .{ .symbol = 8, .bits = 5, .baseline = 32 },
248 .{ .symbol = 9, .bits = 5, .baseline = 32 },
249 .{ .symbol = 11, .bits = 5, .baseline = 32 },
250 .{ .symbol = 12, .bits = 5, .baseline = 32 },
251 .{ .symbol = 15, .bits = 6, .baseline = 0 },
252 .{ .symbol = 17, .bits = 5, .baseline = 32 },
253 .{ .symbol = 18, .bits = 5, .baseline = 32 },
254 .{ .symbol = 20, .bits = 5, .baseline = 32 },
255 .{ .symbol = 21, .bits = 5, .baseline = 32 },
256 .{ .symbol = 23, .bits = 5, .baseline = 32 },
257 .{ .symbol = 24, .bits = 5, .baseline = 32 },
258 .{ .symbol = 35, .bits = 6, .baseline = 0 },
259 .{ .symbol = 34, .bits = 6, .baseline = 0 },
260 .{ .symbol = 33, .bits = 6, .baseline = 0 },
261 .{ .symbol = 32, .bits = 6, .baseline = 0 },
262 },
263 };
264
265 pub const predefined_match_fse_table = Table{
266 .fse = &[64]Table.Fse{
267 .{ .symbol = 0, .bits = 6, .baseline = 0 },
268 .{ .symbol = 1, .bits = 4, .baseline = 0 },
269 .{ .symbol = 2, .bits = 5, .baseline = 32 },
270 .{ .symbol = 3, .bits = 5, .baseline = 0 },
271 .{ .symbol = 5, .bits = 5, .baseline = 0 },
272 .{ .symbol = 6, .bits = 5, .baseline = 0 },
273 .{ .symbol = 8, .bits = 5, .baseline = 0 },
274 .{ .symbol = 10, .bits = 6, .baseline = 0 },
275 .{ .symbol = 13, .bits = 6, .baseline = 0 },
276 .{ .symbol = 16, .bits = 6, .baseline = 0 },
277 .{ .symbol = 19, .bits = 6, .baseline = 0 },
278 .{ .symbol = 22, .bits = 6, .baseline = 0 },
279 .{ .symbol = 25, .bits = 6, .baseline = 0 },
280 .{ .symbol = 28, .bits = 6, .baseline = 0 },
281 .{ .symbol = 31, .bits = 6, .baseline = 0 },
282 .{ .symbol = 33, .bits = 6, .baseline = 0 },
283 .{ .symbol = 35, .bits = 6, .baseline = 0 },
284 .{ .symbol = 37, .bits = 6, .baseline = 0 },
285 .{ .symbol = 39, .bits = 6, .baseline = 0 },
286 .{ .symbol = 41, .bits = 6, .baseline = 0 },
287 .{ .symbol = 43, .bits = 6, .baseline = 0 },
288 .{ .symbol = 45, .bits = 6, .baseline = 0 },
289 .{ .symbol = 1, .bits = 4, .baseline = 16 },
290 .{ .symbol = 2, .bits = 4, .baseline = 0 },
291 .{ .symbol = 3, .bits = 5, .baseline = 32 },
292 .{ .symbol = 4, .bits = 5, .baseline = 0 },
293 .{ .symbol = 6, .bits = 5, .baseline = 32 },
294 .{ .symbol = 7, .bits = 5, .baseline = 0 },
295 .{ .symbol = 9, .bits = 6, .baseline = 0 },
296 .{ .symbol = 12, .bits = 6, .baseline = 0 },
297 .{ .symbol = 15, .bits = 6, .baseline = 0 },
298 .{ .symbol = 18, .bits = 6, .baseline = 0 },
299 .{ .symbol = 21, .bits = 6, .baseline = 0 },
300 .{ .symbol = 24, .bits = 6, .baseline = 0 },
301 .{ .symbol = 27, .bits = 6, .baseline = 0 },
302 .{ .symbol = 30, .bits = 6, .baseline = 0 },
303 .{ .symbol = 32, .bits = 6, .baseline = 0 },
304 .{ .symbol = 34, .bits = 6, .baseline = 0 },
305 .{ .symbol = 36, .bits = 6, .baseline = 0 },
306 .{ .symbol = 38, .bits = 6, .baseline = 0 },
307 .{ .symbol = 40, .bits = 6, .baseline = 0 },
308 .{ .symbol = 42, .bits = 6, .baseline = 0 },
309 .{ .symbol = 44, .bits = 6, .baseline = 0 },
310 .{ .symbol = 1, .bits = 4, .baseline = 32 },
311 .{ .symbol = 1, .bits = 4, .baseline = 48 },
312 .{ .symbol = 2, .bits = 4, .baseline = 16 },
313 .{ .symbol = 4, .bits = 5, .baseline = 32 },
314 .{ .symbol = 5, .bits = 5, .baseline = 32 },
315 .{ .symbol = 7, .bits = 5, .baseline = 32 },
316 .{ .symbol = 8, .bits = 5, .baseline = 32 },
317 .{ .symbol = 11, .bits = 6, .baseline = 0 },
318 .{ .symbol = 14, .bits = 6, .baseline = 0 },
319 .{ .symbol = 17, .bits = 6, .baseline = 0 },
320 .{ .symbol = 20, .bits = 6, .baseline = 0 },
321 .{ .symbol = 23, .bits = 6, .baseline = 0 },
322 .{ .symbol = 26, .bits = 6, .baseline = 0 },
323 .{ .symbol = 29, .bits = 6, .baseline = 0 },
324 .{ .symbol = 52, .bits = 6, .baseline = 0 },
325 .{ .symbol = 51, .bits = 6, .baseline = 0 },
326 .{ .symbol = 50, .bits = 6, .baseline = 0 },
327 .{ .symbol = 49, .bits = 6, .baseline = 0 },
328 .{ .symbol = 48, .bits = 6, .baseline = 0 },
329 .{ .symbol = 47, .bits = 6, .baseline = 0 },
330 .{ .symbol = 46, .bits = 6, .baseline = 0 },
331 },
332 };
333
334 pub const predefined_offset_fse_table = Table{
335 .fse = &[32]Table.Fse{
336 .{ .symbol = 0, .bits = 5, .baseline = 0 },
337 .{ .symbol = 6, .bits = 4, .baseline = 0 },
338 .{ .symbol = 9, .bits = 5, .baseline = 0 },
339 .{ .symbol = 15, .bits = 5, .baseline = 0 },
340 .{ .symbol = 21, .bits = 5, .baseline = 0 },
341 .{ .symbol = 3, .bits = 5, .baseline = 0 },
342 .{ .symbol = 7, .bits = 4, .baseline = 0 },
343 .{ .symbol = 12, .bits = 5, .baseline = 0 },
344 .{ .symbol = 18, .bits = 5, .baseline = 0 },
345 .{ .symbol = 23, .bits = 5, .baseline = 0 },
346 .{ .symbol = 5, .bits = 5, .baseline = 0 },
347 .{ .symbol = 8, .bits = 4, .baseline = 0 },
348 .{ .symbol = 14, .bits = 5, .baseline = 0 },
349 .{ .symbol = 20, .bits = 5, .baseline = 0 },
350 .{ .symbol = 2, .bits = 5, .baseline = 0 },
351 .{ .symbol = 7, .bits = 4, .baseline = 16 },
352 .{ .symbol = 11, .bits = 5, .baseline = 0 },
353 .{ .symbol = 17, .bits = 5, .baseline = 0 },
354 .{ .symbol = 22, .bits = 5, .baseline = 0 },
355 .{ .symbol = 4, .bits = 5, .baseline = 0 },
356 .{ .symbol = 8, .bits = 4, .baseline = 16 },
357 .{ .symbol = 13, .bits = 5, .baseline = 0 },
358 .{ .symbol = 19, .bits = 5, .baseline = 0 },
359 .{ .symbol = 1, .bits = 5, .baseline = 0 },
360 .{ .symbol = 6, .bits = 4, .baseline = 16 },
361 .{ .symbol = 10, .bits = 5, .baseline = 0 },
362 .{ .symbol = 16, .bits = 5, .baseline = 0 },
363 .{ .symbol = 28, .bits = 5, .baseline = 0 },
364 .{ .symbol = 27, .bits = 5, .baseline = 0 },
365 .{ .symbol = 26, .bits = 5, .baseline = 0 },
366 .{ .symbol = 25, .bits = 5, .baseline = 0 },
367 .{ .symbol = 24, .bits = 5, .baseline = 0 },
368 },
369 };
370 pub const start_repeated_offset_1 = 1;
371 pub const start_repeated_offset_2 = 4;
372 pub const start_repeated_offset_3 = 8;
373
374 pub const table_accuracy_log_max = struct {
375 pub const literal = 9;
376 pub const match = 9;
377 pub const offset = 8;
378 };
379
380 pub const table_symbol_count_max = struct {
381 pub const literal = 36;
382 pub const match = 53;
383 pub const offset = 32;
384 };
385
386 pub const default_accuracy_log = struct {
387 pub const literal = 6;
388 pub const match = 6;
389 pub const offset = 5;
390 };
391 pub const table_size_max = struct {
392 pub const literal = 1 << table_accuracy_log_max.literal;
393 pub const match = 1 << table_accuracy_log_max.match;
394 pub const offset = 1 << table_accuracy_log_max.match;
395 };
396};
397
398test {
399 const testing = @import("std").testing;
400 testing.refAllDeclsRecursive(@This());
401}
lib/std/hash.zig+5
...@@ -32,6 +32,10 @@ pub const CityHash64 = cityhash.CityHash64;...@@ -32,6 +32,10 @@ pub const CityHash64 = cityhash.CityHash64;
32const wyhash = @import("hash/wyhash.zig");32const wyhash = @import("hash/wyhash.zig");
33pub const Wyhash = wyhash.Wyhash;33pub const Wyhash = wyhash.Wyhash;
3434
35const xxhash = @import("hash/xxhash.zig");
36pub const XxHash64 = xxhash.XxHash64;
37pub const XxHash32 = xxhash.XxHash32;
38
35test "hash" {39test "hash" {
36 _ = adler;40 _ = adler;
37 _ = auto_hash;41 _ = auto_hash;
...@@ -40,4 +44,5 @@ test "hash" {...@@ -40,4 +44,5 @@ test "hash" {
40 _ = murmur;44 _ = murmur;
41 _ = cityhash;45 _ = cityhash;
42 _ = wyhash;46 _ = wyhash;
47 _ = xxhash;
43}48}
lib/std/hash/xxhash.zig created+268
...@@ -0,0 +1,268 @@
1const std = @import("std");
2const mem = std.mem;
3const expectEqual = std.testing.expectEqual;
4
5const rotl = std.math.rotl;
6
7pub const XxHash64 = struct {
8 acc1: u64,
9 acc2: u64,
10 acc3: u64,
11 acc4: u64,
12
13 seed: u64,
14 buf: [32]u8,
15 buf_len: usize,
16 byte_count: usize,
17
18 const prime_1 = 0x9E3779B185EBCA87; // 0b1001111000110111011110011011000110000101111010111100101010000111
19 const prime_2 = 0xC2B2AE3D27D4EB4F; // 0b1100001010110010101011100011110100100111110101001110101101001111
20 const prime_3 = 0x165667B19E3779F9; // 0b0001011001010110011001111011000110011110001101110111100111111001
21 const prime_4 = 0x85EBCA77C2B2AE63; // 0b1000010111101011110010100111011111000010101100101010111001100011
22 const prime_5 = 0x27D4EB2F165667C5; // 0b0010011111010100111010110010111100010110010101100110011111000101
23
24 pub fn init(seed: u64) XxHash64 {
25 return XxHash64{
26 .seed = seed,
27 .acc1 = seed +% prime_1 +% prime_2,
28 .acc2 = seed +% prime_2,
29 .acc3 = seed,
30 .acc4 = seed -% prime_1,
31 .buf = undefined,
32 .buf_len = 0,
33 .byte_count = 0,
34 };
35 }
36
37 pub fn update(self: *XxHash64, input: []const u8) void {
38 if (input.len < 32 - self.buf_len) {
39 mem.copy(u8, self.buf[self.buf_len..], input);
40 self.buf_len += input.len;
41 return;
42 }
43
44 var i: usize = 0;
45
46 if (self.buf_len > 0) {
47 i = 32 - self.buf_len;
48 mem.copy(u8, self.buf[self.buf_len..], input[0..i]);
49 self.processStripe(&self.buf);
50 self.buf_len = 0;
51 }
52
53 while (i + 32 <= input.len) : (i += 32) {
54 self.processStripe(input[i..][0..32]);
55 }
56
57 const remaining_bytes = input[i..];
58 mem.copy(u8, &self.buf, remaining_bytes);
59 self.buf_len = remaining_bytes.len;
60 }
61
62 inline fn processStripe(self: *XxHash64, buf: *const [32]u8) void {
63 self.acc1 = round(self.acc1, mem.readIntLittle(u64, buf[0..8]));
64 self.acc2 = round(self.acc2, mem.readIntLittle(u64, buf[8..16]));
65 self.acc3 = round(self.acc3, mem.readIntLittle(u64, buf[16..24]));
66 self.acc4 = round(self.acc4, mem.readIntLittle(u64, buf[24..32]));
67 self.byte_count += 32;
68 }
69
70 inline fn round(acc: u64, lane: u64) u64 {
71 const a = acc +% (lane *% prime_2);
72 const b = rotl(u64, a, 31);
73 return b *% prime_1;
74 }
75
76 pub fn final(self: *XxHash64) u64 {
77 var acc: u64 = undefined;
78
79 if (self.byte_count < 32) {
80 acc = self.seed +% prime_5;
81 } else {
82 acc = rotl(u64, self.acc1, 1) +% rotl(u64, self.acc2, 7) +%
83 rotl(u64, self.acc3, 12) +% rotl(u64, self.acc4, 18);
84 acc = mergeAccumulator(acc, self.acc1);
85 acc = mergeAccumulator(acc, self.acc2);
86 acc = mergeAccumulator(acc, self.acc3);
87 acc = mergeAccumulator(acc, self.acc4);
88 }
89
90 acc = acc +% @as(u64, self.byte_count) +% @as(u64, self.buf_len);
91
92 var pos: usize = 0;
93 while (pos + 8 <= self.buf_len) : (pos += 8) {
94 const lane = mem.readIntLittle(u64, self.buf[pos..][0..8]);
95 acc ^= round(0, lane);
96 acc = rotl(u64, acc, 27) *% prime_1;
97 acc +%= prime_4;
98 }
99
100 if (pos + 4 <= self.buf_len) {
101 const lane = @as(u64, mem.readIntLittle(u32, self.buf[pos..][0..4]));
102 acc ^= lane *% prime_1;
103 acc = rotl(u64, acc, 23) *% prime_2;
104 acc +%= prime_3;
105 pos += 4;
106 }
107
108 while (pos < self.buf_len) : (pos += 1) {
109 const lane = @as(u64, self.buf[pos]);
110 acc ^= lane *% prime_5;
111 acc = rotl(u64, acc, 11) *% prime_1;
112 }
113
114 acc ^= acc >> 33;
115 acc *%= prime_2;
116 acc ^= acc >> 29;
117 acc *%= prime_3;
118 acc ^= acc >> 32;
119
120 return acc;
121 }
122
123 inline fn mergeAccumulator(acc: u64, other: u64) u64 {
124 const a = acc ^ round(0, other);
125 const b = a *% prime_1;
126 return b +% prime_4;
127 }
128
129 pub fn hash(input: []const u8) u64 {
130 var hasher = XxHash64.init(0);
131 hasher.update(input);
132 return hasher.final();
133 }
134};
135
136pub const XxHash32 = struct {
137 acc1: u32,
138 acc2: u32,
139 acc3: u32,
140 acc4: u32,
141
142 seed: u32,
143 buf: [16]u8,
144 buf_len: usize,
145 byte_count: usize,
146
147 const prime_1 = 0x9E3779B1; // 0b10011110001101110111100110110001
148 const prime_2 = 0x85EBCA77; // 0b10000101111010111100101001110111
149 const prime_3 = 0xC2B2AE3D; // 0b11000010101100101010111000111101
150 const prime_4 = 0x27D4EB2F; // 0b00100111110101001110101100101111
151 const prime_5 = 0x165667B1; // 0b00010110010101100110011110110001
152
153 pub fn init(seed: u32) XxHash32 {
154 return XxHash32{
155 .seed = seed,
156 .acc1 = seed +% prime_1 +% prime_2,
157 .acc2 = seed +% prime_2,
158 .acc3 = seed,
159 .acc4 = seed -% prime_1,
160 .buf = undefined,
161 .buf_len = 0,
162 .byte_count = 0,
163 };
164 }
165
166 pub fn update(self: *XxHash32, input: []const u8) void {
167 if (input.len < 16 - self.buf_len) {
168 mem.copy(u8, self.buf[self.buf_len..], input);
169 self.buf_len += input.len;
170 return;
171 }
172
173 var i: usize = 0;
174
175 if (self.buf_len > 0) {
176 i = 16 - self.buf_len;
177 mem.copy(u8, self.buf[self.buf_len..], input[0..i]);
178 self.processStripe(&self.buf);
179 self.buf_len = 0;
180 }
181
182 while (i + 16 <= input.len) : (i += 16) {
183 self.processStripe(input[i..][0..16]);
184 }
185
186 const remaining_bytes = input[i..];
187 mem.copy(u8, &self.buf, remaining_bytes);
188 self.buf_len = remaining_bytes.len;
189 }
190
191 inline fn processStripe(self: *XxHash32, buf: *const [16]u8) void {
192 self.acc1 = round(self.acc1, mem.readIntLittle(u32, buf[0..4]));
193 self.acc2 = round(self.acc2, mem.readIntLittle(u32, buf[4..8]));
194 self.acc3 = round(self.acc3, mem.readIntLittle(u32, buf[8..12]));
195 self.acc4 = round(self.acc4, mem.readIntLittle(u32, buf[12..16]));
196 self.byte_count += 16;
197 }
198
199 inline fn round(acc: u32, lane: u32) u32 {
200 const a = acc +% (lane *% prime_2);
201 const b = rotl(u32, a, 13);
202 return b *% prime_1;
203 }
204
205 pub fn final(self: *XxHash32) u32 {
206 var acc: u32 = undefined;
207
208 if (self.byte_count < 16) {
209 acc = self.seed +% prime_5;
210 } else {
211 acc = rotl(u32, self.acc1, 1) +% rotl(u32, self.acc2, 7) +%
212 rotl(u32, self.acc3, 12) +% rotl(u32, self.acc4, 18);
213 }
214
215 acc = acc +% @intCast(u32, self.byte_count) +% @intCast(u32, self.buf_len);
216
217 var pos: usize = 0;
218 while (pos + 4 <= self.buf_len) : (pos += 4) {
219 const lane = mem.readIntLittle(u32, self.buf[pos..][0..4]);
220 acc +%= lane *% prime_3;
221 acc = rotl(u32, acc, 17) *% prime_4;
222 }
223
224 while (pos < self.buf_len) : (pos += 1) {
225 const lane = @as(u32, self.buf[pos]);
226 acc +%= lane *% prime_5;
227 acc = rotl(u32, acc, 11) *% prime_1;
228 }
229
230 acc ^= acc >> 15;
231 acc *%= prime_2;
232 acc ^= acc >> 13;
233 acc *%= prime_3;
234 acc ^= acc >> 16;
235
236 return acc;
237 }
238
239 pub fn hash(input: []const u8) u32 {
240 var hasher = XxHash32.init(0);
241 hasher.update(input);
242 return hasher.final();
243 }
244};
245
246test "xxhash64" {
247 const hash = XxHash64.hash;
248
249 try expectEqual(hash(""), 0xef46db3751d8e999);
250 try expectEqual(hash("a"), 0xd24ec4f1a98c6e5b);
251 try expectEqual(hash("abc"), 0x44bc2cf5ad770999);
252 try expectEqual(hash("message digest"), 0x066ed728fceeb3be);
253 try expectEqual(hash("abcdefghijklmnopqrstuvwxyz"), 0xcfe1f278fa89835c);
254 try expectEqual(hash("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0xaaa46907d3047814);
255 try expectEqual(hash("12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0xe04a477f19ee145d);
256}
257
258test "xxhash32" {
259 const hash = XxHash32.hash;
260
261 try expectEqual(hash(""), 0x02cc5d05);
262 try expectEqual(hash("a"), 0x550d7456);
263 try expectEqual(hash("abc"), 0x32d153ff);
264 try expectEqual(hash("message digest"), 0x7c948494);
265 try expectEqual(hash("abcdefghijklmnopqrstuvwxyz"), 0x63a14d5f);
266 try expectEqual(hash("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x9c285e64);
267 try expectEqual(hash("12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x9c05f475);
268}
lib/std/std.zig+1
...@@ -31,6 +31,7 @@ pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceE...@@ -31,6 +31,7 @@ pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceE
31pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;31pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;
32pub const PriorityDequeue = @import("priority_dequeue.zig").PriorityDequeue;32pub const PriorityDequeue = @import("priority_dequeue.zig").PriorityDequeue;
33pub const Progress = @import("Progress.zig");33pub const Progress = @import("Progress.zig");
34pub const RingBuffer = @import("RingBuffer.zig");
34pub const SegmentedList = @import("segmented_list.zig").SegmentedList;35pub const SegmentedList = @import("segmented_list.zig").SegmentedList;
35pub const SemanticVersion = @import("SemanticVersion.zig");36pub const SemanticVersion = @import("SemanticVersion.zig");
36pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;37pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;