Commit 61cb5143872ec2f3ae9c2942a03e18968bf27761 | zig

author	Dominic <4678790+dweiller@users.noreply.github.com> 2023-01-21 19:10:47+11:00
committer	Dominic <4678790+dweiller@users.noreply.github.com> 2023-02-20 09:09:05+11:00
log	61cb5143872ec2f3ae9c2942a03e18968bf27761
tree	c30a0c2c323a8db3d4004ed22da6e7f7efb70ed4
parent	19984d8751894608dc3cd2271c8e2e40c476d5df

std.compress: add zstandard decompressor

8 files changed, 4697 insertions(+), 0 deletions(-)

build.zig
lib
- std
  - compress.zig
  - compress
    testdata
    rfc8478.txt
    rfc8478.txt.zst.19
    rfc8478.txt.zst.3
    zstandard.zig
    zstandard
    decompress.zig
    types.zig

build.zig+3

...	...	@@ -113,8 +113,11 @@ pub fn build(b: *std.Build) !void {
113	113	".gz",
114	114	".z.0",
115	115	".z.9",
	116	".zstd.3",
	117	".zstd.19",
116	118	"rfc1951.txt",
117	119	"rfc1952.txt",
	120	"rfc8478.txt",
118	121	// exclude files from lib/std/compress/deflate/testdata
119	122	".expect",
120	123	".expect-noinput",

lib/std/compress.zig+2

...	...	@@ -6,6 +6,7 @@ pub const lzma = @import("compress/lzma.zig");
6	6	pub const lzma2 = @import("compress/lzma2.zig");
7	7	pub const xz = @import("compress/xz.zig");
8	8	pub const zlib = @import("compress/zlib.zig");
	9	pub const zstandard = @import("compress/zstandard.zig");
9	10
10	11	pub fn HashedReader(
11	12	comptime ReaderType: anytype,
...	...	@@ -44,4 +45,5 @@ test {
44	45	_ = lzma2;
45	46	_ = xz;
46	47	_ = zlib;
	48	_ = zstandard;
47	49	}

lib/std/compress/testdata/rfc8478.txt created+3027

...	...	@@ -0,0 +1,3027 @@
	1
	2
	3
	4
	5
	6
	7	Internet Engineering Task Force (IETF) Y. Collet
	8	Request for Comments: 8478 M. Kucherawy, Ed.
	9	Category: Informational Facebook
	10	ISSN: 2070-1721 October 2018
	11
	12
	13	Zstandard Compression and the application/zstd Media Type
	14
	15	Abstract
	16
	17	Zstandard, or "zstd" (pronounced "zee standard"), is a data
	18	compression mechanism. This document describes the mechanism and
	19	registers a media type and content encoding to be used when
	20	transporting zstd-compressed content via Multipurpose Internet Mail
	21	Extensions (MIME).
	22
	23	Despite use of the word "standard" as part of its name, readers are
	24	advised that this document is not an Internet Standards Track
	25	specification; it is being published for informational purposes only.
	26
	27	Status of This Memo
	28
	29	This document is not an Internet Standards Track specification; it is
	30	published for informational purposes.
	31
	32	This document is a product of the Internet Engineering Task Force
	33	(IETF). It represents the consensus of the IETF community. It has
	34	received public review and has been approved for publication by the
	35	Internet Engineering Steering Group (IESG). Not all documents
	36	approved by the IESG are candidates for any level of Internet
	37	Standard; see Section 2 of RFC 7841.
	38
	39	Information about the current status of this document, any errata,
	40	and how to provide feedback on it may be obtained at
	41	https://www.rfc-editor.org/info/rfc8478.
	42
	43
	44
	45
	46
	47
	48
	49
	50
	51
	52
	53
	54
	55
	56
	57
	58	Collet & Kucherawy Informational [Page 1]
	59
	60	RFC 8478 application/zstd October 2018
	61
	62
	63	Copyright Notice
	64
	65	Copyright (c) 2018 IETF Trust and the persons identified as the
	66	document authors. All rights reserved.
	67
	68	This document is subject to BCP 78 and the IETF Trust's Legal
	69	Provisions Relating to IETF Documents
	70	(https://trustee.ietf.org/license-info) in effect on the date of
	71	publication of this document. Please review these documents
	72	carefully, as they describe your rights and restrictions with respect
	73	to this document. Code Components extracted from this document must
	74	include Simplified BSD License text as described in Section 4.e of
	75	the Trust Legal Provisions and are provided without warranty as
	76	described in the Simplified BSD License.
	77
	78
	79
	80
	81
	82
	83
	84
	85
	86
	87
	88
	89
	90
	91
	92
	93
	94
	95
	96
	97
	98
	99
	100
	101
	102
	103
	104
	105
	106
	107
	108
	109
	110
	111
	112
	113
	114	Collet & Kucherawy Informational [Page 2]
	115
	116	RFC 8478 application/zstd October 2018
	117
	118
	119	Table of Contents
	120
	121	1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 4
	122	2. Definitions . . . . . . . . . . . . . . . . . . . . . . . . . 4
	123	3. Compression Algorithm . . . . . . . . . . . . . . . . . . . . 5
	124	3.1. Frames . . . . . . . . . . . . . . . . . . . . . . . . . 6
	125	3.1.1. Zstandard Frames . . . . . . . . . . . . . . . . . . 6
	126	3.1.1.1. Frame Header . . . . . . . . . . . . . . . . . . 7
	127	3.1.1.2. Blocks . . . . . . . . . . . . . . . . . . . . . 12
	128	3.1.1.3. Compressed Blocks . . . . . . . . . . . . . . . . 14
	129	3.1.1.4. Sequence Execution . . . . . . . . . . . . . . . 28
	130	3.1.1.5. Repeat Offsets . . . . . . . . . . . . . . . . . 29
	131	3.1.2. Skippable Frames . . . . . . . . . . . . . . . . . . 30
	132	4. Entropy Encoding . . . . . . . . . . . . . . . . . . . . . . 30
	133	4.1. FSE . . . . . . . . . . . . . . . . . . . . . . . . . . . 31
	134	4.1.1. FSE Table Description . . . . . . . . . . . . . . . . 31
	135	4.2. Huffman Coding . . . . . . . . . . . . . . . . . . . . . 34
	136	4.2.1. Huffman Tree Description . . . . . . . . . . . . . . 35
	137	4.2.1.1. Huffman Tree Header . . . . . . . . . . . . . . . 36
	138	4.2.1.2. FSE Compression of Huffman Weights . . . . . . . 37
	139	4.2.1.3. Conversion from Weights to Huffman Prefix Codes . 38
	140	4.2.2. Huffman-Coded Streams . . . . . . . . . . . . . . . . 39
	141	5. Dictionary Format . . . . . . . . . . . . . . . . . . . . . . 40
	142	6. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 42
	143	6.1. The 'application/zstd' Media Type . . . . . . . . . . . . 42
	144	6.2. Content Encoding . . . . . . . . . . . . . . . . . . . . 43
	145	6.3. Dictionaries . . . . . . . . . . . . . . . . . . . . . . 43
	146	7. Security Considerations . . . . . . . . . . . . . . . . . . . 43
	147	8. Implementation Status . . . . . . . . . . . . . . . . . . . . 44
	148	9. References . . . . . . . . . . . . . . . . . . . . . . . . . 45
	149	9.1. Normative References . . . . . . . . . . . . . . . . . . 45
	150	9.2. Informative References . . . . . . . . . . . . . . . . . 45
	151	Appendix A. Decoding Tables for Predefined Codes . . . . . . . . 46
	152	A.1. Literal Length Code Table . . . . . . . . . . . . . . . . 46
	153	A.2. Match Length Code Table . . . . . . . . . . . . . . . . . 49
	154	A.3. Offset Code Table . . . . . . . . . . . . . . . . . . . . 52
	155	Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . . . 53
	156	Authors' Addresses . . . . . . . . . . . . . . . . . . . . . . . 54
	157
	158
	159
	160
	161
	162
	163
	164
	165
	166
	167
	168
	169
	170	Collet & Kucherawy Informational [Page 3]
	171
	172	RFC 8478 application/zstd October 2018
	173
	174
	175	1. Introduction
	176
	177	Zstandard, or "zstd" (pronounced "zee standard"), is a data
	178	compression mechanism, akin to gzip [RFC1952].
	179
	180	Despite use of the word "standard" as part of its name, readers are
	181	advised that this document is not an Internet Standards Track
	182	specification; it is being published for informational purposes only.
	183
	184	This document describes the Zstandard format. Also, to enable the
	185	transport of a data object compressed with Zstandard, this document
	186	registers a media type that can be used to identify such content when
	187	it is used in a payload encoded using Multipurpose Internet Mail
	188	Extensions (MIME).
	189
	190	2. Definitions
	191
	192	Some terms used elsewhere in this document are defined here for
	193	clarity.
	194
	195	uncompressed: Describes an arbitrary set of bytes in their original
	196	form, prior to being subjected to compression.
	197
	198	compress, compression: The act of processing a set of bytes via the
	199	compression mechanism described here.
	200
	201	compressed: Describes the result of passing a set of bytes through
	202	this mechanism. The original input has thus been compressed.
	203
	204	decompress, decompression: The act of processing a set of bytes
	205	through the inverse of the compression mechanism described here,
	206	in an attempt to recover the original set of bytes prior to
	207	compression.
	208
	209	decompressed: Describes the result of passing a set of bytes through
	210	the reverse of this mechanism. When this is successful, the
	211	decompressed payload and the uncompressed payload are
	212	indistinguishable.
	213
	214	encode: The process of translating data from one form to another;
	215	this may include compression or it may refer to other translations
	216	done as part of this specification.
	217
	218	decode: The reverse of "encode"; describes a process of reversing a
	219	prior encoding to recover the original content.
	220
	221
	222
	223
	224
	225
	226	Collet & Kucherawy Informational [Page 4]
	227
	228	RFC 8478 application/zstd October 2018
	229
	230
	231	frame: Content compressed by Zstandard is transformed into a
	232	Zstandard frame. Multiple frames can be appended into a single
	233	file or stream. A frame is completely independent, has a defined
	234	beginning and end, and has a set of parameters that tells the
	235	decoder how to decompress it.
	236
	237	block: A frame encapsulates one or multiple blocks. Each block
	238	contains arbitrary content, which is described by its header, and
	239	has a guaranteed maximum content size that depends upon frame
	240	parameters. Unlike frames, each block depends on previous blocks
	241	for proper decoding. However, each block can be decompressed
	242	without waiting for its successor, allowing streaming operations.
	243
	244	natural order: A sequence or ordering of objects or values that is
	245	typical of that type of object or value. A set of unique
	246	integers, for example, is in "natural order" if when progressing
	247	from one element in the set or sequence to the next, there is
	248	never a decrease in value.
	249
	250	The naming convention for identifiers within the specification is
	251	Mixed_Case_With_Underscores. Identifiers inside square brackets
	252	indicate that the identifier is optional in the presented context.
	253
	254	3. Compression Algorithm
	255
	256	This section describes the Zstandard algorithm.
	257
	258	The purpose of this document is to define a lossless compressed data
	259	format that is a) independent of the CPU type, operating system, file
	260	system, and character set and b) is suitable for file compression and
	261	pipe and streaming compression, using the Zstandard algorithm. The
	262	text of the specification assumes a basic background in programming
	263	at the level of bits and other primitive data representations.
	264
	265	The data can be produced or consumed, even for an arbitrarily long
	266	sequentially presented input data stream, using only an a priori
	267	bounded amount of intermediate storage, and hence can be used in data
	268	communications. The format uses the Zstandard compression method,
	269	and an optional xxHash-64 checksum method [XXHASH], for detection of
	270	data corruption.
	271
	272	The data format defined by this specification does not attempt to
	273	allow random access to compressed data.
	274
	275	Unless otherwise indicated below, a compliant compressor must produce
	276	data sets that conform to the specifications presented here.
	277	However, it does not need to support all options.
	278
	279
	280
	281
	282	Collet & Kucherawy Informational [Page 5]
	283
	284	RFC 8478 application/zstd October 2018
	285
	286
	287	A compliant decompressor must be able to decompress at least one
	288	working set of parameters that conforms to the specifications
	289	presented here. It may also ignore informative fields, such as the
	290	checksum. Whenever it does not support a parameter defined in the
	291	compressed stream, it must produce a non-ambiguous error code and
	292	associated error message explaining which parameter is unsupported.
	293
	294	This specification is intended for use by implementers of software to
	295	compress data into Zstandard format and/or decompress data from
	296	Zstandard format. The Zstandard format is supported by an open
	297	source reference implementation, written in portable C, and available
	298	at [ZSTD].
	299
	300	3.1. Frames
	301
	302	Zstandard compressed data is made up of one or more frames. Each
	303	frame is independent and can be decompressed independently of other
	304	frames. The decompressed content of multiple concatenated frames is
	305	the concatenation of each frame's decompressed content.
	306
	307	There are two frame formats defined for Zstandard: Zstandard frames
	308	and skippable frames. Zstandard frames contain compressed data,
	309	while skippable frames contain custom user metadata.
	310
	311	3.1.1. Zstandard Frames
	312
	313	The structure of a single Zstandard frame is as follows:
	314
	315	+--------------------+------------+
	316	\| Magic_Number \| 4 bytes \|
	317	+--------------------+------------+
	318	\| Frame_Header \| 2-14 bytes \|
	319	+--------------------+------------+
	320	\| Data_Block \| n bytes \|
	321	+--------------------+------------+
	322	\| [More Data_Blocks] \| \|
	323	+--------------------+------------+
	324	\| [Content_Checksum] \| 0-4 bytes \|
	325	+--------------------+------------+
	326
	327	Magic_Number: 4 bytes, little-endian format. Value: 0xFD2FB528.
	328
	329	Frame_Header: 2 to 14 bytes, detailed in Section 3.1.1.1.
	330
	331	Data_Block: Detailed in Section 3.1.1.2. This is where data
	332	appears.
	333
	334
	335
	336
	337
	338	Collet & Kucherawy Informational [Page 6]
	339
	340	RFC 8478 application/zstd October 2018
	341
	342
	343	Content_Checksum: An optional 32-bit checksum, only present if
	344	Content_Checksum_Flag is set. The content checksum is the result
	345	of the XXH64() hash function [XXHASH] digesting the original
	346	(decoded) data as input, and a seed of zero. The low 4 bytes of
	347	the checksum are stored in little-endian format.
	348
	349	The magic number was selected to be less probable to find at the
	350	beginning of an arbitrary file. It avoids trivial patterns (0x00,
	351	0xFF, repeated bytes, increasing bytes, etc.), contains byte values
	352	outside of ASCII range, and doesn't map into UTF-8 space, all of
	353	which reduce the likelihood of its appearance at the top of a text
	354	file.
	355
	356	3.1.1.1. Frame Header
	357
	358	The frame header has a variable size, with a minimum of 2 bytes and
	359	up to 14 bytes depending on optional parameters. The structure of
	360	Frame_Header is as follows:
	361
	362	+-------------------------+-----------+
	363	\| Frame_Header_Descriptor \| 1 byte \|
	364	+-------------------------+-----------+
	365	\| [Window_Descriptor] \| 0-1 byte \|
	366	+-------------------------+-----------+
	367	\| [Dictionary_ID] \| 0-4 bytes \|
	368	+-------------------------+-----------+
	369	\| [Frame_Content_Size] \| 0-8 bytes \|
	370	+-------------------------+-----------+
	371
	372
	373
	374
	375
	376
	377
	378
	379
	380
	381
	382
	383
	384
	385
	386
	387
	388
	389
	390
	391
	392
	393
	394	Collet & Kucherawy Informational [Page 7]
	395
	396	RFC 8478 application/zstd October 2018
	397
	398
	399	3.1.1.1.1. Frame_Header_Descriptor
	400
	401	The first header's byte is called the Frame_Header_Descriptor. It
	402	describes which other fields are present. Decoding this byte is
	403	enough to tell the size of Frame_Header.
	404
	405	+------------+-------------------------+
	406	\| Bit Number \| Field Name \|
	407	+------------+-------------------------+
	408	\| 7-6 \| Frame_Content_Size_Flag \|
	409	+------------+-------------------------+
	410	\| 5 \| Single_Segment_Flag \|
	411	+------------+-------------------------+
	412	\| 4 \| (unused) \|
	413	+------------+-------------------------+
	414	\| 3 \| (reserved) \|
	415	+------------+-------------------------+
	416	\| 2 \| Content_Checksum_Flag \|
	417	+------------+-------------------------+
	418	\| 1-0 \| Dictionary_ID_Flag \|
	419	+------------+-------------------------+
	420
	421	In this table, bit 7 is the highest bit, while bit 0 is the lowest
	422	one.
	423
	424	3.1.1.1.1.1. Frame_Content_Size_Flag
	425
	426	This is a 2-bit flag (equivalent to Frame_Header_Descriptor right-
	427	shifted 6 bits) specifying whether Frame_Content_Size (the
	428	decompressed data size) is provided within the header. Flag_Value
	429	provides FCS_Field_Size, which is the number of bytes used by
	430	Frame_Content_Size according to the following table:
	431
	432	+----------------+--------+---+---+---+
	433	\| Flag_Value \| 0 \| 1 \| 2 \| 3 \|
	434	+----------------+--------+---+---+---+
	435	\| FCS_Field_Size \| 0 or 1 \| 2 \| 4 \| 8 \|
	436	+----------------+--------+---+---+---+
	437
	438	When Flag_Value is 0, FCS_Field_Size depends on Single_Segment_Flag:
	439	If Single_Segment_Flag is set, FCS_Field_Size is 1. Otherwise,
	440	FCS_Field_Size is 0; Frame_Content_Size is not provided.
	441
	442
	443
	444
	445
	446
	447
	448
	449
	450	Collet & Kucherawy Informational [Page 8]
	451
	452	RFC 8478 application/zstd October 2018
	453
	454
	455	3.1.1.1.1.2. Single_Segment_Flag
	456
	457	If this flag is set, data must be regenerated within a single
	458	continuous memory segment.
	459
	460	In this case, Window_Descriptor byte is skipped, but
	461	Frame_Content_Size is necessarily present. As a consequence, the
	462	decoder must allocate a memory segment of size equal or larger than
	463	Frame_Content_Size.
	464
	465	In order to protect the decoder from unreasonable memory
	466	requirements, a decoder is allowed to reject a compressed frame that
	467	requests a memory size beyond the decoder's authorized range.
	468
	469	For broader compatibility, decoders are recommended to support memory
	470	sizes of at least 8 MB. This is only a recommendation; each decoder
	471	is free to support higher or lower limits, depending on local
	472	limitations.
	473
	474	3.1.1.1.1.3. Unused Bit
	475
	476	A decoder compliant with this specification version shall not
	477	interpret this bit. It might be used in a future version, to signal
	478	a property that is not mandatory to properly decode the frame. An
	479	encoder compliant with this specification must set this bit to zero.
	480
	481	3.1.1.1.1.4. Reserved Bit
	482
	483	This bit is reserved for some future feature. Its value must be
	484	zero. A decoder compliant with this specification version must
	485	ensure it is not set. This bit may be used in a future revision, to
	486	signal a feature that must be interpreted to decode the frame
	487	correctly.
	488
	489	3.1.1.1.1.5. Content_Checksum_Flag
	490
	491	If this flag is set, a 32-bit Content_Checksum will be present at the
	492	frame's end. See the description of Content_Checksum above.
	493
	494
	495
	496
	497
	498
	499
	500
	501
	502
	503
	504
	505
	506	Collet & Kucherawy Informational [Page 9]
	507
	508	RFC 8478 application/zstd October 2018
	509
	510
	511	3.1.1.1.1.6. Dictionary_ID_Flag
	512
	513	This is a 2-bit flag (= Frame_Header_Descriptor & 0x3) indicating
	514	whether a dictionary ID is provided within the header. It also
	515	specifies the size of this field as DID_Field_Size:
	516
	517	+----------------+---+---+---+---+
	518	\| Flag_Value \| 0 \| 1 \| 2 \| 3 \|
	519	+----------------+---+---+---+---+
	520	\| DID_Field_Size \| 0 \| 1 \| 2 \| 4 \|
	521	+----------------+---+---+---+---+
	522
	523	3.1.1.1.2. Window Descriptor
	524
	525	This provides guarantees about the minimum memory buffer required to
	526	decompress a frame. This information is important for decoders to
	527	allocate enough memory.
	528
	529	The Window_Descriptor byte is optional. When Single_Segment_Flag is
	530	set, Window_Descriptor is not present. In this case, Window_Size is
	531	Frame_Content_Size, which can be any value from 0 to 2^64-1 bytes (16
	532	ExaBytes).
	533
	534	+------------+----------+----------+
	535	\| Bit Number \| 7-3 \| 2-0 \|
	536	+------------+----------+----------+
	537	\| Field Name \| Exponent \| Mantissa \|
	538	+------------+----------+----------+
	539
	540	The minimum memory buffer size is called Window_Size. It is
	541	described by the following formulae:
	542
	543	windowLog = 10 + Exponent;
	544	windowBase = 1 << windowLog;
	545	windowAdd = (windowBase / 8) * Mantissa;
	546	Window_Size = windowBase + windowAdd;
	547
	548	The minimum Window_Size is 1 KB. The maximum Window_Size is (1<<41)
	549	+ 7*(1<<38) bytes, which is 3.75 TB.
	550
	551	In general, larger Window_Size values tend to improve the compression
	552	ratio, but at the cost of increased memory usage.
	553
	554	To properly decode compressed data, a decoder will need to allocate a
	555	buffer of at least Window_Size bytes.
	556
	557
	558
	559
	560
	561
	562	Collet & Kucherawy Informational [Page 10]
	563
	564	RFC 8478 application/zstd October 2018
	565
	566
	567	In order to protect decoders from unreasonable memory requirements, a
	568	decoder is allowed to reject a compressed frame that requests a
	569	memory size beyond decoder's authorized range.
	570
	571	For improved interoperability, it's recommended for decoders to
	572	support values of Window_Size up to 8 MB and for encoders not to
	573	generate frames requiring a Window_Size larger than 8 MB. It's
	574	merely a recommendation though, and decoders are free to support
	575	larger or lower limits, depending on local limitations.
	576
	577	3.1.1.1.3. Dictionary_ID
	578
	579	This is a variable size field, which contains the ID of the
	580	dictionary required to properly decode the frame. This field is
	581	optional. When it's not present, it's up to the decoder to know
	582	which dictionary to use.
	583
	584	Dictionary_ID field size is provided by DID_Field_Size.
	585	DID_Field_Size is directly derived from the value of
	586	Dictionary_ID_Flag. One byte can represent an ID 0-255; 2 bytes can
	587	represent an ID 0-65535; 4 bytes can represent an ID 0-4294967295.
	588	Format is little-endian.
	589
	590	It is permitted to represent a small ID (for example, 13) with a
	591	large 4-byte dictionary ID, even if it is less efficient.
	592
	593	Within private environments, any dictionary ID can be used. However,
	594	for frames and dictionaries distributed in public space,
	595	Dictionary_ID must be attributed carefully. The following ranges are
	596	reserved for use only with dictionaries that have been registered
	597	with IANA (see Section 6.3):
	598
	599	low range: <= 32767
	600	high range: >= (1 << 31)
	601
	602	Any other value for Dictionary_ID can be used by private arrangement
	603	between participants.
	604
	605	Any payload presented for decompression that references an
	606	unregistered reserved dictionary ID results in an error.
	607
	608
	609
	610
	611
	612
	613
	614
	615
	616
	617
	618	Collet & Kucherawy Informational [Page 11]
	619
	620	RFC 8478 application/zstd October 2018
	621
	622
	623	3.1.1.1.4. Frame Content Size
	624
	625	This is the original (uncompressed) size. This information is
	626	optional. Frame_Content_Size uses a variable number of bytes,
	627	provided by FCS_Field_Size. FCS_Field_Size is provided by the value
	628	of Frame_Content_Size_Flag. FCS_Field_Size can be equal to 0 (not
	629	present), 1, 2, 4, or 8 bytes.
	630
	631	+----------------+--------------+
	632	\| FCS Field Size \| Range \|
	633	+----------------+--------------+
	634	\| 0 \| unknown \|
	635	+----------------+--------------+
	636	\| 1 \| 0 - 255 \|
	637	+----------------+--------------+
	638	\| 2 \| 256 - 65791 \|
	639	+----------------+--------------+
	640	\| 4 \| 0 - 2^32 - 1 \|
	641	+----------------+--------------+
	642	\| 8 \| 0 - 2^64 - 1 \|
	643	+----------------+--------------+
	644
	645	Frame_Content_Size format is little-endian. When FCS_Field_Size is
	646	1, 4, or 8 bytes, the value is read directly. When FCS_Field_Size is
	647	2, the offset of 256 is added. It's allowed to represent a small
	648	size (for example 18) using any compatible variant.
	649
	650	3.1.1.2. Blocks
	651
	652	After Magic_Number and Frame_Header, there are some number of blocks.
	653	Each frame must have at least 1 block, but there is no upper limit on
	654	the number of blocks per frame.
	655
	656	The structure of a block is as follows:
	657
	658	+--------------+---------------+
	659	\| Block_Header \| Block_Content \|
	660	+--------------+---------------+
	661	\| 3 bytes \| n bytes \|
	662	+--------------+---------------+
	663
	664
	665
	666
	667
	668
	669
	670
	671
	672
	673
	674	Collet & Kucherawy Informational [Page 12]
	675
	676	RFC 8478 application/zstd October 2018
	677
	678
	679	Block_Header uses 3 bytes, written using little-endian convention.
	680	It contains three fields:
	681
	682	+------------+------------+------------+
	683	\| Last_Block \| Block_Type \| Block_Size \|
	684	+------------+------------+------------+
	685	\| bit 0 \| bits 1-2 \| bits 3-23 \|
	686	+------------+------------+------------+
	687
	688	3.1.1.2.1. Last_Block
	689
	690	The lowest bit (Last_Block) signals whether this block is the last
	691	one. The frame will end after this last block. It may be followed
	692	by an optional Content_Checksum (see Section 3.1.1).
	693
	694	3.1.1.2.2. Block_Type
	695
	696	The next 2 bits represent the Block_Type. There are four block
	697	types:
	698
	699	+-----------+------------------+
	700	\| Value \| Block_Type \|
	701	+-----------+------------------+
	702	\| 0 \| Raw_Block \|
	703	+-----------+------------------+
	704	\| 1 \| RLE_Block \|
	705	+-----------+------------------+
	706	\| 2 \| Compressed_Block \|
	707	+-----------+------------------+
	708	\| 3 \| Reserved \|
	709	+-----------+------------------+
	710
	711	Raw_Block: This is an uncompressed block. Block_Content contains
	712	Block_Size bytes.
	713
	714	RLE_Block: This is a single byte, repeated Block_Size times.
	715	Block_Content consists of a single byte. On the decompression
	716	side, this byte must be repeated Block_Size times.
	717
	718	Compressed_Block: This is a compressed block as described in
	719	Section 3.1.1.3. Block_Size is the length of Block_Content,
	720	namely the compressed data. The decompressed size is not known,
	721	but its maximum possible value is guaranteed (see below).
	722
	723	Reserved: This is not a block. This value cannot be used with the
	724	current specification. If such a value is present, it is
	725	considered to be corrupt data.
	726
	727
	728
	729
	730	Collet & Kucherawy Informational [Page 13]
	731
	732	RFC 8478 application/zstd October 2018
	733
	734
	735	3.1.1.2.3. Block_Size
	736
	737	The upper 21 bits of Block_Header represent the Block_Size.
	738	Block_Size is the size of the block excluding the header. A block
	739	can contain any number of bytes (even zero), up to
	740	Block_Maximum_Decompressed_Size, which is the smallest of:
	741
	742	o Window_Size
	743
	744	o 128 KB
	745
	746	A Compressed_Block has the extra restriction that Block_Size is
	747	always strictly less than the decompressed size. If this condition
	748	cannot be respected, the block must be sent uncompressed instead
	749	(i.e., treated as a Raw_Block).
	750
	751	3.1.1.3. Compressed Blocks
	752
	753	To decompress a compressed block, the compressed size must be
	754	provided from the Block_Size field within Block_Header.
	755
	756	A compressed block consists of two sections: a Literals
	757	Section (Section 3.1.1.3.1) and a
	758	Sequences_Section (Section 3.1.1.3.2). The results of the two
	759	sections are then combined to produce the decompressed data in
	760	Sequence Execution (Section 3.1.1.4).
	761
	762	To decode a compressed block, the following elements are necessary:
	763
	764	o Previous decoded data, up to a distance of Window_Size, or the
	765	beginning of the Frame, whichever is smaller. Single_Segment_Flag
	766	will be set in the latter case.
	767
	768	o List of "recent offsets" from the previous Compressed_Block.
	769
	770	o The previous Huffman tree, required by Treeless_Literals_Block
	771	type.
	772
	773	o Previous Finite State Entropy (FSE) decoding tables, required by
	774	Repeat_Mode, for each symbol type (literals lengths, match
	775	lengths, offsets).
	776
	777	Note that decoding tables are not always from the previous
	778	Compressed_Block:
	779
	780	o Every decoding table can come from a dictionary.
	781
	782
	783
	784
	785
	786	Collet & Kucherawy Informational [Page 14]
	787
	788	RFC 8478 application/zstd October 2018
	789
	790
	791	o The Huffman tree comes from the previous
	792	Compressed_Literals_Block.
	793
	794	3.1.1.3.1. Literals_Section_Header
	795
	796	All literals are regrouped in the first part of the block. They can
	797	be decoded first and then copied during Sequence Execution (see
	798	Section 3.1.1.4), or they can be decoded on the flow during Sequence
	799	Execution.
	800
	801	Literals can be stored uncompressed or compressed using Huffman
	802	prefix codes. When compressed, an optional tree description can be
	803	present, followed by 1 or 4 streams.
	804
	805	+----------------------------+
	806	\| Literals_Section_Header \|
	807	+----------------------------+
	808	\| [Huffman_Tree_Description] \|
	809	+----------------------------+
	810	\| [Jump_Table] \|
	811	+----------------------------+
	812	\| Stream_1 \|
	813	+----------------------------+
	814	\| [Stream_2] \|
	815	+----------------------------+
	816	\| [Stream_3] \|
	817	+----------------------------+
	818	\| [Stream_4] \|
	819	+----------------------------+
	820
	821	3.1.1.3.1.1. Literals_Section_Header
	822
	823	This field describes how literals are packed. It's a byte-aligned
	824	variable-size bit field, ranging from 1 to 5 bytes, using little-
	825	endian convention.
	826
	827	+---------------------+-----------+
	828	\| Literals_Block_Type \| 2 bits \|
	829	+---------------------+-----------+
	830	\| Size_Format \| 1-2 bits \|
	831	+---------------------+-----------+
	832	\| Regenerated_Size \| 5-20 bits \|
	833	+---------------------+-----------+
	834	\| [Compressed_Size] \| 0-18 bits \|
	835	+---------------------+-----------+
	836
	837	In this representation, bits at the top are the lowest bits.
	838
	839
	840
	841
	842	Collet & Kucherawy Informational [Page 15]
	843
	844	RFC 8478 application/zstd October 2018
	845
	846
	847	The Literals_Block_Type field uses the two lowest bits of the first
	848	byte, describing four different block types:
	849
	850	+---------------------------+-------+
	851	\| Literals_Block_Type \| Value \|
	852	+---------------------------+-------+
	853	\| Raw_Literals_Block \| 0 \|
	854	+---------------------------+-------+
	855	\| RLE_Literals_Block \| 1 \|
	856	+---------------------------+-------+
	857	\| Compressed_Literals_Block \| 2 \|
	858	+---------------------------+-------+
	859	\| Treeless_Literals_Block \| 3 \|
	860	+---------------------------+-------+
	861
	862	Raw_Literals_Block: Literals are stored uncompressed.
	863	Literals_Section_Content is Regenerated_Size.
	864
	865	RLE_Literals_Block: Literals consist of a single-byte value repeated
	866	Regenerated_Size times. Literals_Section_Content is 1.
	867
	868	Compressed_Literals_Block: This is a standard Huffman-compressed
	869	block, starting with a Huffman tree description. See details
	870	below. Literals_Section_Content is Compressed_Size.
	871
	872	Treeless_Literals_Block: This is a Huffman-compressed block, using
	873	the Huffman tree from the previous Compressed_Literals_Block, or a
	874	dictionary if there is no previous Huffman-compressed literals
	875	block. Huffman_Tree_Description will be skipped. Note that if
	876	this mode is triggered without any previous Huffman-table in the
	877	frame (or dictionary, per Section 5), it should be treated as data
	878	corruption. Literals_Section_Content is Compressed_Size.
	879
	880	The Size_Format is divided into two families:
	881
	882	o For Raw_Literals_Block and RLE_Literals_Block, it's only necessary
	883	to decode Regenerated_Size. There is no Compressed_Size field.
	884
	885	o For Compressed_Block and Treeless_Literals_Block, it's required to
	886	decode both Compressed_Size and Regenerated_Size (the decompressed
	887	size). It's also necessary to decode the number of streams (1 or
	888	4).
	889
	890	For values spanning several bytes, the convention is little endian.
	891
	892	Size_Format for Raw_Literals_Block and RLE_Literals_Block uses 1 or 2
	893	bits. Its value is (Literals_Section_Header[0]>>2) & 0x3.
	894
	895
	896
	897
	898	Collet & Kucherawy Informational [Page 16]
	899
	900	RFC 8478 application/zstd October 2018
	901
	902
	903	Size_Format == 00 or 10: Size_Format uses 1 bit. Regenerated_Size
	904	uses 5 bits (value 0-31). Literals_Section_Header uses 1 byte.
	905	Regenerated_Size = Literal_Section_Header[0]>>3.
	906
	907	Size_Format == 01: Size_Format uses 2 bits. Regenerated_Size uses
	908	12 bits (values 0-4095). Literals_Section_Header uses 2 bytes.
	909	Regenerated_Size = (Literals_Section_Header[0]>>4) +
	910	(Literals_Section_Header[1]<<4).
	911
	912	Size_Format == 11: Size_Format uses 2 bits. Regenerated_Size uses
	913	20 bits (values 0-1048575). Literals_Section_Header uses 3 bytes.
	914	Regenerated_Size = (Literals_Section_Header[0]>>4) +
	915	(Literals_Section_Header[1]<<4) + (Literals_Section_Header[2]<<12)
	916
	917	Only Stream_1 is present for these cases. Note that it is permitted
	918	to represent a short value (for example, 13) using a long format,
	919	even if it's less efficient.
	920
	921	Size_Format for Compressed_Literals_Block and Treeless_Literals_Block
	922	always uses 2 bits.
	923
	924	Size_Format == 00: A single stream. Both Regenerated_Size and
	925	Compressed_Size use 10 bits (values 0-1023).
	926	Literals_Section_Header uses 3 bytes.
	927
	928	Size_Format == 01: 4 streams. Both Regenerated_Size and
	929	Compressed_Size use 10 bits (values 0-1023).
	930	Literals_Section_Header uses 3 bytes.
	931
	932	Size_Format == 10: 4 streams. Both Regenerated_Size and
	933	Compressed_Size use 14 bits (values 0-16383).
	934	Literals_Section_Header uses 4 bytes.
	935
	936	Size_Format == 11: 4 streams. Both Regenerated_Size and
	937	Compressed_Size use 18 bits (values 0-262143).
	938	Literals_Section_Header uses 5 bytes.
	939
	940	Both the Compressed_Size and Regenerated_Size fields follow little-
	941	endian convention. Note that Compressed_Size includes the size of
	942	the Huffman_Tree_Description when it is present.
	943
	944	3.1.1.3.1.2. Raw_Literals_Block
	945
	946	The data in Stream_1 is Regenerated_Size bytes long. It contains the
	947	raw literals data to be used during Sequence Execution
	948	(Section 3.1.1.3.2).
	949
	950
	951
	952
	953
	954	Collet & Kucherawy Informational [Page 17]
	955
	956	RFC 8478 application/zstd October 2018
	957
	958
	959	3.1.1.3.1.3. RLE_Literals_Block
	960
	961	Stream_1 consists of a single byte that should be repeated
	962	Regenerated_Size times to generate the decoded literals.
	963
	964	3.1.1.3.1.4. Compressed_Literals_Block and Treeless_Literals_Block
	965
	966	Both of these modes contain Huffman-encoded data. For
	967	Treeless_Literals_Block, the Huffman table comes from the previously
	968	compressed literals block, or from a dictionary; see Section 5.
	969
	970	3.1.1.3.1.5. Huffman_Tree_Description
	971
	972	This section is only present when the Literals_Block_Type type is
	973	Compressed_Literals_Block (2). The format of
	974	Huffman_Tree_Description can be found in Section 4.2.1. The size of
	975	Huffman_Tree_Description is determined during the decoding process.
	976	It must be used to determine where streams begin.
	977
	978	Total_Streams_Size = Compressed_Size
	979	- Huffman_Tree_Description_Size
	980
	981	3.1.1.3.1.6. Jump_Table
	982
	983	The Jump_Table is only present when there are 4 Huffman-coded
	984	streams.
	985
	986	(Reminder: Huffman-compressed data consists of either 1 or 4 Huffman-
	987	coded streams.)
	988
	989	If only 1 stream is present, it is a single bitstream occupying the
	990	entire remaining portion of the literals block, encoded as described
	991	within Section 4.2.2.
	992
	993	If there are 4 streams, Literals_Section_Header only provides enough
	994	information to know the decompressed and compressed sizes of all 4
	995	streams combined. The decompressed size of each stream is equal to
	996	(Regenerated_Size+3)/4, except for the last stream, which may be up
	997	to 3 bytes smaller, to reach a total decompressed size as specified
	998	in Regenerated_Size.
	999
	1000	The compressed size of each stream is provided explicitly in the
	1001	Jump_Table. The Jump_Table is 6 bytes long and consists of three
	1002	2-byte little-endian fields, describing the compressed sizes of the
	1003	first 3 streams. Stream4_Size is computed from Total_Streams_Size
	1004	minus sizes of other streams.
	1005
	1006
	1007
	1008
	1009
	1010	Collet & Kucherawy Informational [Page 18]
	1011
	1012	RFC 8478 application/zstd October 2018
	1013
	1014
	1015	Stream4_Size = Total_Streams_Size - 6
	1016	- Stream1_Size - Stream2_Size
	1017	- Stream3_Size
	1018
	1019	Note that if Stream1_Size + Stream2_Size + Stream3_Size exceeds
	1020	Total_Streams_Size, the data are considered corrupted.
	1021
	1022	Each of these 4 bitstreams is then decoded independently as a
	1023	Huffman-Coded stream, as described in Section 4.2.2.
	1024
	1025	3.1.1.3.2. Sequences_Section
	1026
	1027	A compressed block is a succession of sequences. A sequence is a
	1028	literal copy command, followed by a match copy command. A literal
	1029	copy command specifies a length. It is the number of bytes to be
	1030	copied (or extracted) from the Literals Section. A match copy
	1031	command specifies an offset and a length.
	1032
	1033	When all sequences are decoded, if there are literals left in the
	1034	literals section, these bytes are added at the end of the block.
	1035
	1036	This is described in more detail in Section 3.1.1.4.
	1037
	1038	The Sequences_Section regroups all symbols required to decode
	1039	commands. There are three symbol types: literals lengths, offsets,
	1040	and match lengths. They are encoded together, interleaved, in a
	1041	single "bitstream".
	1042
	1043	The Sequences_Section starts by a header, followed by optional
	1044	probability tables for each symbol type, followed by the bitstream.
	1045
	1046	Sequences_Section_Header
	1047	[Literals_Length_Table]
	1048	[Offset_Table]
	1049	[Match_Length_Table]
	1050	bitStream
	1051
	1052	To decode the Sequences_Section, it's necessary to know its size.
	1053	This size is deduced from the size of the Literals_Section:
	1054	Sequences_Section_Size = Block_Size - Literals_Section_Header -
	1055	Literals_Section_Content
	1056
	1057
	1058
	1059
	1060
	1061
	1062
	1063
	1064
	1065
	1066	Collet & Kucherawy Informational [Page 19]
	1067
	1068	RFC 8478 application/zstd October 2018
	1069
	1070
	1071	3.1.1.3.2.1. Sequences_Section_Header
	1072
	1073	This header consists of two items:
	1074
	1075	o Number_of_Sequences
	1076
	1077	o Symbol_Compression_Modes
	1078
	1079	Number_of_Sequences is a variable size field using between 1 and 3
	1080	bytes. If the first byte is "byte0":
	1081
	1082	o if (byte0 == 0): there are no sequences. The sequence section
	1083	stops here. Decompressed content is defined entirely as Literals
	1084	Section content. The FSE tables used in Repeat_Mode are not
	1085	updated.
	1086
	1087	o if (byte0 < 128): Number_of_Sequences = byte0. Uses 1 byte.
	1088
	1089	o if (byte0 < 255): Number_of_Sequences = ((byte0 - 128) << 8) +
	1090	byte1. Uses 2 bytes.
	1091
	1092	o if (byte0 == 255): Number_of_Sequences = byte1 + (byte2 << 8) +
	1093	0x7F00. Uses 3 bytes.
	1094
	1095	Symbol_Compression_Modes is a single byte, defining the compression
	1096	mode of each symbol type.
	1097
	1098	+-------------+----------------------+
	1099	\| Bit Number \| Field Name \|
	1100	+-------------+----------------------+
	1101	\| 7-6 \| Literal_Lengths_Mode \|
	1102	+-------------+----------------------+
	1103	\| 5-4 \| Offsets_Mode \|
	1104	+-------------+----------------------+
	1105	\| 3-2 \| Match_Lengths_Mode \|
	1106	+-------------+----------------------+
	1107	\| 1-0 \| Reserved \|
	1108	+-------------+----------------------+
	1109
	1110	The last field, Reserved, must be all zeroes.
	1111
	1112
	1113
	1114
	1115
	1116
	1117
	1118
	1119
	1120
	1121
	1122	Collet & Kucherawy Informational [Page 20]
	1123
	1124	RFC 8478 application/zstd October 2018
	1125
	1126
	1127	Literals_Lengths_Mode, Offsets_Mode, and Match_Lengths_Mode define
	1128	the Compression_Mode of literals lengths, offsets, and match lengths
	1129	symbols, respectively. They follow the same enumeration:
	1130
	1131	+-------+---------------------+
	1132	\| Value \| Compression_Mode \|
	1133	+-------+---------------------+
	1134	\| 0 \| Predefined_Mode \|
	1135	+-------+---------------------+
	1136	\| 1 \| RLE_Mode \|
	1137	+-------+---------------------+
	1138	\| 2 \| FSE_Compressed_Mode \|
	1139	+-------+---------------------+
	1140	\| 3 \| Repeat_Mode \|
	1141	+-------+---------------------+
	1142
	1143	Predefined_Mode: A predefined FSE (see Section 4.1) distribution
	1144	table is used, as defined in Section 3.1.1.3.2.2. No distribution
	1145	table will be present.
	1146
	1147	RLE_Mode: The table description consists of a single byte, which
	1148	contains the symbol's value. This symbol will be used for all
	1149	sequences.
	1150
	1151	FSE_Compressed_Mode: Standard FSE compression. A distribution table
	1152	will be present. The format of this distribution table is
	1153	described in Section 4.1.1. Note that the maximum allowed
	1154	accuracy log for literals length and match length tables is 9, and
	1155	the maximum accuracy log for the offsets table is 8. This mode
	1156	must not be used when only one symbol is present; RLE_Mode should
	1157	be used instead (although any other mode will work).
	1158
	1159	Repeat_Mode: The table used in the previous Compressed_Block with
	1160	Number_Of_Sequences > 0 will be used again, or if this is the
	1161	first block, the table in the dictionary will be used. Note that
	1162	this includes RLE_Mode, so if Repeat_Mode follows RLE_Mode, the
	1163	same symbol will be repeated. It also includes Predefined_Mode,
	1164	in which case Repeat_Mode will have the same outcome as
	1165	Predefined_Mode. No distribution table will be present. If this
	1166	mode is used without any previous sequence table in the frame (or
	1167	dictionary; see Section 5) to repeat, this should be treated as
	1168	corruption.
	1169
	1170
	1171
	1172
	1173
	1174
	1175
	1176
	1177
	1178	Collet & Kucherawy Informational [Page 21]
	1179
	1180	RFC 8478 application/zstd October 2018
	1181
	1182
	1183	3.1.1.3.2.1.1. Sequence Codes for Lengths and Offsets
	1184
	1185	Each symbol is a code in its own context, which specifies Baseline
	1186	and Number_of_Bits to add. Codes are FSE compressed and interleaved
	1187	with raw additional bits in the same bitstream.
	1188
	1189	Literals length codes are values ranging from 0 to 35 inclusive.
	1190	They define lengths from 0 to 131071 bytes. The literals length is
	1191	equal to the decoded Baseline plus the result of reading
	1192	Number_of_Bits bits from the bitstream, as a little-endian value.
	1193
	1194
	1195
	1196
	1197
	1198
	1199
	1200
	1201
	1202
	1203
	1204
	1205
	1206
	1207
	1208
	1209
	1210
	1211
	1212
	1213
	1214
	1215
	1216
	1217
	1218
	1219
	1220
	1221
	1222
	1223
	1224
	1225
	1226
	1227
	1228
	1229
	1230
	1231
	1232
	1233
	1234	Collet & Kucherawy Informational [Page 22]
	1235
	1236	RFC 8478 application/zstd October 2018
	1237
	1238
	1239	+----------------------+----------+----------------+
	1240	\| Literals_Length_Code \| Baseline \| Number_of_Bits \|
	1241	+----------------------+----------+----------------+
	1242	\| 0-15 \| length \| 0 \|
	1243	+----------------------+----------+----------------+
	1244	\| 16 \| 16 \| 1 \|
	1245	+----------------------+----------+----------------+
	1246	\| 17 \| 18 \| 1 \|
	1247	+----------------------+----------+----------------+
	1248	\| 18 \| 20 \| 1 \|
	1249	+----------------------+----------+----------------+
	1250	\| 19 \| 22 \| 1 \|
	1251	+----------------------+----------+----------------+
	1252	\| 20 \| 24 \| 2 \|
	1253	+----------------------+----------+----------------+
	1254	\| 21 \| 28 \| 2 \|
	1255	+----------------------+----------+----------------+
	1256	\| 22 \| 32 \| 3 \|
	1257	+----------------------+----------+----------------+
	1258	\| 23 \| 40 \| 3 \|
	1259	+----------------------+----------+----------------+
	1260	\| 24 \| 48 \| 4 \|
	1261	+----------------------+----------+----------------+
	1262	\| 25 \| 64 \| 6 \|
	1263	+----------------------+----------+----------------+
	1264	\| 26 \| 128 \| 7 \|
	1265	+----------------------+----------+----------------+
	1266	\| 27 \| 256 \| 8 \|
	1267	+----------------------+----------+----------------+
	1268	\| 28 \| 512 \| 9 \|
	1269	+----------------------+----------+----------------+
	1270	\| 29 \| 1024 \| 10 \|
	1271	+----------------------+----------+----------------+
	1272	\| 30 \| 2048 \| 11 \|
	1273	+----------------------+----------+----------------+
	1274	\| 31 \| 4096 \| 12 \|
	1275	+----------------------+----------+----------------+
	1276	\| 32 \| 8192 \| 13 \|
	1277	+----------------------+----------+----------------+
	1278	\| 33 \| 16384 \| 14 \|
	1279	+----------------------+----------+----------------+
	1280	\| 34 \| 32768 \| 15 \|
	1281	+----------------------+----------+----------------+
	1282	\| 35 \| 65536 \| 16 \|
	1283	+----------------------+----------+----------------+
	1284
	1285
	1286
	1287
	1288
	1289
	1290	Collet & Kucherawy Informational [Page 23]
	1291
	1292	RFC 8478 application/zstd October 2018
	1293
	1294
	1295	Match length codes are values ranging from 0 to 52 inclusive. They
	1296	define lengths from 3 to 131074 bytes. The match length is equal to
	1297	the decoded Baseline plus the result of reading Number_of_Bits bits
	1298	from the bitstream, as a little-endian value.
	1299
	1300
	1301
	1302
	1303
	1304
	1305
	1306
	1307
	1308
	1309
	1310
	1311
	1312
	1313
	1314
	1315
	1316
	1317
	1318
	1319
	1320
	1321
	1322
	1323
	1324
	1325
	1326
	1327
	1328
	1329
	1330
	1331
	1332
	1333
	1334
	1335
	1336
	1337
	1338
	1339
	1340
	1341
	1342
	1343
	1344
	1345
	1346	Collet & Kucherawy Informational [Page 24]
	1347
	1348	RFC 8478 application/zstd October 2018
	1349
	1350
	1351	+-------------------+-----------------------+----------------+
	1352	\| Match_Length_Code \| Baseline \| Number_of_Bits \|
	1353	+-------------------+-----------------------+----------------+
	1354	\| 0-31 \| Match_Length_Code + 3 \| 0 \|
	1355	+-------------------+-----------------------+----------------+
	1356	\| 32 \| 35 \| 1 \|
	1357	+-------------------+-----------------------+----------------+
	1358	\| 33 \| 37 \| 1 \|
	1359	+-------------------+-----------------------+----------------+
	1360	\| 34 \| 39 \| 1 \|
	1361	+-------------------+-----------------------+----------------+
	1362	\| 35 \| 41 \| 1 \|
	1363	+-------------------+-----------------------+----------------+
	1364	\| 36 \| 43 \| 2 \|
	1365	+-------------------+-----------------------+----------------+
	1366	\| 37 \| 47 \| 2 \|
	1367	+-------------------+-----------------------+----------------+
	1368	\| 38 \| 51 \| 3 \|
	1369	+-------------------+-----------------------+----------------+
	1370	\| 39 \| 59 \| 3 \|
	1371	+-------------------+-----------------------+----------------+
	1372	\| 40 \| 67 \| 4 \|
	1373	+-------------------+-----------------------+----------------+
	1374	\| 41 \| 83 \| 4 \|
	1375	+-------------------+-----------------------+----------------+
	1376	\| 42 \| 99 \| 5 \|
	1377	+-------------------+-----------------------+----------------+
	1378	\| 43 \| 131 \| 7 \|
	1379	+-------------------+-----------------------+----------------+
	1380	\| 44 \| 259 \| 8 \|
	1381	+-------------------+-----------------------+----------------+
	1382	\| 45 \| 515 \| 9 \|
	1383	+-------------------+-----------------------+----------------+
	1384	\| 46 \| 1027 \| 10 \|
	1385	+-------------------+-----------------------+----------------+
	1386	\| 47 \| 2051 \| 11 \|
	1387	+-------------------+-----------------------+----------------+
	1388	\| 48 \| 4099 \| 12 \|
	1389	+-------------------+-----------------------+----------------+
	1390	\| 49 \| 8195 \| 13 \|
	1391	+-------------------+-----------------------+----------------+
	1392	\| 50 \| 16387 \| 14 \|
	1393	+-------------------+-----------------------+----------------+
	1394	\| 51 \| 32771 \| 15 \|
	1395	+-------------------+-----------------------+----------------+
	1396	\| 52 \| 65539 \| 16 \|
	1397	+-------------------+-----------------------+----------------+
	1398
	1399
	1400
	1401
	1402	Collet & Kucherawy Informational [Page 25]
	1403
	1404	RFC 8478 application/zstd October 2018
	1405
	1406
	1407	Offset codes are values ranging from 0 to N.
	1408
	1409	A decoder is free to limit its maximum supported value for N.
	1410	Support for values of at least 22 is recommended. At the time of
	1411	this writing, the reference decoder supports a maximum N value of 31.
	1412
	1413	An offset code is also the number of additional bits to read in
	1414	little-endian fashion and can be translated into an Offset_Value
	1415	using the following formulas:
	1416
	1417	Offset_Value = (1 << offsetCode) + readNBits(offsetCode);
	1418	if (Offset_Value > 3) Offset = Offset_Value - 3;
	1419
	1420	This means that maximum Offset_Value is (2^(N+1))-1, supporting back-
	1421	reference distance up to (2^(N+1))-4, but it is limited by the
	1422	maximum back-reference distance (see Section 3.1.1.1.2).
	1423
	1424	Offset_Value from 1 to 3 are special: they define "repeat codes".
	1425	This is described in more detail in Section 3.1.1.5.
	1426
	1427	3.1.1.3.2.1.2. Decoding Sequences
	1428
	1429	FSE bitstreams are read in reverse of the direction they are written.
	1430	In zstd, the compressor writes bits forward into a block, and the
	1431	decompressor must read the bitstream backwards.
	1432
	1433	To find the start of the bitstream, it is therefore necessary to know
	1434	the offset of the last byte of the block, which can be found by
	1435	counting Block_Size bytes after the block header.
	1436
	1437	After writing the last bit containing information, the compressor
	1438	writes a single 1 bit and then fills the byte with 0-7 zero bits of
	1439	padding. The last byte of the compressed bitstream cannot be zero
	1440	for that reason.
	1441
	1442	When decompressing, the last byte containing the padding is the first
	1443	byte to read. The decompressor needs to skip 0-7 initial zero bits
	1444	until the first 1 bit occurs. Afterwards, the useful part of the
	1445	bitstream begins.
	1446
	1447	FSE decoding requires a 'state' to be carried from symbol to symbol.
	1448	For more explanation on FSE decoding, see Section 4.1.
	1449
	1450	For sequence decoding, a separate state keeps track of each literal
	1451	lengths, offsets, and match lengths symbols. Some FSE primitives are
	1452	also used. For more details on the operation of these primitives,
	1453	see Section 4.1.
	1454
	1455
	1456
	1457
	1458	Collet & Kucherawy Informational [Page 26]
	1459
	1460	RFC 8478 application/zstd October 2018
	1461
	1462
	1463	The bitstream starts with initial FSE state values, each using the
	1464	required number of bits in their respective accuracy, decoded
	1465	previously from their normalized distribution. It starts with
	1466	Literals_Length_State, followed by Offset_State, and finally
	1467	Match_Length_State.
	1468
	1469	Note that all values are read backward, so the 'start' of the
	1470	bitstream is at the highest position in memory, immediately before
	1471	the last 1 bit for padding.
	1472
	1473	After decoding the starting states, a single sequence is decoded
	1474	Number_Of_Sequences times. These sequences are decoded in order from
	1475	first to last. Since the compressor writes the bitstream in the
	1476	forward direction, this means the compressor must encode the
	1477	sequences starting with the last one and ending with the first.
	1478
	1479	For each of the symbol types, the FSE state can be used to determine
	1480	the appropriate code. The code then defines the Baseline and
	1481	Number_of_Bits to read for each type. The description of the codes
	1482	for how to determine these values can be found in
	1483	Section 3.1.1.3.2.1.
	1484
	1485	Decoding starts by reading the Number_of_Bits required to decode
	1486	offset. It does the same for Match_Length and then for
	1487	Literals_Length. This sequence is then used for Sequence Execution
	1488	(see Section 3.1.1.4).
	1489
	1490	If it is not the last sequence in the block, the next operation is to
	1491	update states. Using the rules pre-calculated in the decoding
	1492	tables, Literals_Length_State is updated, followed by
	1493	Match_Length_State, and then Offset_State. See Section 4.1 for
	1494	details on how to update states from the bitstream.
	1495
	1496	This operation will be repeated Number_of_Sequences times. At the
	1497	end, the bitstream shall be entirely consumed; otherwise, the
	1498	bitstream is considered corrupted.
	1499
	1500	3.1.1.3.2.2. Default Distributions
	1501
	1502	If Predefined_Mode is selected for a symbol type, its FSE decoding
	1503	table is generated from a predefined distribution table defined here.
	1504	For details on how to convert this distribution into a decoding
	1505	table, see Section 4.1.
	1506
	1507
	1508
	1509
	1510
	1511
	1512
	1513
	1514	Collet & Kucherawy Informational [Page 27]
	1515
	1516	RFC 8478 application/zstd October 2018
	1517
	1518
	1519	3.1.1.3.2.2.1. Literals Length
	1520
	1521	The decoding table uses an accuracy log of 6 bits (64 states).
	1522
	1523	short literalsLength_defaultDistribution[36] =
	1524	{ 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
	1525	2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
	1526	-1,-1,-1,-1
	1527	};
	1528
	1529	3.1.1.3.2.2.2. Match Length
	1530
	1531	The decoding table uses an accuracy log of 6 bits (64 states).
	1532
	1533	short matchLengths_defaultDistribution[53] =
	1534	{ 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
	1535	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1536	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
	1537	-1,-1,-1,-1,-1
	1538	};
	1539
	1540	3.1.1.3.2.2.3. Offset Codes
	1541
	1542	The decoding table uses an accuracy log of 5 bits (32 states), and
	1543	supports a maximum N value of 28, allowing offset values up to
	1544	536,870,908.
	1545
	1546	If any sequence in the compressed block requires a larger offset than
	1547	this, it's not possible to use the default distribution to represent
	1548	it.
	1549
	1550	short offsetCodes_defaultDistribution[29] =
	1551	{ 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
	1552	1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1
	1553	};
	1554
	1555	3.1.1.4. Sequence Execution
	1556
	1557	Once literals and sequences have been decoded, they are combined to
	1558	produce the decoded content of a block.
	1559
	1560	Each sequence consists of a tuple of (literals_length, offset_value,
	1561	match_length), decoded as described in the
	1562	Sequences_Section (Section 3.1.1.3.2). To execute a sequence, first
	1563	copy literals_length bytes from the decoded literals to the output.
	1564
	1565
	1566
	1567
	1568
	1569
	1570	Collet & Kucherawy Informational [Page 28]
	1571
	1572	RFC 8478 application/zstd October 2018
	1573
	1574
	1575	Then, match_length bytes are copied from previous decoded data. The
	1576	offset to copy from is determined by offset_value:
	1577
	1578	o if Offset_Value > 3, then the offset is Offset_Value - 3;
	1579
	1580	o if Offset_Value is from 1-3, the offset is a special repeat offset
	1581	value. See Section 3.1.1.5 for how the offset is determined in
	1582	this case.
	1583
	1584	The offset is defined as from the current position (after copying the
	1585	literals), so an offset of 6 and a match length of 3 means that 3
	1586	bytes should be copied from 6 bytes back. Note that all offsets
	1587	leading to previously decoded data must be smaller than Window_Size
	1588	defined in Frame_Header_Descriptor (Section 3.1.1.1.1).
	1589
	1590	3.1.1.5. Repeat Offsets
	1591
	1592	As seen above, the first three values define a repeated offset; we
	1593	will call them Repeated_Offset1, Repeated_Offset2, and
	1594	Repeated_Offset3. They are sorted in recency order, with
	1595	Repeated_Offset1 meaning "most recent one".
	1596
	1597	If offset_value is 1, then the offset used is Repeated_Offset1, etc.
	1598
	1599	There is one exception: When the current sequence's literals_length
	1600	is 0, repeated offsets are shifted by 1, so an offset_value of 1
	1601	means Repeated_Offset2, an offset_value of 2 means Repeated_Offset3,
	1602	and an offset_value of 3 means Repeated_Offset1 - 1_byte.
	1603
	1604	For the first block, the starting offset history is populated with
	1605	the following values: Repeated_Offset1 (1), Repeated_Offset2 (4), and
	1606	Repeated_Offset3 (8), unless a dictionary is used, in which case they
	1607	come from the dictionary.
	1608
	1609	Then each block gets its starting offset history from the ending
	1610	values of the most recent Compressed_Block. Note that blocks that
	1611	are not Compressed_Block are skipped; they do not contribute to
	1612	offset history.
	1613
	1614	The newest offset takes the lead in offset history, shifting others
	1615	back (up to its previous place if it was already present). This
	1616	means that when Repeated_Offset1 (most recent) is used, history is
	1617	unmodified. When Repeated_Offset2 is used, it is swapped with
	1618	Repeated_Offset1. If any other offset is used, it becomes
	1619	Repeated_Offset1, and the rest are shifted back by 1.
	1620
	1621
	1622
	1623
	1624
	1625
	1626	Collet & Kucherawy Informational [Page 29]
	1627
	1628	RFC 8478 application/zstd October 2018
	1629
	1630
	1631	3.1.2. Skippable Frames
	1632
	1633	+--------------+------------+-----------+
	1634	\| Magic_Number \| Frame_Size \| User_Data \|
	1635	+--------------+------------+-----------+
	1636	\| 4 bytes \| 4 bytes \| n bytes \|
	1637	+--------------+------------+-----------+
	1638
	1639	Skippable frames allow the insertion of user-defined metadata into a
	1640	flow of concatenated frames.
	1641
	1642	Skippable frames defined in this specification are compatible with
	1643	skippable frames in [LZ4].
	1644
	1645	From a compliant decoder perspective, skippable frames simply need to
	1646	be skipped, and their content ignored, resuming decoding after the
	1647	skippable frame.
	1648
	1649	It should be noted that a skippable frame can be used to watermark a
	1650	stream of concatenated frames embedding any kind of tracking
	1651	information (even just a Universally Unique Identifier (UUID)).
	1652	Users wary of such possibility should scan the stream of concatenated
	1653	frames in an attempt to detect such frames for analysis or removal.
	1654
	1655	The fields are:
	1656
	1657	Magic_Number: 4 bytes, little-endian format. Value: 0x184D2A5?,
	1658	which means any value from 0x184D2A50 to 0x184D2A5F. All 16
	1659	values are valid to identify a skippable frame. This
	1660	specification does not detail any specific tagging methods for
	1661	skippable frames.
	1662
	1663	Frame_Size: This is the size, in bytes, of the following User_Data
	1664	(without including the magic number nor the size field itself).
	1665	This field is represented using 4 bytes, little-endian format,
	1666	unsigned 32 bits. This means User_Data can't be bigger than
	1667	(2^32-1) bytes.
	1668
	1669	User_Data: This field can be anything. Data will just be skipped by
	1670	the decoder.
	1671
	1672	4. Entropy Encoding
	1673
	1674	Two types of entropy encoding are used by the Zstandard format: FSE
	1675	and Huffman coding. Huffman is used to compress literals, while FSE
	1676	is used for all other symbols (Literals_Length_Code,
	1677	Match_Length_Code, and offset codes) and to compress Huffman headers.
	1678
	1679
	1680
	1681
	1682	Collet & Kucherawy Informational [Page 30]
	1683
	1684	RFC 8478 application/zstd October 2018
	1685
	1686
	1687	4.1. FSE
	1688
	1689	FSE, short for Finite State Entropy, is an entropy codec based on
	1690	[ANS]. FSE encoding/decoding involves a state that is carried over
	1691	between symbols, so decoding must be done in the opposite direction
	1692	as encoding. Therefore, all FSE bitstreams are read from end to
	1693	beginning. Note that the order of the bits in the stream is not
	1694	reversed; they are simply read in the reverse order from which they
	1695	were written.
	1696
	1697	For additional details on FSE, see Finite State Entropy [FSE].
	1698
	1699	FSE decoding involves a decoding table that has a power of 2 size and
	1700	contains three elements: Symbol, Num_Bits, and Baseline. The base 2
	1701	logarithm of the table size is its Accuracy_Log. An FSE state value
	1702	represents an index in this table.
	1703
	1704	To obtain the initial state value, consume Accuracy_Log bits from the
	1705	stream as a little-endian value. The next symbol in the stream is
	1706	the Symbol indicated in the table for that state. To obtain the next
	1707	state value, the decoder should consume Num_Bits bits from the stream
	1708	as a little-endian value and add it to Baseline.
	1709
	1710	4.1.1. FSE Table Description
	1711
	1712	To decode FSE streams, it is necessary to construct the decoding
	1713	table. The Zstandard format encodes FSE table descriptions as
	1714	described here.
	1715
	1716	An FSE distribution table describes the probabilities of all symbols
	1717	from 0 to the last present one (included) on a normalized scale of
	1718	(1 << Accuracy_Log). Note that there must be two or more symbols
	1719	with non-zero probability.
	1720
	1721	A bitstream is read forward, in little-endian fashion. It is not
	1722	necessary to know its exact size, since the size will be discovered
	1723	and reported by the decoding process. The bitstream starts by
	1724	reporting on which scale it operates. If low4bits designates the
	1725	lowest 4 bits of the first byte, then Accuracy_Log = low4bits + 5.
	1726
	1727
	1728
	1729
	1730
	1731
	1732
	1733
	1734
	1735
	1736
	1737
	1738	Collet & Kucherawy Informational [Page 31]
	1739
	1740	RFC 8478 application/zstd October 2018
	1741
	1742
	1743	This is followed by each symbol value, from 0 to the last present
	1744	one. The number of bits used by each field is variable and depends
	1745	on:
	1746
	1747	Remaining probabilities + 1: For example, presuming an Accuracy_Log
	1748	of 8, and presuming 100 probabilities points have already been
	1749	distributed, the decoder may read any value from 0 to
	1750	(256 - 100 + 1) == 157, inclusive. Therefore, it must read
	1751	log2sup(157) == 8 bits.
	1752
	1753	Value decoded: Small values use 1 fewer bit. For example, presuming
	1754	values from 0 to 157 (inclusive) are possible, 255 - 157 = 98
	1755	values are remaining in an 8-bit field. The first 98 values
	1756	(hence from 0 to 97) use only 7 bits, and values from 98 to 157
	1757	use 8 bits. This is achieved through this scheme:
	1758
	1759	+------------+---------------+-----------+
	1760	\| Value Read \| Value Decoded \| Bits Used \|
	1761	+------------+---------------+-----------+
	1762	\| 0 - 97 \| 0 - 97 \| 7 \|
	1763	+------------+---------------+-----------+
	1764	\| 98 - 127 \| 98 - 127 \| 8 \|
	1765	+------------+---------------+-----------+
	1766	\| 128 - 225 \| 0 - 97 \| 7 \|
	1767	+------------+---------------+-----------+
	1768	\| 226 - 255 \| 128 - 157 \| 8 \|
	1769	+------------+---------------+-----------+
	1770
	1771	Symbol probabilities are read one by one, in order. The probability
	1772	is obtained from Value decoded using the formula P = Value - 1. This
	1773	means the value 0 becomes the negative probability -1. This is a
	1774	special probability that means "less than 1". Its effect on the
	1775	distribution table is described below. For the purpose of
	1776	calculating total allocated probability points, it counts as 1.
	1777
	1778	When a symbol has a probability of zero, it is followed by a 2-bit
	1779	repeat flag. This repeat flag tells how many probabilities of zeroes
	1780	follow the current one. It provides a number ranging from 0 to 3.
	1781	If it is a 3, another 2-bit repeat flag follows, and so on.
	1782
	1783	When the last symbol reaches a cumulated total of
	1784	(1 << Accuracy_Log), decoding is complete. If the last symbol makes
	1785	the cumulated total go above (1 << Accuracy_Log), distribution is
	1786	considered corrupted.
	1787
	1788
	1789
	1790
	1791
	1792
	1793
	1794	Collet & Kucherawy Informational [Page 32]
	1795
	1796	RFC 8478 application/zstd October 2018
	1797
	1798
	1799	Finally, the decoder can tell how many bytes were used in this
	1800	process and how many symbols are present. The bitstream consumes a
	1801	round number of bytes. Any remaining bit within the last byte is
	1802	simply unused.
	1803
	1804	The distribution of normalized probabilities is enough to create a
	1805	unique decoding table. The table has a size of (1 << Accuracy_Log).
	1806	Each cell describes the symbol decoded and instructions to get the
	1807	next state.
	1808
	1809	Symbols are scanned in their natural order for "less than 1"
	1810	probabilities as described above. Symbols with this probability are
	1811	being attributed a single cell, starting from the end of the table
	1812	and retreating. These symbols define a full state reset, reading
	1813	Accuracy_Log bits.
	1814
	1815	All remaining symbols are allocated in their natural order. Starting
	1816	from symbol 0 and table position 0, each symbol gets allocated as
	1817	many cells as its probability. Cell allocation is spread, not
	1818	linear; each successor position follows this rule:
	1819
	1820	position += (tableSize >> 1) + (tableSize >> 3) + 3;
	1821	position &= tableSize - 1;
	1822
	1823	A position is skipped if it is already occupied by a "less than 1"
	1824	probability symbol. Position does not reset between symbols; it
	1825	simply iterates through each position in the table, switching to the
	1826	next symbol when enough states have been allocated to the current
	1827	one.
	1828
	1829	The result is a list of state values. Each state will decode the
	1830	current symbol.
	1831
	1832	To get the Number_of_Bits and Baseline required for the next state,
	1833	it is first necessary to sort all states in their natural order. The
	1834	lower states will need 1 more bit than higher ones. The process is
	1835	repeated for each symbol.
	1836
	1837	For example, presuming a symbol has a probability of 5, it receives
	1838	five state values. States are sorted in natural order. The next
	1839	power of 2 is 8. The space of probabilities is divided into 8 equal
	1840	parts. Presuming the Accuracy_Log is 7, this defines 128 states, and
	1841	each share (divided by 8) is 16 in size. In order to reach 8, 8 - 5
	1842	= 3 lowest states will count "double", doubling the number of shares
	1843	(32 in width), requiring 1 more bit in the process.
	1844
	1845
	1846
	1847
	1848
	1849
	1850	Collet & Kucherawy Informational [Page 33]
	1851
	1852	RFC 8478 application/zstd October 2018
	1853
	1854
	1855	Baseline is assigned starting from the higher states using fewer
	1856	bits, and proceeding naturally, then resuming at the first state,
	1857	each taking its allocated width from Baseline.
	1858
	1859	+----------------+-------+-------+--------+------+-------+
	1860	\| state order \| 0 \| 1 \| 2 \| 3 \| 4 \|
	1861	+----------------+-------+-------+--------+------+-------+
	1862	\| width \| 32 \| 32 \| 32 \| 16 \| 16 \|
	1863	+----------------+-------+-------+--------+------+-------+
	1864	\| Number_of_Bits \| 5 \| 5 \| 5 \| 4 \| 4 \|
	1865	+----------------+-------+-------+--------+------+-------+
	1866	\| range number \| 2 \| 4 \| 6 \| 0 \| 1 \|
	1867	+----------------+-------+-------+--------+------+-------+
	1868	\| Baseline \| 32 \| 64 \| 96 \| 0 \| 16 \|
	1869	+----------------+-------+-------+--------+------+-------+
	1870	\| range \| 32-63 \| 64-95 \| 96-127 \| 0-15 \| 16-31 \|
	1871	+----------------+-------+-------+--------+------+-------+
	1872
	1873	The next state is determined from the current state by reading the
	1874	required Number_of_Bits and adding the specified Baseline.
	1875
	1876	See Appendix A for the results of this process that are applied to
	1877	the default distributions.
	1878
	1879	4.2. Huffman Coding
	1880
	1881	Zstandard Huffman-coded streams are read backwards, similar to the
	1882	FSE bitstreams. Therefore, to find the start of the bitstream, it is
	1883	necessary to know the offset of the last byte of the Huffman-coded
	1884	stream.
	1885
	1886	After writing the last bit containing information, the compressor
	1887	writes a single 1 bit and then fills the byte with 0-7 0 bits of
	1888	padding. The last byte of the compressed bitstream cannot be 0 for
	1889	that reason.
	1890
	1891	When decompressing, the last byte containing the padding is the first
	1892	byte to read. The decompressor needs to skip 0-7 initial 0 bits and
	1893	the first 1 bit that occurs. Afterwards, the useful part of the
	1894	bitstream begins.
	1895
	1896	The bitstream contains Huffman-coded symbols in little-endian order,
	1897	with the codes defined by the method below.
	1898
	1899
	1900
	1901
	1902
	1903
	1904
	1905
	1906	Collet & Kucherawy Informational [Page 34]
	1907
	1908	RFC 8478 application/zstd October 2018
	1909
	1910
	1911	4.2.1. Huffman Tree Description
	1912
	1913	Prefix coding represents symbols from an a priori known alphabet by
	1914	bit sequences (codewords), one codeword for each symbol, in a manner
	1915	such that different symbols may be represented by bit sequences of
	1916	different lengths, but a parser can always parse an encoded string
	1917	unambiguously symbol by symbol.
	1918
	1919	Given an alphabet with known symbol frequencies, the Huffman
	1920	algorithm allows the construction of an optimal prefix code using the
	1921	fewest bits of any possible prefix codes for that alphabet.
	1922
	1923	The prefix code must not exceed a maximum code length. More bits
	1924	improve accuracy but yield a larger header size and require more
	1925	memory or more complex decoding operations. This specification
	1926	limits the maximum code length to 11 bits.
	1927
	1928	All literal values from zero (included) to the last present one
	1929	(excluded) are represented by Weight with values from 0 to
	1930	Max_Number_of_Bits. Transformation from Weight to Number_of_Bits
	1931	follows this pseudocode:
	1932
	1933	if Weight == 0
	1934	Number_of_Bits = 0
	1935	else
	1936	Number_of_Bits = Max_Number_of_Bits + 1 - Weight
	1937
	1938	The last symbol's Weight is deduced from previously decoded ones, by
	1939	completing to the nearest power of 2. This power of 2 gives
	1940	Max_Number_of_Bits the depth of the current tree.
	1941
	1942	For example, presume the following Huffman tree must be described:
	1943
	1944	+---------------+----------------+
	1945	\| Literal Value \| Number_of_Bits \|
	1946	+---------------+----------------+
	1947	\| 0 \| 1 \|
	1948	+---------------+----------------+
	1949	\| 1 \| 2 \|
	1950	+---------------+----------------+
	1951	\| 2 \| 3 \|
	1952	+---------------+----------------+
	1953	\| 3 \| 0 \|
	1954	+---------------+----------------+
	1955	\| 4 \| 4 \|
	1956	+---------------+----------------+
	1957	\| 5 \| 4 \|
	1958	+---------------+----------------+
	1959
	1960
	1961
	1962	Collet & Kucherawy Informational [Page 35]
	1963
	1964	RFC 8478 application/zstd October 2018
	1965
	1966
	1967	The tree depth is 4, since its longest element uses 4 bits. (The
	1968	longest elements are those with the smallest frequencies.) Value 5
	1969	will not be listed as it can be determined from the values for 0-4,
	1970	nor will values above 5 as they are all 0. Values from 0 to 4 will
	1971	be listed using Weight instead of Number_of_Bits. The pseudocode to
	1972	determine Weight is:
	1973
	1974	if Number_of_Bits == 0
	1975	Weight = 0
	1976	else
	1977	Weight = Max_Number_of_Bits + 1 - Number_of_Bits
	1978
	1979	It gives the following series of weights:
	1980
	1981	+---------------+--------+
	1982	\| Literal Value \| Weight \|
	1983	+---------------+--------+
	1984	\| 0 \| 4 \|
	1985	+---------------+--------+
	1986	\| 1 \| 3 \|
	1987	+---------------+--------+
	1988	\| 2 \| 2 \|
	1989	+---------------+--------+
	1990	\| 3 \| 0 \|
	1991	+---------------+--------+
	1992	\| 4 \| 1 \|
	1993	+---------------+--------+
	1994
	1995	The decoder will do the inverse operation: having collected weights
	1996	of literals from 0 to 4, it knows the last literal, 5, is present
	1997	with a non-zero Weight. The Weight of 5 can be determined by
	1998	advancing to the next power of 2. The sum of 2^(Weight-1) (excluding
	1999	0's) is 15. The nearest power of 2 is 16. Therefore,
	2000	Max_Number_of_Bits = 4 and Weight[5] = 16 - 15 = 1.
	2001
	2002	4.2.1.1. Huffman Tree Header
	2003
	2004	This is a single byte value (0-255), which describes how the series
	2005	of weights is encoded.
	2006
	2007	headerByte < 128: The series of weights is compressed using FSE (see
	2008	below). The length of the FSE-compressed series is equal to
	2009	headerByte (0-127).
	2010
	2011
	2012
	2013
	2014
	2015
	2016
	2017
	2018	Collet & Kucherawy Informational [Page 36]
	2019
	2020	RFC 8478 application/zstd October 2018
	2021
	2022
	2023	headerByte >= 128: This is a direct representation, where each
	2024	Weight is written directly as a 4-bit field (0-15). They are
	2025	encoded forward, 2 weights to a byte with the first weight taking
	2026	the top 4 bits and the second taking the bottom 4; for example,
	2027	the following operations could be used to read the weights:
	2028
	2029	Weight[0] = (Byte[0] >> 4)
	2030	Weight[1] = (Byte[0] & 0xf),
	2031	etc.
	2032
	2033	The full representation occupies ceiling(Number_of_Symbols/2)
	2034	bytes, meaning it uses only full bytes even if Number_of_Symbols
	2035	is odd. Number_of_Symbols = headerByte - 127. Note that maximum
	2036	Number_of_Symbols is 255 - 127 = 128. If any literal has a value
	2037	over 128, raw header mode is not possible, and it is necessary to
	2038	use FSE compression.
	2039
	2040	4.2.1.2. FSE Compression of Huffman Weights
	2041
	2042	In this case, the series of Huffman weights is compressed using FSE
	2043	compression. It is a single bitstream with two interleaved states,
	2044	sharing a single distribution table.
	2045
	2046	To decode an FSE bitstream, it is necessary to know its compressed
	2047	size. Compressed size is provided by headerByte. It's also
	2048	necessary to know its maximum possible decompressed size, which is
	2049	255, since literal values span from 0 to 255, and the last symbol's
	2050	Weight is not represented.
	2051
	2052	An FSE bitstream starts by a header, describing probabilities
	2053	distribution. It will create a decoding table. For a list of
	2054	Huffman weights, the maximum accuracy log is 6 bits. For more
	2055	details, see Section 4.1.1.
	2056
	2057	The Huffman header compression uses two states, which share the same
	2058	FSE distribution table. The first state (State1) encodes the even-
	2059	numbered index symbols, and the second (State2) encodes the odd-
	2060	numbered index symbols. State1 is initialized first, and then
	2061	State2, and they take turns decoding a single symbol and updating
	2062	their state. For more details on these FSE operations, see
	2063	Section 4.1.
	2064
	2065	The number of symbols to be decoded is determined by tracking the
	2066	bitStream overflow condition: If updating state after decoding a
	2067	symbol would require more bits than remain in the stream, it is
	2068	assumed that extra bits are zero. Then, symbols for each of the
	2069	final states are decoded and the process is complete.
	2070
	2071
	2072
	2073
	2074	Collet & Kucherawy Informational [Page 37]
	2075
	2076	RFC 8478 application/zstd October 2018
	2077
	2078
	2079	4.2.1.3. Conversion from Weights to Huffman Prefix Codes
	2080
	2081	All present symbols will now have a Weight value. It is possible to
	2082	transform weights into Number_of_Bits, using this formula:
	2083
	2084	if Weight > 0
	2085	Number_of_Bits = Max_Number_of_Bits + 1 - Weight
	2086	else
	2087	Number_of_Bits = 0
	2088
	2089	Symbols are sorted by Weight. Within the same Weight, symbols keep
	2090	natural sequential order. Symbols with a Weight of zero are removed.
	2091	Then, starting from the lowest Weight, prefix codes are distributed
	2092	in sequential order.
	2093
	2094	For example, assume the following list of weights has been decoded:
	2095
	2096	+---------+--------+
	2097	\| Literal \| Weight \|
	2098	+---------+--------+
	2099	\| 0 \| 4 \|
	2100	+---------+--------+
	2101	\| 1 \| 3 \|
	2102	+---------+--------+
	2103	\| 2 \| 2 \|
	2104	+---------+--------+
	2105	\| 3 \| 0 \|
	2106	+---------+--------+
	2107	\| 4 \| 1 \|
	2108	+---------+--------+
	2109	\| 5 \| 1 \|
	2110	+---------+--------+
	2111
	2112
	2113
	2114
	2115
	2116
	2117
	2118
	2119
	2120
	2121
	2122
	2123
	2124
	2125
	2126
	2127
	2128
	2129
	2130	Collet & Kucherawy Informational [Page 38]
	2131
	2132	RFC 8478 application/zstd October 2018
	2133
	2134
	2135	Sorting by weight and then the natural sequential order yields the
	2136	following distribution:
	2137
	2138	+---------+--------+----------------+--------------+
	2139	\| Literal \| Weight \| Number_Of_Bits \| Prefix Codes \|
	2140	+---------+--------+----------------\|--------------+
	2141	\| 3 \| 0 \| 0 \| N/A \|
	2142	+---------+--------+----------------\|--------------+
	2143	\| 4 \| 1 \| 4 \| 0000 \|
	2144	+---------+--------+----------------\|--------------+
	2145	\| 5 \| 1 \| 4 \| 0001 \|
	2146	+---------+--------+----------------\|--------------+
	2147	\| 2 \| 2 \| 3 \| 001 \|
	2148	+---------+--------+----------------\|--------------+
	2149	\| 1 \| 3 \| 2 \| 01 \|
	2150	+---------+--------+----------------\|--------------+
	2151	\| 0 \| 4 \| 1 \| 1 \|
	2152	+---------+--------+----------------\|--------------+
	2153
	2154	4.2.2. Huffman-Coded Streams
	2155
	2156	Given a Huffman decoding table, it is possible to decode a Huffman-
	2157	coded stream.
	2158
	2159	Each bitstream must be read backward, which starts from the end and
	2160	goes up to the beginning. Therefore, it is necessary to know the
	2161	size of each bitstream.
	2162
	2163	It is also necessary to know exactly which bit is the last. This is
	2164	detected by a final bit flag: the highest bit of the last byte is a
	2165	final-bit-flag. Consequently, a last byte of 0 is not possible. And
	2166	the final-bit-flag itself is not part of the useful bitstream.
	2167	Hence, the last byte contains between 0 and 7 useful bits.
	2168
	2169	Starting from the end, it is possible to read the bitstream in a
	2170	little-endian fashion, keeping track of already used bits. Since the
	2171	bitstream is encoded in reverse order, starting from the end, read
	2172	symbols in forward order.
	2173
	2174
	2175
	2176
	2177
	2178
	2179
	2180
	2181
	2182
	2183
	2184
	2185
	2186	Collet & Kucherawy Informational [Page 39]
	2187
	2188	RFC 8478 application/zstd October 2018
	2189
	2190
	2191	For example, if the literal sequence "0145" was encoded using the
	2192	above prefix code, it would be encoded (in reverse order) as:
	2193
	2194	+---------+----------+
	2195	\| Symbol \| Encoding \|
	2196	+---------+----------+
	2197	\| 5 \| 0000 \|
	2198	+---------+----------+
	2199	\| 4 \| 0001 \|
	2200	+---------+----------+
	2201	\| 1 \| 01 \|
	2202	+---------+----------+
	2203	\| 0 \| 1 \|
	2204	+---------+----------+
	2205	\| Padding \| 00001 \|
	2206	+---------+----------+
	2207
	2208	This results in the following 2-byte bitstream:
	2209
	2210	00010000 00001101
	2211
	2212	Here is an alternative representation with the symbol codes separated
	2213	by underscores:
	2214
	2215	0001_0000 00001_1_01
	2216
	2217	Reading the highest Max_Number_of_Bits bits, it's possible to compare
	2218	the extracted value to the decoding table, determining the symbol to
	2219	decode and number of bits to discard.
	2220
	2221	The process continues reading up to the required number of symbols
	2222	per stream. If a bitstream is not entirely and exactly consumed,
	2223	hence reaching exactly its beginning position with all bits consumed,
	2224	the decoding process is considered faulty.
	2225
	2226	5. Dictionary Format
	2227
	2228	Zstandard is compatible with "raw content" dictionaries, free of any
	2229	format restriction, except that they must be at least 8 bytes. These
	2230	dictionaries function as if they were just the content part of a
	2231	formatted dictionary.
	2232
	2233	However, dictionaries created by "zstd --train" in the reference
	2234	implementation follow a specific format, described here.
	2235
	2236	Dictionaries are not included in the compressed content but rather
	2237	are provided out of band. That is, the Dictionary_ID identifies
	2238	which should be used, but this specification does not describe the
	2239
	2240
	2241
	2242	Collet & Kucherawy Informational [Page 40]
	2243
	2244	RFC 8478 application/zstd October 2018
	2245
	2246
	2247	mechanism by which the dictionary is obtained prior to use during
	2248	compression or decompression.
	2249
	2250	A dictionary has a size, defined either by a buffer limit or a file
	2251	size. The general format is:
	2252
	2253	+--------------+---------------+----------------+---------+
	2254	\| Magic_Number \| Dictionary_ID \| Entropy_Tables \| Content \|
	2255	+--------------+---------------+----------------+---------+
	2256
	2257	Magic_Number: 4 bytes ID, value 0xEC30A437, little-endian format.
	2258
	2259	Dictionary_ID: 4 bytes, stored in little-endian format.
	2260	Dictionary_ID can be any value, except 0 (which means no
	2261	Dictionary_ID). It is used by decoders to check if they use the
	2262	correct dictionary. If the frame is going to be distributed in a
	2263	private environment, any Dictionary_ID can be used. However, for
	2264	public distribution of compressed frames, the following ranges are
	2265	reserved and shall not be used:
	2266
	2267	low range: <= 32767
	2268	high range: >= (2^31)
	2269
	2270	Entropy_Tables: Follow the same format as the tables in compressed
	2271	blocks. See the relevant FSE and Huffman sections for how to
	2272	decode these tables. They are stored in the following order:
	2273	Huffman table for literals, FSE table for offsets, FSE table for
	2274	match lengths, and FSE table for literals lengths. These tables
	2275	populate the Repeat Stats literals mode and Repeat distribution
	2276	mode for sequence decoding. It is finally followed by 3 offset
	2277	values, populating repeat offsets (instead of using {1,4,8}),
	2278	stored in order, 4-bytes little-endian each, for a total of 12
	2279	bytes. Each repeat offset must have a value less than the
	2280	dictionary size.
	2281
	2282	Content: The rest of the dictionary is its content. The content
	2283	acts as a "past" in front of data to be compressed or
	2284	decompressed, so it can be referenced in sequence commands. As
	2285	long as the amount of data decoded from this frame is less than or
	2286	equal to Window_Size, sequence commands may specify offsets longer
	2287	than the total length of decoded output so far to reference back
	2288	to the dictionary, even parts of the dictionary with offsets
	2289	larger than Window_Size. After the total output has surpassed
	2290	Window_Size, however, this is no longer allowed, and the
	2291	dictionary is no longer accessible.
	2292
	2293
	2294
	2295
	2296
	2297
	2298	Collet & Kucherawy Informational [Page 41]
	2299
	2300	RFC 8478 application/zstd October 2018
	2301
	2302
	2303	6. IANA Considerations
	2304
	2305	IANA has made two registrations, as described below.
	2306
	2307	6.1. The 'application/zstd' Media Type
	2308
	2309	The 'application/zstd' media type identifies a block of data that is
	2310	compressed using zstd compression. The data is a stream of bytes as
	2311	described in this document. IANA has added the following to the
	2312	"Media Types" registry:
	2313
	2314	Type name: application
	2315
	2316	Subtype name: zstd
	2317
	2318	Required parameters: N/A
	2319
	2320	Optional parameters: N/A
	2321
	2322	Encoding considerations: binary
	2323
	2324	Security considerations: See Section 7 of RFC 8478
	2325
	2326	Interoperability considerations: N/A
	2327
	2328	Published specification: RFC 8478
	2329
	2330	Applications that use this media type: anywhere data size is an
	2331	issue
	2332
	2333	Additional information:
	2334
	2335	Magic number(s): 4 bytes, little-endian format.
	2336	Value: 0xFD2FB528
	2337
	2338	File extension(s): zst
	2339
	2340	Macintosh file type code(s): N/A
	2341
	2342	For further information: See [ZSTD]
	2343
	2344	Intended usage: common
	2345
	2346	Restrictions on usage: N/A
	2347
	2348	Author: Murray S. Kucherawy
	2349
	2350	Change Controller: IETF
	2351
	2352
	2353
	2354	Collet & Kucherawy Informational [Page 42]
	2355
	2356	RFC 8478 application/zstd October 2018
	2357
	2358
	2359	Provisional registration: no
	2360
	2361	6.2. Content Encoding
	2362
	2363	IANA has added the following entry to the "HTTP Content Coding
	2364	Registry" within the "Hypertext Transfer Protocol (HTTP) Parameters"
	2365	registry:
	2366
	2367	Name: zstd
	2368
	2369	Description: A stream of bytes compressed using the Zstandard
	2370	protocol
	2371
	2372	Pointer to specification text: RFC 8478
	2373
	2374	6.3. Dictionaries
	2375
	2376	Work in progress includes development of dictionaries that will
	2377	optimize compression and decompression of particular types of data.
	2378	Specification of such dictionaries for public use will necessitate
	2379	registration of a code point from the reserved range described in
	2380	Section 3.1.1.1.3 and its association with a specific dictionary.
	2381
	2382	However, there are at present no such dictionaries published for
	2383	public use, so this document makes no immediate request of IANA to
	2384	create such a registry.
	2385
	2386	7. Security Considerations
	2387
	2388	Any data compression method involves the reduction of redundancy in
	2389	the data. Zstandard is no exception, and the usual precautions
	2390	apply.
	2391
	2392	One should never compress a message whose content must remain secret
	2393	with a message generated by a third party. Such a compression can be
	2394	used to guess the content of the secret message through analysis of
	2395	entropy reduction. This was demonstrated in the Compression Ratio
	2396	Info-leak Made Easy (CRIME) attack [CRIME], for example.
	2397
	2398	A decoder has to demonstrate capabilities to detect and prevent any
	2399	kind of data tampering in the compressed frame from triggering system
	2400	faults, such as reading or writing beyond allowed memory ranges.
	2401	This can be guaranteed by either the implementation language or
	2402	careful bound checkings. Of particular note is the encoding of
	2403	Number_of_Sequences values that cause the decoder to read into the
	2404	block header (and beyond), as well as the indication of a
	2405	Frame_Content_Size that is smaller than the actual decompressed data,
	2406	in an attempt to trigger a buffer overflow. It is highly recommended
	2407
	2408
	2409
	2410	Collet & Kucherawy Informational [Page 43]
	2411
	2412	RFC 8478 application/zstd October 2018
	2413
	2414
	2415	to fuzz-test (i.e., provide invalid, unexpected, or random input and
	2416	verify safe operation of) decoder implementations to test and harden
	2417	their capability to detect bad frames and deal with them without any
	2418	adverse system side effect.
	2419
	2420	An attacker may provide correctly formed compressed frames with
	2421	unreasonable memory requirements. A decoder must always control
	2422	memory requirements and enforce some (system-specific) limits in
	2423	order to protect memory usage from such scenarios.
	2424
	2425	Compression can be optimized by training a dictionary on a variety of
	2426	related content payloads. This dictionary must then be available at
	2427	the decoder for decompression of the payload to be possible. While
	2428	this document does not specify how to acquire a dictionary for a
	2429	given compressed payload, it is worth noting that third-party
	2430	dictionaries may interact unexpectedly with a decoder, leading to
	2431	possible memory or other resource exhaustion attacks. We expect such
	2432	topics to be discussed in further detail in the Security
	2433	Considerations section of a forthcoming RFC for dictionary
	2434	acquisition and transmission, but highlight this issue now out of an
	2435	abundance of caution.
	2436
	2437	As discussed in Section 3.1.2, it is possible to store arbitrary user
	2438	metadata in skippable frames. While such frames are ignored during
	2439	decompression of the data, they can be used as a watermark to track
	2440	the path of the compressed payload.
	2441
	2442	8. Implementation Status
	2443
	2444	Source code for a C language implementation of a Zstandard-compliant
	2445	library is available at [ZSTD-GITHUB]. This implementation is
	2446	considered to be the reference implementation and is production
	2447	ready; it implements the full range of the specification. It is
	2448	routinely tested against security hazards and widely deployed within
	2449	Facebook infrastructure.
	2450
	2451	The reference version is optimized for speed and is highly portable.
	2452	It has been proven to run safely on multiple architectures (e.g.,
	2453	x86, x64, ARM, MIPS, PowerPC, IA64) featuring 32- or 64-bit
	2454	addressing schemes, a little- or big-endian storage scheme, a number
	2455	of different operating systems (e.g., UNIX (including Linux, BSD,
	2456	OS-X, and Solaris) and Windows), and a number of compilers (e.g.,
	2457	gcc, clang, visual, and icc).
	2458
	2459
	2460
	2461
	2462
	2463
	2464
	2465
	2466	Collet & Kucherawy Informational [Page 44]
	2467
	2468	RFC 8478 application/zstd October 2018
	2469
	2470
	2471	9. References
	2472
	2473	9.1. Normative References
	2474
	2475	[ZSTD] "Zstandard", <http://www.zstd.net>.
	2476
	2477	9.2. Informative References
	2478
	2479	[ANS] Duda, J., "Asymmetric numeral systems: entropy coding
	2480	combining speed of Huffman coding with compression rate of
	2481	arithmetic coding", January 2014,
	2482	<https://arxiv.org/pdf/1311.2540>.
	2483
	2484	[CRIME] "CRIME", June 2018, <https://en.wikipedia.org/w/
	2485	index.php?title=CRIME&oldid=844538656>.
	2486
	2487	[FSE] "FiniteStateEntropy", commit 6efa78a, June 2018,
	2488	<https://github.com/Cyan4973/FiniteStateEntropy/>.
	2489
	2490	[LZ4] "LZ4 Frame Format Description", commit d03224b, January
	2491	2018, <https://github.com/lz4/lz4/blob/master/doc/
	2492	lz4_Frame_format.md>.
	2493
	2494	[RFC1952] Deutsch, P., "GZIP file format specification version 4.3",
	2495	RFC 1952, DOI 10.17487/RFC1952, May 1996,
	2496	<https://www.rfc-editor.org/info/rfc1952>.
	2497
	2498	[XXHASH] "XXHASH Algorithm", <http://www.xxhash.org>.
	2499
	2500	[ZSTD-GITHUB]
	2501	"zstd", commit 8514bd8, August 2018,
	2502	<https://github.com/facebook/zstd>.
	2503
	2504
	2505
	2506
	2507
	2508
	2509
	2510
	2511
	2512
	2513
	2514
	2515
	2516
	2517
	2518
	2519
	2520
	2521
	2522	Collet & Kucherawy Informational [Page 45]
	2523
	2524	RFC 8478 application/zstd October 2018
	2525
	2526
	2527	Appendix A. Decoding Tables for Predefined Codes
	2528
	2529	This appendix contains FSE decoding tables for the predefined literal
	2530	length, match length, and offset codes. The tables have been
	2531	constructed using the algorithm as given above in Section 4.1.1. The
	2532	tables here can be used as examples to crosscheck that an
	2533	implementation has built its decoding tables correctly.
	2534
	2535	A.1. Literal Length Code Table
	2536
	2537	+-------+--------+----------------+------+
	2538	\| State \| Symbol \| Number_Of_Bits \| Base \|
	2539	+-------+--------+----------------+------+
	2540	\| 0 \| 0 \| 0 \| 0 \|
	2541	+-------+--------+----------------+------+
	2542	\| 0 \| 0 \| 4 \| 0 \|
	2543	+-------+--------+----------------+------+
	2544	\| 1 \| 0 \| 4 \| 16 \|
	2545	+-------+--------+----------------+------+
	2546	\| 2 \| 1 \| 5 \| 32 \|
	2547	+-------+--------+----------------+------+
	2548	\| 3 \| 3 \| 5 \| 0 \|
	2549	+-------+--------+----------------+------+
	2550	\| 4 \| 4 \| 5 \| 0 \|
	2551	+-------+--------+----------------+------+
	2552	\| 5 \| 6 \| 5 \| 0 \|
	2553	+-------+--------+----------------+------+
	2554	\| 6 \| 7 \| 5 \| 0 \|
	2555	+-------+--------+----------------+------+
	2556	\| 7 \| 9 \| 5 \| 0 \|
	2557	+-------+--------+----------------+------+
	2558	\| 8 \| 10 \| 5 \| 0 \|
	2559	+-------+--------+----------------+------+
	2560	\| 9 \| 12 \| 5 \| 0 \|
	2561	+-------+--------+----------------+------+
	2562	\| 10 \| 14 \| 6 \| 0 \|
	2563	+-------+--------+----------------+------+
	2564	\| 11 \| 16 \| 5 \| 0 \|
	2565	+-------+--------+----------------+------+
	2566	\| 12 \| 18 \| 5 \| 0 \|
	2567	+-------+--------+----------------+------+
	2568	\| 13 \| 19 \| 5 \| 0 \|
	2569	+-------+--------+----------------+------+
	2570	\| 14 \| 21 \| 5 \| 0 \|
	2571	+-------+--------+----------------+------+
	2572	\| 15 \| 22 \| 5 \| 0 \|
	2573	+-------+--------+----------------+------+
	2574	\| 16 \| 24 \| 5 \| 0 \|
	2575
	2576
	2577
	2578	Collet & Kucherawy Informational [Page 46]
	2579
	2580	RFC 8478 application/zstd October 2018
	2581
	2582
	2583	+-------+--------+----------------+------+
	2584	\| 17 \| 25 \| 5 \| 32 \|
	2585	+-------+--------+----------------+------+
	2586	\| 18 \| 26 \| 5 \| 0 \|
	2587	+-------+--------+----------------+------+
	2588	\| 19 \| 27 \| 6 \| 0 \|
	2589	+-------+--------+----------------+------+
	2590	\| 20 \| 29 \| 6 \| 0 \|
	2591	+-------+--------+----------------+------+
	2592	\| 21 \| 31 \| 6 \| 0 \|
	2593	+-------+--------+----------------+------+
	2594	\| 22 \| 0 \| 4 \| 32 \|
	2595	+-------+--------+----------------+------+
	2596	\| 23 \| 1 \| 4 \| 0 \|
	2597	+-------+--------+----------------+------+
	2598	\| 24 \| 2 \| 5 \| 0 \|
	2599	+-------+--------+----------------+------+
	2600	\| 25 \| 4 \| 5 \| 32 \|
	2601	+-------+--------+----------------+------+
	2602	\| 26 \| 5 \| 5 \| 0 \|
	2603	+-------+--------+----------------+------+
	2604	\| 27 \| 7 \| 5 \| 32 \|
	2605	+-------+--------+----------------+------+
	2606	\| 28 \| 8 \| 5 \| 0 \|
	2607	+-------+--------+----------------+------+
	2608	\| 29 \| 10 \| 5 \| 32 \|
	2609	+-------+--------+----------------+------+
	2610	\| 30 \| 11 \| 5 \| 0 \|
	2611	+-------+--------+----------------+------+
	2612	\| 31 \| 13 \| 6 \| 0 \|
	2613	+-------+--------+----------------+------+
	2614	\| 32 \| 16 \| 5 \| 32 \|
	2615	+-------+--------+----------------+------+
	2616	\| 33 \| 17 \| 5 \| 0 \|
	2617	+-------+--------+----------------+------+
	2618	\| 34 \| 19 \| 5 \| 32 \|
	2619	+-------+--------+----------------+------+
	2620	\| 35 \| 20 \| 5 \| 0 \|
	2621	+-------+--------+----------------+------+
	2622	\| 36 \| 22 \| 5 \| 32 \|
	2623	+-------+--------+----------------+------+
	2624	\| 37 \| 23 \| 5 \| 0 \|
	2625	+-------+--------+----------------+------+
	2626	\| 38 \| 25 \| 4 \| 0 \|
	2627	+-------+--------+----------------+------+
	2628	\| 39 \| 25 \| 4 \| 16 \|
	2629	+-------+--------+----------------+------+
	2630	\| 40 \| 26 \| 5 \| 32 \|
	2631
	2632
	2633
	2634	Collet & Kucherawy Informational [Page 47]
	2635
	2636	RFC 8478 application/zstd October 2018
	2637
	2638
	2639	+-------+--------+----------------+------+
	2640	\| 41 \| 28 \| 6 \| 0 \|
	2641	+-------+--------+----------------+------+
	2642	\| 42 \| 30 \| 6 \| 0 \|
	2643	+-------+--------+----------------+------+
	2644	\| 43 \| 0 \| 4 \| 48 \|
	2645	+-------+--------+----------------+------+
	2646	\| 44 \| 1 \| 4 \| 16 \|
	2647	+-------+--------+----------------+------+
	2648	\| 45 \| 2 \| 5 \| 32 \|
	2649	+-------+--------+----------------+------+
	2650	\| 46 \| 3 \| 5 \| 32 \|
	2651	+-------+--------+----------------+------+
	2652	\| 47 \| 5 \| 5 \| 32 \|
	2653	+-------+--------+----------------+------+
	2654	\| 48 \| 6 \| 5 \| 32 \|
	2655	+-------+--------+----------------+------+
	2656	\| 49 \| 8 \| 5 \| 32 \|
	2657	+-------+--------+----------------+------+
	2658	\| 50 \| 9 \| 5 \| 32 \|
	2659	+-------+--------+----------------+------+
	2660	\| 51 \| 11 \| 5 \| 32 \|
	2661	+-------+--------+----------------+------+
	2662	\| 52 \| 12 \| 5 \| 32 \|
	2663	+-------+--------+----------------+------+
	2664	\| 53 \| 15 \| 6 \| 0 \|
	2665	+-------+--------+----------------+------+
	2666	\| 54 \| 17 \| 5 \| 32 \|
	2667	+-------+--------+----------------+------+
	2668	\| 55 \| 18 \| 5 \| 32 \|
	2669	+-------+--------+----------------+------+
	2670	\| 56 \| 20 \| 5 \| 32 \|
	2671	+-------+--------+----------------+------+
	2672	\| 57 \| 21 \| 5 \| 32 \|
	2673	+-------+--------+----------------+------+
	2674	\| 58 \| 23 \| 5 \| 32 \|
	2675	+-------+--------+----------------+------+
	2676	\| 59 \| 24 \| 5 \| 32 \|
	2677	+-------+--------+----------------+------+
	2678	\| 60 \| 35 \| 6 \| 0 \|
	2679	+-------+--------+----------------+------+
	2680	\| 61 \| 34 \| 6 \| 0 \|
	2681	+-------+--------+----------------+------+
	2682	\| 62 \| 33 \| 6 \| 0 \|
	2683	+-------+--------+----------------+------+
	2684	\| 63 \| 32 \| 6 \| 0 \|
	2685	+-------+--------+----------------+------+
	2686
	2687
	2688
	2689
	2690	Collet & Kucherawy Informational [Page 48]
	2691
	2692	RFC 8478 application/zstd October 2018
	2693
	2694
	2695	A.2. Match Length Code Table
	2696
	2697	+-------+--------+----------------+------+
	2698	\| State \| Symbol \| Number_Of_Bits \| Base \|
	2699	+-------+--------+----------------+------+
	2700	\| 0 \| 0 \| 0 \| 0 \|
	2701	+-------+--------+----------------+------+
	2702	\| 0 \| 0 \| 6 \| 0 \|
	2703	+-------+--------+----------------+------+
	2704	\| 1 \| 1 \| 4 \| 0 \|
	2705	+-------+--------+----------------+------+
	2706	\| 2 \| 2 \| 5 \| 32 \|
	2707	+-------+--------+----------------+------+
	2708	\| 3 \| 3 \| 5 \| 0 \|
	2709	+-------+--------+----------------+------+
	2710	\| 4 \| 5 \| 5 \| 0 \|
	2711	+-------+--------+----------------+------+
	2712	\| 5 \| 6 \| 5 \| 0 \|
	2713	+-------+--------+----------------+------+
	2714	\| 6 \| 8 \| 5 \| 0 \|
	2715	+-------+--------+----------------+------+
	2716	\| 7 \| 10 \| 6 \| 0 \|
	2717	+-------+--------+----------------+------+
	2718	\| 8 \| 13 \| 6 \| 0 \|
	2719	+-------+--------+----------------+------+
	2720	\| 9 \| 16 \| 6 \| 0 \|
	2721	+-------+--------+----------------+------+
	2722	\| 10 \| 19 \| 6 \| 0 \|
	2723	+-------+--------+----------------+------+
	2724	\| 11 \| 22 \| 6 \| 0 \|
	2725	+-------+--------+----------------+------+
	2726	\| 12 \| 25 \| 6 \| 0 \|
	2727	+-------+--------+----------------+------+
	2728	\| 13 \| 28 \| 6 \| 0 \|
	2729	+-------+--------+----------------+------+
	2730	\| 14 \| 31 \| 6 \| 0 \|
	2731	+-------+--------+----------------+------+
	2732	\| 15 \| 33 \| 6 \| 0 \|
	2733	+-------+--------+----------------+------+
	2734	\| 16 \| 35 \| 6 \| 0 \|
	2735	+-------+--------+----------------+------+
	2736	\| 17 \| 37 \| 6 \| 0 \|
	2737	+-------+--------+----------------+------+
	2738	\| 18 \| 39 \| 6 \| 0 \|
	2739	+-------+--------+----------------+------+
	2740	\| 19 \| 41 \| 6 \| 0 \|
	2741	+-------+--------+----------------+------+
	2742	\| 20 \| 43 \| 6 \| 0 \|
	2743
	2744
	2745
	2746	Collet & Kucherawy Informational [Page 49]
	2747
	2748	RFC 8478 application/zstd October 2018
	2749
	2750
	2751	+-------+--------+----------------+------+
	2752	\| 21 \| 45 \| 6 \| 0 \|
	2753	+-------+--------+----------------+------+
	2754	\| 22 \| 1 \| 4 \| 16 \|
	2755	+-------+--------+----------------+------+
	2756	\| 23 \| 2 \| 4 \| 0 \|
	2757	+-------+--------+----------------+------+
	2758	\| 24 \| 3 \| 5 \| 32 \|
	2759	+-------+--------+----------------+------+
	2760	\| 25 \| 4 \| 5 \| 0 \|
	2761	+-------+--------+----------------+------+
	2762	\| 26 \| 6 \| 5 \| 32 \|
	2763	+-------+--------+----------------+------+
	2764	\| 27 \| 7 \| 5 \| 0 \|
	2765	+-------+--------+----------------+------+
	2766	\| 28 \| 9 \| 6 \| 0 \|
	2767	+-------+--------+----------------+------+
	2768	\| 29 \| 12 \| 6 \| 0 \|
	2769	+-------+--------+----------------+------+
	2770	\| 30 \| 15 \| 6 \| 0 \|
	2771	+-------+--------+----------------+------+
	2772	\| 31 \| 18 \| 6 \| 0 \|
	2773	+-------+--------+----------------+------+
	2774	\| 32 \| 21 \| 6 \| 0 \|
	2775	+-------+--------+----------------+------+
	2776	\| 33 \| 24 \| 6 \| 0 \|
	2777	+-------+--------+----------------+------+
	2778	\| 34 \| 27 \| 6 \| 0 \|
	2779	+-------+--------+----------------+------+
	2780	\| 35 \| 30 \| 6 \| 0 \|
	2781	+-------+--------+----------------+------+
	2782	\| 36 \| 32 \| 6 \| 0 \|
	2783	+-------+--------+----------------+------+
	2784	\| 37 \| 34 \| 6 \| 0 \|
	2785	+-------+--------+----------------+------+
	2786	\| 38 \| 36 \| 6 \| 0 \|
	2787	+-------+--------+----------------+------+
	2788	\| 39 \| 38 \| 6 \| 0 \|
	2789	+-------+--------+----------------+------+
	2790	\| 40 \| 40 \| 6 \| 0 \|
	2791	+-------+--------+----------------+------+
	2792	\| 41 \| 42 \| 6 \| 0 \|
	2793	+-------+--------+----------------+------+
	2794	\| 42 \| 44 \| 6 \| 0 \|
	2795	+-------+--------+----------------+------+
	2796	\| 43 \| 1 \| 4 \| 32 \|
	2797	+-------+--------+----------------+------+
	2798	\| 44 \| 1 \| 4 \| 48 \|
	2799
	2800
	2801
	2802	Collet & Kucherawy Informational [Page 50]
	2803
	2804	RFC 8478 application/zstd October 2018
	2805
	2806
	2807	+-------+--------+----------------+------+
	2808	\| 45 \| 2 \| 4 \| 16 \|
	2809	+-------+--------+----------------+------+
	2810	\| 46 \| 4 \| 5 \| 32 \|
	2811	+-------+--------+----------------+------+
	2812	\| 47 \| 5 \| 5 \| 32 \|
	2813	+-------+--------+----------------+------+
	2814	\| 48 \| 7 \| 5 \| 32 \|
	2815	+-------+--------+----------------+------+
	2816	\| 49 \| 8 \| 5 \| 32 \|
	2817	+-------+--------+----------------+------+
	2818	\| 50 \| 11 \| 6 \| 0 \|
	2819	+-------+--------+----------------+------+
	2820	\| 51 \| 14 \| 6 \| 0 \|
	2821	+-------+--------+----------------+------+
	2822	\| 52 \| 17 \| 6 \| 0 \|
	2823	+-------+--------+----------------+------+
	2824	\| 53 \| 20 \| 6 \| 0 \|
	2825	+-------+--------+----------------+------+
	2826	\| 54 \| 23 \| 6 \| 0 \|
	2827	+-------+--------+----------------+------+
	2828	\| 55 \| 26 \| 6 \| 0 \|
	2829	+-------+--------+----------------+------+
	2830	\| 56 \| 29 \| 6 \| 0 \|
	2831	+-------+--------+----------------+------+
	2832	\| 57 \| 52 \| 6 \| 0 \|
	2833	+-------+--------+----------------+------+
	2834	\| 58 \| 51 \| 6 \| 0 \|
	2835	+-------+--------+----------------+------+
	2836	\| 59 \| 50 \| 6 \| 0 \|
	2837	+-------+--------+----------------+------+
	2838	\| 60 \| 49 \| 6 \| 0 \|
	2839	+-------+--------+----------------+------+
	2840	\| 61 \| 48 \| 6 \| 0 \|
	2841	+-------+--------+----------------+------+
	2842	\| 62 \| 47 \| 6 \| 0 \|
	2843	+-------+--------+----------------+------+
	2844	\| 63 \| 46 \| 6 \| 0 \|
	2845	+-------+--------+----------------+------+
	2846
	2847
	2848
	2849
	2850
	2851
	2852
	2853
	2854
	2855
	2856
	2857
	2858	Collet & Kucherawy Informational [Page 51]
	2859
	2860	RFC 8478 application/zstd October 2018
	2861
	2862
	2863	A.3. Offset Code Table
	2864
	2865	+-------+--------+----------------+------+
	2866	\| State \| Symbol \| Number_Of_Bits \| Base \|
	2867	+-------+--------+----------------+------+
	2868	\| 0 \| 0 \| 0 \| 0 \|
	2869	+-------+--------+----------------+------+
	2870	\| 0 \| 0 \| 5 \| 0 \|
	2871	+-------+--------+----------------+------+
	2872	\| 1 \| 6 \| 4 \| 0 \|
	2873	+-------+--------+----------------+------+
	2874	\| 2 \| 9 \| 5 \| 0 \|
	2875	+-------+--------+----------------+------+
	2876	\| 3 \| 15 \| 5 \| 0 \|
	2877	+-------+--------+----------------+------+
	2878	\| 4 \| 21 \| 5 \| 0 \|
	2879	+-------+--------+----------------+------+
	2880	\| 5 \| 3 \| 5 \| 0 \|
	2881	+-------+--------+----------------+------+
	2882	\| 6 \| 7 \| 4 \| 0 \|
	2883	+-------+--------+----------------+------+
	2884	\| 7 \| 12 \| 5 \| 0 \|
	2885	+-------+--------+----------------+------+
	2886	\| 8 \| 18 \| 5 \| 0 \|
	2887	+-------+--------+----------------+------+
	2888	\| 9 \| 23 \| 5 \| 0 \|
	2889	+-------+--------+----------------+------+
	2890	\| 10 \| 5 \| 5 \| 0 \|
	2891	+-------+--------+----------------+------+
	2892	\| 11 \| 8 \| 4 \| 0 \|
	2893	+-------+--------+----------------+------+
	2894	\| 12 \| 14 \| 5 \| 0 \|
	2895	+-------+--------+----------------+------+
	2896	\| 13 \| 20 \| 5 \| 0 \|
	2897	+-------+--------+----------------+------+
	2898	\| 14 \| 2 \| 5 \| 0 \|
	2899	+-------+--------+----------------+------+
	2900	\| 15 \| 7 \| 4 \| 16 \|
	2901	+-------+--------+----------------+------+
	2902	\| 16 \| 11 \| 5 \| 0 \|
	2903	+-------+--------+----------------+------+
	2904	\| 17 \| 17 \| 5 \| 0 \|
	2905	+-------+--------+----------------+------+
	2906	\| 18 \| 22 \| 5 \| 0 \|
	2907	+-------+--------+----------------+------+
	2908	\| 19 \| 4 \| 5 \| 0 \|
	2909	+-------+--------+----------------+------+
	2910	\| 20 \| 8 \| 4 \| 16 \|
	2911
	2912
	2913
	2914	Collet & Kucherawy Informational [Page 52]
	2915
	2916	RFC 8478 application/zstd October 2018
	2917
	2918
	2919	+-------+--------+----------------+------+
	2920	\| 21 \| 13 \| 5 \| 0 \|
	2921	+-------+--------+----------------+------+
	2922	\| 22 \| 19 \| 5 \| 0 \|
	2923	+-------+--------+----------------+------+
	2924	\| 23 \| 1 \| 5 \| 0 \|
	2925	+-------+--------+----------------+------+
	2926	\| 24 \| 6 \| 4 \| 16 \|
	2927	+-------+--------+----------------+------+
	2928	\| 25 \| 10 \| 5 \| 0 \|
	2929	+-------+--------+----------------+------+
	2930	\| 26 \| 16 \| 5 \| 0 \|
	2931	+-------+--------+----------------+------+
	2932	\| 27 \| 28 \| 5 \| 0 \|
	2933	+-------+--------+----------------+------+
	2934	\| 28 \| 27 \| 5 \| 0 \|
	2935	+-------+--------+----------------+------+
	2936	\| 29 \| 26 \| 5 \| 0 \|
	2937	+-------+--------+----------------+------+
	2938	\| 30 \| 25 \| 5 \| 0 \|
	2939	+-------+--------+----------------+------+
	2940	\| 31 \| 24 \| 5 \| 0 \|
	2941	+-------+--------+----------------+------+
	2942
	2943	Acknowledgments
	2944
	2945	zstd was developed by Yann Collet.
	2946
	2947	Bobo Bose-Kolanu, Felix Handte, Kyle Nekritz, Nick Terrell, and David
	2948	Schleimer provided helpful feedback during the development of this
	2949	document.
	2950
	2951
	2952
	2953
	2954
	2955
	2956
	2957
	2958
	2959
	2960
	2961
	2962
	2963
	2964
	2965
	2966
	2967
	2968
	2969
	2970	Collet & Kucherawy Informational [Page 53]
	2971
	2972	RFC 8478 application/zstd October 2018
	2973
	2974
	2975	Authors' Addresses
	2976
	2977	Yann Collet
	2978	Facebook
	2979	1 Hacker Way
	2980	Menlo Park, CA 94025
	2981	United States of America
	2982
	2983	Email: cyan@fb.com
	2984
	2985
	2986	Murray S. Kucherawy (editor)
	2987	Facebook
	2988	1 Hacker Way
	2989	Menlo Park, CA 94025
	2990	United States of America
	2991
	2992	Email: msk@fb.com
	2993
	2994
	2995
	2996
	2997
	2998
	2999
	3000
	3001
	3002
	3003
	3004
	3005
	3006
	3007
	3008
	3009
	3010
	3011
	3012
	3013
	3014
	3015
	3016
	3017
	3018
	3019
	3020
	3021
	3022
	3023
	3024
	3025
	3026	Collet & Kucherawy Informational [Page 54]
	3027

lib/std/compress/testdata/rfc8478.txt.zst.19 created

Binary files /dev/null and b/lib/std/compress/testdata/rfc8478.txt.zst.19 differ

lib/std/compress/testdata/rfc8478.txt.zst.3 created

Binary files /dev/null and b/lib/std/compress/testdata/rfc8478.txt.zst.3 differ

lib/std/compress/zstandard.zig created+22

...	...	@@ -0,0 +1,22 @@
	1	const std = @import("std");
	2
	3	pub const decompress = @import("zstandard/decompress.zig");
	4
	5	test "decompression" {
	6	const uncompressed = @embedFile("testdata/rfc8478.txt");
	7	const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
	8	const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
	9
	10	var buffer = try std.testing.allocator.alloc(u8, uncompressed.len);
	11	defer std.testing.allocator.free(buffer);
	12
	13	const res3 = try decompress.decodeFrame(buffer, compressed3, true);
	14	try std.testing.expectEqual(compressed3.len, res3.read_count);
	15	try std.testing.expectEqual(uncompressed.len, res3.write_count);
	16	try std.testing.expectEqualSlices(u8, uncompressed, buffer);
	17
	18	const res19 = try decompress.decodeFrame(buffer, compressed19, true);
	19	try std.testing.expectEqual(compressed19.len, res19.read_count);
	20	try std.testing.expectEqual(uncompressed.len, res19.write_count);
	21	try std.testing.expectEqualSlices(u8, uncompressed, buffer);
	22	}

lib/std/compress/zstandard/decompress.zig created+1249

...	...	@@ -0,0 +1,1249 @@
	1	const std = @import("std");
	2	const assert = std.debug.assert;
	3
	4	const types = @import("types.zig");
	5	const frame = types.frame;
	6	const Literals = types.compressed_block.Literals;
	7	const Sequences = types.compressed_block.Sequences;
	8	const Table = types.compressed_block.Table;
	9
	10	const readInt = std.mem.readIntLittle;
	11	const readIntSlice = std.mem.readIntSliceLittle;
	12	fn readVarInt(comptime T: type, bytes: []const u8) T {
	13	return std.mem.readVarInt(T, bytes, .Little);
	14	}
	15
	16	const log = std.log.scoped(.Decompress);
	17
	18	fn isSkippableMagic(magic: u32) bool {
	19	return frame.Skippable.magic_number_min <= magic and magic <= frame.Skippable.magic_number_max;
	20	}
	21
	22	pub fn getFrameDecompressedSize(src: []const u8) !?usize {
	23	switch (try frameType(src)) {
	24	.zstandard => {
	25	const header = try decodeZStandardHeader(src[4..], null);
	26	return header.content_size;
	27	},
	28	.skippable => return 0,
	29	}
	30	}
	31
	32	pub fn frameType(src: []const u8) !frame.Kind {
	33	const magic = readInt(u32, src[0..4]);
	34	return if (magic == frame.ZStandard.magic_number)
	35	.zstandard
	36	else if (isSkippableMagic(magic))
	37	.skippable
	38	else
	39	error.BadMagic;
	40	}
	41
	42	const ReadWriteCount = struct {
	43	read_count: usize,
	44	write_count: usize,
	45	};
	46
	47	pub fn decodeFrame(dest: []u8, src: []const u8, verify_checksum: bool) !ReadWriteCount {
	48	return switch (try frameType(src)) {
	49	.zstandard => decodeZStandardFrame(dest, src, verify_checksum),
	50	.skippable => ReadWriteCount{
	51	.read_count = try skippableFrameSize(src[0..8]) + 8,
	52	.write_count = 0,
	53	},
	54	};
	55	}
	56
	57	const DecodeState = struct {
	58	repeat_offsets: [3]u32,
	59
	60	offset: StateData(8),
	61	match: StateData(9),
	62	literal: StateData(9),
	63
	64	offset_fse_buffer: []Table.Fse,
	65	match_fse_buffer: []Table.Fse,
	66	literal_fse_buffer: []Table.Fse,
	67
	68	literal_written_count: usize,
	69
	70	literal_stream_reader: ReverseBitReader(ReversedByteReader.Reader),
	71	literal_stream_bytes: ReversedByteReader,
	72	literal_stream_index: usize,
	73	huffman_tree: Literals.HuffmanTree,
	74
	75	fn StateData(comptime max_accuracy_log: comptime_int) type {
	76	return struct {
	77	state: State,
	78	table: Table,
	79	accuracy_log: u8,
	80
	81	const State = std.meta.Int(.unsigned, max_accuracy_log);
	82	};
	83	}
	84
	85	fn readInitialState(self: *DecodeState, bit_reader: anytype) !void {
	86	self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
	87	self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
	88	self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
	89	log.debug("initial decoder state: literal = {d}, offset = {d} match = {d}", .{
	90	self.literal.state,
	91	self.offset.state,
	92	self.match.state,
	93	});
	94	}
	95
	96	fn updateRepeatOffset(self: *DecodeState, offset: u32) void {
	97	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1]);
	98	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
	99	self.repeat_offsets[0] = offset;
	100	}
	101
	102	fn useRepeatOffset(self: *DecodeState, index: usize) u32 {
	103	if (index == 1)
	104	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
	105	else if (index == 2) {
	106	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
	107	std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
	108	}
	109	return self.repeat_offsets[0];
	110	}
	111
	112	const DataType = enum { offset, match, literal };
	113
	114	fn updateState(self: *DecodeState, comptime choice: DataType, bit_reader: anytype) !void {
	115	switch (@field(self, @tagName(choice)).table) {
	116	.rle => {},
	117	.fse => \|table\| {
	118	const data = table[@field(self, @tagName(choice)).state];
	119	const T = @TypeOf(@field(self, @tagName(choice))).State;
	120	const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
	121	const next_state = data.baseline + bits_summand;
	122	@field(self, @tagName(choice)).state = @intCast(@TypeOf(@field(self, @tagName(choice))).State, next_state);
	123	},
	124	}
	125	}
	126
	127	fn updateFseTable(
	128	self: *DecodeState,
	129	src: []const u8,
	130	comptime choice: DataType,
	131	mode: Sequences.Header.Mode,
	132	first_compressed_block: bool,
	133	) !usize {
	134	const field_name = @tagName(choice);
	135	switch (mode) {
	136	.predefined => {
	137	@field(self, field_name).accuracy_log = @field(types.compressed_block.default_accuracy_log, field_name);
	138	@field(self, field_name).table = @field(types.compressed_block, "predefined_" ++ field_name ++ "_fse_table");
	139	return 0;
	140	},
	141	.rle => {
	142	@field(self, field_name).accuracy_log = 0;
	143	@field(self, field_name).table = .{ .rle = src[0] };
	144	return 1;
	145	},
	146	.fse => {
	147	var stream = std.io.fixedBufferStream(src);
	148	var counting_reader = std.io.countingReader(stream.reader());
	149	var bit_reader = bitReader(counting_reader.reader());
	150
	151	const table_size = try decodeFseTable(
	152	&bit_reader,
	153	@field(types.compressed_block.table_symbol_count_max, field_name),
	154	@field(types.compressed_block.table_accuracy_log_max, field_name),
	155	@field(self, field_name ++ "_fse_buffer"),
	156	);
	157	@field(self, field_name).table = .{ .fse = @field(self, field_name ++ "_fse_buffer")[0..table_size] };
	158	@field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
	159	log.debug("decoded fse " ++ field_name ++ " table '{}'", .{
	160	std.fmt.fmtSliceHexUpper(src[0..counting_reader.bytes_read]),
	161	});
	162	dumpFseTable(field_name, @field(self, field_name).table.fse);
	163	return counting_reader.bytes_read;
	164	},
	165	.repeat => return if (first_compressed_block) error.RepeatModeFirst else 0,
	166	}
	167	}
	168
	169	const Sequence = struct {
	170	literal_length: u32,
	171	match_length: u32,
	172	offset: u32,
	173	};
	174
	175	fn nextSequence(self: *DecodeState, bit_reader: anytype) !Sequence {
	176	const raw_code = self.getCode(.offset);
	177	const offset_code = std.math.cast(u5, raw_code) orelse {
	178	log.err("got offset code of {d}", .{raw_code});
	179	return error.OffsetCodeTooLarge;
	180	};
	181	const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
	182
	183	const match_code = self.getCode(.match);
	184	const match = types.compressed_block.match_length_code_table[match_code];
	185	const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
	186
	187	const literal_code = self.getCode(.literal);
	188	const literal = types.compressed_block.literals_length_code_table[literal_code];
	189	const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
	190
	191	const offset = if (offset_value > 3) offset: {
	192	const offset = offset_value - 3;
	193	self.updateRepeatOffset(offset);
	194	break :offset offset;
	195	} else offset: {
	196	if (literal_length == 0) {
	197	if (offset_value == 3) {
	198	const offset = self.repeat_offsets[0] - 1;
	199	self.updateRepeatOffset(offset);
	200	break :offset offset;
	201	}
	202	break :offset self.useRepeatOffset(offset_value);
	203	}
	204	break :offset self.useRepeatOffset(offset_value - 1);
	205	};
	206
	207	log.debug("sequence = ({d}, {d}, {d})", .{ literal_length, offset, match_length });
	208	return .{
	209	.literal_length = literal_length,
	210	.match_length = match_length,
	211	.offset = offset,
	212	};
	213	}
	214
	215	fn executeSequenceSlice(self: *DecodeState, dest: []u8, write_pos: usize, literals: Literals, sequence: Sequence) !void {
	216	try self.decodeLiteralsInto(dest[write_pos..], literals, sequence.literal_length);
	217
	218	// TODO: should we validate offset against max_window_size?
	219	assert(sequence.offset <= write_pos + sequence.literal_length);
	220	const copy_start = write_pos + sequence.literal_length - sequence.offset;
	221	const copy_end = copy_start + sequence.match_length;
	222	// NOTE: we ignore the usage message for std.mem.copy and copy with dest.ptr >= src.ptr
	223	// to allow repeats
	224	std.mem.copy(u8, dest[write_pos + sequence.literal_length ..], dest[copy_start..copy_end]);
	225	}
	226
	227	fn decodeSequenceSlice(
	228	self: *DecodeState,
	229	dest: []u8,
	230	write_pos: usize,
	231	literals: Literals,
	232	bit_reader: anytype,
	233	last_sequence: bool,
	234	) !usize {
	235	const sequence = try self.nextSequence(bit_reader);
	236	try self.executeSequenceSlice(dest, write_pos, literals, sequence);
	237	log.debug("sequence decompressed into '{x}'", .{
	238	std.fmt.fmtSliceHexUpper(dest[write_pos .. write_pos + sequence.literal_length + sequence.match_length]),
	239	});
	240	if (!last_sequence) {
	241	try self.updateState(.literal, bit_reader);
	242	try self.updateState(.match, bit_reader);
	243	try self.updateState(.offset, bit_reader);
	244	}
	245	return sequence.match_length + sequence.literal_length;
	246	}
	247
	248	fn nextLiteralMultiStream(self: *DecodeState, literals: Literals) !void {
	249	self.literal_stream_index += 1;
	250	try self.initLiteralStream(literals.streams.four[self.literal_stream_index]);
	251	}
	252
	253	fn initLiteralStream(self: *DecodeState, bytes: []const u8) !void {
	254	log.debug("initing literal stream: {}", .{std.fmt.fmtSliceHexUpper(bytes)});
	255	self.literal_stream_bytes = reversedByteReader(bytes);
	256	self.literal_stream_reader = reverseBitReader(self.literal_stream_bytes.reader());
	257	while (0 == try self.literal_stream_reader.readBitsNoEof(u1, 1)) {}
	258	}
	259
	260	fn decodeLiteralsInto(self: *DecodeState, dest: []u8, literals: Literals, len: usize) !void {
	261	if (self.literal_written_count + len > literals.header.regenerated_size) return error.MalformedLiteralsLength;
	262	switch (literals.header.block_type) {
	263	.raw => {
	264	const literal_data = literals.streams.one[self.literal_written_count .. self.literal_written_count + len];
	265	std.mem.copy(u8, dest, literal_data);
	266	self.literal_written_count += len;
	267	},
	268	.rle => {
	269	var i: usize = 0;
	270	while (i < len) : (i += 1) {
	271	dest[i] = literals.streams.one[0];
	272	}
	273	log.debug("rle: {}", .{std.fmt.fmtSliceHexUpper(dest[0..len])});
	274	self.literal_written_count += len;
	275	},
	276	.compressed, .treeless => {
	277	// const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
	278	const huffman_tree = self.huffman_tree;
	279	const max_bit_count = huffman_tree.max_bit_count;
	280	const starting_bit_count = Literals.HuffmanTree.weightToBitCount(
	281	huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
	282	max_bit_count,
	283	);
	284	var bits_read: u4 = 0;
	285	var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
	286	var bit_count_to_read: u4 = starting_bit_count;
	287	var i: usize = 0;
	288	while (i < len) : (i += 1) {
	289	var prefix: u16 = 0;
	290	while (true) {
	291	const new_bits = self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch \|err\|
	292	switch (err) {
	293	error.EndOfStream => if (literals.streams == .four and self.literal_stream_index < 3) bits: {
	294	try self.nextLiteralMultiStream(literals);
	295	break :bits try self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read);
	296	} else {
	297	return error.UnexpectedEndOfLiteralStream;
	298	},
	299	};
	300	prefix <<= bit_count_to_read;
	301	prefix \|= new_bits;
	302	bits_read += bit_count_to_read;
	303	const result = try huffman_tree.query(huffman_tree_index, prefix);
	304
	305	switch (result) {
	306	.symbol => \|sym\| {
	307	dest[i] = sym;
	308	bit_count_to_read = starting_bit_count;
	309	bits_read = 0;
	310	huffman_tree_index = huffman_tree.symbol_count_minus_one;
	311	break;
	312	},
	313	.index => \|index\| {
	314	huffman_tree_index = index;
	315	const bit_count = Literals.HuffmanTree.weightToBitCount(
	316	huffman_tree.nodes[index].weight,
	317	max_bit_count,
	318	);
	319	bit_count_to_read = bit_count - bits_read;
	320	},
	321	}
	322	}
	323	}
	324	self.literal_written_count += len;
	325	},
	326	}
	327	}
	328
	329	fn getCode(self: *DecodeState, comptime choice: DataType) u32 {
	330	return switch (@field(self, @tagName(choice)).table) {
	331	.rle => \|value\| value,
	332	.fse => \|table\| table[@field(self, @tagName(choice)).state].symbol,
	333	};
	334	}
	335	};
	336
	337	const literal_table_size_max = 1 << types.compressed_block.table_accuracy_log_max.literal;
	338	const match_table_size_max = 1 << types.compressed_block.table_accuracy_log_max.match;
	339	const offset_table_size_max = 1 << types.compressed_block.table_accuracy_log_max.match;
	340
	341	pub fn decodeZStandardFrame(dest: []u8, src: []const u8, verify_checksum: bool) !ReadWriteCount {
	342	assert(readInt(u32, src[0..4]) == frame.ZStandard.magic_number);
	343	var consumed_count: usize = 4;
	344
	345	const frame_header = try decodeZStandardHeader(src[consumed_count..], &consumed_count);
	346
	347	if (frame_header.descriptor.dictionary_id_flag != 0) return error.DictionaryIdFlagUnsupported;
	348
	349	const content_size = frame_header.content_size orelse return error.UnknownContentSizeUnsupported;
	350	// const window_size = frameWindowSize(header) orelse return error.WindowSizeUnknown;
	351	if (dest.len < content_size) return error.ContentTooLarge;
	352
	353	const should_compute_checksum = frame_header.descriptor.content_checksum_flag and verify_checksum;
	354	var hash_state = if (should_compute_checksum) std.hash.XxHash64.init(0) else undefined;
	355
	356	// TODO: block_maximum_size should be @min(1 << 17, window_size);
	357	const written_count = try decodeFrameBlocks(
	358	dest,
	359	src[consumed_count..],
	360	&consumed_count,
	361	if (should_compute_checksum) &hash_state else null,
	362	);
	363
	364	if (frame_header.descriptor.content_checksum_flag) {
	365	const checksum = readIntSlice(u32, src[consumed_count .. consumed_count + 4]);
	366	consumed_count += 4;
	367	if (verify_checksum) {
	368	const hash = hash_state.final();
	369	const hash_low_bytes = hash & 0xFFFFFFFF;
	370	if (checksum != hash_low_bytes) {
	371	std.log.err("expected checksum {x}, got {x} (full hash {x})", .{ checksum, hash_low_bytes, hash });
	372	return error.ChecksumFailure;
	373	}
	374	}
	375	}
	376	return ReadWriteCount{ .read_count = consumed_count, .write_count = written_count };
	377	}
	378
	379	pub fn decodeFrameBlocks(dest: []u8, src: []const u8, consumed_count: usize, hash: ?std.hash.XxHash64) !usize {
	380	// These tables take 7680 bytes
	381	var literal_fse_data: [literal_table_size_max]Table.Fse = undefined;
	382	var match_fse_data: [match_table_size_max]Table.Fse = undefined;
	383	var offset_fse_data: [offset_table_size_max]Table.Fse = undefined;
	384
	385	var block_header = decodeBlockHeader(src[0..3]);
	386	var bytes_read: usize = 3;
	387	var decode_state = DecodeState{
	388	.repeat_offsets = .{
	389	types.compressed_block.start_repeated_offset_1,
	390	types.compressed_block.start_repeated_offset_2,
	391	types.compressed_block.start_repeated_offset_3,
	392	},
	393
	394	.offset = undefined,
	395	.match = undefined,
	396	.literal = undefined,
	397
	398	.literal_fse_buffer = &literal_fse_data,
	399	.match_fse_buffer = &match_fse_data,
	400	.offset_fse_buffer = &offset_fse_data,
	401
	402	.literal_written_count = 0,
	403	.literal_stream_reader = undefined,
	404	.literal_stream_bytes = undefined,
	405	.literal_stream_index = undefined,
	406	.huffman_tree = undefined,
	407	};
	408	var first_compressed_block = true;
	409	var first_compressed_literals = true;
	410	var written_count: usize = 0;
	411	while (true) : ({
	412	block_header = decodeBlockHeader(src[bytes_read..][0..3]);
	413	bytes_read += 3;
	414	}) {
	415	const written_size = try decodeBlock(
	416	dest,
	417	src[bytes_read..],
	418	block_header,
	419	&decode_state,
	420	&first_compressed_block,
	421	&first_compressed_literals,
	422	&bytes_read,
	423	written_count,
	424	);
	425	if (hash) \|hash_state\| hash_state.update(dest[written_count .. written_count + written_size]);
	426	written_count += written_size;
	427	if (block_header.last_block) break;
	428	}
	429	consumed_count.* += bytes_read;
	430	return written_count;
	431	}
	432
	433	pub fn decodeBlock(
	434	dest: []u8,
	435	src: []const u8,
	436	block_header: frame.ZStandard.Block.Header,
	437	decode_state: *DecodeState,
	438	first_compressed_block: *bool,
	439	first_compressed_literals: *bool,
	440	consumed_count: *usize,
	441	written_count: usize,
	442	) !usize {
	443	const block_maximum_size = 1 << 17; // 128KiB
	444	const block_size = block_header.block_size;
	445	if (block_maximum_size < block_size) return error.BlockSizeOverMaximum;
	446	// TODO: we probably want to enable safety for release-fast and release-small (or insert custom checks)
	447	switch (block_header.block_type) {
	448	.raw => {
	449	log.debug("writing raw block - size {d}", .{block_size});
	450	const data = src[0..block_size];
	451	std.mem.copy(u8, dest[written_count..], data);
	452	consumed_count.* += block_size;
	453	return block_size;
	454	},
	455	.rle => {
	456	log.debug("writing rle block - '{x}'x{d}", .{ src[0], block_size });
	457	var write_pos: usize = written_count;
	458	while (write_pos < block_size + written_count) : (write_pos += 1) {
	459	dest[write_pos] = src[0];
	460	}
	461	consumed_count.* += 1;
	462	return block_size;
	463	},
	464	.compressed => {
	465	var bytes_read: usize = 0;
	466	const literals = try decodeLiteralsSection(src, &bytes_read);
	467	const sequences_header = try decodeSequencesHeader(src[bytes_read..], &bytes_read);
	468
	469	if (first_compressed_literals.* and literals.header.block_type == .treeless)
	470	return error.TreelessLiteralsFirst;
	471
	472	if (literals.huffman_tree) \|tree\| {
	473	decode_state.huffman_tree = tree;
	474	first_compressed_literals.* = false;
	475	}
	476
	477	switch (literals.header.block_type) {
	478	.raw, .rle => {},
	479	.compressed, .treeless => {
	480	decode_state.literal_stream_index = 0;
	481	switch (literals.streams) {
	482	.one => \|slice\| try decode_state.initLiteralStream(slice),
	483	.four => \|streams\| try decode_state.initLiteralStream(streams[0]),
	484	}
	485	},
	486	}
	487
	488	if (sequences_header.sequence_count > 0) {
	489	bytes_read += try decode_state.updateFseTable(
	490	src[bytes_read..],
	491	.literal,
	492	sequences_header.literal_lengths,
	493	first_compressed_block.*,
	494	);
	495
	496	bytes_read += try decode_state.updateFseTable(
	497	src[bytes_read..],
	498	.offset,
	499	sequences_header.offsets,
	500	first_compressed_block.*,
	501	);
	502
	503	bytes_read += try decode_state.updateFseTable(
	504	src[bytes_read..],
	505	.match,
	506	sequences_header.match_lengths,
	507	first_compressed_block.*,
	508	);
	509	first_compressed_block.* = false;
	510	}
	511
	512	var bytes_written: usize = 0;
	513	if (sequences_header.sequence_count > 0) {
	514	const bit_stream_bytes = src[bytes_read..block_size];
	515	var reverse_byte_reader = reversedByteReader(bit_stream_bytes);
	516	var bit_stream = reverseBitReader(reverse_byte_reader.reader());
	517
	518	while (0 == try bit_stream.readBitsNoEof(u1, 1)) {}
	519	try decode_state.readInitialState(&bit_stream);
	520
	521	var i: usize = 0;
	522	while (i < sequences_header.sequence_count) : (i += 1) {
	523	log.debug("decoding sequence {d}", .{i});
	524	const decompressed_size = try decode_state.decodeSequenceSlice(
	525	dest,
	526	written_count + bytes_written,
	527	literals,
	528	&bit_stream,
	529	i == sequences_header.sequence_count - 1,
	530	);
	531	bytes_written += decompressed_size;
	532	}
	533
	534	bytes_read += bit_stream_bytes.len;
	535	}
	536
	537	if (decode_state.literal_written_count < literals.header.regenerated_size) {
	538	log.debug("decoding remaining literals", .{});
	539	const len = literals.header.regenerated_size - decode_state.literal_written_count;
	540	try decode_state.decodeLiteralsInto(dest[written_count + bytes_written ..], literals, len);
	541	log.debug("remaining decoded literals at {d}: {}", .{
	542	written_count,
	543	std.fmt.fmtSliceHexUpper(dest[written_count .. written_count + len]),
	544	});
	545	bytes_written += len;
	546	}
	547
	548	decode_state.literal_written_count = 0;
	549	assert(bytes_read == block_header.block_size);
	550	consumed_count.* += bytes_read;
	551	return bytes_written;
	552	},
	553	.reserved => return error.FrameContainsReservedBlock,
	554	}
	555	}
	556
	557	pub fn decodeSkippableHeader(src: *const [8]u8) frame.Skippable.Header {
	558	const magic = readInt(u32, src[0..4]);
	559	assert(isSkippableMagic(magic));
	560	const frame_size = readInt(u32, src[4..8]);
	561	return .{
	562	.magic_number = magic,
	563	.frame_size = frame_size,
	564	};
	565	}
	566
	567	pub fn skippableFrameSize(src: *const [8]u8) !usize {
	568	assert(isSkippableMagic(readInt(u32, src[0..4])));
	569	const frame_size = readInt(u32, src[4..8]);
	570	return frame_size;
	571	}
	572
	573	pub fn frameWindowSize(header: frame.ZStandard.Header) ?u64 {
	574	if (header.window_descriptor) \|descriptor\| {
	575	const exponent = (descriptor & 0b11111000) >> 3;
	576	const mantissa = descriptor & 0b00000111;
	577	const window_log = 10 + exponent;
	578	const window_base = @as(u64, 1) << @intCast(u6, window_log);
	579	const window_add = (window_base / 8) * mantissa;
	580	return window_base + window_add;
	581	} else return header.content_size;
	582	}
	583
	584	pub fn decodeZStandardHeader(src: []const u8, consumed_count: ?*usize) !frame.ZStandard.Header {
	585	const descriptor = @bitCast(frame.ZStandard.Header.Descriptor, src[0]);
	586
	587	if (descriptor.unused) return error.UnusedBitSet;
	588	if (descriptor.reserved) return error.ReservedBitSet;
	589
	590	var bytes_read_count: usize = 1;
	591
	592	var window_descriptor: ?u8 = null;
	593	if (!descriptor.single_segment_flag) {
	594	window_descriptor = src[bytes_read_count];
	595	bytes_read_count += 1;
	596	}
	597
	598	var dictionary_id: ?u32 = null;
	599	if (descriptor.dictionary_id_flag > 0) {
	600	// if flag is 3 we field_size = 4, else field_size = flag
	601	const field_size = (@as(u3, 1) << descriptor.dictionary_id_flag) >> 1;
	602	dictionary_id = readVarInt(u32, src[bytes_read_count .. bytes_read_count + field_size]);
	603	bytes_read_count += field_size;
	604	}
	605
	606	var content_size: ?u64 = null;
	607	if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) {
	608	const field_size = @as(u4, 1) << descriptor.content_size_flag;
	609	content_size = readVarInt(u64, src[bytes_read_count .. bytes_read_count + field_size]);
	610	if (field_size == 2) content_size.? += 256;
	611	bytes_read_count += field_size;
	612	}
	613
	614	if (consumed_count) \|p\| p.* += bytes_read_count;
	615
	616	const header = frame.ZStandard.Header{
	617	.descriptor = descriptor,
	618	.window_descriptor = window_descriptor,
	619	.dictionary_id = dictionary_id,
	620	.content_size = content_size,
	621	};
	622	log.debug(
	623	"decoded ZStandard frame header {x}: " ++
	624	"desc = (d={d},c={},r={},u={},s={},cs={d}), win_desc = {?x}, dict_id = {?x}, content_size = {?d}",
	625	.{
	626	std.fmt.fmtSliceHexUpper(src[0..bytes_read_count]),
	627	header.descriptor.dictionary_id_flag,
	628	header.descriptor.content_checksum_flag,
	629	header.descriptor.reserved,
	630	header.descriptor.unused,
	631	header.descriptor.single_segment_flag,
	632	header.descriptor.content_size_flag,
	633	header.window_descriptor,
	634	header.dictionary_id,
	635	header.content_size,
	636	},
	637	);
	638	return header;
	639	}
	640
	641	pub fn decodeBlockHeader(src: *const [3]u8) frame.ZStandard.Block.Header {
	642	const last_block = src[0] & 1 == 1;
	643	const block_type = @intToEnum(frame.ZStandard.Block.Type, (src[0] & 0b110) >> 1);
	644	const block_size = ((src[0] & 0b11111000) >> 3) + (@as(u21, src[1]) << 5) + (@as(u21, src[2]) << 13);
	645	log.debug("decoded block header {}: last = {}, type = {s}, size = {d}", .{
	646	std.fmt.fmtSliceHexUpper(src),
	647	last_block,
	648	@tagName(block_type),
	649	block_size,
	650	});
	651	return .{
	652	.last_block = last_block,
	653	.block_type = block_type,
	654	.block_size = block_size,
	655	};
	656	}
	657
	658	pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals {
	659	// TODO: we probably want to enable safety for release-fast and release-small (or insert custom checks)
	660	var bytes_read: usize = 0;
	661	const header = decodeLiteralsHeader(src, &bytes_read);
	662	switch (header.block_type) {
	663	.raw => {
	664	const stream = src[bytes_read .. bytes_read + header.regenerated_size];
	665	consumed_count.* += header.regenerated_size + bytes_read;
	666	return Literals{
	667	.header = header,
	668	.huffman_tree = null,
	669	.streams = .{ .one = stream },
	670	};
	671	},
	672	.rle => {
	673	const stream = src[bytes_read .. bytes_read + 1];
	674	consumed_count.* += 1 + bytes_read;
	675	return Literals{
	676	.header = header,
	677	.huffman_tree = null,
	678	.streams = .{ .one = stream },
	679	};
	680	},
	681	.compressed, .treeless => {
	682	const huffman_tree_start = bytes_read;
	683	const huffman_tree = if (header.block_type == .compressed)
	684	try decodeHuffmanTree(src[bytes_read..], &bytes_read)
	685	else
	686	null;
	687	const huffman_tree_size = bytes_read - huffman_tree_start;
	688	const total_streams_size = @as(usize, header.compressed_size.?) - huffman_tree_size;
	689	log.debug("huffman tree size = {}, total streams size = {}", .{ huffman_tree_size, total_streams_size });
	690	if (huffman_tree) \|tree\| dumpHuffmanTree(tree);
	691
	692	if (header.size_format == 0) {
	693	const stream = src[bytes_read .. bytes_read + total_streams_size];
	694	bytes_read += total_streams_size;
	695	consumed_count.* += bytes_read;
	696	return Literals{
	697	.header = header,
	698	.huffman_tree = huffman_tree,
	699	.streams = .{ .one = stream },
	700	};
	701	}
	702
	703	const stream_data = src[bytes_read .. bytes_read + total_streams_size];
	704
	705	log.debug("jump table: {}", .{std.fmt.fmtSliceHexUpper(stream_data[0..6])});
	706	const stream_1_length = @as(usize, readInt(u16, stream_data[0..2]));
	707	const stream_2_length = @as(usize, readInt(u16, stream_data[2..4]));
	708	const stream_3_length = @as(usize, readInt(u16, stream_data[4..6]));
	709	const stream_4_length = (total_streams_size - 6) - (stream_1_length + stream_2_length + stream_3_length);
	710
	711	const stream_1_start = 6;
	712	const stream_2_start = stream_1_start + stream_1_length;
	713	const stream_3_start = stream_2_start + stream_2_length;
	714	const stream_4_start = stream_3_start + stream_3_length;
	715
	716	consumed_count.* += total_streams_size + bytes_read;
	717
	718	return Literals{
	719	.header = header,
	720	.huffman_tree = huffman_tree,
	721	.streams = .{ .four = .{
	722	stream_data[stream_1_start .. stream_1_start + stream_1_length],
	723	stream_data[stream_2_start .. stream_2_start + stream_2_length],
	724	stream_data[stream_3_start .. stream_3_start + stream_3_length],
	725	stream_data[stream_4_start .. stream_4_start + stream_4_length],
	726	} },
	727	};
	728	},
	729	}
	730	}
	731
	732	fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanTree {
	733	var bytes_read: usize = 0;
	734	bytes_read += 1;
	735	const header = src[0];
	736	var symbol_count: usize = undefined;
	737	var weights: [256]u4 = undefined;
	738	var max_number_of_bits: u4 = undefined;
	739	if (header < 128) {
	740	// FSE compressed weigths
	741	const compressed_size = header;
	742	var stream = std.io.fixedBufferStream(src[1 .. compressed_size + 1]);
	743	var counting_reader = std.io.countingReader(stream.reader());
	744	var bit_reader = bitReader(counting_reader.reader());
	745
	746	var entries: [1 << 6]Table.Fse = undefined;
	747	const table_size = try decodeFseTable(&bit_reader, 256, 6, &entries);
	748	const accuracy_log = std.math.log2_int_ceil(usize, table_size);
	749
	750	var huff_data = src[1 + counting_reader.bytes_read .. compressed_size + 1];
	751	var huff_data_bytes = reversedByteReader(huff_data);
	752	var huff_bits = reverseBitReader(huff_data_bytes.reader());
	753	while (0 == try huff_bits.readBitsNoEof(u1, 1)) {}
	754
	755	dumpFseTable("huffman", entries[0..table_size]);
	756
	757	var i: usize = 0;
	758	var even_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
	759	var odd_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
	760
	761	while (i < 255) {
	762	const even_data = entries[even_state];
	763	var read_bits: usize = 0;
	764	const even_bits = try huff_bits.readBits(u32, even_data.bits, &read_bits);
	765	weights[i] = @intCast(u4, even_data.symbol);
	766	i += 1;
	767	if (read_bits < even_data.bits) {
	768	weights[i] = @intCast(u4, entries[odd_state].symbol);
	769	log.debug("overflow condition: setting weights[{d}] = {d}", .{ i, weights[i] });
	770	i += 1;
	771	break;
	772	}
	773	even_state = even_data.baseline + even_bits;
	774
	775	read_bits = 0;
	776	const odd_data = entries[odd_state];
	777	const odd_bits = try huff_bits.readBits(u32, odd_data.bits, &read_bits);
	778	weights[i] = @intCast(u4, odd_data.symbol);
	779	i += 1;
	780	if (read_bits < odd_data.bits) {
	781	if (i == 256) return error.MalformedHuffmanTree;
	782	weights[i] = @intCast(u4, entries[even_state].symbol);
	783	log.debug("overflow condition: setting weights[{d}] = {d}", .{ i, weights[i] });
	784	i += 1;
	785	break;
	786	}
	787	odd_state = odd_data.baseline + odd_bits;
	788	} else return error.MalformedHuffmanTree;
	789
	790	symbol_count = i + 1; // stream contains all but the last symbol
	791	bytes_read += compressed_size;
	792	} else {
	793	const encoded_symbol_count = header - 127;
	794	symbol_count = encoded_symbol_count + 1;
	795	log.debug("huffman tree symbol count = {d}", .{symbol_count});
	796	const weights_byte_count = (encoded_symbol_count + 1) / 2;
	797	log.debug("decoding direct huffman tree: {}\|{}", .{
	798	std.fmt.fmtSliceHexUpper(src[0..1]),
	799	std.fmt.fmtSliceHexUpper(src[1 .. weights_byte_count + 1]),
	800	});
	801	if (src.len < weights_byte_count) return error.MalformedHuffmanTree;
	802	var i: usize = 0;
	803	while (i < weights_byte_count) : (i += 1) {
	804	weights[2 * i] = @intCast(u4, src[i + 1] >> 4);
	805	weights[2 * i + 1] = @intCast(u4, src[i + 1] & 0xF);
	806	log.debug("weights[{d}] = {d}", .{ 2 * i, weights[2 * i] });
	807	log.debug("weights[{d}] = {d}", .{ 2 * i + 1, weights[2 * i + 1] });
	808	}
	809	bytes_read += weights_byte_count;
	810	}
	811	var weight_power_sum: u16 = 0;
	812	for (weights[0 .. symbol_count - 1]) \|value\| {
	813	if (value > 0) {
	814	weight_power_sum += @as(u16, 1) << (value - 1);
	815	}
	816	}
	817	log.debug("weight power sum = {d}", .{weight_power_sum});
	818
	819	// advance to next power of two (even if weight_power_sum is a power of 2)
	820	max_number_of_bits = @intCast(u4, std.math.log2_int(u16, weight_power_sum) + 1);
	821	const next_power_of_two = @as(u16, 1) << max_number_of_bits;
	822	weights[symbol_count - 1] = @intCast(u4, std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1);
	823	log.debug("weights[{d}] = {d}", .{ symbol_count - 1, weights[symbol_count - 1] });
	824
	825	var weight_sorted_prefixed_symbols: [256]Literals.HuffmanTree.PrefixedSymbol = undefined;
	826	for (weight_sorted_prefixed_symbols[0..symbol_count]) \|_, i\| {
	827	weight_sorted_prefixed_symbols[i] = .{
	828	.symbol = @intCast(u8, i),
	829	.weight = undefined,
	830	.prefix = undefined,
	831	};
	832	}
	833
	834	std.sort.sort(
	835	Literals.HuffmanTree.PrefixedSymbol,
	836	weight_sorted_prefixed_symbols[0..symbol_count],
	837	weights,
	838	lessThanByWeight,
	839	);
	840
	841	var prefix: u16 = 0;
	842	var prefixed_symbol_count: usize = 0;
	843	var sorted_index: usize = 0;
	844	while (sorted_index < symbol_count) {
	845	var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
	846	const weight = weights[symbol];
	847	if (weight == 0) {
	848	sorted_index += 1;
	849	continue;
	850	}
	851
	852	while (sorted_index < symbol_count) : ({
	853	sorted_index += 1;
	854	prefixed_symbol_count += 1;
	855	prefix += 1;
	856	}) {
	857	symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
	858	if (weights[symbol] != weight) {
	859	prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
	860	break;
	861	}
	862	weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
	863	weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
	864	weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
	865	}
	866	}
	867	consumed_count.* += bytes_read;
	868	const tree = Literals.HuffmanTree{
	869	.max_bit_count = max_number_of_bits,
	870	.symbol_count_minus_one = @intCast(u8, prefixed_symbol_count - 1),
	871	.nodes = weight_sorted_prefixed_symbols,
	872	};
	873	log.debug("decoded huffman tree {}:", .{std.fmt.fmtSliceHexUpper(src[0..bytes_read])});
	874	return tree;
	875	}
	876
	877	fn lessThanByWeight(
	878	weights: [256]u4,
	879	lhs: Literals.HuffmanTree.PrefixedSymbol,
	880	rhs: Literals.HuffmanTree.PrefixedSymbol,
	881	) bool {
	882	// NOTE: this function relies on the use of a stable sorting algorithm,
	883	// otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
	884	// should be added
	885	return weights[lhs.symbol] < weights[rhs.symbol];
	886	}
	887
	888	pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) Literals.Header {
	889	// TODO: we probably want to enable safety for release-fast and release-small (or insert custom checks)
	890	const start = consumed_count.*;
	891	const byte0 = src[0];
	892	const block_type = @intToEnum(Literals.BlockType, byte0 & 0b11);
	893	const size_format = @intCast(u2, (byte0 & 0b1100) >> 2);
	894	var regenerated_size: u20 = undefined;
	895	var compressed_size: ?u18 = null;
	896	switch (block_type) {
	897	.raw, .rle => {
	898	switch (size_format) {
	899	0, 2 => {
	900	regenerated_size = byte0 >> 3;
	901	consumed_count.* += 1;
	902	},
	903	1 => {
	904	regenerated_size = (byte0 >> 4) +
	905	(@as(u20, src[consumed_count.* + 1]) << 4);
	906	consumed_count.* += 2;
	907	},
	908	3 => {
	909	regenerated_size = (byte0 >> 4) +
	910	(@as(u20, src[consumed_count.* + 1]) << 4) +
	911	(@as(u20, src[consumed_count.* + 2]) << 12);
	912	consumed_count.* += 3;
	913	},
	914	}
	915	},
	916	.compressed, .treeless => {
	917	const byte1 = src[1];
	918	const byte2 = src[2];
	919	switch (size_format) {
	920	0, 1 => {
	921	regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
	922	compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
	923	consumed_count.* += 3;
	924	},
	925	2 => {
	926	const byte3 = src[3];
	927	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
	928	compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
	929	consumed_count.* += 4;
	930	},
	931	3 => {
	932	const byte3 = src[3];
	933	const byte4 = src[4];
	934	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
	935	compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
	936	consumed_count.* += 5;
	937	},
	938	}
	939	},
	940	}
	941	log.debug(
	942	"decoded literals section header '{}': type = {s}, size_format = {}, regen_size = {d}, compressed size = {?d}",
	943	.{
	944	std.fmt.fmtSliceHexUpper(src[0 .. consumed_count.* - start]),
	945	@tagName(block_type),
	946	size_format,
	947	regenerated_size,
	948	compressed_size,
	949	},
	950	);
	951	return Literals.Header{
	952	.block_type = block_type,
	953	.size_format = size_format,
	954	.regenerated_size = regenerated_size,
	955	.compressed_size = compressed_size,
	956	};
	957	}
	958
	959	fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Header {
	960	var sequence_count: u24 = undefined;
	961
	962	var bytes_read: usize = 0;
	963	const byte0 = src[0];
	964	if (byte0 == 0) {
	965	bytes_read += 1;
	966	log.debug("decoded sequences header '{}': sequence count = 0", .{std.fmt.fmtSliceHexUpper(src[0..bytes_read])});
	967	consumed_count.* += bytes_read;
	968	return Sequences.Header{
	969	.sequence_count = 0,
	970	.offsets = undefined,
	971	.match_lengths = undefined,
	972	.literal_lengths = undefined,
	973	};
	974	} else if (byte0 < 128) {
	975	sequence_count = byte0;
	976	bytes_read += 1;
	977	} else if (byte0 < 255) {
	978	sequence_count = (@as(u24, (byte0 - 128)) << 8) + src[1];
	979	bytes_read += 2;
	980	} else {
	981	sequence_count = src[1] + (@as(u24, src[2]) << 8) + 0x7F00;
	982	bytes_read += 3;
	983	}
	984
	985	const compression_modes = src[bytes_read];
	986	bytes_read += 1;
	987
	988	consumed_count.* += bytes_read;
	989	const matches_mode = @intToEnum(Sequences.Header.Mode, (compression_modes & 0b00001100) >> 2);
	990	const offsets_mode = @intToEnum(Sequences.Header.Mode, (compression_modes & 0b00110000) >> 4);
	991	const literal_mode = @intToEnum(Sequences.Header.Mode, (compression_modes & 0b11000000) >> 6);
	992	log.debug("decoded sequences header '{}': (sc={d},o={s},m={s},l={s})", .{
	993	std.fmt.fmtSliceHexUpper(src[0..bytes_read]),
	994	sequence_count,
	995	@tagName(offsets_mode),
	996	@tagName(matches_mode),
	997	@tagName(literal_mode),
	998	});
	999	if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
	1000
	1001	return Sequences.Header{
	1002	.sequence_count = sequence_count,
	1003	.offsets = offsets_mode,
	1004	.match_lengths = matches_mode,
	1005	.literal_lengths = literal_mode,
	1006	};
	1007	}
	1008
	1009	fn buildFseTable(values: []const u16, entries: []Table.Fse) !void {
	1010	const total_probability = @intCast(u16, entries.len);
	1011	const accuracy_log = std.math.log2_int(u16, total_probability);
	1012	assert(total_probability <= 1 << 9);
	1013
	1014	var less_than_one_count: usize = 0;
	1015	for (values) \|value, i\| {
	1016	if (value == 0) {
	1017	entries[entries.len - 1 - less_than_one_count] = Table.Fse{
	1018	.symbol = @intCast(u8, i),
	1019	.baseline = 0,
	1020	.bits = accuracy_log,
	1021	};
	1022	less_than_one_count += 1;
	1023	}
	1024	}
	1025
	1026	var position: usize = 0;
	1027	var temp_states: [1 << 9]u16 = undefined;
	1028	for (values) \|value, symbol\| {
	1029	if (value == 0 or value == 1) continue;
	1030	const probability = value - 1;
	1031
	1032	const state_share_dividend = try std.math.ceilPowerOfTwo(u16, probability);
	1033	const share_size = @divExact(total_probability, state_share_dividend);
	1034	const double_state_count = state_share_dividend - probability;
	1035	const single_state_count = probability - double_state_count;
	1036	const share_size_log = std.math.log2_int(u16, share_size);
	1037
	1038	var i: u16 = 0;
	1039	while (i < probability) : (i += 1) {
	1040	temp_states[i] = @intCast(u16, position);
	1041	position += (entries.len >> 1) + (entries.len >> 3) + 3;
	1042	position &= entries.len - 1;
	1043	while (position >= entries.len - less_than_one_count) {
	1044	position += (entries.len >> 1) + (entries.len >> 3) + 3;
	1045	position &= entries.len - 1;
	1046	}
	1047	}
	1048	std.sort.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
	1049	i = 0;
	1050	while (i < probability) : (i += 1) {
	1051	entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
	1052	.symbol = @intCast(u8, symbol),
	1053	.bits = share_size_log + 1,
	1054	.baseline = single_state_count * share_size + i * 2 * share_size,
	1055	} else Table.Fse{
	1056	.symbol = @intCast(u8, symbol),
	1057	.bits = share_size_log,
	1058	.baseline = (i - double_state_count) * share_size,
	1059	};
	1060	}
	1061	}
	1062	}
	1063
	1064	fn decodeFseTable(
	1065	bit_reader: anytype,
	1066	expected_symbol_count: usize,
	1067	max_accuracy_log: u4,
	1068	entries: []Table.Fse,
	1069	) !usize {
	1070	log.debug("decoding fse table {d} {d}", .{ max_accuracy_log, expected_symbol_count });
	1071
	1072	const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
	1073	log.debug("accuracy_log_biased = {d}", .{accuracy_log_biased});
	1074	if (accuracy_log_biased > max_accuracy_log -\| 5) return error.MalformedAccuracyLog;
	1075	const accuracy_log = accuracy_log_biased + 5;
	1076
	1077	var values: [256]u16 = undefined;
	1078	var value_count: usize = 0;
	1079
	1080	const total_probability = @as(u16, 1) << accuracy_log;
	1081	log.debug("total probability = {d}", .{total_probability});
	1082	var accumulated_probability: u16 = 0;
	1083
	1084	while (accumulated_probability < total_probability) {
	1085	// WARNING: The RFC in poorly worded, and would suggest std.math.log2_int_ceil is correct here,
	1086	// but power of two (remaining probabilities + 1) need max bits set to 1 more.
	1087	const max_bits = @intCast(u4, std.math.log2_int(u16, total_probability - accumulated_probability + 1)) + 1;
	1088	const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
	1089
	1090	const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
	1091
	1092	const value = if (small < cutoff)
	1093	small
	1094	else value: {
	1095	const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
	1096	break :value if (value_read < @as(u16, 1) << (max_bits - 1))
	1097	value_read
	1098	else
	1099	value_read - cutoff;
	1100	};
	1101
	1102	accumulated_probability += if (value != 0) value - 1 else 1;
	1103
	1104	values[value_count] = value;
	1105	value_count += 1;
	1106
	1107	if (value == 1) {
	1108	while (true) {
	1109	const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
	1110	var i: usize = 0;
	1111	while (i < repeat_flag) : (i += 1) {
	1112	values[value_count] = 1;
	1113	value_count += 1;
	1114	}
	1115	if (repeat_flag < 3) break;
	1116	}
	1117	}
	1118	}
	1119	bit_reader.alignToByte();
	1120
	1121	// TODO: check there are at least 2 non-zero probabilities
	1122
	1123	if (accumulated_probability != total_probability) return error.MalformedFseTable;
	1124	if (value_count > expected_symbol_count) return error.MalformedFseTable;
	1125
	1126	const table_size = total_probability;
	1127
	1128	try buildFseTable(values[0..value_count], entries[0..table_size]);
	1129	return table_size;
	1130	}
	1131
	1132	const ReversedByteReader = struct {
	1133	remaining_bytes: usize,
	1134	bytes: []const u8,
	1135
	1136	const Reader = std.io.Reader(*ReversedByteReader, error{}, readFn);
	1137
	1138	fn reader(self: *ReversedByteReader) Reader {
	1139	return .{ .context = self };
	1140	}
	1141	};
	1142
	1143	fn readFn(ctx: *ReversedByteReader, buffer: []u8) !usize {
	1144	if (ctx.remaining_bytes == 0) return 0;
	1145	const byte_index = ctx.remaining_bytes - 1;
	1146	buffer[0] = ctx.bytes[byte_index];
	1147	// buffer[0] = @bitReverse(ctx.bytes[byte_index]);
	1148	ctx.remaining_bytes = byte_index;
	1149	return 1;
	1150	}
	1151
	1152	fn reversedByteReader(bytes: []const u8) ReversedByteReader {
	1153	return ReversedByteReader{
	1154	.remaining_bytes = bytes.len,
	1155	.bytes = bytes,
	1156	};
	1157	}
	1158
	1159	fn ReverseBitReader(comptime Reader: type) type {
	1160	return struct {
	1161	underlying: std.io.BitReader(.Big, Reader),
	1162
	1163	fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: usize) !U {
	1164	return self.underlying.readBitsNoEof(U, num_bits);
	1165	}
	1166
	1167	fn readBits(self: @This(), comptime U: type, num_bits: usize, out_bits: usize) !U {
	1168	return try self.underlying.readBits(U, num_bits, out_bits);
	1169	}
	1170
	1171	fn alignToByte(self: *@This()) void {
	1172	self.underlying.alignToByte();
	1173	}
	1174	};
	1175	}
	1176
	1177	fn reverseBitReader(reader: anytype) ReverseBitReader(@TypeOf(reader)) {
	1178	return .{ .underlying = std.io.bitReader(.Big, reader) };
	1179	}
	1180
	1181	fn BitReader(comptime Reader: type) type {
	1182	return struct {
	1183	underlying: std.io.BitReader(.Little, Reader),
	1184
	1185	fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: usize) !U {
	1186	return self.underlying.readBitsNoEof(U, num_bits);
	1187	}
	1188
	1189	fn readBits(self: @This(), comptime U: type, num_bits: usize, out_bits: usize) !U {
	1190	return self.underlying.readBits(U, num_bits, out_bits);
	1191	}
	1192
	1193	fn alignToByte(self: *@This()) void {
	1194	self.underlying.alignToByte();
	1195	}
	1196	};
	1197	}
	1198
	1199	fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {
	1200	return .{ .underlying = std.io.bitReader(.Little, reader) };
	1201	}
	1202
	1203	test {
	1204	std.testing.refAllDecls(@This());
	1205	}
	1206
	1207	test buildFseTable {
	1208	const literals_length_default_values = [36]u16{
	1209	5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
	1210	3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
	1211	0, 0, 0, 0,
	1212	};
	1213
	1214	const match_lengths_default_values = [53]u16{
	1215	2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
	1216	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
	1217	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
	1218	0, 0, 0, 0, 0,
	1219	};
	1220
	1221	const offset_codes_default_values = [29]u16{
	1222	2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
	1223	2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
	1224	};
	1225
	1226	var entries: [64]Table.Fse = undefined;
	1227	try buildFseTable(&literals_length_default_values, &entries);
	1228	try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_literal_fse_table.fse, &entries);
	1229
	1230	try buildFseTable(&match_lengths_default_values, &entries);
	1231	try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_match_fse_table.fse, &entries);
	1232
	1233	try buildFseTable(&offset_codes_default_values, entries[0..32]);
	1234	try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_offset_fse_table.fse, entries[0..32]);
	1235	}
	1236
	1237	fn dumpFseTable(prefix: []const u8, table: []const Table.Fse) void {
	1238	log.debug("{s} fse table:", .{prefix});
	1239	for (table) \|entry, i\| {
	1240	log.debug("state = {d} symbol = {d} bl = {d}, bits = {d}", .{ i, entry.symbol, entry.baseline, entry.bits });
	1241	}
	1242	}
	1243
	1244	fn dumpHuffmanTree(tree: Literals.HuffmanTree) void {
	1245	log.debug("Huffman tree: max bit count = {}, symbol count = {}", .{ tree.max_bit_count, tree.symbol_count_minus_one + 1 });
	1246	for (tree.nodes[0 .. tree.symbol_count_minus_one + 1]) \|node\| {
	1247	log.debug("symbol = {[symbol]d}, prefix = {[prefix]d}, weight = {[weight]d}", node);
	1248	}
	1249	}

lib/std/compress/zstandard/types.zig created+394

...	...	@@ -0,0 +1,394 @@
	1	pub const frame = struct {
	2	pub const Kind = enum { zstandard, skippable };
	3
	4	pub const ZStandard = struct {
	5	pub const magic_number = 0xFD2FB528;
	6
	7	header: Header,
	8	data_blocks: []Block,
	9	checksum: ?u32,
	10
	11	pub const Header = struct {
	12	descriptor: Descriptor,
	13	window_descriptor: ?u8,
	14	dictionary_id: ?u32,
	15	content_size: ?u64,
	16
	17	pub const Descriptor = packed struct {
	18	dictionary_id_flag: u2,
	19	content_checksum_flag: bool,
	20	reserved: bool,
	21	unused: bool,
	22	single_segment_flag: bool,
	23	content_size_flag: u2,
	24	};
	25	};
	26
	27	pub const Block = struct {
	28	pub const Header = struct {
	29	last_block: bool,
	30	block_type: Block.Type,
	31	block_size: u21,
	32	};
	33
	34	pub const Type = enum(u2) {
	35	raw,
	36	rle,
	37	compressed,
	38	reserved,
	39	};
	40	};
	41	};
	42
	43	pub const Skippable = struct {
	44	pub const magic_number_min = 0x184D2A50;
	45	pub const magic_number_max = 0x184D2A5F;
	46
	47	pub const Header = struct {
	48	magic_number: u32,
	49	frame_size: u32,
	50	};
	51	};
	52	};
	53
	54	pub const compressed_block = struct {
	55	pub const Literals = struct {
	56	header: Header,
	57	huffman_tree: ?HuffmanTree,
	58	streams: Streams,
	59
	60	pub const Streams = union(enum) {
	61	one: []const u8,
	62	four: [4][]const u8,
	63	};
	64
	65	pub const Header = struct {
	66	block_type: BlockType,
	67	size_format: u2,
	68	regenerated_size: u20,
	69	compressed_size: ?u18,
	70	};
	71
	72	pub const BlockType = enum(u2) {
	73	raw,
	74	rle,
	75	compressed,
	76	treeless,
	77	};
	78
	79	pub const HuffmanTree = struct {
	80	max_bit_count: u4,
	81	symbol_count_minus_one: u8,
	82	nodes: [256]PrefixedSymbol,
	83
	84	pub const PrefixedSymbol = struct {
	85	symbol: u8,
	86	prefix: u16,
	87	weight: u4,
	88	};
	89
	90	pub const Result = union(enum) {
	91	symbol: u8,
	92	index: usize,
	93	};
	94
	95	pub fn query(self: HuffmanTree, index: usize, prefix: u16) !Result {
	96	var node = self.nodes[index];
	97	const weight = node.weight;
	98	var i: usize = index;
	99	while (node.weight == weight) {
	100	if (node.prefix == prefix) return Result{ .symbol = node.symbol };
	101	if (i == 0) return error.PrefixNotFound;
	102	i -= 1;
	103	node = self.nodes[i];
	104	}
	105	return Result{ .index = i };
	106	}
	107
	108	pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
	109	return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
	110	}
	111	};
	112
	113	pub const StreamCount = enum { one, four };
	114	pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
	115	return switch (block_type) {
	116	.raw, .rle => .one,
	117	.compressed, .treeless => if (size_format == 0) .one else .four,
	118	};
	119	}
	120	};
	121
	122	pub const Sequences = struct {
	123	header: Sequences.Header,
	124	literals_length_table: Table,
	125	offset_table: Table,
	126	match_length_table: Table,
	127
	128	pub const Header = struct {
	129	sequence_count: u24,
	130	match_lengths: Mode,
	131	offsets: Mode,
	132	literal_lengths: Mode,
	133
	134	pub const Mode = enum(u2) {
	135	predefined,
	136	rle,
	137	fse,
	138	repeat,
	139	};
	140	};
	141	};
	142
	143	pub const Table = union(enum) {
	144	fse: []const Fse,
	145	rle: u8,
	146
	147	pub const Fse = struct {
	148	symbol: u8,
	149	baseline: u16,
	150	bits: u8,
	151	};
	152	};
	153
	154	pub const literals_length_code_table = [36]struct { u32, u5 }{
	155	.{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
	156	.{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
	157	.{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
	158	.{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
	159	.{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
	160	.{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
	161	.{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
	162	.{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
	163	.{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
	164	};
	165
	166	pub const match_length_code_table = [53]struct { u32, u5 }{
	167	.{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 },
	168	.{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 },
	169	.{ 19, 0 }, .{ 20, 0 }, .{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
	170	.{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 }, .{ 33, 0 }, .{ 34, 0 },
	171	.{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 }, .{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 },
	172	.{ 67, 4 }, .{ 83, 4 }, .{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
	173	.{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
	174	};
	175
	176	pub const literals_length_default_distribution = [36]i16{
	177	4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
	178	2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
	179	-1, -1, -1, -1,
	180	};
	181
	182	pub const match_lengths_default_distribution = [53]i16{
	183	1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
	184	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	185	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
	186	-1, -1, -1, -1, -1,
	187	};
	188
	189	pub const offset_codes_default_distribution = [29]i16{
	190	1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
	191	1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
	192	};
	193
	194	pub const predefined_literal_fse_table = Table{
	195	.fse = &[64]Table.Fse{
	196	.{ .symbol = 0, .bits = 4, .baseline = 0 },
	197	.{ .symbol = 0, .bits = 4, .baseline = 16 },
	198	.{ .symbol = 1, .bits = 5, .baseline = 32 },
	199	.{ .symbol = 3, .bits = 5, .baseline = 0 },
	200	.{ .symbol = 4, .bits = 5, .baseline = 0 },
	201	.{ .symbol = 6, .bits = 5, .baseline = 0 },
	202	.{ .symbol = 7, .bits = 5, .baseline = 0 },
	203	.{ .symbol = 9, .bits = 5, .baseline = 0 },
	204	.{ .symbol = 10, .bits = 5, .baseline = 0 },
	205	.{ .symbol = 12, .bits = 5, .baseline = 0 },
	206	.{ .symbol = 14, .bits = 6, .baseline = 0 },
	207	.{ .symbol = 16, .bits = 5, .baseline = 0 },
	208	.{ .symbol = 18, .bits = 5, .baseline = 0 },
	209	.{ .symbol = 19, .bits = 5, .baseline = 0 },
	210	.{ .symbol = 21, .bits = 5, .baseline = 0 },
	211	.{ .symbol = 22, .bits = 5, .baseline = 0 },
	212	.{ .symbol = 24, .bits = 5, .baseline = 0 },
	213	.{ .symbol = 25, .bits = 5, .baseline = 32 },
	214	.{ .symbol = 26, .bits = 5, .baseline = 0 },
	215	.{ .symbol = 27, .bits = 6, .baseline = 0 },
	216	.{ .symbol = 29, .bits = 6, .baseline = 0 },
	217	.{ .symbol = 31, .bits = 6, .baseline = 0 },
	218	.{ .symbol = 0, .bits = 4, .baseline = 32 },
	219	.{ .symbol = 1, .bits = 4, .baseline = 0 },
	220	.{ .symbol = 2, .bits = 5, .baseline = 0 },
	221	.{ .symbol = 4, .bits = 5, .baseline = 32 },
	222	.{ .symbol = 5, .bits = 5, .baseline = 0 },
	223	.{ .symbol = 7, .bits = 5, .baseline = 32 },
	224	.{ .symbol = 8, .bits = 5, .baseline = 0 },
	225	.{ .symbol = 10, .bits = 5, .baseline = 32 },
	226	.{ .symbol = 11, .bits = 5, .baseline = 0 },
	227	.{ .symbol = 13, .bits = 6, .baseline = 0 },
	228	.{ .symbol = 16, .bits = 5, .baseline = 32 },
	229	.{ .symbol = 17, .bits = 5, .baseline = 0 },
	230	.{ .symbol = 19, .bits = 5, .baseline = 32 },
	231	.{ .symbol = 20, .bits = 5, .baseline = 0 },
	232	.{ .symbol = 22, .bits = 5, .baseline = 32 },
	233	.{ .symbol = 23, .bits = 5, .baseline = 0 },
	234	.{ .symbol = 25, .bits = 4, .baseline = 0 },
	235	.{ .symbol = 25, .bits = 4, .baseline = 16 },
	236	.{ .symbol = 26, .bits = 5, .baseline = 32 },
	237	.{ .symbol = 28, .bits = 6, .baseline = 0 },
	238	.{ .symbol = 30, .bits = 6, .baseline = 0 },
	239	.{ .symbol = 0, .bits = 4, .baseline = 48 },
	240	.{ .symbol = 1, .bits = 4, .baseline = 16 },
	241	.{ .symbol = 2, .bits = 5, .baseline = 32 },
	242	.{ .symbol = 3, .bits = 5, .baseline = 32 },
	243	.{ .symbol = 5, .bits = 5, .baseline = 32 },
	244	.{ .symbol = 6, .bits = 5, .baseline = 32 },
	245	.{ .symbol = 8, .bits = 5, .baseline = 32 },
	246	.{ .symbol = 9, .bits = 5, .baseline = 32 },
	247	.{ .symbol = 11, .bits = 5, .baseline = 32 },
	248	.{ .symbol = 12, .bits = 5, .baseline = 32 },
	249	.{ .symbol = 15, .bits = 6, .baseline = 0 },
	250	.{ .symbol = 17, .bits = 5, .baseline = 32 },
	251	.{ .symbol = 18, .bits = 5, .baseline = 32 },
	252	.{ .symbol = 20, .bits = 5, .baseline = 32 },
	253	.{ .symbol = 21, .bits = 5, .baseline = 32 },
	254	.{ .symbol = 23, .bits = 5, .baseline = 32 },
	255	.{ .symbol = 24, .bits = 5, .baseline = 32 },
	256	.{ .symbol = 35, .bits = 6, .baseline = 0 },
	257	.{ .symbol = 34, .bits = 6, .baseline = 0 },
	258	.{ .symbol = 33, .bits = 6, .baseline = 0 },
	259	.{ .symbol = 32, .bits = 6, .baseline = 0 },
	260	},
	261	};
	262
	263	pub const predefined_match_fse_table = Table{
	264	.fse = &[64]Table.Fse{
	265	.{ .symbol = 0, .bits = 6, .baseline = 0 },
	266	.{ .symbol = 1, .bits = 4, .baseline = 0 },
	267	.{ .symbol = 2, .bits = 5, .baseline = 32 },
	268	.{ .symbol = 3, .bits = 5, .baseline = 0 },
	269	.{ .symbol = 5, .bits = 5, .baseline = 0 },
	270	.{ .symbol = 6, .bits = 5, .baseline = 0 },
	271	.{ .symbol = 8, .bits = 5, .baseline = 0 },
	272	.{ .symbol = 10, .bits = 6, .baseline = 0 },
	273	.{ .symbol = 13, .bits = 6, .baseline = 0 },
	274	.{ .symbol = 16, .bits = 6, .baseline = 0 },
	275	.{ .symbol = 19, .bits = 6, .baseline = 0 },
	276	.{ .symbol = 22, .bits = 6, .baseline = 0 },
	277	.{ .symbol = 25, .bits = 6, .baseline = 0 },
	278	.{ .symbol = 28, .bits = 6, .baseline = 0 },
	279	.{ .symbol = 31, .bits = 6, .baseline = 0 },
	280	.{ .symbol = 33, .bits = 6, .baseline = 0 },
	281	.{ .symbol = 35, .bits = 6, .baseline = 0 },
	282	.{ .symbol = 37, .bits = 6, .baseline = 0 },
	283	.{ .symbol = 39, .bits = 6, .baseline = 0 },
	284	.{ .symbol = 41, .bits = 6, .baseline = 0 },
	285	.{ .symbol = 43, .bits = 6, .baseline = 0 },
	286	.{ .symbol = 45, .bits = 6, .baseline = 0 },
	287	.{ .symbol = 1, .bits = 4, .baseline = 16 },
	288	.{ .symbol = 2, .bits = 4, .baseline = 0 },
	289	.{ .symbol = 3, .bits = 5, .baseline = 32 },
	290	.{ .symbol = 4, .bits = 5, .baseline = 0 },
	291	.{ .symbol = 6, .bits = 5, .baseline = 32 },
	292	.{ .symbol = 7, .bits = 5, .baseline = 0 },
	293	.{ .symbol = 9, .bits = 6, .baseline = 0 },
	294	.{ .symbol = 12, .bits = 6, .baseline = 0 },
	295	.{ .symbol = 15, .bits = 6, .baseline = 0 },
	296	.{ .symbol = 18, .bits = 6, .baseline = 0 },
	297	.{ .symbol = 21, .bits = 6, .baseline = 0 },
	298	.{ .symbol = 24, .bits = 6, .baseline = 0 },
	299	.{ .symbol = 27, .bits = 6, .baseline = 0 },
	300	.{ .symbol = 30, .bits = 6, .baseline = 0 },
	301	.{ .symbol = 32, .bits = 6, .baseline = 0 },
	302	.{ .symbol = 34, .bits = 6, .baseline = 0 },
	303	.{ .symbol = 36, .bits = 6, .baseline = 0 },
	304	.{ .symbol = 38, .bits = 6, .baseline = 0 },
	305	.{ .symbol = 40, .bits = 6, .baseline = 0 },
	306	.{ .symbol = 42, .bits = 6, .baseline = 0 },
	307	.{ .symbol = 44, .bits = 6, .baseline = 0 },
	308	.{ .symbol = 1, .bits = 4, .baseline = 32 },
	309	.{ .symbol = 1, .bits = 4, .baseline = 48 },
	310	.{ .symbol = 2, .bits = 4, .baseline = 16 },
	311	.{ .symbol = 4, .bits = 5, .baseline = 32 },
	312	.{ .symbol = 5, .bits = 5, .baseline = 32 },
	313	.{ .symbol = 7, .bits = 5, .baseline = 32 },
	314	.{ .symbol = 8, .bits = 5, .baseline = 32 },
	315	.{ .symbol = 11, .bits = 6, .baseline = 0 },
	316	.{ .symbol = 14, .bits = 6, .baseline = 0 },
	317	.{ .symbol = 17, .bits = 6, .baseline = 0 },
	318	.{ .symbol = 20, .bits = 6, .baseline = 0 },
	319	.{ .symbol = 23, .bits = 6, .baseline = 0 },
	320	.{ .symbol = 26, .bits = 6, .baseline = 0 },
	321	.{ .symbol = 29, .bits = 6, .baseline = 0 },
	322	.{ .symbol = 52, .bits = 6, .baseline = 0 },
	323	.{ .symbol = 51, .bits = 6, .baseline = 0 },
	324	.{ .symbol = 50, .bits = 6, .baseline = 0 },
	325	.{ .symbol = 49, .bits = 6, .baseline = 0 },
	326	.{ .symbol = 48, .bits = 6, .baseline = 0 },
	327	.{ .symbol = 47, .bits = 6, .baseline = 0 },
	328	.{ .symbol = 46, .bits = 6, .baseline = 0 },
	329	},
	330	};
	331
	332	pub const predefined_offset_fse_table = Table{
	333	.fse = &[32]Table.Fse{
	334	.{ .symbol = 0, .bits = 5, .baseline = 0 },
	335	.{ .symbol = 6, .bits = 4, .baseline = 0 },
	336	.{ .symbol = 9, .bits = 5, .baseline = 0 },
	337	.{ .symbol = 15, .bits = 5, .baseline = 0 },
	338	.{ .symbol = 21, .bits = 5, .baseline = 0 },
	339	.{ .symbol = 3, .bits = 5, .baseline = 0 },
	340	.{ .symbol = 7, .bits = 4, .baseline = 0 },
	341	.{ .symbol = 12, .bits = 5, .baseline = 0 },
	342	.{ .symbol = 18, .bits = 5, .baseline = 0 },
	343	.{ .symbol = 23, .bits = 5, .baseline = 0 },
	344	.{ .symbol = 5, .bits = 5, .baseline = 0 },
	345	.{ .symbol = 8, .bits = 4, .baseline = 0 },
	346	.{ .symbol = 14, .bits = 5, .baseline = 0 },
	347	.{ .symbol = 20, .bits = 5, .baseline = 0 },
	348	.{ .symbol = 2, .bits = 5, .baseline = 0 },
	349	.{ .symbol = 7, .bits = 4, .baseline = 16 },
	350	.{ .symbol = 11, .bits = 5, .baseline = 0 },
	351	.{ .symbol = 17, .bits = 5, .baseline = 0 },
	352	.{ .symbol = 22, .bits = 5, .baseline = 0 },
	353	.{ .symbol = 4, .bits = 5, .baseline = 0 },
	354	.{ .symbol = 8, .bits = 4, .baseline = 16 },
	355	.{ .symbol = 13, .bits = 5, .baseline = 0 },
	356	.{ .symbol = 19, .bits = 5, .baseline = 0 },
	357	.{ .symbol = 1, .bits = 5, .baseline = 0 },
	358	.{ .symbol = 6, .bits = 4, .baseline = 16 },
	359	.{ .symbol = 10, .bits = 5, .baseline = 0 },
	360	.{ .symbol = 16, .bits = 5, .baseline = 0 },
	361	.{ .symbol = 28, .bits = 5, .baseline = 0 },
	362	.{ .symbol = 27, .bits = 5, .baseline = 0 },
	363	.{ .symbol = 26, .bits = 5, .baseline = 0 },
	364	.{ .symbol = 25, .bits = 5, .baseline = 0 },
	365	.{ .symbol = 24, .bits = 5, .baseline = 0 },
	366	},
	367	};
	368	pub const start_repeated_offset_1 = 1;
	369	pub const start_repeated_offset_2 = 4;
	370	pub const start_repeated_offset_3 = 8;
	371
	372	pub const table_accuracy_log_max = struct {
	373	pub const literal = 9;
	374	pub const match = 9;
	375	pub const offset = 8;
	376	};
	377
	378	pub const table_symbol_count_max = struct {
	379	pub const literal = 36;
	380	pub const match = 53;
	381	pub const offset = 32;
	382	};
	383
	384	pub const default_accuracy_log = struct {
	385	pub const literal = 6;
	386	pub const match = 6;
	387	pub const offset = 5;
	388	};
	389	};
	390
	391	test {
	392	const testing = @import("std").testing;
	393	testing.refAllDeclsRecursive(@This());
	394	}