Commit 66e49d93b7b3815a5213b39d19b3c5f39c21ffc3 | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-07-25 20:02:53-07:00
committer	GitHub <noreply@github.com> 2025-07-25 20:02:53-07:00
log	66e49d93b7b3815a5213b39d19b3c5f39c21ffc3
tree	311d2907194f6d87ba62823622250260b18a349c
parent	9e11727c7c88289554251646166f844376dcbbe9
parent	733b2082565a86daac6f2b22a1a27f0b60621b4e
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #24559 from ziglang/zstd

std: rework zstd for new I/O API

20 files changed, 2188 insertions(+), 3072 deletions(-)

lib
- std
  - Io
    DeprecatedReader.zig
    Reader.zig
    Writer.zig
  - compress.zig
  - compress
    xz.zig
    xz
    block.zig
    zstandard.zig
    zstandard
    decode
    block.zig
    fse.zig
    huffman.zig
    decompress.zig
    readers.zig
    types.zig
    zstd.zig
    zstd
    Decompress.zig
  - fs
    File.zig
  - http
    Server.zig
  - posix.zig
src
- Package
  - Fetch.zig
  - Fetch
    git.zig

lib/std/Io/DeprecatedReader.zig+2-2

...	@@ -373,11 +373,11 @@ pub fn discard(self: Self) anyerror!u64 {	...	@@ -373,11 +373,11 @@ pub fn discard(self: Self) anyerror!u64 {
373	}	373	}
374		374
375	/// Helper for bridging to the new `Reader` API while upgrading.	375	/// Helper for bridging to the new `Reader` API while upgrading.
376	pub fn adaptToNewApi(self: *const Self) Adapter {	376	pub fn adaptToNewApi(self: *const Self, buffer: []u8) Adapter {
377	return .{	377	return .{
378	.derp_reader = self.*,	378	.derp_reader = self.*,
379	.new_interface = .{	379	.new_interface = .{
380	.buffer = &.{},	380	.buffer = buffer,
381	.vtable = &.{ .stream = Adapter.stream },	381	.vtable = &.{ .stream = Adapter.stream },
382	.seek = 0,	382	.seek = 0,
383	.end = 0,	383	.end = 0,

lib/std/Io/Reader.zig+28-2

...	@@ -185,6 +185,32 @@ pub fn streamExact64(r: Reader, w: Writer, n: u64) StreamError!void {	...	@@ -185,6 +185,32 @@ pub fn streamExact64(r: Reader, w: Writer, n: u64) StreamError!void {
185	while (remaining != 0) remaining -= try r.stream(w, .limited64(remaining));	185	while (remaining != 0) remaining -= try r.stream(w, .limited64(remaining));
186	}	186	}
187		187
		188	/// "Pump" exactly `n` bytes from the reader to the writer.
		189	///
		190	/// When draining `w`, ensures that at least `preserve_len` bytes remain
		191	/// buffered.
		192	///
		193	/// Asserts `Writer.buffer` capacity exceeds `preserve_len`.
		194	pub fn streamExactPreserve(r: Reader, w: Writer, preserve_len: usize, n: usize) StreamError!void {
		195	if (w.end + n <= w.buffer.len) {
		196	@branchHint(.likely);
		197	return streamExact(r, w, n);
		198	}
		199	// If `n` is large, we can ignore `preserve_len` up to a point.
		200	var remaining = n;
		201	while (remaining > preserve_len) {
		202	assert(remaining != 0);
		203	remaining -= try r.stream(w, .limited(remaining - preserve_len));
		204	if (w.end + remaining <= w.buffer.len) return streamExact(r, w, remaining);
		205	}
		206	// All the next bytes received must be preserved.
		207	if (preserve_len < w.end) {
		208	@memmove(w.buffer[0..preserve_len], w.buffer[w.end - preserve_len ..][0..preserve_len]);
		209	w.end = preserve_len;
		210	}
		211	return streamExact(r, w, remaining);
		212	}
		213
188	/// "Pump" data from the reader to the writer, handling `error.EndOfStream` as	214	/// "Pump" data from the reader to the writer, handling `error.EndOfStream` as
189	/// a success case.	215	/// a success case.
190	///	216	///
...	@@ -240,7 +266,7 @@ pub fn allocRemaining(r: *Reader, gpa: Allocator, limit: Limit) LimitedAllocErro	...	@@ -240,7 +266,7 @@ pub fn allocRemaining(r: *Reader, gpa: Allocator, limit: Limit) LimitedAllocErro
240	/// such case, the next byte that would be read will be the first one to exceed	266	/// such case, the next byte that would be read will be the first one to exceed
241	/// `limit`, and all preceeding bytes have been appended to `list`.	267	/// `limit`, and all preceeding bytes have been appended to `list`.
242	///	268	///
243	/// Asserts `buffer` has nonzero capacity.	269	/// If `limit` is not `Limit.unlimited`, asserts `buffer` has nonzero capacity.
244	///	270	///
245	/// See also:	271	/// See also:
246	/// * `allocRemaining`	272	/// * `allocRemaining`
...	@@ -251,7 +277,7 @@ pub fn appendRemaining(	...	@@ -251,7 +277,7 @@ pub fn appendRemaining(
251	list: *std.ArrayListAlignedUnmanaged(u8, alignment),	277	list: *std.ArrayListAlignedUnmanaged(u8, alignment),
252	limit: Limit,	278	limit: Limit,
253	) LimitedAllocError!void {	279	) LimitedAllocError!void {
254	assert(r.buffer.len != 0); // Needed to detect limit exceeded without losing data.	280	if (limit != .unlimited) assert(r.buffer.len != 0); // Needed to detect limit exceeded without losing data.
255	const buffer_contents = r.buffer[r.seek..r.end];	281	const buffer_contents = r.buffer[r.seek..r.end];
256	const copy_len = limit.minInt(buffer_contents.len);	282	const copy_len = limit.minInt(buffer_contents.len);
257	try list.appendSlice(gpa, r.buffer[0..copy_len]);	283	try list.appendSlice(gpa, r.buffer[0..copy_len]);

lib/std/Io/Writer.zig+70-23

 @@ -256,10 +256,10 @@ test "fixed buffer flush" {
     try testing.expectEqual(10, buffer[0]);
 }
-/// Calls `VTable.drain` but hides the last `preserve_length` bytes from the
+/// Calls `VTable.drain` but hides the last `preserve_len` bytes from the
 /// implementation, keeping them buffered.
-pub fn drainPreserve(w: *Writer, preserve_length: usize) Error!void {
+pub fn drainPreserve(w: *Writer, preserve_len: usize) Error!void {
-    const temp_end = w.end -| preserve_length;
+    const temp_end = w.end -| preserve_len;
     const preserved = w.buffer[temp_end..w.end];
     w.end = temp_end;
     defer w.end += preserved.len;
 @@ -310,24 +310,38 @@ pub fn writableSliceGreedy(w: *Writer, minimum_length: usize) Error![]u8 {
 }
 /// Asserts the provided buffer has total capacity enough for `minimum_length`
-/// and `preserve_length` combined.
+/// and `preserve_len` combined.
 ///
 /// Does not `advance` the buffer end position.
 ///
-/// When draining the buffer, ensures that at least `preserve_length` bytes
+/// When draining the buffer, ensures that at least `preserve_len` bytes
 /// remain buffered.
 ///
-/// If `preserve_length` is zero, this is equivalent to `writableSliceGreedy`.
+/// If `preserve_len` is zero, this is equivalent to `writableSliceGreedy`.
-pub fn writableSliceGreedyPreserve(w: *Writer, preserve_length: usize, minimum_length: usize) Error![]u8 {
+pub fn writableSliceGreedyPreserve(w: *Writer, preserve_len: usize, minimum_length: usize) Error![]u8 {
-    assert(w.buffer.len >= preserve_length + minimum_length);
+    assert(w.buffer.len >= preserve_len + minimum_length);
     while (w.buffer.len - w.end < minimum_length) {
-        try drainPreserve(w, preserve_length);
+        try drainPreserve(w, preserve_len);
     } else {
         @branchHint(.likely);
         return w.buffer[w.end..];
     }
 }
+/// Asserts the provided buffer has total capacity enough for `len`.
+///
+/// Advances the buffer end position by `len`.
+///
+/// When draining the buffer, ensures that at least `preserve_len` bytes
+/// remain buffered.
+///
+/// If `preserve_len` is zero, this is equivalent to `writableSlice`.
+pub fn writableSlicePreserve(w: *Writer, preserve_len: usize, len: usize) Error![]u8 {
+    const big_slice = try w.writableSliceGreedyPreserve(preserve_len, len);
+    advance(w, len);
+    return big_slice[0..len];
+}
 pub const WritableVectorIterator = struct {
     first: []u8,
     middle: []const []u8 = &.{},
 @@ -523,16 +537,16 @@ pub fn write(w: *Writer, bytes: []const u8) Error!usize {
     return w.vtable.drain(w, &.{bytes}, 1);
 }
-/// Asserts `buffer` capacity exceeds `preserve_length`.
+/// Asserts `buffer` capacity exceeds `preserve_len`.
-pub fn writePreserve(w: *Writer, preserve_length: usize, bytes: []const u8) Error!usize {
+pub fn writePreserve(w: *Writer, preserve_len: usize, bytes: []const u8) Error!usize {
-    assert(preserve_length <= w.buffer.len);
+    assert(preserve_len <= w.buffer.len);
     if (w.end + bytes.len <= w.buffer.len) {
         @branchHint(.likely);
         @memcpy(w.buffer[w.end..][0..bytes.len], bytes);
         w.end += bytes.len;
         return bytes.len;
     }
-    const temp_end = w.end -| preserve_length;
+    const temp_end = w.end -| preserve_len;
     const preserved = w.buffer[temp_end..w.end];
     w.end = temp_end;
     defer w.end += preserved.len;
 @@ -552,13 +566,13 @@ pub fn writeAll(w: *Writer, bytes: []const u8) Error!void {
 /// Calls `drain` as many times as necessary such that all of `bytes` are
 /// transferred.
 ///
-/// When draining the buffer, ensures that at least `preserve_length` bytes
+/// When draining the buffer, ensures that at least `preserve_len` bytes
 /// remain buffered.
 ///
-/// Asserts `buffer` capacity exceeds `preserve_length`.
+/// Asserts `buffer` capacity exceeds `preserve_len`.
-pub fn writeAllPreserve(w: *Writer, preserve_length: usize, bytes: []const u8) Error!void {
+pub fn writeAllPreserve(w: *Writer, preserve_len: usize, bytes: []const u8) Error!void {
     var index: usize = 0;
-    while (index < bytes.len) index += try w.writePreserve(preserve_length, bytes[index..]);
+    while (index < bytes.len) index += try w.writePreserve(preserve_len, bytes[index..]);
 }
 /// Renders fmt string with args, calling `writer` with slices of bytes.
 @@ -761,11 +775,11 @@ pub fn writeByte(w: *Writer, byte: u8) Error!void {
     }
 }
-/// When draining the buffer, ensures that at least `preserve_length` bytes
+/// When draining the buffer, ensures that at least `preserve_len` bytes
 /// remain buffered.
-pub fn writeBytePreserve(w: *Writer, preserve_length: usize, byte: u8) Error!void {
+pub fn writeBytePreserve(w: *Writer, preserve_len: usize, byte: u8) Error!void {
     while (w.buffer.len - w.end == 0) {
-        try drainPreserve(w, preserve_length);
+        try drainPreserve(w, preserve_len);
     } else {
         @branchHint(.likely);
         w.buffer[w.end] = byte;
 @@ -788,10 +802,42 @@ test splatByteAll {
     try testing.expectEqualStrings("7" ** 45, aw.writer.buffered());
 }
+pub fn splatBytePreserve(w: *Writer, preserve_len: usize, byte: u8, n: usize) Error!void {
+    const new_end = w.end + n;
+    if (new_end <= w.buffer.len) {
+        @memset(w.buffer[w.end..][0..n], byte);
+        w.end = new_end;
+        return;
+    }
+    // If `n` is large, we can ignore `preserve_len` up to a point.
+    var remaining = n;
+    while (remaining > preserve_len) {
+        assert(remaining != 0);
+        remaining -= try splatByte(w, byte, remaining - preserve_len);
+        if (w.end + remaining <= w.buffer.len) {
+            @memset(w.buffer[w.end..][0..remaining], byte);
+            w.end += remaining;
+            return;
+        }
+    }
+    // All the next bytes received must be preserved.
+    if (preserve_len < w.end) {
+        @memmove(w.buffer[0..preserve_len], w.buffer[w.end - preserve_len ..][0..preserve_len]);
+        w.end = preserve_len;
+    }
+    while (remaining > 0) remaining -= try w.splatByte(byte, remaining);
+}
 /// Writes the same byte many times, allowing short writes.
 ///
 /// Does maximum of one underlying `VTable.drain`.
 pub fn splatByte(w: *Writer, byte: u8, n: usize) Error!usize {
+    if (w.end + n <= w.buffer.len) {
+        @branchHint(.likely);
+        @memset(w.buffer[w.end..][0..n], byte);
+        w.end += n;
+        return n;
+    }
     return writeSplat(w, &.{&.{byte}}, n);
 }
 @@ -801,9 +847,10 @@ pub fn splatBytesAll(w: *Writer, bytes: []const u8, splat: usize) Error!void {
     var remaining_bytes: usize = bytes.len * splat;
     remaining_bytes -= try w.splatBytes(bytes, splat);
     while (remaining_bytes > 0) {
-        const leftover = remaining_bytes % bytes.len;
+        const leftover_splat = remaining_bytes / bytes.len;
-        const buffers: [2][]const u8 = .{ bytes[bytes.len - leftover ..], bytes };
+        const leftover_bytes = remaining_bytes % bytes.len;
-        remaining_bytes -= try w.writeSplat(&buffers, splat);
+        const buffers: [2][]const u8 = .{ bytes[bytes.len - leftover_bytes ..], bytes };
+        remaining_bytes -= try w.writeSplat(&buffers, leftover_splat);
     }
 }

lib/std/compress.zig+2-58

...	@@ -1,75 +1,19 @@	...	@@ -1,75 +1,19 @@
1	//! Compression algorithms.	1	//! Compression algorithms.
2		2
3	const std = @import("std.zig");
4
5	pub const flate = @import("compress/flate.zig");	3	pub const flate = @import("compress/flate.zig");
6	pub const gzip = @import("compress/gzip.zig");	4	pub const gzip = @import("compress/gzip.zig");
7	pub const zlib = @import("compress/zlib.zig");	5	pub const zlib = @import("compress/zlib.zig");
8	pub const lzma = @import("compress/lzma.zig");	6	pub const lzma = @import("compress/lzma.zig");
9	pub const lzma2 = @import("compress/lzma2.zig");	7	pub const lzma2 = @import("compress/lzma2.zig");
10	pub const xz = @import("compress/xz.zig");	8	pub const xz = @import("compress/xz.zig");
11	pub const zstd = @import("compress/zstandard.zig");	9	pub const zstd = @import("compress/zstd.zig");
12
13	pub fn HashedReader(ReaderType: type, HasherType: type) type {
14	return struct {
15	child_reader: ReaderType,
16	hasher: HasherType,
17
18	pub const Error = ReaderType.Error;
19	pub const Reader = std.io.GenericReader(*@This(), Error, read);
20
21	pub fn read(self: *@This(), buf: []u8) Error!usize {
22	const amt = try self.child_reader.read(buf);
23	self.hasher.update(buf[0..amt]);
24	return amt;
25	}
26
27	pub fn reader(self: *@This()) Reader {
28	return .{ .context = self };
29	}
30	};
31	}
32
33	pub fn hashedReader(
34	reader: anytype,
35	hasher: anytype,
36	) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
37	return .{ .child_reader = reader, .hasher = hasher };
38	}
39
40	pub fn HashedWriter(WriterType: type, HasherType: type) type {
41	return struct {
42	child_writer: WriterType,
43	hasher: HasherType,
44
45	pub const Error = WriterType.Error;
46	pub const Writer = std.io.GenericWriter(*@This(), Error, write);
47
48	pub fn write(self: *@This(), buf: []const u8) Error!usize {
49	const amt = try self.child_writer.write(buf);
50	self.hasher.update(buf[0..amt]);
51	return amt;
52	}
53
54	pub fn writer(self: *@This()) Writer {
55	return .{ .context = self };
56	}
57	};
58	}
59
60	pub fn hashedWriter(
61	writer: anytype,
62	hasher: anytype,
63	) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
64	return .{ .child_writer = writer, .hasher = hasher };
65	}
66		10
67	test {	11	test {
		12	_ = flate;
68	_ = lzma;	13	_ = lzma;
69	_ = lzma2;	14	_ = lzma2;
70	_ = xz;	15	_ = xz;
71	_ = zstd;	16	_ = zstd;
72	_ = flate;
73	_ = gzip;	17	_ = gzip;
74	_ = zlib;	18	_ = zlib;
75	}	19	}

lib/std/compress/xz.zig+30-3

...	@@ -47,7 +47,7 @@ pub fn Decompress(comptime ReaderType: type) type {	...	@@ -47,7 +47,7 @@ pub fn Decompress(comptime ReaderType: type) type {
47		47
48	var check: Check = undefined;	48	var check: Check = undefined;
49	const hash_a = blk: {	49	const hash_a = blk: {
50	var hasher = std.compress.hashedReader(source, Crc32.init());	50	var hasher = hashedReader(source, Crc32.init());
51	try readStreamFlags(hasher.reader(), &check);	51	try readStreamFlags(hasher.reader(), &check);
52	break :blk hasher.hasher.final();	52	break :blk hasher.hasher.final();
53	};	53	};
...	@@ -80,7 +80,7 @@ pub fn Decompress(comptime ReaderType: type) type {	...	@@ -80,7 +80,7 @@ pub fn Decompress(comptime ReaderType: type) type {
80	return r;	80	return r;
81		81
82	const index_size = blk: {	82	const index_size = blk: {
83	var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());	83	var hasher = hashedReader(self.in_reader, Crc32.init());
84	hasher.hasher.update(&[1]u8{0x00});	84	hasher.hasher.update(&[1]u8{0x00});
85		85
86	var counter = std.io.countingReader(hasher.reader());	86	var counter = std.io.countingReader(hasher.reader());
...	@@ -115,7 +115,7 @@ pub fn Decompress(comptime ReaderType: type) type {	...	@@ -115,7 +115,7 @@ pub fn Decompress(comptime ReaderType: type) type {
115	const hash_a = try self.in_reader.readInt(u32, .little);	115	const hash_a = try self.in_reader.readInt(u32, .little);
116		116
117	const hash_b = blk: {	117	const hash_b = blk: {
118	var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());	118	var hasher = hashedReader(self.in_reader, Crc32.init());
119	const hashed_reader = hasher.reader();	119	const hashed_reader = hasher.reader();
120		120
121	const backward_size = (@as(u64, try hashed_reader.readInt(u32, .little)) + 1) * 4;	121	const backward_size = (@as(u64, try hashed_reader.readInt(u32, .little)) + 1) * 4;
...	@@ -140,6 +140,33 @@ pub fn Decompress(comptime ReaderType: type) type {	...	@@ -140,6 +140,33 @@ pub fn Decompress(comptime ReaderType: type) type {
140	};	140	};
141	}	141	}
142		142
		143	pub fn HashedReader(ReaderType: type, HasherType: type) type {
		144	return struct {
		145	child_reader: ReaderType,
		146	hasher: HasherType,
		147
		148	pub const Error = ReaderType.Error;
		149	pub const Reader = std.io.GenericReader(*@This(), Error, read);
		150
		151	pub fn read(self: *@This(), buf: []u8) Error!usize {
		152	const amt = try self.child_reader.read(buf);
		153	self.hasher.update(buf[0..amt]);
		154	return amt;
		155	}
		156
		157	pub fn reader(self: *@This()) Reader {
		158	return .{ .context = self };
		159	}
		160	};
		161	}
		162
		163	pub fn hashedReader(
		164	reader: anytype,
		165	hasher: anytype,
		166	) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
		167	return .{ .child_reader = reader, .hasher = hasher };
		168	}
		169
143	test {	170	test {
144	_ = @import("xz/test.zig");	171	_ = @import("xz/test.zig");
145	}	172	}

lib/std/compress/xz/block.zig+1-1

...	@@ -91,7 +91,7 @@ pub fn Decoder(comptime ReaderType: type) type {	...	@@ -91,7 +91,7 @@ pub fn Decoder(comptime ReaderType: type) type {
91		91
92	// Block Header	92	// Block Header
93	{	93	{
94	var header_hasher = std.compress.hashedReader(block_reader, Crc32.init());	94	var header_hasher = xz.hashedReader(block_reader, Crc32.init());
95	const header_reader = header_hasher.reader();	95	const header_reader = header_hasher.reader();
96		96
97	const header_size = @as(u64, try header_reader.readByte()) * 4;	97	const header_size = @as(u64, try header_reader.readByte()) * 4;

lib/std/compress/zstandard.zig deleted-310

...	@@ -1,310 +0,0 @@
1	const std = @import("std");
2	const RingBuffer = std.RingBuffer;
3
4	const types = @import("zstandard/types.zig");
5	pub const frame = types.frame;
6	pub const compressed_block = types.compressed_block;
7
8	pub const decompress = @import("zstandard/decompress.zig");
9
10	pub const DecompressorOptions = struct {
11	verify_checksum: bool = true,
12	window_buffer: []u8,
13
14	/// Recommended amount by the standard. Lower than this may result
15	/// in inability to decompress common streams.
16	pub const default_window_buffer_len = 8 * 1024 * 1024;
17	};
18
19	pub fn Decompressor(comptime ReaderType: type) type {
20	return struct {
21	const Self = @This();
22
23	const table_size_max = types.compressed_block.table_size_max;
24
25	source: std.io.CountingReader(ReaderType),
26	state: enum { NewFrame, InFrame, LastBlock },
27	decode_state: decompress.block.DecodeState,
28	frame_context: decompress.FrameContext,
29	buffer: WindowBuffer,
30	literal_fse_buffer: [table_size_max.literal]types.compressed_block.Table.Fse,
31	match_fse_buffer: [table_size_max.match]types.compressed_block.Table.Fse,
32	offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,
33	literals_buffer: [types.block_size_max]u8,
34	sequence_buffer: [types.block_size_max]u8,
35	verify_checksum: bool,
36	checksum: ?u32,
37	current_frame_decompressed_size: usize,
38
39	const WindowBuffer = struct {
40	data: []u8 = undefined,
41	read_index: usize = 0,
42	write_index: usize = 0,
43	};
44
45	pub const Error = ReaderType.Error \|\| error{
46	ChecksumFailure,
47	DictionaryIdFlagUnsupported,
48	MalformedBlock,
49	MalformedFrame,
50	OutOfMemory,
51	};
52
53	pub const Reader = std.io.GenericReader(*Self, Error, read);
54
55	pub fn init(source: ReaderType, options: DecompressorOptions) Self {
56	return .{
57	.source = std.io.countingReader(source),
58	.state = .NewFrame,
59	.decode_state = undefined,
60	.frame_context = undefined,
61	.buffer = .{ .data = options.window_buffer },
62	.literal_fse_buffer = undefined,
63	.match_fse_buffer = undefined,
64	.offset_fse_buffer = undefined,
65	.literals_buffer = undefined,
66	.sequence_buffer = undefined,
67	.verify_checksum = options.verify_checksum,
68	.checksum = undefined,
69	.current_frame_decompressed_size = undefined,
70	};
71	}
72
73	fn frameInit(self: *Self) !void {
74	const source_reader = self.source.reader();
75	switch (try decompress.decodeFrameHeader(source_reader)) {
76	.skippable => \|header\| {
77	try source_reader.skipBytes(header.frame_size, .{});
78	self.state = .NewFrame;
79	},
80	.zstandard => \|header\| {
81	const frame_context = try decompress.FrameContext.init(
82	header,
83	self.buffer.data.len,
84	self.verify_checksum,
85	);
86
87	const decode_state = decompress.block.DecodeState.init(
88	&self.literal_fse_buffer,
89	&self.match_fse_buffer,
90	&self.offset_fse_buffer,
91	);
92
93	self.decode_state = decode_state;
94	self.frame_context = frame_context;
95
96	self.checksum = null;
97	self.current_frame_decompressed_size = 0;
98
99	self.state = .InFrame;
100	},
101	}
102	}
103
104	pub fn reader(self: *Self) Reader {
105	return .{ .context = self };
106	}
107
108	pub fn read(self: *Self, buffer: []u8) Error!usize {
109	if (buffer.len == 0) return 0;
110
111	var size: usize = 0;
112	while (size == 0) {
113	while (self.state == .NewFrame) {
114	const initial_count = self.source.bytes_read;
115	self.frameInit() catch \|err\| switch (err) {
116	error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
117	error.EndOfStream => return if (self.source.bytes_read == initial_count)
118	0
119	else
120	error.MalformedFrame,
121	else => return error.MalformedFrame,
122	};
123	}
124	size = try self.readInner(buffer);
125	}
126	return size;
127	}
128
129	fn readInner(self: *Self, buffer: []u8) Error!usize {
130	std.debug.assert(self.state != .NewFrame);
131
132	var ring_buffer = RingBuffer{
133	.data = self.buffer.data,
134	.read_index = self.buffer.read_index,
135	.write_index = self.buffer.write_index,
136	};
137	defer {
138	self.buffer.read_index = ring_buffer.read_index;
139	self.buffer.write_index = ring_buffer.write_index;
140	}
141
142	const source_reader = self.source.reader();
143	while (ring_buffer.isEmpty() and self.state != .LastBlock) {
144	const header_bytes = source_reader.readBytesNoEof(3) catch
145	return error.MalformedFrame;
146	const block_header = decompress.block.decodeBlockHeader(&header_bytes);
147
148	decompress.block.decodeBlockReader(
149	&ring_buffer,
150	source_reader,
151	block_header,
152	&self.decode_state,
153	self.frame_context.block_size_max,
154	&self.literals_buffer,
155	&self.sequence_buffer,
156	) catch
157	return error.MalformedBlock;
158
159	if (self.frame_context.content_size) \|size\| {
160	if (self.current_frame_decompressed_size > size) return error.MalformedFrame;
161	}
162
163	const size = ring_buffer.len();
164	self.current_frame_decompressed_size += size;
165
166	if (self.frame_context.hasher_opt) \|*hasher\| {
167	if (size > 0) {
168	const written_slice = ring_buffer.sliceLast(size);
169	hasher.update(written_slice.first);
170	hasher.update(written_slice.second);
171	}
172	}
173	if (block_header.last_block) {
174	self.state = .LastBlock;
175	if (self.frame_context.has_checksum) {
176	const checksum = source_reader.readInt(u32, .little) catch
177	return error.MalformedFrame;
178	if (self.verify_checksum) {
179	if (self.frame_context.hasher_opt) \|*hasher\| {
180	if (checksum != decompress.computeChecksum(hasher))
181	return error.ChecksumFailure;
182	}
183	}
184	}
185	if (self.frame_context.content_size) \|content_size\| {
186	if (content_size != self.current_frame_decompressed_size) {
187	return error.MalformedFrame;
188	}
189	}
190	}
191	}
192
193	const size = @min(ring_buffer.len(), buffer.len);
194	if (size > 0) {
195	ring_buffer.readFirstAssumeLength(buffer, size);
196	}
197	if (self.state == .LastBlock and ring_buffer.len() == 0) {
198	self.state = .NewFrame;
199	}
200	return size;
201	}
202	};
203	}
204
205	pub fn decompressor(reader: anytype, options: DecompressorOptions) Decompressor(@TypeOf(reader)) {
206	return Decompressor(@TypeOf(reader)).init(reader, options);
207	}
208
209	fn testDecompress(data: []const u8) ![]u8 {
210	const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
211	defer std.testing.allocator.free(window_buffer);
212
213	var in_stream = std.io.fixedBufferStream(data);
214	var zstd_stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
215	const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
216	return result;
217	}
218
219	fn testReader(data: []const u8, comptime expected: []const u8) !void {
220	const buf = try testDecompress(data);
221	defer std.testing.allocator.free(buf);
222	try std.testing.expectEqualSlices(u8, expected, buf);
223	}
224
225	test "decompression" {
226	const uncompressed = @embedFile("testdata/rfc8478.txt");
227	const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
228	const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
229
230	const buffer = try std.testing.allocator.alloc(u8, uncompressed.len);
231	defer std.testing.allocator.free(buffer);
232
233	const res3 = try decompress.decode(buffer, compressed3, true);
234	try std.testing.expectEqual(uncompressed.len, res3);
235	try std.testing.expectEqualSlices(u8, uncompressed, buffer);
236
237	@memset(buffer, undefined);
238	const res19 = try decompress.decode(buffer, compressed19, true);
239	try std.testing.expectEqual(uncompressed.len, res19);
240	try std.testing.expectEqualSlices(u8, uncompressed, buffer);
241
242	try testReader(compressed3, uncompressed);
243	try testReader(compressed19, uncompressed);
244	}
245
246	fn expectEqualDecoded(expected: []const u8, input: []const u8) !void {
247	{
248	const result = try decompress.decodeAlloc(std.testing.allocator, input, false, 1 << 23);
249	defer std.testing.allocator.free(result);
250	try std.testing.expectEqualStrings(expected, result);
251	}
252
253	{
254	var buffer = try std.testing.allocator.alloc(u8, 2 * expected.len);
255	defer std.testing.allocator.free(buffer);
256
257	const size = try decompress.decode(buffer, input, false);
258	try std.testing.expectEqualStrings(expected, buffer[0..size]);
259	}
260	}
261
262	fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {
263	const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
264	defer std.testing.allocator.free(window_buffer);
265
266	var in_stream = std.io.fixedBufferStream(input);
267	var stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
268
269	const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
270	defer std.testing.allocator.free(result);
271
272	try std.testing.expectEqualStrings(expected, result);
273	}
274
275	test "zero sized block" {
276	const input_raw =
277	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
278	"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
279	"\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
280
281	const input_rle =
282	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
283	"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
284	"\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
285	"\xaa"; // block_content
286
287	try expectEqualDecoded("", input_raw);
288	try expectEqualDecoded("", input_rle);
289	try expectEqualDecodedStreaming("", input_raw);
290	try expectEqualDecodedStreaming("", input_rle);
291	}
292
293	test "declared raw literals size too large" {
294	const input_raw =
295	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
296	"\x00\x00" ++ // frame header: everything unset, window descriptor zero
297	"\x95\x00\x00" ++ // block header with: last_block set, block_type compressed, block_size 18
298	"\xbc\xf3\xae" ++ // literals section header with: type raw, size_format 3, regenerated_size 716603
299	"\xa5\x9f\xe3"; // some bytes of literal content - the content is shorter than regenerated_size
300
301	// Note that the regenerated_size in the above input is larger than block maximum size, so the
302	// block can't be valid as it is a raw literals block.
303
304	var fbs = std.io.fixedBufferStream(input_raw);
305	var window: [1024]u8 = undefined;
306	var stream = decompressor(fbs.reader(), .{ .window_buffer = &window });
307
308	var buf: [1024]u8 = undefined;
309	try std.testing.expectError(error.MalformedBlock, stream.read(&buf));
310	}

lib/std/compress/zstandard/decode/block.zig deleted-1149

...	@@ -1,1149 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const RingBuffer = std.RingBuffer;
4
5	const types = @import("../types.zig");
6	const frame = types.frame;
7	const Table = types.compressed_block.Table;
8	const LiteralsSection = types.compressed_block.LiteralsSection;
9	const SequencesSection = types.compressed_block.SequencesSection;
10
11	const huffman = @import("huffman.zig");
12	const readers = @import("../readers.zig");
13
14	const decodeFseTable = @import("fse.zig").decodeFseTable;
15
16	pub const Error = error{
17	BlockSizeOverMaximum,
18	MalformedBlockSize,
19	ReservedBlock,
20	MalformedRleBlock,
21	MalformedCompressedBlock,
22	};
23
24	pub const DecodeState = struct {
25	repeat_offsets: [3]u32,
26
27	offset: StateData(8),
28	match: StateData(9),
29	literal: StateData(9),
30
31	offset_fse_buffer: []Table.Fse,
32	match_fse_buffer: []Table.Fse,
33	literal_fse_buffer: []Table.Fse,
34
35	fse_tables_undefined: bool,
36
37	literal_stream_reader: readers.ReverseBitReader,
38	literal_stream_index: usize,
39	literal_streams: LiteralsSection.Streams,
40	literal_header: LiteralsSection.Header,
41	huffman_tree: ?LiteralsSection.HuffmanTree,
42
43	literal_written_count: usize,
44	written_count: usize = 0,
45
46	fn StateData(comptime max_accuracy_log: comptime_int) type {
47	return struct {
48	state: State,
49	table: Table,
50	accuracy_log: u8,
51
52	const State = std.meta.Int(.unsigned, max_accuracy_log);
53	};
54	}
55
56	pub fn init(
57	literal_fse_buffer: []Table.Fse,
58	match_fse_buffer: []Table.Fse,
59	offset_fse_buffer: []Table.Fse,
60	) DecodeState {
61	return DecodeState{
62	.repeat_offsets = .{
63	types.compressed_block.start_repeated_offset_1,
64	types.compressed_block.start_repeated_offset_2,
65	types.compressed_block.start_repeated_offset_3,
66	},
67
68	.offset = undefined,
69	.match = undefined,
70	.literal = undefined,
71
72	.literal_fse_buffer = literal_fse_buffer,
73	.match_fse_buffer = match_fse_buffer,
74	.offset_fse_buffer = offset_fse_buffer,
75
76	.fse_tables_undefined = true,
77
78	.literal_written_count = 0,
79	.literal_header = undefined,
80	.literal_streams = undefined,
81	.literal_stream_reader = undefined,
82	.literal_stream_index = undefined,
83	.huffman_tree = null,
84
85	.written_count = 0,
86	};
87	}
88
89	/// Prepare the decoder to decode a compressed block. Loads the literals
90	/// stream and Huffman tree from `literals` and reads the FSE tables from
91	/// `source`.
92	///
93	/// Errors returned:
94	/// - `error.BitStreamHasNoStartBit` if the (reversed) literal bitstream's
95	/// first byte does not have any bits set
96	/// - `error.TreelessLiteralsFirst` `literals` is a treeless literals
97	/// section and the decode state does not have a Huffman tree from a
98	/// previous block
99	/// - `error.RepeatModeFirst` on the first call if one of the sequence FSE
100	/// tables is set to repeat mode
101	/// - `error.MalformedAccuracyLog` if an FSE table has an invalid accuracy
102	/// - `error.MalformedFseTable` if there are errors decoding an FSE table
103	/// - `error.EndOfStream` if `source` ends before all FSE tables are read
104	pub fn prepare(
105	self: *DecodeState,
106	source: anytype,
107	literals: LiteralsSection,
108	sequences_header: SequencesSection.Header,
109	) !void {
110	self.literal_written_count = 0;
111	self.literal_header = literals.header;
112	self.literal_streams = literals.streams;
113
114	if (literals.huffman_tree) \|tree\| {
115	self.huffman_tree = tree;
116	} else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
117	return error.TreelessLiteralsFirst;
118	}
119
120	switch (literals.header.block_type) {
121	.raw, .rle => {},
122	.compressed, .treeless => {
123	self.literal_stream_index = 0;
124	switch (literals.streams) {
125	.one => \|slice\| try self.initLiteralStream(slice),
126	.four => \|streams\| try self.initLiteralStream(streams[0]),
127	}
128	},
129	}
130
131	if (sequences_header.sequence_count > 0) {
132	try self.updateFseTable(source, .literal, sequences_header.literal_lengths);
133	try self.updateFseTable(source, .offset, sequences_header.offsets);
134	try self.updateFseTable(source, .match, sequences_header.match_lengths);
135	self.fse_tables_undefined = false;
136	}
137	}
138
139	/// Read initial FSE states for sequence decoding.
140	///
141	/// Errors returned:
142	/// - `error.EndOfStream` if `bit_reader` does not contain enough bits.
143	pub fn readInitialFseState(self: DecodeState, bit_reader: readers.ReverseBitReader) error{EndOfStream}!void {
144	self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
145	self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
146	self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
147	}
148
149	fn updateRepeatOffset(self: *DecodeState, offset: u32) void {
150	self.repeat_offsets[2] = self.repeat_offsets[1];
151	self.repeat_offsets[1] = self.repeat_offsets[0];
152	self.repeat_offsets[0] = offset;
153	}
154
155	fn useRepeatOffset(self: *DecodeState, index: usize) u32 {
156	if (index == 1)
157	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
158	else if (index == 2) {
159	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
160	std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
161	}
162	return self.repeat_offsets[0];
163	}
164
165	const DataType = enum { offset, match, literal };
166
167	fn updateState(
168	self: *DecodeState,
169	comptime choice: DataType,
170	bit_reader: *readers.ReverseBitReader,
171	) error{ MalformedFseBits, EndOfStream }!void {
172	switch (@field(self, @tagName(choice)).table) {
173	.rle => {},
174	.fse => \|table\| {
175	const data = table[@field(self, @tagName(choice)).state];
176	const T = @TypeOf(@field(self, @tagName(choice))).State;
177	const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
178	const next_state = std.math.cast(
179	@TypeOf(@field(self, @tagName(choice))).State,
180	data.baseline + bits_summand,
181	) orelse return error.MalformedFseBits;
182	@field(self, @tagName(choice)).state = next_state;
183	},
184	}
185	}
186
187	const FseTableError = error{
188	MalformedFseTable,
189	MalformedAccuracyLog,
190	RepeatModeFirst,
191	EndOfStream,
192	};
193
194	fn updateFseTable(
195	self: *DecodeState,
196	source: anytype,
197	comptime choice: DataType,
198	mode: SequencesSection.Header.Mode,
199	) !void {
200	const field_name = @tagName(choice);
201	switch (mode) {
202	.predefined => {
203	@field(self, field_name).accuracy_log =
204	@field(types.compressed_block.default_accuracy_log, field_name);
205
206	@field(self, field_name).table =
207	@field(types.compressed_block, "predefined_" ++ field_name ++ "_fse_table");
208	},
209	.rle => {
210	@field(self, field_name).accuracy_log = 0;
211	@field(self, field_name).table = .{ .rle = try source.readByte() };
212	},
213	.fse => {
214	var bit_reader = readers.bitReader(source);
215
216	const table_size = try decodeFseTable(
217	&bit_reader,
218	@field(types.compressed_block.table_symbol_count_max, field_name),
219	@field(types.compressed_block.table_accuracy_log_max, field_name),
220	@field(self, field_name ++ "_fse_buffer"),
221	);
222	@field(self, field_name).table = .{
223	.fse = @field(self, field_name ++ "_fse_buffer")[0..table_size],
224	};
225	@field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
226	},
227	.repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
228	}
229	}
230
231	const Sequence = struct {
232	literal_length: u32,
233	match_length: u32,
234	offset: u32,
235	};
236
237	fn nextSequence(
238	self: *DecodeState,
239	bit_reader: *readers.ReverseBitReader,
240	) error{ InvalidBitStream, EndOfStream }!Sequence {
241	const raw_code = self.getCode(.offset);
242	const offset_code = std.math.cast(u5, raw_code) orelse {
243	return error.InvalidBitStream;
244	};
245	const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
246
247	const match_code = self.getCode(.match);
248	if (match_code >= types.compressed_block.match_length_code_table.len)
249	return error.InvalidBitStream;
250	const match = types.compressed_block.match_length_code_table[match_code];
251	const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
252
253	const literal_code = self.getCode(.literal);
254	if (literal_code >= types.compressed_block.literals_length_code_table.len)
255	return error.InvalidBitStream;
256	const literal = types.compressed_block.literals_length_code_table[literal_code];
257	const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
258
259	const offset = if (offset_value > 3) offset: {
260	const offset = offset_value - 3;
261	self.updateRepeatOffset(offset);
262	break :offset offset;
263	} else offset: {
264	if (literal_length == 0) {
265	if (offset_value == 3) {
266	const offset = self.repeat_offsets[0] - 1;
267	self.updateRepeatOffset(offset);
268	break :offset offset;
269	}
270	break :offset self.useRepeatOffset(offset_value);
271	}
272	break :offset self.useRepeatOffset(offset_value - 1);
273	};
274
275	if (offset == 0) return error.InvalidBitStream;
276
277	return .{
278	.literal_length = literal_length,
279	.match_length = match_length,
280	.offset = offset,
281	};
282	}
283
284	fn executeSequenceSlice(
285	self: *DecodeState,
286	dest: []u8,
287	write_pos: usize,
288	sequence: Sequence,
289	) (error{MalformedSequence} \|\| DecodeLiteralsError)!void {
290	if (sequence.offset > write_pos + sequence.literal_length) return error.MalformedSequence;
291
292	try self.decodeLiteralsSlice(dest[write_pos..], sequence.literal_length);
293	const copy_start = write_pos + sequence.literal_length - sequence.offset;
294	for (
295	dest[write_pos + sequence.literal_length ..][0..sequence.match_length],
296	dest[copy_start..][0..sequence.match_length],
297	) \|d, s\| d. = s;
298	self.written_count += sequence.match_length;
299	}
300
301	fn executeSequenceRingBuffer(
302	self: *DecodeState,
303	dest: *RingBuffer,
304	sequence: Sequence,
305	) (error{MalformedSequence} \|\| DecodeLiteralsError)!void {
306	if (sequence.offset > @min(dest.data.len, self.written_count + sequence.literal_length))
307	return error.MalformedSequence;
308
309	try self.decodeLiteralsRingBuffer(dest, sequence.literal_length);
310	const copy_start = dest.write_index + dest.data.len - sequence.offset;
311	const copy_slice = dest.sliceAt(copy_start, sequence.match_length);
312	dest.writeSliceForwardsAssumeCapacity(copy_slice.first);
313	dest.writeSliceForwardsAssumeCapacity(copy_slice.second);
314	self.written_count += sequence.match_length;
315	}
316
317	const DecodeSequenceError = error{
318	InvalidBitStream,
319	EndOfStream,
320	MalformedSequence,
321	MalformedFseBits,
322	} \|\| DecodeLiteralsError;
323
324	/// Decode one sequence from `bit_reader` into `dest`, written starting at
325	/// `write_pos` and update FSE states if `last_sequence` is `false`.
326	/// `prepare()` must be called for the block before attempting to decode
327	/// sequences.
328	///
329	/// Errors returned:
330	/// - `error.MalformedSequence` if the decompressed sequence would be
331	/// longer than `sequence_size_limit` or the sequence's offset is too
332	/// large
333	/// - `error.UnexpectedEndOfLiteralStream` if the decoder state's literal
334	/// streams do not contain enough literals for the sequence (this may
335	/// mean the literal stream or the sequence is malformed).
336	/// - `error.InvalidBitStream` if the FSE sequence bitstream is malformed
337	/// - `error.EndOfStream` if `bit_reader` does not contain enough bits
338	/// - `error.DestTooSmall` if `dest` is not large enough to holde the
339	/// decompressed sequence
340	pub fn decodeSequenceSlice(
341	self: *DecodeState,
342	dest: []u8,
343	write_pos: usize,
344	bit_reader: *readers.ReverseBitReader,
345	sequence_size_limit: usize,
346	last_sequence: bool,
347	) (error{DestTooSmall} \|\| DecodeSequenceError)!usize {
348	const sequence = try self.nextSequence(bit_reader);
349	const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
350	if (sequence_length > sequence_size_limit) return error.MalformedSequence;
351	if (sequence_length > dest[write_pos..].len) return error.DestTooSmall;
352
353	try self.executeSequenceSlice(dest, write_pos, sequence);
354	if (!last_sequence) {
355	try self.updateState(.literal, bit_reader);
356	try self.updateState(.match, bit_reader);
357	try self.updateState(.offset, bit_reader);
358	}
359	return sequence_length;
360	}
361
362	/// Decode one sequence from `bit_reader` into `dest`; see
363	/// `decodeSequenceSlice`.
364	pub fn decodeSequenceRingBuffer(
365	self: *DecodeState,
366	dest: *RingBuffer,
367	bit_reader: anytype,
368	sequence_size_limit: usize,
369	last_sequence: bool,
370	) DecodeSequenceError!usize {
371	const sequence = try self.nextSequence(bit_reader);
372	const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
373	if (sequence_length > sequence_size_limit) return error.MalformedSequence;
374
375	try self.executeSequenceRingBuffer(dest, sequence);
376	if (!last_sequence) {
377	try self.updateState(.literal, bit_reader);
378	try self.updateState(.match, bit_reader);
379	try self.updateState(.offset, bit_reader);
380	}
381	return sequence_length;
382	}
383
384	fn nextLiteralMultiStream(
385	self: *DecodeState,
386	) error{BitStreamHasNoStartBit}!void {
387	self.literal_stream_index += 1;
388	try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
389	}
390
391	fn initLiteralStream(self: *DecodeState, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
392	try self.literal_stream_reader.init(bytes);
393	}
394
395	fn isLiteralStreamEmpty(self: *DecodeState) bool {
396	switch (self.literal_streams) {
397	.one => return self.literal_stream_reader.isEmpty(),
398	.four => return self.literal_stream_index == 3 and self.literal_stream_reader.isEmpty(),
399	}
400	}
401
402	const LiteralBitsError = error{
403	BitStreamHasNoStartBit,
404	UnexpectedEndOfLiteralStream,
405	};
406	fn readLiteralsBits(
407	self: *DecodeState,
408	bit_count_to_read: u16,
409	) LiteralBitsError!u16 {
410	return self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch bits: {
411	if (self.literal_streams == .four and self.literal_stream_index < 3) {
412	try self.nextLiteralMultiStream();
413	break :bits self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch
414	return error.UnexpectedEndOfLiteralStream;
415	} else {
416	return error.UnexpectedEndOfLiteralStream;
417	}
418	};
419	}
420
421	const DecodeLiteralsError = error{
422	MalformedLiteralsLength,
423	NotFound,
424	} \|\| LiteralBitsError;
425
426	/// Decode `len` bytes of literals into `dest`.
427	///
428	/// Errors returned:
429	/// - `error.MalformedLiteralsLength` if the number of literal bytes
430	/// decoded by `self` plus `len` is greater than the regenerated size of
431	/// `literals`
432	/// - `error.UnexpectedEndOfLiteralStream` and `error.NotFound` if there
433	/// are problems decoding Huffman compressed literals
434	pub fn decodeLiteralsSlice(
435	self: *DecodeState,
436	dest: []u8,
437	len: usize,
438	) DecodeLiteralsError!void {
439	if (self.literal_written_count + len > self.literal_header.regenerated_size)
440	return error.MalformedLiteralsLength;
441
442	switch (self.literal_header.block_type) {
443	.raw => {
444	const literal_data = self.literal_streams.one[self.literal_written_count..][0..len];
445	@memcpy(dest[0..len], literal_data);
446	self.literal_written_count += len;
447	self.written_count += len;
448	},
449	.rle => {
450	for (0..len) \|i\| {
451	dest[i] = self.literal_streams.one[0];
452	}
453	self.literal_written_count += len;
454	self.written_count += len;
455	},
456	.compressed, .treeless => {
457	// const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
458	const huffman_tree = self.huffman_tree orelse unreachable;
459	const max_bit_count = huffman_tree.max_bit_count;
460	const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
461	huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
462	max_bit_count,
463	);
464	var bits_read: u4 = 0;
465	var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
466	var bit_count_to_read: u4 = starting_bit_count;
467	for (0..len) \|i\| {
468	var prefix: u16 = 0;
469	while (true) {
470	const new_bits = self.readLiteralsBits(bit_count_to_read) catch \|err\| {
471	return err;
472	};
473	prefix <<= bit_count_to_read;
474	prefix \|= new_bits;
475	bits_read += bit_count_to_read;
476	const result = huffman_tree.query(huffman_tree_index, prefix) catch \|err\| {
477	return err;
478	};
479
480	switch (result) {
481	.symbol => \|sym\| {
482	dest[i] = sym;
483	bit_count_to_read = starting_bit_count;
484	bits_read = 0;
485	huffman_tree_index = huffman_tree.symbol_count_minus_one;
486	break;
487	},
488	.index => \|index\| {
489	huffman_tree_index = index;
490	const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
491	huffman_tree.nodes[index].weight,
492	max_bit_count,
493	);
494	bit_count_to_read = bit_count - bits_read;
495	},
496	}
497	}
498	}
499	self.literal_written_count += len;
500	self.written_count += len;
501	},
502	}
503	}
504
505	/// Decode literals into `dest`; see `decodeLiteralsSlice()`.
506	pub fn decodeLiteralsRingBuffer(
507	self: *DecodeState,
508	dest: *RingBuffer,
509	len: usize,
510	) DecodeLiteralsError!void {
511	if (self.literal_written_count + len > self.literal_header.regenerated_size)
512	return error.MalformedLiteralsLength;
513
514	switch (self.literal_header.block_type) {
515	.raw => {
516	const literals_end = self.literal_written_count + len;
517	const literal_data = self.literal_streams.one[self.literal_written_count..literals_end];
518	dest.writeSliceAssumeCapacity(literal_data);
519	self.literal_written_count += len;
520	self.written_count += len;
521	},
522	.rle => {
523	for (0..len) \|_\| {
524	dest.writeAssumeCapacity(self.literal_streams.one[0]);
525	}
526	self.literal_written_count += len;
527	self.written_count += len;
528	},
529	.compressed, .treeless => {
530	// const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
531	const huffman_tree = self.huffman_tree orelse unreachable;
532	const max_bit_count = huffman_tree.max_bit_count;
533	const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
534	huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
535	max_bit_count,
536	);
537	var bits_read: u4 = 0;
538	var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
539	var bit_count_to_read: u4 = starting_bit_count;
540	for (0..len) \|_\| {
541	var prefix: u16 = 0;
542	while (true) {
543	const new_bits = try self.readLiteralsBits(bit_count_to_read);
544	prefix <<= bit_count_to_read;
545	prefix \|= new_bits;
546	bits_read += bit_count_to_read;
547	const result = try huffman_tree.query(huffman_tree_index, prefix);
548
549	switch (result) {
550	.symbol => \|sym\| {
551	dest.writeAssumeCapacity(sym);
552	bit_count_to_read = starting_bit_count;
553	bits_read = 0;
554	huffman_tree_index = huffman_tree.symbol_count_minus_one;
555	break;
556	},
557	.index => \|index\| {
558	huffman_tree_index = index;
559	const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
560	huffman_tree.nodes[index].weight,
561	max_bit_count,
562	);
563	bit_count_to_read = bit_count - bits_read;
564	},
565	}
566	}
567	}
568	self.literal_written_count += len;
569	self.written_count += len;
570	},
571	}
572	}
573
574	fn getCode(self: *DecodeState, comptime choice: DataType) u32 {
575	return switch (@field(self, @tagName(choice)).table) {
576	.rle => \|value\| value,
577	.fse => \|table\| table[@field(self, @tagName(choice)).state].symbol,
578	};
579	}
580	};
581
582	/// Decode a single block from `src` into `dest`. The beginning of `src` must be
583	/// the start of the block content (i.e. directly after the block header).
584	/// Increments `consumed_count` by the number of bytes read from `src` to decode
585	/// the block and returns the decompressed size of the block.
586	///
587	/// Errors returned:
588	///
589	/// - `error.BlockSizeOverMaximum` if block's size is larger than 1 << 17 or
590	/// `dest[written_count..].len`
591	/// - `error.MalformedBlockSize` if `src.len` is smaller than the block size
592	/// and the block is a raw or compressed block
593	/// - `error.ReservedBlock` if the block is a reserved block
594	/// - `error.MalformedRleBlock` if the block is an RLE block and `src.len < 1`
595	/// - `error.MalformedCompressedBlock` if there are errors decoding a
596	/// compressed block
597	/// - `error.DestTooSmall` is `dest` is not large enough to hold the
598	/// decompressed block
599	pub fn decodeBlock(
600	dest: []u8,
601	src: []const u8,
602	block_header: frame.Zstandard.Block.Header,
603	decode_state: *DecodeState,
604	consumed_count: *usize,
605	block_size_max: usize,
606	written_count: usize,
607	) (error{DestTooSmall} \|\| Error)!usize {
608	const block_size = block_header.block_size;
609	if (block_size_max < block_size) return error.BlockSizeOverMaximum;
610	switch (block_header.block_type) {
611	.raw => {
612	if (src.len < block_size) return error.MalformedBlockSize;
613	if (dest[written_count..].len < block_size) return error.DestTooSmall;
614	@memcpy(dest[written_count..][0..block_size], src[0..block_size]);
615	consumed_count.* += block_size;
616	decode_state.written_count += block_size;
617	return block_size;
618	},
619	.rle => {
620	if (src.len < 1) return error.MalformedRleBlock;
621	if (dest[written_count..].len < block_size) return error.DestTooSmall;
622	for (written_count..block_size + written_count) \|write_pos\| {
623	dest[write_pos] = src[0];
624	}
625	consumed_count.* += 1;
626	decode_state.written_count += block_size;
627	return block_size;
628	},
629	.compressed => {
630	if (src.len < block_size) return error.MalformedBlockSize;
631	var bytes_read: usize = 0;
632	const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
633	return error.MalformedCompressedBlock;
634	var fbs = std.io.fixedBufferStream(src[bytes_read..block_size]);
635	const fbs_reader = fbs.reader();
636	const sequences_header = decodeSequencesHeader(fbs_reader) catch
637	return error.MalformedCompressedBlock;
638
639	decode_state.prepare(fbs_reader, literals, sequences_header) catch
640	return error.MalformedCompressedBlock;
641
642	bytes_read += fbs.pos;
643
644	var bytes_written: usize = 0;
645	{
646	const bit_stream_bytes = src[bytes_read..block_size];
647	var bit_stream: readers.ReverseBitReader = undefined;
648	bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
649
650	if (sequences_header.sequence_count > 0) {
651	decode_state.readInitialFseState(&bit_stream) catch
652	return error.MalformedCompressedBlock;
653
654	var sequence_size_limit = block_size_max;
655	for (0..sequences_header.sequence_count) \|i\| {
656	const write_pos = written_count + bytes_written;
657	const decompressed_size = decode_state.decodeSequenceSlice(
658	dest,
659	write_pos,
660	&bit_stream,
661	sequence_size_limit,
662	i == sequences_header.sequence_count - 1,
663	) catch \|err\| switch (err) {
664	error.DestTooSmall => return error.DestTooSmall,
665	else => return error.MalformedCompressedBlock,
666	};
667	bytes_written += decompressed_size;
668	sequence_size_limit -= decompressed_size;
669	}
670	}
671
672	if (!bit_stream.isEmpty()) {
673	return error.MalformedCompressedBlock;
674	}
675	}
676
677	if (decode_state.literal_written_count < literals.header.regenerated_size) {
678	const len = literals.header.regenerated_size - decode_state.literal_written_count;
679	if (len > dest[written_count + bytes_written ..].len) return error.DestTooSmall;
680	decode_state.decodeLiteralsSlice(dest[written_count + bytes_written ..], len) catch
681	return error.MalformedCompressedBlock;
682	bytes_written += len;
683	}
684
685	switch (decode_state.literal_header.block_type) {
686	.treeless, .compressed => {
687	if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
688	},
689	.raw, .rle => {},
690	}
691
692	consumed_count.* += block_size;
693	return bytes_written;
694	},
695	.reserved => return error.ReservedBlock,
696	}
697	}
698
699	/// Decode a single block from `src` into `dest`; see `decodeBlock()`. Returns
700	/// the size of the decompressed block, which can be used with `dest.sliceLast()`
701	/// to get the decompressed bytes. `error.BlockSizeOverMaximum` is returned if
702	/// the block's compressed or decompressed size is larger than `block_size_max`.
703	pub fn decodeBlockRingBuffer(
704	dest: *RingBuffer,
705	src: []const u8,
706	block_header: frame.Zstandard.Block.Header,
707	decode_state: *DecodeState,
708	consumed_count: *usize,
709	block_size_max: usize,
710	) Error!usize {
711	const block_size = block_header.block_size;
712	if (block_size_max < block_size) return error.BlockSizeOverMaximum;
713	switch (block_header.block_type) {
714	.raw => {
715	if (src.len < block_size) return error.MalformedBlockSize;
716	// dest may have length zero if block_size == 0, causing division by zero in
717	// writeSliceAssumeCapacity()
718	if (block_size > 0) {
719	const data = src[0..block_size];
720	dest.writeSliceAssumeCapacity(data);
721	consumed_count.* += block_size;
722	decode_state.written_count += block_size;
723	}
724	return block_size;
725	},
726	.rle => {
727	if (src.len < 1) return error.MalformedRleBlock;
728	for (0..block_size) \|_\| {
729	dest.writeAssumeCapacity(src[0]);
730	}
731	consumed_count.* += 1;
732	decode_state.written_count += block_size;
733	return block_size;
734	},
735	.compressed => {
736	if (src.len < block_size) return error.MalformedBlockSize;
737	var bytes_read: usize = 0;
738	const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
739	return error.MalformedCompressedBlock;
740	var fbs = std.io.fixedBufferStream(src[bytes_read..block_size]);
741	const fbs_reader = fbs.reader();
742	const sequences_header = decodeSequencesHeader(fbs_reader) catch
743	return error.MalformedCompressedBlock;
744
745	decode_state.prepare(fbs_reader, literals, sequences_header) catch
746	return error.MalformedCompressedBlock;
747
748	bytes_read += fbs.pos;
749
750	var bytes_written: usize = 0;
751	{
752	const bit_stream_bytes = src[bytes_read..block_size];
753	var bit_stream: readers.ReverseBitReader = undefined;
754	bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
755
756	if (sequences_header.sequence_count > 0) {
757	decode_state.readInitialFseState(&bit_stream) catch
758	return error.MalformedCompressedBlock;
759
760	var sequence_size_limit = block_size_max;
761	for (0..sequences_header.sequence_count) \|i\| {
762	const decompressed_size = decode_state.decodeSequenceRingBuffer(
763	dest,
764	&bit_stream,
765	sequence_size_limit,
766	i == sequences_header.sequence_count - 1,
767	) catch return error.MalformedCompressedBlock;
768	bytes_written += decompressed_size;
769	sequence_size_limit -= decompressed_size;
770	}
771	}
772
773	if (!bit_stream.isEmpty()) {
774	return error.MalformedCompressedBlock;
775	}
776	}
777
778	if (decode_state.literal_written_count < literals.header.regenerated_size) {
779	const len = literals.header.regenerated_size - decode_state.literal_written_count;
780	decode_state.decodeLiteralsRingBuffer(dest, len) catch
781	return error.MalformedCompressedBlock;
782	bytes_written += len;
783	}
784
785	switch (decode_state.literal_header.block_type) {
786	.treeless, .compressed => {
787	if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
788	},
789	.raw, .rle => {},
790	}
791
792	consumed_count.* += block_size;
793	if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
794	return bytes_written;
795	},
796	.reserved => return error.ReservedBlock,
797	}
798	}
799
800	/// Decode a single block from `source` into `dest`. Literal and sequence data
801	/// from the block is copied into `literals_buffer` and `sequence_buffer`, which
802	/// must be large enough or `error.LiteralsBufferTooSmall` and
803	/// `error.SequenceBufferTooSmall` are returned (the maximum block size is an
804	/// upper bound for the size of both buffers). See `decodeBlock`
805	/// and `decodeBlockRingBuffer` for function that can decode a block without
806	/// these extra copies. `error.EndOfStream` is returned if `source` does not
807	/// contain enough bytes.
808	pub fn decodeBlockReader(
809	dest: *RingBuffer,
810	source: anytype,
811	block_header: frame.Zstandard.Block.Header,
812	decode_state: *DecodeState,
813	block_size_max: usize,
814	literals_buffer: []u8,
815	sequence_buffer: []u8,
816	) !void {
817	const block_size = block_header.block_size;
818	var block_reader_limited = std.io.limitedReader(source, block_size);
819	const block_reader = block_reader_limited.reader();
820	if (block_size_max < block_size) return error.BlockSizeOverMaximum;
821	switch (block_header.block_type) {
822	.raw => {
823	if (block_size == 0) return;
824	const slice = dest.sliceAt(dest.write_index, block_size);
825	try source.readNoEof(slice.first);
826	try source.readNoEof(slice.second);
827	dest.write_index = dest.mask2(dest.write_index + block_size);
828	decode_state.written_count += block_size;
829	},
830	.rle => {
831	const byte = try source.readByte();
832	for (0..block_size) \|_\| {
833	dest.writeAssumeCapacity(byte);
834	}
835	decode_state.written_count += block_size;
836	},
837	.compressed => {
838	const literals = try decodeLiteralsSection(block_reader, literals_buffer);
839	const sequences_header = try decodeSequencesHeader(block_reader);
840
841	try decode_state.prepare(block_reader, literals, sequences_header);
842
843	var bytes_written: usize = 0;
844	{
845	const size = try block_reader.readAll(sequence_buffer);
846	var bit_stream: readers.ReverseBitReader = undefined;
847	try bit_stream.init(sequence_buffer[0..size]);
848
849	if (sequences_header.sequence_count > 0) {
850	if (sequence_buffer.len < block_reader_limited.bytes_left)
851	return error.SequenceBufferTooSmall;
852
853	decode_state.readInitialFseState(&bit_stream) catch
854	return error.MalformedCompressedBlock;
855
856	var sequence_size_limit = block_size_max;
857	for (0..sequences_header.sequence_count) \|i\| {
858	const decompressed_size = decode_state.decodeSequenceRingBuffer(
859	dest,
860	&bit_stream,
861	sequence_size_limit,
862	i == sequences_header.sequence_count - 1,
863	) catch return error.MalformedCompressedBlock;
864	sequence_size_limit -= decompressed_size;
865	bytes_written += decompressed_size;
866	}
867	}
868
869	if (!bit_stream.isEmpty()) {
870	return error.MalformedCompressedBlock;
871	}
872	}
873
874	if (decode_state.literal_written_count < literals.header.regenerated_size) {
875	const len = literals.header.regenerated_size - decode_state.literal_written_count;
876	decode_state.decodeLiteralsRingBuffer(dest, len) catch
877	return error.MalformedCompressedBlock;
878	bytes_written += len;
879	}
880
881	switch (decode_state.literal_header.block_type) {
882	.treeless, .compressed => {
883	if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
884	},
885	.raw, .rle => {},
886	}
887
888	if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
889	if (block_reader_limited.bytes_left != 0) return error.MalformedCompressedBlock;
890	decode_state.literal_written_count = 0;
891	},
892	.reserved => return error.ReservedBlock,
893	}
894	}
895
896	/// Decode the header of a block.
897	pub fn decodeBlockHeader(src: *const [3]u8) frame.Zstandard.Block.Header {
898	const last_block = src[0] & 1 == 1;
899	const block_type = @as(frame.Zstandard.Block.Type, @enumFromInt((src[0] & 0b110) >> 1));
900	const block_size = ((src[0] & 0b11111000) >> 3) + (@as(u21, src[1]) << 5) + (@as(u21, src[2]) << 13);
901	return .{
902	.last_block = last_block,
903	.block_type = block_type,
904	.block_size = block_size,
905	};
906	}
907
908	/// Decode the header of a block.
909	///
910	/// Errors returned:
911	/// - `error.EndOfStream` if `src.len < 3`
912	pub fn decodeBlockHeaderSlice(src: []const u8) error{EndOfStream}!frame.Zstandard.Block.Header {
913	if (src.len < 3) return error.EndOfStream;
914	return decodeBlockHeader(src[0..3]);
915	}
916
917	/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
918	/// number of bytes the section uses.
919	///
920	/// Errors returned:
921	/// - `error.MalformedLiteralsHeader` if the header is invalid
922	/// - `error.MalformedLiteralsSection` if there are decoding errors
923	/// - `error.MalformedAccuracyLog` if compressed literals have invalid
924	/// accuracy
925	/// - `error.MalformedFseTable` if compressed literals have invalid FSE table
926	/// - `error.MalformedHuffmanTree` if there are errors decoding a Huffamn tree
927	/// - `error.EndOfStream` if there are not enough bytes in `src`
928	pub fn decodeLiteralsSectionSlice(
929	src: []const u8,
930	consumed_count: *usize,
931	) (error{ MalformedLiteralsHeader, MalformedLiteralsSection, EndOfStream } \|\| huffman.Error)!LiteralsSection {
932	var bytes_read: usize = 0;
933	const header = header: {
934	var fbs = std.io.fixedBufferStream(src);
935	defer bytes_read = fbs.pos;
936	break :header decodeLiteralsHeader(fbs.reader()) catch return error.MalformedLiteralsHeader;
937	};
938	switch (header.block_type) {
939	.raw => {
940	if (src.len < bytes_read + header.regenerated_size) return error.MalformedLiteralsSection;
941	const stream = src[bytes_read..][0..header.regenerated_size];
942	consumed_count.* += header.regenerated_size + bytes_read;
943	return LiteralsSection{
944	.header = header,
945	.huffman_tree = null,
946	.streams = .{ .one = stream },
947	};
948	},
949	.rle => {
950	if (src.len < bytes_read + 1) return error.MalformedLiteralsSection;
951	const stream = src[bytes_read..][0..1];
952	consumed_count.* += 1 + bytes_read;
953	return LiteralsSection{
954	.header = header,
955	.huffman_tree = null,
956	.streams = .{ .one = stream },
957	};
958	},
959	.compressed, .treeless => {
960	const huffman_tree_start = bytes_read;
961	const huffman_tree = if (header.block_type == .compressed)
962	try huffman.decodeHuffmanTreeSlice(src[bytes_read..], &bytes_read)
963	else
964	null;
965	const huffman_tree_size = bytes_read - huffman_tree_start;
966	const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
967	return error.MalformedLiteralsSection;
968
969	if (src.len < bytes_read + total_streams_size) return error.MalformedLiteralsSection;
970	const stream_data = src[bytes_read .. bytes_read + total_streams_size];
971
972	const streams = try decodeStreams(header.size_format, stream_data);
973	consumed_count.* += bytes_read + total_streams_size;
974	return LiteralsSection{
975	.header = header,
976	.huffman_tree = huffman_tree,
977	.streams = streams,
978	};
979	},
980	}
981	}
982
983	/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
984	/// number of bytes the section uses. See `decodeLiterasSectionSlice()`.
985	pub fn decodeLiteralsSection(
986	source: anytype,
987	buffer: []u8,
988	) !LiteralsSection {
989	const header = try decodeLiteralsHeader(source);
990	switch (header.block_type) {
991	.raw => {
992	if (buffer.len < header.regenerated_size) return error.LiteralsBufferTooSmall;
993	try source.readNoEof(buffer[0..header.regenerated_size]);
994	return LiteralsSection{
995	.header = header,
996	.huffman_tree = null,
997	.streams = .{ .one = buffer },
998	};
999	},
1000	.rle => {
1001	buffer[0] = try source.readByte();
1002	return LiteralsSection{
1003	.header = header,
1004	.huffman_tree = null,
1005	.streams = .{ .one = buffer[0..1] },
1006	};
1007	},
1008	.compressed, .treeless => {
1009	var counting_reader = std.io.countingReader(source);
1010	const huffman_tree = if (header.block_type == .compressed)
1011	try huffman.decodeHuffmanTree(counting_reader.reader(), buffer)
1012	else
1013	null;
1014	const huffman_tree_size = @as(usize, @intCast(counting_reader.bytes_read));
1015	const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
1016	return error.MalformedLiteralsSection;
1017
1018	if (total_streams_size > buffer.len) return error.LiteralsBufferTooSmall;
1019	try source.readNoEof(buffer[0..total_streams_size]);
1020	const stream_data = buffer[0..total_streams_size];
1021
1022	const streams = try decodeStreams(header.size_format, stream_data);
1023	return LiteralsSection{
1024	.header = header,
1025	.huffman_tree = huffman_tree,
1026	.streams = streams,
1027	};
1028	},
1029	}
1030	}
1031
1032	fn decodeStreams(size_format: u2, stream_data: []const u8) !LiteralsSection.Streams {
1033	if (size_format == 0) {
1034	return .{ .one = stream_data };
1035	}
1036
1037	if (stream_data.len < 6) return error.MalformedLiteralsSection;
1038
1039	const stream_1_length: usize = std.mem.readInt(u16, stream_data[0..2], .little);
1040	const stream_2_length: usize = std.mem.readInt(u16, stream_data[2..4], .little);
1041	const stream_3_length: usize = std.mem.readInt(u16, stream_data[4..6], .little);
1042
1043	const stream_1_start = 6;
1044	const stream_2_start = stream_1_start + stream_1_length;
1045	const stream_3_start = stream_2_start + stream_2_length;
1046	const stream_4_start = stream_3_start + stream_3_length;
1047
1048	if (stream_data.len < stream_4_start) return error.MalformedLiteralsSection;
1049
1050	return .{ .four = .{
1051	stream_data[stream_1_start .. stream_1_start + stream_1_length],
1052	stream_data[stream_2_start .. stream_2_start + stream_2_length],
1053	stream_data[stream_3_start .. stream_3_start + stream_3_length],
1054	stream_data[stream_4_start..],
1055	} };
1056	}
1057
1058	/// Decode a literals section header.
1059	///
1060	/// Errors returned:
1061	/// - `error.EndOfStream` if there are not enough bytes in `source`
1062	pub fn decodeLiteralsHeader(source: anytype) !LiteralsSection.Header {
1063	const byte0 = try source.readByte();
1064	const block_type = @as(LiteralsSection.BlockType, @enumFromInt(byte0 & 0b11));
1065	const size_format = @as(u2, @intCast((byte0 & 0b1100) >> 2));
1066	var regenerated_size: u20 = undefined;
1067	var compressed_size: ?u18 = null;
1068	switch (block_type) {
1069	.raw, .rle => {
1070	switch (size_format) {
1071	0, 2 => {
1072	regenerated_size = byte0 >> 3;
1073	},
1074	1 => regenerated_size = (byte0 >> 4) + (@as(u20, try source.readByte()) << 4),
1075	3 => regenerated_size = (byte0 >> 4) +
1076	(@as(u20, try source.readByte()) << 4) +
1077	(@as(u20, try source.readByte()) << 12),
1078	}
1079	},
1080	.compressed, .treeless => {
1081	const byte1 = try source.readByte();
1082	const byte2 = try source.readByte();
1083	switch (size_format) {
1084	0, 1 => {
1085	regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
1086	compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
1087	},
1088	2 => {
1089	const byte3 = try source.readByte();
1090	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
1091	compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
1092	},
1093	3 => {
1094	const byte3 = try source.readByte();
1095	const byte4 = try source.readByte();
1096	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
1097	compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
1098	},
1099	}
1100	},
1101	}
1102	return LiteralsSection.Header{
1103	.block_type = block_type,
1104	.size_format = size_format,
1105	.regenerated_size = regenerated_size,
1106	.compressed_size = compressed_size,
1107	};
1108	}
1109
1110	/// Decode a sequences section header.
1111	///
1112	/// Errors returned:
1113	/// - `error.ReservedBitSet` if the reserved bit is set
1114	/// - `error.EndOfStream` if there are not enough bytes in `source`
1115	pub fn decodeSequencesHeader(
1116	source: anytype,
1117	) !SequencesSection.Header {
1118	var sequence_count: u24 = undefined;
1119
1120	const byte0 = try source.readByte();
1121	if (byte0 == 0) {
1122	return SequencesSection.Header{
1123	.sequence_count = 0,
1124	.offsets = undefined,
1125	.match_lengths = undefined,
1126	.literal_lengths = undefined,
1127	};
1128	} else if (byte0 < 128) {
1129	sequence_count = byte0;
1130	} else if (byte0 < 255) {
1131	sequence_count = (@as(u24, (byte0 - 128)) << 8) + try source.readByte();
1132	} else {
1133	sequence_count = (try source.readByte()) + (@as(u24, try source.readByte()) << 8) + 0x7F00;
1134	}
1135
1136	const compression_modes = try source.readByte();
1137
1138	const matches_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00001100) >> 2));
1139	const offsets_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00110000) >> 4));
1140	const literal_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b11000000) >> 6));
1141	if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
1142
1143	return SequencesSection.Header{
1144	.sequence_count = sequence_count,
1145	.offsets = offsets_mode,
1146	.match_lengths = matches_mode,
1147	.literal_lengths = literal_mode,
1148	};
1149	}

lib/std/compress/zstandard/decode/fse.zig deleted-153

...	@@ -1,153 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3
4	const types = @import("../types.zig");
5	const Table = types.compressed_block.Table;
6
7	pub fn decodeFseTable(
8	bit_reader: anytype,
9	expected_symbol_count: usize,
10	max_accuracy_log: u4,
11	entries: []Table.Fse,
12	) !usize {
13	const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
14	if (accuracy_log_biased > max_accuracy_log -\| 5) return error.MalformedAccuracyLog;
15	const accuracy_log = accuracy_log_biased + 5;
16
17	var values: [256]u16 = undefined;
18	var value_count: usize = 0;
19
20	const total_probability = @as(u16, 1) << accuracy_log;
21	var accumulated_probability: u16 = 0;
22
23	while (accumulated_probability < total_probability) {
24	// WARNING: The RFC is poorly worded, and would suggest std.math.log2_int_ceil is correct here,
25	// but power of two (remaining probabilities + 1) need max bits set to 1 more.
26	const max_bits = std.math.log2_int(u16, total_probability - accumulated_probability + 1) + 1;
27	const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
28
29	const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
30
31	const value = if (small < cutoff)
32	small
33	else value: {
34	const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
35	break :value if (value_read < @as(u16, 1) << (max_bits - 1))
36	value_read
37	else
38	value_read - cutoff;
39	};
40
41	accumulated_probability += if (value != 0) value - 1 else 1;
42
43	values[value_count] = value;
44	value_count += 1;
45
46	if (value == 1) {
47	while (true) {
48	const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
49	if (repeat_flag + value_count > 256) return error.MalformedFseTable;
50	for (0..repeat_flag) \|_\| {
51	values[value_count] = 1;
52	value_count += 1;
53	}
54	if (repeat_flag < 3) break;
55	}
56	}
57	if (value_count == 256) break;
58	}
59	bit_reader.alignToByte();
60
61	if (value_count < 2) return error.MalformedFseTable;
62	if (accumulated_probability != total_probability) return error.MalformedFseTable;
63	if (value_count > expected_symbol_count) return error.MalformedFseTable;
64
65	const table_size = total_probability;
66
67	try buildFseTable(values[0..value_count], entries[0..table_size]);
68	return table_size;
69	}
70
71	fn buildFseTable(values: []const u16, entries: []Table.Fse) !void {
72	const total_probability = @as(u16, @intCast(entries.len));
73	const accuracy_log = std.math.log2_int(u16, total_probability);
74	assert(total_probability <= 1 << 9);
75
76	var less_than_one_count: usize = 0;
77	for (values, 0..) \|value, i\| {
78	if (value == 0) {
79	entries[entries.len - 1 - less_than_one_count] = Table.Fse{
80	.symbol = @as(u8, @intCast(i)),
81	.baseline = 0,
82	.bits = accuracy_log,
83	};
84	less_than_one_count += 1;
85	}
86	}
87
88	var position: usize = 0;
89	var temp_states: [1 << 9]u16 = undefined;
90	for (values, 0..) \|value, symbol\| {
91	if (value == 0 or value == 1) continue;
92	const probability = value - 1;
93
94	const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
95	return error.MalformedFseTable;
96	const share_size = @divExact(total_probability, state_share_dividend);
97	const double_state_count = state_share_dividend - probability;
98	const single_state_count = probability - double_state_count;
99	const share_size_log = std.math.log2_int(u16, share_size);
100
101	for (0..probability) \|i\| {
102	temp_states[i] = @as(u16, @intCast(position));
103	position += (entries.len >> 1) + (entries.len >> 3) + 3;
104	position &= entries.len - 1;
105	while (position >= entries.len - less_than_one_count) {
106	position += (entries.len >> 1) + (entries.len >> 3) + 3;
107	position &= entries.len - 1;
108	}
109	}
110	std.mem.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
111	for (0..probability) \|i\| {
112	entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
113	.symbol = @as(u8, @intCast(symbol)),
114	.bits = share_size_log + 1,
115	.baseline = single_state_count * share_size + @as(u16, @intCast(i)) * 2 * share_size,
116	} else Table.Fse{
117	.symbol = @as(u8, @intCast(symbol)),
118	.bits = share_size_log,
119	.baseline = (@as(u16, @intCast(i)) - double_state_count) * share_size,
120	};
121	}
122	}
123	}
124
125	test buildFseTable {
126	const literals_length_default_values = [36]u16{
127	5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
128	3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
129	0, 0, 0, 0,
130	};
131
132	const match_lengths_default_values = [53]u16{
133	2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
134	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
135	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
136	0, 0, 0, 0, 0,
137	};
138
139	const offset_codes_default_values = [29]u16{
140	2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
141	2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
142	};
143
144	var entries: [64]Table.Fse = undefined;
145	try buildFseTable(&literals_length_default_values, &entries);
146	try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_literal_fse_table.fse, &entries);
147
148	try buildFseTable(&match_lengths_default_values, &entries);
149	try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_match_fse_table.fse, &entries);
150
151	try buildFseTable(&offset_codes_default_values, entries[0..32]);
152	try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_offset_fse_table.fse, entries[0..32]);
153	}

lib/std/compress/zstandard/decode/huffman.zig deleted-234

...	@@ -1,234 +0,0 @@
1	const std = @import("std");
2
3	const types = @import("../types.zig");
4	const LiteralsSection = types.compressed_block.LiteralsSection;
5	const Table = types.compressed_block.Table;
6
7	const readers = @import("../readers.zig");
8
9	const decodeFseTable = @import("fse.zig").decodeFseTable;
10
11	pub const Error = error{
12	MalformedHuffmanTree,
13	MalformedFseTable,
14	MalformedAccuracyLog,
15	EndOfStream,
16	};
17
18	fn decodeFseHuffmanTree(
19	source: anytype,
20	compressed_size: usize,
21	buffer: []u8,
22	weights: *[256]u4,
23	) !usize {
24	var stream = std.io.limitedReader(source, compressed_size);
25	var bit_reader = readers.bitReader(stream.reader());
26
27	var entries: [1 << 6]Table.Fse = undefined;
28	const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch \|err\| switch (err) {
29	error.MalformedAccuracyLog, error.MalformedFseTable => \|e\| return e,
30	error.EndOfStream => return error.MalformedFseTable,
31	else => \|e\| return e,
32	};
33	const accuracy_log = std.math.log2_int_ceil(usize, table_size);
34
35	const amount = try stream.reader().readAll(buffer);
36	var huff_bits: readers.ReverseBitReader = undefined;
37	huff_bits.init(buffer[0..amount]) catch return error.MalformedHuffmanTree;
38
39	return assignWeights(&huff_bits, accuracy_log, &entries, weights);
40	}
41
42	fn decodeFseHuffmanTreeSlice(src: []const u8, compressed_size: usize, weights: *[256]u4) !usize {
43	if (src.len < compressed_size) return error.MalformedHuffmanTree;
44	var stream = std.io.fixedBufferStream(src[0..compressed_size]);
45	var counting_reader = std.io.countingReader(stream.reader());
46	var bit_reader = readers.bitReader(counting_reader.reader());
47
48	var entries: [1 << 6]Table.Fse = undefined;
49	const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch \|err\| switch (err) {
50	error.MalformedAccuracyLog, error.MalformedFseTable => \|e\| return e,
51	error.EndOfStream => return error.MalformedFseTable,
52	};
53	const accuracy_log = std.math.log2_int_ceil(usize, table_size);
54
55	const start_index = std.math.cast(usize, counting_reader.bytes_read) orelse
56	return error.MalformedHuffmanTree;
57	const huff_data = src[start_index..compressed_size];
58	var huff_bits: readers.ReverseBitReader = undefined;
59	huff_bits.init(huff_data) catch return error.MalformedHuffmanTree;
60
61	return assignWeights(&huff_bits, accuracy_log, &entries, weights);
62	}
63
64	fn assignWeights(
65	huff_bits: *readers.ReverseBitReader,
66	accuracy_log: u16,
67	entries: *[1 << 6]Table.Fse,
68	weights: *[256]u4,
69	) !usize {
70	var i: usize = 0;
71	var even_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
72	var odd_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
73
74	while (i < 254) {
75	const even_data = entries[even_state];
76	var read_bits: u16 = 0;
77	const even_bits = huff_bits.readBits(u32, even_data.bits, &read_bits) catch unreachable;
78	weights[i] = std.math.cast(u4, even_data.symbol) orelse return error.MalformedHuffmanTree;
79	i += 1;
80	if (read_bits < even_data.bits) {
81	weights[i] = std.math.cast(u4, entries[odd_state].symbol) orelse return error.MalformedHuffmanTree;
82	i += 1;
83	break;
84	}
85	even_state = even_data.baseline + even_bits;
86
87	read_bits = 0;
88	const odd_data = entries[odd_state];
89	const odd_bits = huff_bits.readBits(u32, odd_data.bits, &read_bits) catch unreachable;
90	weights[i] = std.math.cast(u4, odd_data.symbol) orelse return error.MalformedHuffmanTree;
91	i += 1;
92	if (read_bits < odd_data.bits) {
93	if (i == 255) return error.MalformedHuffmanTree;
94	weights[i] = std.math.cast(u4, entries[even_state].symbol) orelse return error.MalformedHuffmanTree;
95	i += 1;
96	break;
97	}
98	odd_state = odd_data.baseline + odd_bits;
99	} else return error.MalformedHuffmanTree;
100
101	if (!huff_bits.isEmpty()) {
102	return error.MalformedHuffmanTree;
103	}
104
105	return i + 1; // stream contains all but the last symbol
106	}
107
108	fn decodeDirectHuffmanTree(source: anytype, encoded_symbol_count: usize, weights: *[256]u4) !usize {
109	const weights_byte_count = (encoded_symbol_count + 1) / 2;
110	for (0..weights_byte_count) \|i\| {
111	const byte = try source.readByte();
112	weights[2 * i] = @as(u4, @intCast(byte >> 4));
113	weights[2 * i + 1] = @as(u4, @intCast(byte & 0xF));
114	}
115	return encoded_symbol_count + 1;
116	}
117
118	fn assignSymbols(weight_sorted_prefixed_symbols: []LiteralsSection.HuffmanTree.PrefixedSymbol, weights: [256]u4) usize {
119	for (0..weight_sorted_prefixed_symbols.len) \|i\| {
120	weight_sorted_prefixed_symbols[i] = .{
121	.symbol = @as(u8, @intCast(i)),
122	.weight = undefined,
123	.prefix = undefined,
124	};
125	}
126
127	std.mem.sort(
128	LiteralsSection.HuffmanTree.PrefixedSymbol,
129	weight_sorted_prefixed_symbols,
130	weights,
131	lessThanByWeight,
132	);
133
134	var prefix: u16 = 0;
135	var prefixed_symbol_count: usize = 0;
136	var sorted_index: usize = 0;
137	const symbol_count = weight_sorted_prefixed_symbols.len;
138	while (sorted_index < symbol_count) {
139	var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
140	const weight = weights[symbol];
141	if (weight == 0) {
142	sorted_index += 1;
143	continue;
144	}
145
146	while (sorted_index < symbol_count) : ({
147	sorted_index += 1;
148	prefixed_symbol_count += 1;
149	prefix += 1;
150	}) {
151	symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
152	if (weights[symbol] != weight) {
153	prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
154	break;
155	}
156	weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
157	weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
158	weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
159	}
160	}
161	return prefixed_symbol_count;
162	}
163
164	fn buildHuffmanTree(weights: *[256]u4, symbol_count: usize) error{MalformedHuffmanTree}!LiteralsSection.HuffmanTree {
165	var weight_power_sum_big: u32 = 0;
166	for (weights[0 .. symbol_count - 1]) \|value\| {
167	weight_power_sum_big += (@as(u16, 1) << value) >> 1;
168	}
169	if (weight_power_sum_big >= 1 << 11) return error.MalformedHuffmanTree;
170	const weight_power_sum = @as(u16, @intCast(weight_power_sum_big));
171
172	// advance to next power of two (even if weight_power_sum is a power of 2)
173	// TODO: is it valid to have weight_power_sum == 0?
174	const max_number_of_bits = if (weight_power_sum == 0) 1 else std.math.log2_int(u16, weight_power_sum) + 1;
175	const next_power_of_two = @as(u16, 1) << max_number_of_bits;
176	weights[symbol_count - 1] = std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1;
177
178	var weight_sorted_prefixed_symbols: [256]LiteralsSection.HuffmanTree.PrefixedSymbol = undefined;
179	const prefixed_symbol_count = assignSymbols(weight_sorted_prefixed_symbols[0..symbol_count], weights.*);
180	const tree = LiteralsSection.HuffmanTree{
181	.max_bit_count = max_number_of_bits,
182	.symbol_count_minus_one = @as(u8, @intCast(prefixed_symbol_count - 1)),
183	.nodes = weight_sorted_prefixed_symbols,
184	};
185	return tree;
186	}
187
188	pub fn decodeHuffmanTree(
189	source: anytype,
190	buffer: []u8,
191	) (@TypeOf(source).Error \|\| Error)!LiteralsSection.HuffmanTree {
192	const header = try source.readByte();
193	var weights: [256]u4 = undefined;
194	const symbol_count = if (header < 128)
195	// FSE compressed weights
196	try decodeFseHuffmanTree(source, header, buffer, &weights)
197	else
198	try decodeDirectHuffmanTree(source, header - 127, &weights);
199
200	return buildHuffmanTree(&weights, symbol_count);
201	}
202
203	pub fn decodeHuffmanTreeSlice(
204	src: []const u8,
205	consumed_count: *usize,
206	) Error!LiteralsSection.HuffmanTree {
207	if (src.len == 0) return error.MalformedHuffmanTree;
208	const header = src[0];
209	var bytes_read: usize = 1;
210	var weights: [256]u4 = undefined;
211	const symbol_count = if (header < 128) count: {
212	// FSE compressed weights
213	bytes_read += header;
214	break :count try decodeFseHuffmanTreeSlice(src[1..], header, &weights);
215	} else count: {
216	var fbs = std.io.fixedBufferStream(src[1..]);
217	defer bytes_read += fbs.pos;
218	break :count try decodeDirectHuffmanTree(fbs.reader(), header - 127, &weights);
219	};
220
221	consumed_count.* += bytes_read;
222	return buildHuffmanTree(&weights, symbol_count);
223	}
224
225	fn lessThanByWeight(
226	weights: [256]u4,
227	lhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
228	rhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
229	) bool {
230	// NOTE: this function relies on the use of a stable sorting algorithm,
231	// otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
232	// should be added
233	return weights[lhs.symbol] < weights[rhs.symbol];
234	}

lib/std/compress/zstandard/decompress.zig deleted-633

...	@@ -1,633 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const Allocator = std.mem.Allocator;
4	const RingBuffer = std.RingBuffer;
5
6	const types = @import("types.zig");
7	const frame = types.frame;
8	const LiteralsSection = types.compressed_block.LiteralsSection;
9	const SequencesSection = types.compressed_block.SequencesSection;
10	const SkippableHeader = types.frame.Skippable.Header;
11	const ZstandardHeader = types.frame.Zstandard.Header;
12	const Table = types.compressed_block.Table;
13
14	pub const block = @import("decode/block.zig");
15
16	const readers = @import("readers.zig");
17
18	/// Returns `true` is `magic` is a valid magic number for a skippable frame
19	pub fn isSkippableMagic(magic: u32) bool {
20	return frame.Skippable.magic_number_min <= magic and magic <= frame.Skippable.magic_number_max;
21	}
22
23	/// Returns the kind of frame at the beginning of `source`.
24	///
25	/// Errors returned:
26	/// - `error.BadMagic` if `source` begins with bytes not equal to the
27	/// Zstandard frame magic number, or outside the range of magic numbers for
28	/// skippable frames.
29	/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
30	pub fn decodeFrameType(source: anytype) error{ BadMagic, EndOfStream }!frame.Kind {
31	const magic = try source.readInt(u32, .little);
32	return frameType(magic);
33	}
34
35	/// Returns the kind of frame associated to `magic`.
36	///
37	/// Errors returned:
38	/// - `error.BadMagic` if `magic` is not a valid magic number.
39	pub fn frameType(magic: u32) error{BadMagic}!frame.Kind {
40	return if (magic == frame.Zstandard.magic_number)
41	.zstandard
42	else if (isSkippableMagic(magic))
43	.skippable
44	else
45	error.BadMagic;
46	}
47
48	pub const FrameHeader = union(enum) {
49	zstandard: ZstandardHeader,
50	skippable: SkippableHeader,
51	};
52
53	pub const HeaderError = error{ BadMagic, EndOfStream, ReservedBitSet };
54
55	/// Returns the header of the frame at the beginning of `source`.
56	///
57	/// Errors returned:
58	/// - `error.BadMagic` if `source` begins with bytes not equal to the
59	/// Zstandard frame magic number, or outside the range of magic numbers for
60	/// skippable frames.
61	/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
62	/// - `error.ReservedBitSet` if the frame is a Zstandard frame and any of the
63	/// reserved bits are set
64	pub fn decodeFrameHeader(source: anytype) (@TypeOf(source).Error \|\| HeaderError)!FrameHeader {
65	const magic = try source.readInt(u32, .little);
66	const frame_type = try frameType(magic);
67	switch (frame_type) {
68	.zstandard => return FrameHeader{ .zstandard = try decodeZstandardHeader(source) },
69	.skippable => return FrameHeader{
70	.skippable = .{
71	.magic_number = magic,
72	.frame_size = try source.readInt(u32, .little),
73	},
74	},
75	}
76	}
77
78	pub const ReadWriteCount = struct {
79	read_count: usize,
80	write_count: usize,
81	};
82
83	/// Decodes frames from `src` into `dest`; returns the length of the result.
84	/// The stream should not have extra trailing bytes - either all bytes in `src`
85	/// will be decoded, or an error will be returned. An error will be returned if
86	/// a Zstandard frame in `src` does not declare its content size.
87	///
88	/// Errors returned:
89	/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
90	/// uses a dictionary
91	/// - `error.MalformedFrame` if a frame in `src` is invalid
92	/// - `error.UnknownContentSizeUnsupported` if a frame in `src` does not
93	/// declare its content size
94	pub fn decode(dest: []u8, src: []const u8, verify_checksum: bool) error{
95	MalformedFrame,
96	UnknownContentSizeUnsupported,
97	DictionaryIdFlagUnsupported,
98	}!usize {
99	var write_count: usize = 0;
100	var read_count: usize = 0;
101	while (read_count < src.len) {
102	const counts = decodeFrame(dest, src[read_count..], verify_checksum) catch \|err\| {
103	switch (err) {
104	error.UnknownContentSizeUnsupported => return error.UnknownContentSizeUnsupported,
105	error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
106	else => return error.MalformedFrame,
107	}
108	};
109	read_count += counts.read_count;
110	write_count += counts.write_count;
111	}
112	return write_count;
113	}
114
115	/// Decodes a stream of frames from `src`; returns the decoded bytes. The stream
116	/// should not have extra trailing bytes - either all bytes in `src` will be
117	/// decoded, or an error will be returned.
118	///
119	/// Errors returned:
120	/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
121	/// uses a dictionary
122	/// - `error.MalformedFrame` if a frame in `src` is invalid
123	/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
124	pub fn decodeAlloc(
125	allocator: Allocator,
126	src: []const u8,
127	verify_checksum: bool,
128	window_size_max: usize,
129	) error{ DictionaryIdFlagUnsupported, MalformedFrame, OutOfMemory }![]u8 {
130	var result = std.ArrayList(u8).init(allocator);
131	errdefer result.deinit();
132
133	var read_count: usize = 0;
134	while (read_count < src.len) {
135	read_count += decodeFrameArrayList(
136	allocator,
137	&result,
138	src[read_count..],
139	verify_checksum,
140	window_size_max,
141	) catch \|err\| switch (err) {
142	error.OutOfMemory => return error.OutOfMemory,
143	error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
144	else => return error.MalformedFrame,
145	};
146	}
147	return result.toOwnedSlice();
148	}
149
150	/// Decodes the frame at the start of `src` into `dest`. Returns the number of
151	/// bytes read from `src` and written to `dest`. This function can only decode
152	/// frames that declare the decompressed content size.
153	///
154	/// Errors returned:
155	/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
156	/// number for a Zstandard or skippable frame
157	/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
158	/// uncompressed content size
159	/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
160	/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
161	/// size declared by the frame header
162	/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
163	/// that is larger than `std.math.maxInt(usize)`
164	/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
165	/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
166	/// contains a checksum that does not match the checksum of the decompressed
167	/// data
168	/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
169	/// are set
170	/// - `error.EndOfStream` if `src` does not contain a complete frame
171	/// - `error.BadContentSize` if the content size declared by the frame does
172	/// not equal the actual size of decompressed data
173	/// - an error in `block.Error` if there are errors decoding a block
174	/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
175	/// size greater than `src.len`
176	pub fn decodeFrame(
177	dest: []u8,
178	src: []const u8,
179	verify_checksum: bool,
180	) (error{
181	BadMagic,
182	UnknownContentSizeUnsupported,
183	ContentTooLarge,
184	ContentSizeTooLarge,
185	WindowSizeUnknown,
186	DictionaryIdFlagUnsupported,
187	SkippableSizeTooLarge,
188	} \|\| FrameError)!ReadWriteCount {
189	var fbs = std.io.fixedBufferStream(src);
190	switch (try decodeFrameType(fbs.reader())) {
191	.zstandard => return decodeZstandardFrame(dest, src, verify_checksum),
192	.skippable => {
193	const content_size = try fbs.reader().readInt(u32, .little);
194	if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;
195	const read_count = @as(usize, content_size) + 8;
196	if (read_count > src.len) return error.SkippableSizeTooLarge;
197	return ReadWriteCount{
198	.read_count = read_count,
199	.write_count = 0,
200	};
201	},
202	}
203	}
204
205	/// Decodes the frame at the start of `src` into `dest`. Returns the number of
206	/// bytes read from `src`.
207	///
208	/// Errors returned:
209	/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
210	/// number for a Zstandard or skippable frame
211	/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
212	/// - `error.WindowTooLarge` if the window size is larger than
213	/// `window_size_max`
214	/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
215	/// that is larger than `std.math.maxInt(usize)`
216	/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
217	/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
218	/// contains a checksum that does not match the checksum of the decompressed
219	/// data
220	/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
221	/// are set
222	/// - `error.EndOfStream` if `src` does not contain a complete frame
223	/// - `error.BadContentSize` if the content size declared by the frame does
224	/// not equal the actual size of decompressed data
225	/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
226	/// - an error in `block.Error` if there are errors decoding a block
227	/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
228	/// size greater than `src.len`
229	pub fn decodeFrameArrayList(
230	allocator: Allocator,
231	dest: *std.ArrayList(u8),
232	src: []const u8,
233	verify_checksum: bool,
234	window_size_max: usize,
235	) (error{ BadMagic, OutOfMemory, SkippableSizeTooLarge } \|\| FrameContext.Error \|\| FrameError)!usize {
236	var fbs = std.io.fixedBufferStream(src);
237	const reader = fbs.reader();
238	const magic = try reader.readInt(u32, .little);
239	switch (try frameType(magic)) {
240	.zstandard => return decodeZstandardFrameArrayList(
241	allocator,
242	dest,
243	src,
244	verify_checksum,
245	window_size_max,
246	),
247	.skippable => {
248	const content_size = try fbs.reader().readInt(u32, .little);
249	if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;
250	const read_count = @as(usize, content_size) + 8;
251	if (read_count > src.len) return error.SkippableSizeTooLarge;
252	return read_count;
253	},
254	}
255	}
256
257	/// Returns the frame checksum corresponding to the data fed into `hasher`
258	pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 {
259	const hash = hasher.final();
260	return @as(u32, @intCast(hash & 0xFFFFFFFF));
261	}
262
263	const FrameError = error{
264	ChecksumFailure,
265	BadContentSize,
266	EndOfStream,
267	ReservedBitSet,
268	} \|\| block.Error;
269
270	/// Decode a Zstandard frame from `src` into `dest`, returning the number of
271	/// bytes read from `src` and written to `dest`. The first four bytes of `src`
272	/// must be the magic number for a Zstandard frame.
273	///
274	/// Error returned:
275	/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
276	/// uncompressed content size
277	/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
278	/// size declared by the frame header
279	/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
280	/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
281	/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
282	/// that is larger than `std.math.maxInt(usize)`
283	/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
284	/// contains a checksum that does not match the checksum of the decompressed
285	/// data
286	/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
287	/// - `error.EndOfStream` if `src` does not contain a complete frame
288	/// - an error in `block.Error` if there are errors decoding a block
289	/// - `error.BadContentSize` if the content size declared by the frame does
290	/// not equal the actual size of decompressed data
291	pub fn decodeZstandardFrame(
292	dest: []u8,
293	src: []const u8,
294	verify_checksum: bool,
295	) (error{
296	UnknownContentSizeUnsupported,
297	ContentTooLarge,
298	ContentSizeTooLarge,
299	WindowSizeUnknown,
300	DictionaryIdFlagUnsupported,
301	} \|\| FrameError)!ReadWriteCount {
302	assert(std.mem.readInt(u32, src[0..4], .little) == frame.Zstandard.magic_number);
303	var consumed_count: usize = 4;
304
305	var frame_context = context: {
306	var fbs = std.io.fixedBufferStream(src[consumed_count..]);
307	const source = fbs.reader();
308	const frame_header = try decodeZstandardHeader(source);
309	consumed_count += fbs.pos;
310	break :context FrameContext.init(
311	frame_header,
312	std.math.maxInt(usize),
313	verify_checksum,
314	) catch \|err\| switch (err) {
315	error.WindowTooLarge => unreachable,
316	inline else => \|e\| return e,
317	};
318	};
319	const counts = try decodeZStandardFrameBlocks(
320	dest,
321	src[consumed_count..],
322	&frame_context,
323	);
324	return ReadWriteCount{
325	.read_count = counts.read_count + consumed_count,
326	.write_count = counts.write_count,
327	};
328	}
329
330	pub fn decodeZStandardFrameBlocks(
331	dest: []u8,
332	src: []const u8,
333	frame_context: *FrameContext,
334	) (error{ ContentTooLarge, UnknownContentSizeUnsupported } \|\| FrameError)!ReadWriteCount {
335	const content_size = frame_context.content_size orelse
336	return error.UnknownContentSizeUnsupported;
337	if (dest.len < content_size) return error.ContentTooLarge;
338
339	var consumed_count: usize = 0;
340	const written_count = decodeFrameBlocksInner(
341	dest[0..content_size],
342	src[consumed_count..],
343	&consumed_count,
344	if (frame_context.hasher_opt) \|*hasher\| hasher else null,
345	frame_context.block_size_max,
346	) catch \|err\| switch (err) {
347	error.DestTooSmall => return error.BadContentSize,
348	inline else => \|e\| return e,
349	};
350
351	if (written_count != content_size) return error.BadContentSize;
352	if (frame_context.has_checksum) {
353	if (src.len < consumed_count + 4) return error.EndOfStream;
354	const checksum = std.mem.readInt(u32, src[consumed_count..][0..4], .little);
355	consumed_count += 4;
356	if (frame_context.hasher_opt) \|*hasher\| {
357	if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
358	}
359	}
360	return ReadWriteCount{ .read_count = consumed_count, .write_count = written_count };
361	}
362
363	pub const FrameContext = struct {
364	hasher_opt: ?std.hash.XxHash64,
365	window_size: usize,
366	has_checksum: bool,
367	block_size_max: usize,
368	content_size: ?usize,
369
370	const Error = error{
371	DictionaryIdFlagUnsupported,
372	WindowSizeUnknown,
373	WindowTooLarge,
374	ContentSizeTooLarge,
375	};
376	/// Validates `frame_header` and returns the associated `FrameContext`.
377	///
378	/// Errors returned:
379	/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
380	/// - `error.WindowSizeUnknown` if the frame does not have a valid window
381	/// size
382	/// - `error.WindowTooLarge` if the window size is larger than
383	/// `window_size_max` or `std.math.intMax(usize)`
384	/// - `error.ContentSizeTooLarge` if the frame header indicates a content
385	/// size larger than `std.math.maxInt(usize)`
386	pub fn init(
387	frame_header: ZstandardHeader,
388	window_size_max: usize,
389	verify_checksum: bool,
390	) Error!FrameContext {
391	if (frame_header.descriptor.dictionary_id_flag != 0)
392	return error.DictionaryIdFlagUnsupported;
393
394	const window_size_raw = frameWindowSize(frame_header) orelse return error.WindowSizeUnknown;
395	const window_size = if (window_size_raw > window_size_max)
396	return error.WindowTooLarge
397	else
398	std.math.cast(usize, window_size_raw) orelse return error.WindowTooLarge;
399
400	const should_compute_checksum =
401	frame_header.descriptor.content_checksum_flag and verify_checksum;
402
403	const content_size = if (frame_header.content_size) \|size\|
404	std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
405	else
406	null;
407
408	return .{
409	.hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
410	.window_size = window_size,
411	.has_checksum = frame_header.descriptor.content_checksum_flag,
412	.block_size_max = @min(types.block_size_max, window_size),
413	.content_size = content_size,
414	};
415	}
416	};
417
418	/// Decode a Zstandard from from `src` and return number of bytes read; see
419	/// `decodeZstandardFrame()`. The first four bytes of `src` must be the magic
420	/// number for a Zstandard frame.
421	///
422	/// Errors returned:
423	/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
424	/// - `error.WindowTooLarge` if the window size is larger than
425	/// `window_size_max`
426	/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
427	/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
428	/// that is larger than `std.math.maxInt(usize)`
429	/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
430	/// contains a checksum that does not match the checksum of the decompressed
431	/// data
432	/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
433	/// - `error.EndOfStream` if `src` does not contain a complete frame
434	/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
435	/// - an error in `block.Error` if there are errors decoding a block
436	/// - `error.BadContentSize` if the content size declared by the frame does
437	/// not equal the size of decompressed data
438	pub fn decodeZstandardFrameArrayList(
439	allocator: Allocator,
440	dest: *std.ArrayList(u8),
441	src: []const u8,
442	verify_checksum: bool,
443	window_size_max: usize,
444	) (error{OutOfMemory} \|\| FrameContext.Error \|\| FrameError)!usize {
445	assert(std.mem.readInt(u32, src[0..4], .little) == frame.Zstandard.magic_number);
446	var consumed_count: usize = 4;
447
448	var frame_context = context: {
449	var fbs = std.io.fixedBufferStream(src[consumed_count..]);
450	const source = fbs.reader();
451	const frame_header = try decodeZstandardHeader(source);
452	consumed_count += fbs.pos;
453	break :context try FrameContext.init(frame_header, window_size_max, verify_checksum);
454	};
455
456	consumed_count += try decodeZstandardFrameBlocksArrayList(
457	allocator,
458	dest,
459	src[consumed_count..],
460	&frame_context,
461	);
462	return consumed_count;
463	}
464
465	pub fn decodeZstandardFrameBlocksArrayList(
466	allocator: Allocator,
467	dest: *std.ArrayList(u8),
468	src: []const u8,
469	frame_context: *FrameContext,
470	) (error{OutOfMemory} \|\| FrameError)!usize {
471	const initial_len = dest.items.len;
472
473	var ring_buffer = try RingBuffer.init(allocator, frame_context.window_size);
474	defer ring_buffer.deinit(allocator);
475
476	// These tables take 7680 bytes
477	var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
478	var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;
479	var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined;
480
481	var block_header = try block.decodeBlockHeaderSlice(src);
482	var consumed_count: usize = 3;
483	var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);
484	while (true) : ({
485	block_header = try block.decodeBlockHeaderSlice(src[consumed_count..]);
486	consumed_count += 3;
487	}) {
488	const written_size = try block.decodeBlockRingBuffer(
489	&ring_buffer,
490	src[consumed_count..],
491	block_header,
492	&decode_state,
493	&consumed_count,
494	frame_context.block_size_max,
495	);
496	if (frame_context.content_size) \|size\| {
497	if (dest.items.len - initial_len > size) {
498	return error.BadContentSize;
499	}
500	}
501	if (written_size > 0) {
502	const written_slice = ring_buffer.sliceLast(written_size);
503	try dest.appendSlice(written_slice.first);
504	try dest.appendSlice(written_slice.second);
505	if (frame_context.hasher_opt) \|*hasher\| {
506	hasher.update(written_slice.first);
507	hasher.update(written_slice.second);
508	}
509	}
510	if (block_header.last_block) break;
511	}
512	if (frame_context.content_size) \|size\| {
513	if (dest.items.len - initial_len != size) {
514	return error.BadContentSize;
515	}
516	}
517
518	if (frame_context.has_checksum) {
519	if (src.len < consumed_count + 4) return error.EndOfStream;
520	const checksum = std.mem.readInt(u32, src[consumed_count..][0..4], .little);
521	consumed_count += 4;
522	if (frame_context.hasher_opt) \|*hasher\| {
523	if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
524	}
525	}
526	return consumed_count;
527	}
528
529	fn decodeFrameBlocksInner(
530	dest: []u8,
531	src: []const u8,
532	consumed_count: *usize,
533	hash: ?*std.hash.XxHash64,
534	block_size_max: usize,
535	) (error{ EndOfStream, DestTooSmall } \|\| block.Error)!usize {
536	// These tables take 7680 bytes
537	var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
538	var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;
539	var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined;
540
541	var block_header = try block.decodeBlockHeaderSlice(src);
542	var bytes_read: usize = 3;
543	defer consumed_count.* += bytes_read;
544	var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);
545	var count: usize = 0;
546	while (true) : ({
547	block_header = try block.decodeBlockHeaderSlice(src[bytes_read..]);
548	bytes_read += 3;
549	}) {
550	const written_size = try block.decodeBlock(
551	dest,
552	src[bytes_read..],
553	block_header,
554	&decode_state,
555	&bytes_read,
556	block_size_max,
557	count,
558	);
559	if (hash) \|hash_state\| hash_state.update(dest[count .. count + written_size]);
560	count += written_size;
561	if (block_header.last_block) break;
562	}
563	return count;
564	}
565
566	/// Decode the header of a skippable frame. The first four bytes of `src` must
567	/// be a valid magic number for a skippable frame.
568	pub fn decodeSkippableHeader(src: *const [8]u8) SkippableHeader {
569	const magic = std.mem.readInt(u32, src[0..4], .little);
570	assert(isSkippableMagic(magic));
571	const frame_size = std.mem.readInt(u32, src[4..8], .little);
572	return .{
573	.magic_number = magic,
574	.frame_size = frame_size,
575	};
576	}
577
578	/// Returns the window size required to decompress a frame, or `null` if it
579	/// cannot be determined (which indicates a malformed frame header).
580	pub fn frameWindowSize(header: ZstandardHeader) ?u64 {
581	if (header.window_descriptor) \|descriptor\| {
582	const exponent = (descriptor & 0b11111000) >> 3;
583	const mantissa = descriptor & 0b00000111;
584	const window_log = 10 + exponent;
585	const window_base = @as(u64, 1) << @as(u6, @intCast(window_log));
586	const window_add = (window_base / 8) * mantissa;
587	return window_base + window_add;
588	} else return header.content_size;
589	}
590
591	/// Decode the header of a Zstandard frame.
592	///
593	/// Errors returned:
594	/// - `error.ReservedBitSet` if any of the reserved bits of the header are set
595	/// - `error.EndOfStream` if `source` does not contain a complete header
596	pub fn decodeZstandardHeader(
597	source: anytype,
598	) (@TypeOf(source).Error \|\| error{ EndOfStream, ReservedBitSet })!ZstandardHeader {
599	const descriptor = @as(ZstandardHeader.Descriptor, @bitCast(try source.readByte()));
600
601	if (descriptor.reserved) return error.ReservedBitSet;
602
603	var window_descriptor: ?u8 = null;
604	if (!descriptor.single_segment_flag) {
605	window_descriptor = try source.readByte();
606	}
607
608	var dictionary_id: ?u32 = null;
609	if (descriptor.dictionary_id_flag > 0) {
610	// if flag is 3 then field_size = 4, else field_size = flag
611	const field_size = (@as(u4, 1) << descriptor.dictionary_id_flag) >> 1;
612	dictionary_id = try source.readVarInt(u32, .little, field_size);
613	}
614
615	var content_size: ?u64 = null;
616	if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) {
617	const field_size = @as(u4, 1) << descriptor.content_size_flag;
618	content_size = try source.readVarInt(u64, .little, field_size);
619	if (field_size == 2) content_size.? += 256;
620	}
621
622	const header = ZstandardHeader{
623	.descriptor = descriptor,
624	.window_descriptor = window_descriptor,
625	.dictionary_id = dictionary_id,
626	.content_size = content_size,
627	};
628	return header;
629	}
630
631	test {
632	std.testing.refAllDecls(@This());
633	}

lib/std/compress/zstandard/readers.zig deleted-82

...	@@ -1,82 +0,0 @@
1	const std = @import("std");
2
3	pub const ReversedByteReader = struct {
4	remaining_bytes: usize,
5	bytes: []const u8,
6
7	const Reader = std.io.GenericReader(*ReversedByteReader, error{}, readFn);
8
9	pub fn init(bytes: []const u8) ReversedByteReader {
10	return .{
11	.bytes = bytes,
12	.remaining_bytes = bytes.len,
13	};
14	}
15
16	pub fn reader(self: *ReversedByteReader) Reader {
17	return .{ .context = self };
18	}
19
20	fn readFn(ctx: *ReversedByteReader, buffer: []u8) !usize {
21	if (ctx.remaining_bytes == 0) return 0;
22	const byte_index = ctx.remaining_bytes - 1;
23	buffer[0] = ctx.bytes[byte_index];
24	// buffer[0] = @bitReverse(ctx.bytes[byte_index]);
25	ctx.remaining_bytes = byte_index;
26	return 1;
27	}
28	};
29
30	/// A bit reader for reading the reversed bit streams used to encode
31	/// FSE compressed data.
32	pub const ReverseBitReader = struct {
33	byte_reader: ReversedByteReader,
34	bit_reader: std.io.BitReader(.big, ReversedByteReader.Reader),
35
36	pub fn init(self: *ReverseBitReader, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
37	self.byte_reader = ReversedByteReader.init(bytes);
38	self.bit_reader = std.io.bitReader(.big, self.byte_reader.reader());
39	if (bytes.len == 0) return;
40	var i: usize = 0;
41	while (i < 8 and 0 == self.readBitsNoEof(u1, 1) catch unreachable) : (i += 1) {}
42	if (i == 8) return error.BitStreamHasNoStartBit;
43	}
44
45	pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: u16) error{EndOfStream}!U {
46	return self.bit_reader.readBitsNoEof(U, num_bits);
47	}
48
49	pub fn readBits(self: @This(), comptime U: type, num_bits: u16, out_bits: u16) error{}!U {
50	return try self.bit_reader.readBits(U, num_bits, out_bits);
51	}
52
53	pub fn alignToByte(self: *@This()) void {
54	self.bit_reader.alignToByte();
55	}
56
57	pub fn isEmpty(self: ReverseBitReader) bool {
58	return self.byte_reader.remaining_bytes == 0 and self.bit_reader.count == 0;
59	}
60	};
61
62	pub fn BitReader(comptime Reader: type) type {
63	return struct {
64	underlying: std.io.BitReader(.little, Reader),
65
66	pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: u16) !U {
67	return self.underlying.readBitsNoEof(U, num_bits);
68	}
69
70	pub fn readBits(self: @This(), comptime U: type, num_bits: u16, out_bits: u16) !U {
71	return self.underlying.readBits(U, num_bits, out_bits);
72	}
73
74	pub fn alignToByte(self: *@This()) void {
75	self.underlying.alignToByte();
76	}
77	};
78	}
79
80	pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {
81	return .{ .underlying = std.io.bitReader(.little, reader) };
82	}

lib/std/compress/zstandard/types.zig deleted-403

...	@@ -1,403 +0,0 @@
1	pub const block_size_max = 1 << 17;
2
3	pub const frame = struct {
4	pub const Kind = enum { zstandard, skippable };
5
6	pub const Zstandard = struct {
7	pub const magic_number = 0xFD2FB528;
8
9	header: Header,
10	data_blocks: []Block,
11	checksum: ?u32,
12
13	pub const Header = struct {
14	descriptor: Descriptor,
15	window_descriptor: ?u8,
16	dictionary_id: ?u32,
17	content_size: ?u64,
18
19	pub const Descriptor = packed struct {
20	dictionary_id_flag: u2,
21	content_checksum_flag: bool,
22	reserved: bool,
23	unused: bool,
24	single_segment_flag: bool,
25	content_size_flag: u2,
26	};
27	};
28
29	pub const Block = struct {
30	pub const Header = struct {
31	last_block: bool,
32	block_type: Block.Type,
33	block_size: u21,
34	};
35
36	pub const Type = enum(u2) {
37	raw,
38	rle,
39	compressed,
40	reserved,
41	};
42	};
43	};
44
45	pub const Skippable = struct {
46	pub const magic_number_min = 0x184D2A50;
47	pub const magic_number_max = 0x184D2A5F;
48
49	pub const Header = struct {
50	magic_number: u32,
51	frame_size: u32,
52	};
53	};
54	};
55
56	pub const compressed_block = struct {
57	pub const LiteralsSection = struct {
58	header: Header,
59	huffman_tree: ?HuffmanTree,
60	streams: Streams,
61
62	pub const Streams = union(enum) {
63	one: []const u8,
64	four: [4][]const u8,
65	};
66
67	pub const Header = struct {
68	block_type: BlockType,
69	size_format: u2,
70	regenerated_size: u20,
71	compressed_size: ?u18,
72	};
73
74	pub const BlockType = enum(u2) {
75	raw,
76	rle,
77	compressed,
78	treeless,
79	};
80
81	pub const HuffmanTree = struct {
82	max_bit_count: u4,
83	symbol_count_minus_one: u8,
84	nodes: [256]PrefixedSymbol,
85
86	pub const PrefixedSymbol = struct {
87	symbol: u8,
88	prefix: u16,
89	weight: u4,
90	};
91
92	pub const Result = union(enum) {
93	symbol: u8,
94	index: usize,
95	};
96
97	pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{NotFound}!Result {
98	var node = self.nodes[index];
99	const weight = node.weight;
100	var i: usize = index;
101	while (node.weight == weight) {
102	if (node.prefix == prefix) return Result{ .symbol = node.symbol };
103	if (i == 0) return error.NotFound;
104	i -= 1;
105	node = self.nodes[i];
106	}
107	return Result{ .index = i };
108	}
109
110	pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
111	return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
112	}
113	};
114
115	pub const StreamCount = enum { one, four };
116	pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
117	return switch (block_type) {
118	.raw, .rle => .one,
119	.compressed, .treeless => if (size_format == 0) .one else .four,
120	};
121	}
122	};
123
124	pub const SequencesSection = struct {
125	header: SequencesSection.Header,
126	literals_length_table: Table,
127	offset_table: Table,
128	match_length_table: Table,
129
130	pub const Header = struct {
131	sequence_count: u24,
132	match_lengths: Mode,
133	offsets: Mode,
134	literal_lengths: Mode,
135
136	pub const Mode = enum(u2) {
137	predefined,
138	rle,
139	fse,
140	repeat,
141	};
142	};
143	};
144
145	pub const Table = union(enum) {
146	fse: []const Fse,
147	rle: u8,
148
149	pub const Fse = struct {
150	symbol: u8,
151	baseline: u16,
152	bits: u8,
153	};
154	};
155
156	pub const literals_length_code_table = [36]struct { u32, u5 }{
157	.{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
158	.{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
159	.{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
160	.{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
161	.{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
162	.{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
163	.{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
164	.{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
165	.{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
166	};
167
168	pub const match_length_code_table = [53]struct { u32, u5 }{
169	.{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
170	.{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
171	.{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
172	.{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
173	.{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
174	.{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
175	.{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
176	.{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
177	.{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
178	};
179
180	pub const literals_length_default_distribution = [36]i16{
181	4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
182	2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
183	-1, -1, -1, -1,
184	};
185
186	pub const match_lengths_default_distribution = [53]i16{
187	1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
188	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
189	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
190	-1, -1, -1, -1, -1,
191	};
192
193	pub const offset_codes_default_distribution = [29]i16{
194	1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
195	1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
196	};
197
198	pub const predefined_literal_fse_table = Table{
199	.fse = &[64]Table.Fse{
200	.{ .symbol = 0, .bits = 4, .baseline = 0 },
201	.{ .symbol = 0, .bits = 4, .baseline = 16 },
202	.{ .symbol = 1, .bits = 5, .baseline = 32 },
203	.{ .symbol = 3, .bits = 5, .baseline = 0 },
204	.{ .symbol = 4, .bits = 5, .baseline = 0 },
205	.{ .symbol = 6, .bits = 5, .baseline = 0 },
206	.{ .symbol = 7, .bits = 5, .baseline = 0 },
207	.{ .symbol = 9, .bits = 5, .baseline = 0 },
208	.{ .symbol = 10, .bits = 5, .baseline = 0 },
209	.{ .symbol = 12, .bits = 5, .baseline = 0 },
210	.{ .symbol = 14, .bits = 6, .baseline = 0 },
211	.{ .symbol = 16, .bits = 5, .baseline = 0 },
212	.{ .symbol = 18, .bits = 5, .baseline = 0 },
213	.{ .symbol = 19, .bits = 5, .baseline = 0 },
214	.{ .symbol = 21, .bits = 5, .baseline = 0 },
215	.{ .symbol = 22, .bits = 5, .baseline = 0 },
216	.{ .symbol = 24, .bits = 5, .baseline = 0 },
217	.{ .symbol = 25, .bits = 5, .baseline = 32 },
218	.{ .symbol = 26, .bits = 5, .baseline = 0 },
219	.{ .symbol = 27, .bits = 6, .baseline = 0 },
220	.{ .symbol = 29, .bits = 6, .baseline = 0 },
221	.{ .symbol = 31, .bits = 6, .baseline = 0 },
222	.{ .symbol = 0, .bits = 4, .baseline = 32 },
223	.{ .symbol = 1, .bits = 4, .baseline = 0 },
224	.{ .symbol = 2, .bits = 5, .baseline = 0 },
225	.{ .symbol = 4, .bits = 5, .baseline = 32 },
226	.{ .symbol = 5, .bits = 5, .baseline = 0 },
227	.{ .symbol = 7, .bits = 5, .baseline = 32 },
228	.{ .symbol = 8, .bits = 5, .baseline = 0 },
229	.{ .symbol = 10, .bits = 5, .baseline = 32 },
230	.{ .symbol = 11, .bits = 5, .baseline = 0 },
231	.{ .symbol = 13, .bits = 6, .baseline = 0 },
232	.{ .symbol = 16, .bits = 5, .baseline = 32 },
233	.{ .symbol = 17, .bits = 5, .baseline = 0 },
234	.{ .symbol = 19, .bits = 5, .baseline = 32 },
235	.{ .symbol = 20, .bits = 5, .baseline = 0 },
236	.{ .symbol = 22, .bits = 5, .baseline = 32 },
237	.{ .symbol = 23, .bits = 5, .baseline = 0 },
238	.{ .symbol = 25, .bits = 4, .baseline = 0 },
239	.{ .symbol = 25, .bits = 4, .baseline = 16 },
240	.{ .symbol = 26, .bits = 5, .baseline = 32 },
241	.{ .symbol = 28, .bits = 6, .baseline = 0 },
242	.{ .symbol = 30, .bits = 6, .baseline = 0 },
243	.{ .symbol = 0, .bits = 4, .baseline = 48 },
244	.{ .symbol = 1, .bits = 4, .baseline = 16 },
245	.{ .symbol = 2, .bits = 5, .baseline = 32 },
246	.{ .symbol = 3, .bits = 5, .baseline = 32 },
247	.{ .symbol = 5, .bits = 5, .baseline = 32 },
248	.{ .symbol = 6, .bits = 5, .baseline = 32 },
249	.{ .symbol = 8, .bits = 5, .baseline = 32 },
250	.{ .symbol = 9, .bits = 5, .baseline = 32 },
251	.{ .symbol = 11, .bits = 5, .baseline = 32 },
252	.{ .symbol = 12, .bits = 5, .baseline = 32 },
253	.{ .symbol = 15, .bits = 6, .baseline = 0 },
254	.{ .symbol = 17, .bits = 5, .baseline = 32 },
255	.{ .symbol = 18, .bits = 5, .baseline = 32 },
256	.{ .symbol = 20, .bits = 5, .baseline = 32 },
257	.{ .symbol = 21, .bits = 5, .baseline = 32 },
258	.{ .symbol = 23, .bits = 5, .baseline = 32 },
259	.{ .symbol = 24, .bits = 5, .baseline = 32 },
260	.{ .symbol = 35, .bits = 6, .baseline = 0 },
261	.{ .symbol = 34, .bits = 6, .baseline = 0 },
262	.{ .symbol = 33, .bits = 6, .baseline = 0 },
263	.{ .symbol = 32, .bits = 6, .baseline = 0 },
264	},
265	};
266
267	pub const predefined_match_fse_table = Table{
268	.fse = &[64]Table.Fse{
269	.{ .symbol = 0, .bits = 6, .baseline = 0 },
270	.{ .symbol = 1, .bits = 4, .baseline = 0 },
271	.{ .symbol = 2, .bits = 5, .baseline = 32 },
272	.{ .symbol = 3, .bits = 5, .baseline = 0 },
273	.{ .symbol = 5, .bits = 5, .baseline = 0 },
274	.{ .symbol = 6, .bits = 5, .baseline = 0 },
275	.{ .symbol = 8, .bits = 5, .baseline = 0 },
276	.{ .symbol = 10, .bits = 6, .baseline = 0 },
277	.{ .symbol = 13, .bits = 6, .baseline = 0 },
278	.{ .symbol = 16, .bits = 6, .baseline = 0 },
279	.{ .symbol = 19, .bits = 6, .baseline = 0 },
280	.{ .symbol = 22, .bits = 6, .baseline = 0 },
281	.{ .symbol = 25, .bits = 6, .baseline = 0 },
282	.{ .symbol = 28, .bits = 6, .baseline = 0 },
283	.{ .symbol = 31, .bits = 6, .baseline = 0 },
284	.{ .symbol = 33, .bits = 6, .baseline = 0 },
285	.{ .symbol = 35, .bits = 6, .baseline = 0 },
286	.{ .symbol = 37, .bits = 6, .baseline = 0 },
287	.{ .symbol = 39, .bits = 6, .baseline = 0 },
288	.{ .symbol = 41, .bits = 6, .baseline = 0 },
289	.{ .symbol = 43, .bits = 6, .baseline = 0 },
290	.{ .symbol = 45, .bits = 6, .baseline = 0 },
291	.{ .symbol = 1, .bits = 4, .baseline = 16 },
292	.{ .symbol = 2, .bits = 4, .baseline = 0 },
293	.{ .symbol = 3, .bits = 5, .baseline = 32 },
294	.{ .symbol = 4, .bits = 5, .baseline = 0 },
295	.{ .symbol = 6, .bits = 5, .baseline = 32 },
296	.{ .symbol = 7, .bits = 5, .baseline = 0 },
297	.{ .symbol = 9, .bits = 6, .baseline = 0 },
298	.{ .symbol = 12, .bits = 6, .baseline = 0 },
299	.{ .symbol = 15, .bits = 6, .baseline = 0 },
300	.{ .symbol = 18, .bits = 6, .baseline = 0 },
301	.{ .symbol = 21, .bits = 6, .baseline = 0 },
302	.{ .symbol = 24, .bits = 6, .baseline = 0 },
303	.{ .symbol = 27, .bits = 6, .baseline = 0 },
304	.{ .symbol = 30, .bits = 6, .baseline = 0 },
305	.{ .symbol = 32, .bits = 6, .baseline = 0 },
306	.{ .symbol = 34, .bits = 6, .baseline = 0 },
307	.{ .symbol = 36, .bits = 6, .baseline = 0 },
308	.{ .symbol = 38, .bits = 6, .baseline = 0 },
309	.{ .symbol = 40, .bits = 6, .baseline = 0 },
310	.{ .symbol = 42, .bits = 6, .baseline = 0 },
311	.{ .symbol = 44, .bits = 6, .baseline = 0 },
312	.{ .symbol = 1, .bits = 4, .baseline = 32 },
313	.{ .symbol = 1, .bits = 4, .baseline = 48 },
314	.{ .symbol = 2, .bits = 4, .baseline = 16 },
315	.{ .symbol = 4, .bits = 5, .baseline = 32 },
316	.{ .symbol = 5, .bits = 5, .baseline = 32 },
317	.{ .symbol = 7, .bits = 5, .baseline = 32 },
318	.{ .symbol = 8, .bits = 5, .baseline = 32 },
319	.{ .symbol = 11, .bits = 6, .baseline = 0 },
320	.{ .symbol = 14, .bits = 6, .baseline = 0 },
321	.{ .symbol = 17, .bits = 6, .baseline = 0 },
322	.{ .symbol = 20, .bits = 6, .baseline = 0 },
323	.{ .symbol = 23, .bits = 6, .baseline = 0 },
324	.{ .symbol = 26, .bits = 6, .baseline = 0 },
325	.{ .symbol = 29, .bits = 6, .baseline = 0 },
326	.{ .symbol = 52, .bits = 6, .baseline = 0 },
327	.{ .symbol = 51, .bits = 6, .baseline = 0 },
328	.{ .symbol = 50, .bits = 6, .baseline = 0 },
329	.{ .symbol = 49, .bits = 6, .baseline = 0 },
330	.{ .symbol = 48, .bits = 6, .baseline = 0 },
331	.{ .symbol = 47, .bits = 6, .baseline = 0 },
332	.{ .symbol = 46, .bits = 6, .baseline = 0 },
333	},
334	};
335
336	pub const predefined_offset_fse_table = Table{
337	.fse = &[32]Table.Fse{
338	.{ .symbol = 0, .bits = 5, .baseline = 0 },
339	.{ .symbol = 6, .bits = 4, .baseline = 0 },
340	.{ .symbol = 9, .bits = 5, .baseline = 0 },
341	.{ .symbol = 15, .bits = 5, .baseline = 0 },
342	.{ .symbol = 21, .bits = 5, .baseline = 0 },
343	.{ .symbol = 3, .bits = 5, .baseline = 0 },
344	.{ .symbol = 7, .bits = 4, .baseline = 0 },
345	.{ .symbol = 12, .bits = 5, .baseline = 0 },
346	.{ .symbol = 18, .bits = 5, .baseline = 0 },
347	.{ .symbol = 23, .bits = 5, .baseline = 0 },
348	.{ .symbol = 5, .bits = 5, .baseline = 0 },
349	.{ .symbol = 8, .bits = 4, .baseline = 0 },
350	.{ .symbol = 14, .bits = 5, .baseline = 0 },
351	.{ .symbol = 20, .bits = 5, .baseline = 0 },
352	.{ .symbol = 2, .bits = 5, .baseline = 0 },
353	.{ .symbol = 7, .bits = 4, .baseline = 16 },
354	.{ .symbol = 11, .bits = 5, .baseline = 0 },
355	.{ .symbol = 17, .bits = 5, .baseline = 0 },
356	.{ .symbol = 22, .bits = 5, .baseline = 0 },
357	.{ .symbol = 4, .bits = 5, .baseline = 0 },
358	.{ .symbol = 8, .bits = 4, .baseline = 16 },
359	.{ .symbol = 13, .bits = 5, .baseline = 0 },
360	.{ .symbol = 19, .bits = 5, .baseline = 0 },
361	.{ .symbol = 1, .bits = 5, .baseline = 0 },
362	.{ .symbol = 6, .bits = 4, .baseline = 16 },
363	.{ .symbol = 10, .bits = 5, .baseline = 0 },
364	.{ .symbol = 16, .bits = 5, .baseline = 0 },
365	.{ .symbol = 28, .bits = 5, .baseline = 0 },
366	.{ .symbol = 27, .bits = 5, .baseline = 0 },
367	.{ .symbol = 26, .bits = 5, .baseline = 0 },
368	.{ .symbol = 25, .bits = 5, .baseline = 0 },
369	.{ .symbol = 24, .bits = 5, .baseline = 0 },
370	},
371	};
372	pub const start_repeated_offset_1 = 1;
373	pub const start_repeated_offset_2 = 4;
374	pub const start_repeated_offset_3 = 8;
375
376	pub const table_accuracy_log_max = struct {
377	pub const literal = 9;
378	pub const match = 9;
379	pub const offset = 8;
380	};
381
382	pub const table_symbol_count_max = struct {
383	pub const literal = 36;
384	pub const match = 53;
385	pub const offset = 32;
386	};
387
388	pub const default_accuracy_log = struct {
389	pub const literal = 6;
390	pub const match = 6;
391	pub const offset = 5;
392	};
393	pub const table_size_max = struct {
394	pub const literal = 1 << table_accuracy_log_max.literal;
395	pub const match = 1 << table_accuracy_log_max.match;
396	pub const offset = 1 << table_accuracy_log_max.offset;
397	};
398	};
399
400	test {
401	const testing = @import("std").testing;
402	testing.refAllDeclsRecursive(@This());
403	}

lib/std/compress/zstd.zig created+152

...	@@ -0,0 +1,152 @@
1	const std = @import("../std.zig");
2	const assert = std.debug.assert;
3
4	pub const Decompress = @import("zstd/Decompress.zig");
5
6	/// Recommended amount by the standard. Lower than this may result in inability
7	/// to decompress common streams.
8	pub const default_window_len = 8 * 1024 * 1024;
9	pub const block_size_max = 1 << 17;
10
11	pub const literals_length_default_distribution = [36]i16{
12	4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
13	2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
14	-1, -1, -1, -1,
15	};
16
17	pub const match_lengths_default_distribution = [53]i16{
18	1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
19	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
20	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
21	-1, -1, -1, -1, -1,
22	};
23
24	pub const offset_codes_default_distribution = [29]i16{
25	1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
26	1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
27	};
28
29	pub const start_repeated_offset_1 = 1;
30	pub const start_repeated_offset_2 = 4;
31	pub const start_repeated_offset_3 = 8;
32
33	pub const literals_length_code_table = [36]struct { u32, u5 }{
34	.{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
35	.{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
36	.{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
37	.{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
38	.{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
39	.{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
40	.{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
41	.{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
42	.{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
43	};
44
45	pub const match_length_code_table = [53]struct { u32, u5 }{
46	.{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
47	.{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
48	.{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
49	.{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
50	.{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
51	.{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
52	.{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
53	.{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
54	.{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
55	};
56
57	pub const table_accuracy_log_max = struct {
58	pub const literal = 9;
59	pub const match = 9;
60	pub const offset = 8;
61	};
62
63	pub const table_symbol_count_max = struct {
64	pub const literal = 36;
65	pub const match = 53;
66	pub const offset = 32;
67	};
68
69	pub const default_accuracy_log = struct {
70	pub const literal = 6;
71	pub const match = 6;
72	pub const offset = 5;
73	};
74	pub const table_size_max = struct {
75	pub const literal = 1 << table_accuracy_log_max.literal;
76	pub const match = 1 << table_accuracy_log_max.match;
77	pub const offset = 1 << table_accuracy_log_max.offset;
78	};
79
80	fn testDecompress(gpa: std.mem.Allocator, compressed: []const u8) ![]u8 {
81	var out: std.ArrayListUnmanaged(u8) = .empty;
82	defer out.deinit(gpa);
83	try out.ensureUnusedCapacity(gpa, default_window_len);
84
85	var in: std.io.Reader = .fixed(compressed);
86	var zstd_stream: Decompress = .init(&in, &.{}, .{});
87	try zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited);
88
89	return out.toOwnedSlice(gpa);
90	}
91
92	fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void {
93	const gpa = std.testing.allocator;
94	const result = try testDecompress(gpa, compressed);
95	defer gpa.free(result);
96	try std.testing.expectEqualSlices(u8, uncompressed, result);
97	}
98
99	fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void {
100	const gpa = std.testing.allocator;
101
102	var out: std.ArrayListUnmanaged(u8) = .empty;
103	defer out.deinit(gpa);
104	try out.ensureUnusedCapacity(gpa, default_window_len);
105
106	var in: std.io.Reader = .fixed(compressed);
107	var zstd_stream: Decompress = .init(&in, &.{}, .{});
108	try std.testing.expectError(
109	error.ReadFailed,
110	zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited),
111	);
112	try std.testing.expectError(err, zstd_stream.err orelse {});
113	}
114
115	test Decompress {
116	const uncompressed = @embedFile("testdata/rfc8478.txt");
117	const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
118	const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
119
120	try testExpectDecompress(uncompressed, compressed3);
121	try testExpectDecompress(uncompressed, compressed19);
122	}
123
124	test "zero sized raw block" {
125	const input_raw =
126	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
127	"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
128	"\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
129	try testExpectDecompress("", input_raw);
130	}
131
132	test "zero sized rle block" {
133	const input_rle =
134	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
135	"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
136	"\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
137	"\xaa"; // block_content
138	try testExpectDecompress("", input_rle);
139	}
140
141	test "declared raw literals size too large" {
142	const input_raw =
143	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
144	"\x00\x00" ++ // frame header: everything unset, window descriptor zero
145	"\x95\x00\x00" ++ // block header with: last_block set, block_type compressed, block_size 18
146	"\xbc\xf3\xae" ++ // literals section header with: type raw, size_format 3, regenerated_size 716603
147	"\xa5\x9f\xe3"; // some bytes of literal content - the content is shorter than regenerated_size
148
149	// Note that the regenerated_size in the above input is larger than block maximum size, so the
150	// block can't be valid as it is a raw literals block.
151	try testExpectDecompressError(error.MalformedLiteralsSection, input_raw);
152	}

lib/std/compress/zstd/Decompress.zig created+1818

...	@@ -0,0 +1,1818 @@
1	const Decompress = @This();
2	const std = @import("std");
3	const assert = std.debug.assert;
4	const Reader = std.io.Reader;
5	const Limit = std.io.Limit;
6	const zstd = @import("../zstd.zig");
7	const Writer = std.io.Writer;
8
9	input: *Reader,
10	reader: Reader,
11	state: State,
12	verify_checksum: bool,
13	window_len: u32,
14	err: ?Error = null,
15
16	const State = union(enum) {
17	new_frame,
18	in_frame: InFrame,
19	skipping_frame: usize,
20	end,
21
22	const InFrame = struct {
23	frame: Frame,
24	checksum: ?u32,
25	decompressed_size: usize,
26	decode: Frame.Zstandard.Decode,
27	};
28	};
29
30	pub const Options = struct {
31	/// Verifying checksums is not implemented yet and will cause a panic if
32	/// you set this to true.
33	verify_checksum: bool = false,
34	/// Affects the minimum capacity of the provided buffer.
35	window_len: u32 = zstd.default_window_len,
36	};
37
38	pub const Error = error{
39	BadMagic,
40	BlockOversize,
41	ChecksumFailure,
42	ContentOversize,
43	DictionaryIdFlagUnsupported,
44	EndOfStream,
45	HuffmanTreeIncomplete,
46	InvalidBitStream,
47	MalformedAccuracyLog,
48	MalformedBlock,
49	MalformedCompressedBlock,
50	MalformedFrame,
51	MalformedFseBits,
52	MalformedFseTable,
53	MalformedHuffmanTree,
54	MalformedLiteralsHeader,
55	MalformedLiteralsLength,
56	MalformedLiteralsSection,
57	MalformedSequence,
58	MissingStartBit,
59	OutputBufferUndersize,
60	InputBufferUndersize,
61	ReadFailed,
62	RepeatModeFirst,
63	ReservedBitSet,
64	ReservedBlock,
65	SequenceBufferUndersize,
66	TreelessLiteralsFirst,
67	UnexpectedEndOfLiteralStream,
68	WindowOversize,
69	WindowSizeUnknown,
70	};
71
72	/// If buffer that is written to is not big enough, some streams will fail with
73	/// `error.OutputBufferUndersize`. A safe value is `zstd.default_window_len * 2`.
74	pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {
75	return .{
76	.input = input,
77	.state = .new_frame,
78	.verify_checksum = options.verify_checksum,
79	.window_len = options.window_len,
80	.reader = .{
81	.vtable = &.{ .stream = stream },
82	.buffer = buffer,
83	.seek = 0,
84	.end = 0,
85	},
86	};
87	}
88
89	fn stream(r: Reader, w: Writer, limit: Limit) Reader.StreamError!usize {
90	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
91	const in = d.input;
92
93	switch (d.state) {
94	.new_frame => {
95	// Allow error.EndOfStream only on the frame magic.
96	const magic = try in.takeEnumNonexhaustive(Frame.Magic, .little);
97	initFrame(d, w.buffer.len, magic) catch \|err\| {
98	d.err = err;
99	return error.ReadFailed;
100	};
101	return readInFrame(d, w, limit, &d.state.in_frame) catch \|err\| switch (err) {
102	error.ReadFailed => return error.ReadFailed,
103	error.WriteFailed => return error.WriteFailed,
104	else => \|e\| {
105	d.err = e;
106	return error.ReadFailed;
107	},
108	};
109	},
110	.in_frame => \|*in_frame\| {
111	return readInFrame(d, w, limit, in_frame) catch \|err\| switch (err) {
112	error.ReadFailed => return error.ReadFailed,
113	error.WriteFailed => return error.WriteFailed,
114	else => \|e\| {
115	d.err = e;
116	return error.ReadFailed;
117	},
118	};
119	},
120	.skipping_frame => \|*remaining\| {
121	const n = in.discard(.limited(remaining.*)) catch \|err\| {
122	d.err = err;
123	return error.ReadFailed;
124	};
125	remaining.* -= n;
126	if (remaining.* == 0) d.state = .new_frame;
127	return 0;
128	},
129	.end => return error.EndOfStream,
130	}
131	}
132
133	fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {
134	const in = d.input;
135	switch (magic.kind() orelse return error.BadMagic) {
136	.zstandard => {
137	const header = try Frame.Zstandard.Header.decode(in);
138	d.state = .{ .in_frame = .{
139	.frame = try Frame.init(header, window_size_max, d.verify_checksum),
140	.checksum = null,
141	.decompressed_size = 0,
142	.decode = .init,
143	} };
144	},
145	.skippable => {
146	const frame_size = try in.takeInt(u32, .little);
147	d.state = .{ .skipping_frame = frame_size };
148	},
149	}
150	}
151
152	fn readInFrame(d: Decompress, w: Writer, limit: Limit, state: *State.InFrame) !usize {
153	const in = d.input;
154	const window_len = d.window_len;
155
156	const block_header = try in.takeStruct(Frame.Zstandard.Block.Header, .little);
157	const block_size = block_header.size;
158	const frame_block_size_max = state.frame.block_size_max;
159	if (frame_block_size_max < block_size) return error.BlockOversize;
160	if (@intFromEnum(limit) < block_size) return error.OutputBufferUndersize;
161	var bytes_written: usize = 0;
162	switch (block_header.type) {
163	.raw => {
164	try in.streamExactPreserve(w, window_len, block_size);
165	bytes_written = block_size;
166	},
167	.rle => {
168	const byte = try in.takeByte();
169	try w.splatBytePreserve(window_len, byte, block_size);
170	bytes_written = block_size;
171	},
172	.compressed => {
173	var literals_buffer: [zstd.block_size_max]u8 = undefined;
174	var sequence_buffer: [zstd.block_size_max]u8 = undefined;
175	var remaining: Limit = .limited(block_size);
176	const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);
177	const sequences_header = try SequencesSection.Header.decode(in, &remaining);
178
179	const decode = &state.decode;
180	try decode.prepare(in, &remaining, literals, sequences_header);
181
182	{
183	if (sequence_buffer.len < @intFromEnum(remaining))
184	return error.SequenceBufferUndersize;
185	const seq_slice = remaining.slice(&sequence_buffer);
186	try in.readSliceAll(seq_slice);
187	var bit_stream = try ReverseBitReader.init(seq_slice);
188
189	if (sequences_header.sequence_count > 0) {
190	try decode.readInitialFseState(&bit_stream);
191
192	// Ensures the following calls to `decodeSequence` will not flush.
193	if (window_len + frame_block_size_max > w.buffer.len) return error.OutputBufferUndersize;
194	const dest = (try w.writableSliceGreedyPreserve(window_len, frame_block_size_max))[0..frame_block_size_max];
195	const write_pos = dest.ptr - w.buffer.ptr;
196	for (0..sequences_header.sequence_count - 1) \|_\| {
197	bytes_written += try decode.decodeSequence(w.buffer, write_pos + bytes_written, &bit_stream);
198	try decode.updateState(.literal, &bit_stream);
199	try decode.updateState(.match, &bit_stream);
200	try decode.updateState(.offset, &bit_stream);
201	}
202	bytes_written += try decode.decodeSequence(w.buffer, write_pos + bytes_written, &bit_stream);
203	if (bytes_written > dest.len) return error.MalformedSequence;
204	w.advance(bytes_written);
205	}
206
207	if (!bit_stream.isEmpty()) {
208	return error.MalformedCompressedBlock;
209	}
210	}
211
212	if (decode.literal_written_count < literals.header.regenerated_size) {
213	const len = literals.header.regenerated_size - decode.literal_written_count;
214	try decode.decodeLiterals(w, len);
215	decode.literal_written_count += len;
216	bytes_written += len;
217	}
218
219	switch (decode.literal_header.block_type) {
220	.treeless, .compressed => {
221	if (!decode.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
222	},
223	.raw, .rle => {},
224	}
225
226	if (bytes_written > frame_block_size_max) return error.BlockOversize;
227	},
228	.reserved => return error.ReservedBlock,
229	}
230
231	if (state.frame.hasher_opt) \|*hasher\| {
232	if (bytes_written > 0) {
233	_ = hasher;
234	@panic("TODO all those bytes written needed to go through the hasher too");
235	}
236	}
237
238	state.decompressed_size += bytes_written;
239
240	if (block_header.last) {
241	if (state.frame.has_checksum) {
242	const expected_checksum = try in.takeInt(u32, .little);
243	if (state.frame.hasher_opt) \|*hasher\| {
244	const actual_checksum: u32 = @truncate(hasher.final());
245	if (expected_checksum != actual_checksum) return error.ChecksumFailure;
246	}
247	}
248	if (state.frame.content_size) \|content_size\| {
249	if (content_size != state.decompressed_size) {
250	return error.MalformedFrame;
251	}
252	}
253	d.state = .new_frame;
254	} else if (state.frame.content_size) \|content_size\| {
255	if (state.decompressed_size > content_size) return error.MalformedFrame;
256	}
257
258	return bytes_written;
259	}
260
261	pub const Frame = struct {
262	hasher_opt: ?std.hash.XxHash64,
263	window_size: usize,
264	has_checksum: bool,
265	block_size_max: usize,
266	content_size: ?usize,
267
268	pub const Magic = enum(u32) {
269	zstandard = 0xFD2FB528,
270	_,
271
272	pub fn kind(m: Magic) ?Kind {
273	return switch (@intFromEnum(m)) {
274	@intFromEnum(Magic.zstandard) => .zstandard,
275	@intFromEnum(Skippable.magic_min)...@intFromEnum(Skippable.magic_max) => .skippable,
276	else => null,
277	};
278	}
279
280	pub fn isSkippable(m: Magic) bool {
281	return switch (@intFromEnum(m)) {
282	@intFromEnum(Skippable.magic_min)...@intFromEnum(Skippable.magic_max) => true,
283	else => false,
284	};
285	}
286	};
287
288	pub const Kind = enum { zstandard, skippable };
289
290	pub const Zstandard = struct {
291	pub const magic: Magic = .zstandard;
292
293	header: Header,
294	data_blocks: []Block,
295	checksum: ?u32,
296
297	pub const Header = struct {
298	descriptor: Descriptor,
299	window_descriptor: ?u8,
300	dictionary_id: ?u32,
301	content_size: ?u64,
302
303	pub const Descriptor = packed struct {
304	dictionary_id_flag: u2,
305	content_checksum_flag: bool,
306	reserved: bool,
307	unused: bool,
308	single_segment_flag: bool,
309	content_size_flag: u2,
310	};
311
312	pub const DecodeError = Reader.Error \|\| error{ReservedBitSet};
313
314	pub fn decode(in: *Reader) DecodeError!Header {
315	const descriptor: Descriptor = @bitCast(try in.takeByte());
316
317	if (descriptor.reserved) return error.ReservedBitSet;
318
319	const window_descriptor: ?u8 = if (descriptor.single_segment_flag) null else try in.takeByte();
320
321	const dictionary_id: ?u32 = if (descriptor.dictionary_id_flag > 0) d: {
322	// if flag is 3 then field_size = 4, else field_size = flag
323	const field_size = (@as(u4, 1) << descriptor.dictionary_id_flag) >> 1;
324	break :d try in.takeVarInt(u32, .little, field_size);
325	} else null;
326
327	const content_size: ?u64 = if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) c: {
328	const field_size = @as(u4, 1) << descriptor.content_size_flag;
329	const content_size = try in.takeVarInt(u64, .little, field_size);
330	break :c if (field_size == 2) content_size + 256 else content_size;
331	} else null;
332
333	return .{
334	.descriptor = descriptor,
335	.window_descriptor = window_descriptor,
336	.dictionary_id = dictionary_id,
337	.content_size = content_size,
338	};
339	}
340
341	/// Returns the window size required to decompress a frame, or `null` if it
342	/// cannot be determined (which indicates a malformed frame header).
343	pub fn windowSize(header: Header) ?u64 {
344	if (header.window_descriptor) \|descriptor\| {
345	const exponent = (descriptor & 0b11111000) >> 3;
346	const mantissa = descriptor & 0b00000111;
347	const window_log = 10 + exponent;
348	const window_base = @as(u64, 1) << @as(u6, @intCast(window_log));
349	const window_add = (window_base / 8) * mantissa;
350	return window_base + window_add;
351	} else return header.content_size;
352	}
353	};
354
355	pub const Block = struct {
356	pub const Header = packed struct(u24) {
357	last: bool,
358	type: Type,
359	size: u21,
360	};
361
362	pub const Type = enum(u2) {
363	raw,
364	rle,
365	compressed,
366	reserved,
367	};
368	};
369
370	pub const Decode = struct {
371	repeat_offsets: [3]u32,
372
373	offset: StateData(8),
374	match: StateData(9),
375	literal: StateData(9),
376
377	literal_fse_buffer: [zstd.table_size_max.literal]Table.Fse,
378	match_fse_buffer: [zstd.table_size_max.match]Table.Fse,
379	offset_fse_buffer: [zstd.table_size_max.offset]Table.Fse,
380
381	fse_tables_undefined: bool,
382
383	literal_stream_reader: ReverseBitReader,
384	literal_stream_index: usize,
385	literal_streams: LiteralsSection.Streams,
386	literal_header: LiteralsSection.Header,
387	huffman_tree: ?LiteralsSection.HuffmanTree,
388
389	literal_written_count: usize,
390
391	fn StateData(comptime max_accuracy_log: comptime_int) type {
392	return struct {
393	state: @This().State,
394	table: Table,
395	accuracy_log: u8,
396
397	const State = std.meta.Int(.unsigned, max_accuracy_log);
398	};
399	}
400
401	const init: Decode = .{
402	.repeat_offsets = .{
403	zstd.start_repeated_offset_1,
404	zstd.start_repeated_offset_2,
405	zstd.start_repeated_offset_3,
406	},
407
408	.offset = undefined,
409	.match = undefined,
410	.literal = undefined,
411
412	.literal_fse_buffer = undefined,
413	.match_fse_buffer = undefined,
414	.offset_fse_buffer = undefined,
415
416	.fse_tables_undefined = true,
417
418	.literal_written_count = 0,
419	.literal_header = undefined,
420	.literal_streams = undefined,
421	.literal_stream_reader = undefined,
422	.literal_stream_index = undefined,
423	.huffman_tree = null,
424	};
425
426	pub const PrepareError = error{
427	/// the (reversed) literal bitstream's first byte does not have any bits set
428	MissingStartBit,
429	/// `literals` is a treeless literals section and the decode state does not
430	/// have a Huffman tree from a previous block
431	TreelessLiteralsFirst,
432	/// on the first call if one of the sequence FSE tables is set to repeat mode
433	RepeatModeFirst,
434	/// an FSE table has an invalid accuracy
435	MalformedAccuracyLog,
436	/// failed decoding an FSE table
437	MalformedFseTable,
438	/// input stream ends before all FSE tables are read
439	EndOfStream,
440	ReadFailed,
441	InputBufferUndersize,
442	};
443
444	/// Prepare the decoder to decode a compressed block. Loads the
445	/// literals stream and Huffman tree from `literals` and reads the
446	/// FSE tables from `in`.
447	pub fn prepare(
448	self: *Decode,
449	in: *Reader,
450	remaining: *Limit,
451	literals: LiteralsSection,
452	sequences_header: SequencesSection.Header,
453	) PrepareError!void {
454	self.literal_written_count = 0;
455	self.literal_header = literals.header;
456	self.literal_streams = literals.streams;
457
458	if (literals.huffman_tree) \|tree\| {
459	self.huffman_tree = tree;
460	} else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
461	return error.TreelessLiteralsFirst;
462	}
463
464	switch (literals.header.block_type) {
465	.raw, .rle => {},
466	.compressed, .treeless => {
467	self.literal_stream_index = 0;
468	switch (literals.streams) {
469	.one => \|slice\| try self.initLiteralStream(slice),
470	.four => \|streams\| try self.initLiteralStream(streams[0]),
471	}
472	},
473	}
474
475	if (sequences_header.sequence_count > 0) {
476	try self.updateFseTable(in, remaining, .literal, sequences_header.literal_lengths);
477	try self.updateFseTable(in, remaining, .offset, sequences_header.offsets);
478	try self.updateFseTable(in, remaining, .match, sequences_header.match_lengths);
479	self.fse_tables_undefined = false;
480	}
481	}
482
483	/// Read initial FSE states for sequence decoding.
484	pub fn readInitialFseState(self: Decode, bit_reader: ReverseBitReader) error{EndOfStream}!void {
485	self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
486	self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
487	self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
488	}
489
490	fn updateRepeatOffset(self: *Decode, offset: u32) void {
491	self.repeat_offsets[2] = self.repeat_offsets[1];
492	self.repeat_offsets[1] = self.repeat_offsets[0];
493	self.repeat_offsets[0] = offset;
494	}
495
496	fn useRepeatOffset(self: *Decode, index: usize) u32 {
497	if (index == 1)
498	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
499	else if (index == 2) {
500	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
501	std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
502	}
503	return self.repeat_offsets[0];
504	}
505
506	const WhichFse = enum { offset, match, literal };
507
508	/// TODO: don't use `@field`
509	fn updateState(
510	self: *Decode,
511	comptime choice: WhichFse,
512	bit_reader: *ReverseBitReader,
513	) error{ MalformedFseBits, EndOfStream }!void {
514	switch (@field(self, @tagName(choice)).table) {
515	.rle => {},
516	.fse => \|table\| {
517	const data = table[@field(self, @tagName(choice)).state];
518	const T = @TypeOf(@field(self, @tagName(choice))).State;
519	const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
520	const next_state = std.math.cast(
521	@TypeOf(@field(self, @tagName(choice))).State,
522	data.baseline + bits_summand,
523	) orelse return error.MalformedFseBits;
524	@field(self, @tagName(choice)).state = next_state;
525	},
526	}
527	}
528
529	const FseTableError = error{
530	MalformedFseTable,
531	MalformedAccuracyLog,
532	RepeatModeFirst,
533	EndOfStream,
534	};
535
536	/// TODO: don't use `@field`
537	fn updateFseTable(
538	self: *Decode,
539	in: *Reader,
540	remaining: *Limit,
541	comptime choice: WhichFse,
542	mode: SequencesSection.Header.Mode,
543	) !void {
544	const field_name = @tagName(choice);
545	switch (mode) {
546	.predefined => {
547	@field(self, field_name).accuracy_log =
548	@field(zstd.default_accuracy_log, field_name);
549
550	@field(self, field_name).table =
551	@field(Table, "predefined_" ++ field_name);
552	},
553	.rle => {
554	@field(self, field_name).accuracy_log = 0;
555	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
556	@field(self, field_name).table = .{ .rle = try in.takeByte() };
557	},
558	.fse => {
559	const max_table_size = 2048;
560	const peek_len: usize = remaining.minInt(max_table_size);
561	if (in.buffer.len < peek_len) return error.InputBufferUndersize;
562	const limited_buffer = try in.peek(peek_len);
563	var bit_reader: BitReader = .{ .bytes = limited_buffer };
564	const table_size = try Table.decode(
565	&bit_reader,
566	@field(zstd.table_symbol_count_max, field_name),
567	@field(zstd.table_accuracy_log_max, field_name),
568	&@field(self, field_name ++ "_fse_buffer"),
569	);
570	@field(self, field_name).table = .{
571	.fse = (&@field(self, field_name ++ "_fse_buffer"))[0..table_size],
572	};
573	@field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
574	in.toss(bit_reader.index);
575	remaining.* = remaining.subtract(bit_reader.index).?;
576	},
577	.repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
578	}
579	}
580
581	const Sequence = struct {
582	literal_length: u32,
583	match_length: u32,
584	offset: u32,
585	};
586
587	fn nextSequence(
588	self: *Decode,
589	bit_reader: *ReverseBitReader,
590	) error{ InvalidBitStream, EndOfStream }!Sequence {
591	const raw_code = self.getCode(.offset);
592	const offset_code = std.math.cast(u5, raw_code) orelse {
593	return error.InvalidBitStream;
594	};
595	const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
596
597	const match_code = self.getCode(.match);
598	if (match_code >= zstd.match_length_code_table.len)
599	return error.InvalidBitStream;
600	const match = zstd.match_length_code_table[match_code];
601	const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
602
603	const literal_code = self.getCode(.literal);
604	if (literal_code >= zstd.literals_length_code_table.len)
605	return error.InvalidBitStream;
606	const literal = zstd.literals_length_code_table[literal_code];
607	const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
608
609	const offset = if (offset_value > 3) offset: {
610	const offset = offset_value - 3;
611	self.updateRepeatOffset(offset);
612	break :offset offset;
613	} else offset: {
614	if (literal_length == 0) {
615	if (offset_value == 3) {
616	const offset = self.repeat_offsets[0] - 1;
617	self.updateRepeatOffset(offset);
618	break :offset offset;
619	}
620	break :offset self.useRepeatOffset(offset_value);
621	}
622	break :offset self.useRepeatOffset(offset_value - 1);
623	};
624
625	if (offset == 0) return error.InvalidBitStream;
626
627	return .{
628	.literal_length = literal_length,
629	.match_length = match_length,
630	.offset = offset,
631	};
632	}
633
634	/// Decode one sequence from `bit_reader` into `dest`. Updates FSE states
635	/// if `last_sequence` is `false`. Assumes `prepare` called for the block
636	/// before attempting to decode sequences.
637	fn decodeSequence(
638	decode: *Decode,
639	dest: []u8,
640	write_pos: usize,
641	bit_reader: *ReverseBitReader,
642	) !usize {
643	const sequence = try decode.nextSequence(bit_reader);
644	const literal_length: usize = sequence.literal_length;
645	const match_length: usize = sequence.match_length;
646	const sequence_length = literal_length + match_length;
647
648	const copy_start = std.math.sub(usize, write_pos + sequence.literal_length, sequence.offset) catch
649	return error.MalformedSequence;
650
651	if (decode.literal_written_count + literal_length > decode.literal_header.regenerated_size)
652	return error.MalformedLiteralsLength;
653	var sub_bw: Writer = .fixed(dest[write_pos..]);
654	try decodeLiterals(decode, &sub_bw, literal_length);
655	decode.literal_written_count += literal_length;
656	// This is not a @memmove; it intentionally repeats patterns
657	// caused by iterating one byte at a time.
658	for (
659	dest[write_pos + literal_length ..][0..match_length],
660	dest[copy_start..][0..match_length],
661	) \|d, s\| d. = s;
662	return sequence_length;
663	}
664
665	fn nextLiteralMultiStream(self: *Decode) error{MissingStartBit}!void {
666	self.literal_stream_index += 1;
667	try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
668	}
669
670	fn initLiteralStream(self: *Decode, bytes: []const u8) error{MissingStartBit}!void {
671	self.literal_stream_reader = try ReverseBitReader.init(bytes);
672	}
673
674	fn isLiteralStreamEmpty(self: *Decode) bool {
675	switch (self.literal_streams) {
676	.one => return self.literal_stream_reader.isEmpty(),
677	.four => return self.literal_stream_index == 3 and self.literal_stream_reader.isEmpty(),
678	}
679	}
680
681	const LiteralBitsError = error{
682	MissingStartBit,
683	UnexpectedEndOfLiteralStream,
684	};
685	fn readLiteralsBits(
686	self: *Decode,
687	bit_count_to_read: u16,
688	) LiteralBitsError!u16 {
689	return self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch bits: {
690	if (self.literal_streams == .four and self.literal_stream_index < 3) {
691	try self.nextLiteralMultiStream();
692	break :bits self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch
693	return error.UnexpectedEndOfLiteralStream;
694	} else {
695	return error.UnexpectedEndOfLiteralStream;
696	}
697	};
698	}
699
700	/// Decode `len` bytes of literals into `w`.
701	fn decodeLiterals(d: Decode, w: Writer, len: usize) !void {
702	switch (d.literal_header.block_type) {
703	.raw => {
704	try w.writeAll(d.literal_streams.one[d.literal_written_count..][0..len]);
705	},
706	.rle => {
707	try w.splatByteAll(d.literal_streams.one[0], len);
708	},
709	.compressed, .treeless => {
710	if (len > w.buffer.len) return error.OutputBufferUndersize;
711	const buf = try w.writableSlice(len);
712	const huffman_tree = d.huffman_tree.?;
713	const max_bit_count = huffman_tree.max_bit_count;
714	const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
715	huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
716	max_bit_count,
717	);
718	var bits_read: u4 = 0;
719	var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
720	var bit_count_to_read: u4 = starting_bit_count;
721	for (buf) \|*out\| {
722	var prefix: u16 = 0;
723	while (true) {
724	const new_bits = try d.readLiteralsBits(bit_count_to_read);
725	prefix <<= bit_count_to_read;
726	prefix \|= new_bits;
727	bits_read += bit_count_to_read;
728	const result = try huffman_tree.query(huffman_tree_index, prefix);
729
730	switch (result) {
731	.symbol => \|sym\| {
732	out.* = sym;
733	bit_count_to_read = starting_bit_count;
734	bits_read = 0;
735	huffman_tree_index = huffman_tree.symbol_count_minus_one;
736	break;
737	},
738	.index => \|index\| {
739	huffman_tree_index = index;
740	const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
741	huffman_tree.nodes[index].weight,
742	max_bit_count,
743	);
744	bit_count_to_read = bit_count - bits_read;
745	},
746	}
747	}
748	}
749	},
750	}
751	}
752
753	/// TODO: don't use `@field`
754	fn getCode(self: *Decode, comptime choice: WhichFse) u32 {
755	return switch (@field(self, @tagName(choice)).table) {
756	.rle => \|value\| value,
757	.fse => \|table\| table[@field(self, @tagName(choice)).state].symbol,
758	};
759	}
760	};
761	};
762
763	pub const Skippable = struct {
764	pub const magic_min: Magic = @enumFromInt(0x184D2A50);
765	pub const magic_max: Magic = @enumFromInt(0x184D2A5F);
766
767	pub const Header = struct {
768	magic_number: u32,
769	frame_size: u32,
770	};
771	};
772
773	const InitError = error{
774	/// Frame uses a dictionary.
775	DictionaryIdFlagUnsupported,
776	/// Frame does not have a valid window size.
777	WindowSizeUnknown,
778	/// Window size exceeds `window_size_max` or max `usize` value.
779	WindowOversize,
780	/// Frame header indicates a content size exceeding max `usize` value.
781	ContentOversize,
782	};
783
784	/// Validates `frame_header` and returns the associated `Frame`.
785	pub fn init(
786	frame_header: Frame.Zstandard.Header,
787	window_size_max: usize,
788	verify_checksum: bool,
789	) InitError!Frame {
790	if (frame_header.descriptor.dictionary_id_flag != 0)
791	return error.DictionaryIdFlagUnsupported;
792
793	const window_size_raw = frame_header.windowSize() orelse return error.WindowSizeUnknown;
794	const window_size = if (window_size_raw > window_size_max)
795	return error.WindowOversize
796	else
797	std.math.cast(usize, window_size_raw) orelse return error.WindowOversize;
798
799	const should_compute_checksum =
800	frame_header.descriptor.content_checksum_flag and verify_checksum;
801
802	const content_size = if (frame_header.content_size) \|size\|
803	std.math.cast(usize, size) orelse return error.ContentOversize
804	else
805	null;
806
807	return .{
808	.hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
809	.window_size = window_size,
810	.has_checksum = frame_header.descriptor.content_checksum_flag,
811	.block_size_max = @min(zstd.block_size_max, window_size),
812	.content_size = content_size,
813	};
814	}
815	};
816
817	pub const LiteralsSection = struct {
818	header: Header,
819	huffman_tree: ?HuffmanTree,
820	streams: Streams,
821
822	pub const Streams = union(enum) {
823	one: []const u8,
824	four: [4][]const u8,
825
826	fn decode(size_format: u2, stream_data: []const u8) !Streams {
827	if (size_format == 0) {
828	return .{ .one = stream_data };
829	}
830
831	if (stream_data.len < 6) return error.MalformedLiteralsSection;
832
833	const stream_1_length: usize = std.mem.readInt(u16, stream_data[0..2], .little);
834	const stream_2_length: usize = std.mem.readInt(u16, stream_data[2..4], .little);
835	const stream_3_length: usize = std.mem.readInt(u16, stream_data[4..6], .little);
836
837	const stream_1_start = 6;
838	const stream_2_start = stream_1_start + stream_1_length;
839	const stream_3_start = stream_2_start + stream_2_length;
840	const stream_4_start = stream_3_start + stream_3_length;
841
842	if (stream_data.len < stream_4_start) return error.MalformedLiteralsSection;
843
844	return .{ .four = .{
845	stream_data[stream_1_start .. stream_1_start + stream_1_length],
846	stream_data[stream_2_start .. stream_2_start + stream_2_length],
847	stream_data[stream_3_start .. stream_3_start + stream_3_length],
848	stream_data[stream_4_start..],
849	} };
850	}
851	};
852
853	pub const Header = struct {
854	block_type: BlockType,
855	size_format: u2,
856	regenerated_size: u20,
857	compressed_size: ?u18,
858
859	/// Decode a literals section header.
860	pub fn decode(in: Reader, remaining: Limit) !Header {
861	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
862	const byte0 = try in.takeByte();
863	const block_type: BlockType = @enumFromInt(byte0 & 0b11);
864	const size_format: u2 = @intCast((byte0 & 0b1100) >> 2);
865	var regenerated_size: u20 = undefined;
866	var compressed_size: ?u18 = null;
867	switch (block_type) {
868	.raw, .rle => {
869	switch (size_format) {
870	0, 2 => {
871	regenerated_size = byte0 >> 3;
872	},
873	1 => {
874	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
875	regenerated_size = (byte0 >> 4) + (@as(u20, try in.takeByte()) << 4);
876	},
877	3 => {
878	remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
879	regenerated_size = (byte0 >> 4) +
880	(@as(u20, try in.takeByte()) << 4) +
881	(@as(u20, try in.takeByte()) << 12);
882	},
883	}
884	},
885	.compressed, .treeless => {
886	remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
887	const byte1 = try in.takeByte();
888	const byte2 = try in.takeByte();
889	switch (size_format) {
890	0, 1 => {
891	regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
892	compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
893	},
894	2 => {
895	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
896	const byte3 = try in.takeByte();
897	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
898	compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
899	},
900	3 => {
901	remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
902	const byte3 = try in.takeByte();
903	const byte4 = try in.takeByte();
904	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
905	compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
906	},
907	}
908	},
909	}
910	return .{
911	.block_type = block_type,
912	.size_format = size_format,
913	.regenerated_size = regenerated_size,
914	.compressed_size = compressed_size,
915	};
916	}
917	};
918
919	pub const BlockType = enum(u2) {
920	raw,
921	rle,
922	compressed,
923	treeless,
924	};
925
926	pub const HuffmanTree = struct {
927	max_bit_count: u4,
928	symbol_count_minus_one: u8,
929	nodes: [256]PrefixedSymbol,
930
931	pub const PrefixedSymbol = struct {
932	symbol: u8,
933	prefix: u16,
934	weight: u4,
935	};
936
937	pub const Result = union(enum) {
938	symbol: u8,
939	index: usize,
940	};
941
942	pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{HuffmanTreeIncomplete}!Result {
943	var node = self.nodes[index];
944	const weight = node.weight;
945	var i: usize = index;
946	while (node.weight == weight) {
947	if (node.prefix == prefix) return .{ .symbol = node.symbol };
948	if (i == 0) return error.HuffmanTreeIncomplete;
949	i -= 1;
950	node = self.nodes[i];
951	}
952	return .{ .index = i };
953	}
954
955	pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
956	return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
957	}
958
959	pub const DecodeError = Reader.Error \|\| error{
960	MalformedHuffmanTree,
961	MalformedFseTable,
962	MalformedAccuracyLog,
963	EndOfStream,
964	MissingStartBit,
965	};
966
967	pub fn decode(in: Reader, remaining: Limit) HuffmanTree.DecodeError!HuffmanTree {
968	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
969	const header = try in.takeByte();
970	if (header < 128) {
971	return decodeFse(in, remaining, header);
972	} else {
973	return decodeDirect(in, remaining, header - 127);
974	}
975	}
976
977	fn decodeDirect(
978	in: *Reader,
979	remaining: *Limit,
980	encoded_symbol_count: usize,
981	) HuffmanTree.DecodeError!HuffmanTree {
982	var weights: [256]u4 = undefined;
983	const weights_byte_count = (encoded_symbol_count + 1) / 2;
984	remaining.* = remaining.subtract(weights_byte_count) orelse return error.EndOfStream;
985	for (0..weights_byte_count) \|i\| {
986	const byte = try in.takeByte();
987	weights[2 * i] = @as(u4, @intCast(byte >> 4));
988	weights[2 * i + 1] = @as(u4, @intCast(byte & 0xF));
989	}
990	const symbol_count = encoded_symbol_count + 1;
991	return build(&weights, symbol_count);
992	}
993
994	fn decodeFse(
995	in: *Reader,
996	remaining: *Limit,
997	compressed_size: usize,
998	) HuffmanTree.DecodeError!HuffmanTree {
999	var weights: [256]u4 = undefined;
1000	remaining.* = remaining.subtract(compressed_size) orelse return error.EndOfStream;
1001	const compressed_buffer = try in.take(compressed_size);
1002	var bit_reader: BitReader = .{ .bytes = compressed_buffer };
1003	var entries: [1 << 6]Table.Fse = undefined;
1004	const table_size = try Table.decode(&bit_reader, 256, 6, &entries);
1005	const accuracy_log = std.math.log2_int_ceil(usize, table_size);
1006	const remaining_buffer = bit_reader.bytes[bit_reader.index..];
1007	const symbol_count = try assignWeights(remaining_buffer, accuracy_log, &entries, &weights);
1008	return build(&weights, symbol_count);
1009	}
1010
1011	fn assignWeights(
1012	huff_bits_buffer: []const u8,
1013	accuracy_log: u16,
1014	entries: *[1 << 6]Table.Fse,
1015	weights: *[256]u4,
1016	) !usize {
1017	var huff_bits = try ReverseBitReader.init(huff_bits_buffer);
1018
1019	var i: usize = 0;
1020	var even_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
1021	var odd_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
1022
1023	while (i < 254) {
1024	const even_data = entries[even_state];
1025	var read_bits: u16 = 0;
1026	const even_bits = huff_bits.readBits(u32, even_data.bits, &read_bits) catch unreachable;
1027	weights[i] = std.math.cast(u4, even_data.symbol) orelse return error.MalformedHuffmanTree;
1028	i += 1;
1029	if (read_bits < even_data.bits) {
1030	weights[i] = std.math.cast(u4, entries[odd_state].symbol) orelse return error.MalformedHuffmanTree;
1031	i += 1;
1032	break;
1033	}
1034	even_state = even_data.baseline + even_bits;
1035
1036	read_bits = 0;
1037	const odd_data = entries[odd_state];
1038	const odd_bits = huff_bits.readBits(u32, odd_data.bits, &read_bits) catch unreachable;
1039	weights[i] = std.math.cast(u4, odd_data.symbol) orelse return error.MalformedHuffmanTree;
1040	i += 1;
1041	if (read_bits < odd_data.bits) {
1042	if (i == 255) return error.MalformedHuffmanTree;
1043	weights[i] = std.math.cast(u4, entries[even_state].symbol) orelse return error.MalformedHuffmanTree;
1044	i += 1;
1045	break;
1046	}
1047	odd_state = odd_data.baseline + odd_bits;
1048	} else return error.MalformedHuffmanTree;
1049
1050	if (!huff_bits.isEmpty()) {
1051	return error.MalformedHuffmanTree;
1052	}
1053
1054	return i + 1; // stream contains all but the last symbol
1055	}
1056
1057	fn assignSymbols(weight_sorted_prefixed_symbols: []PrefixedSymbol, weights: [256]u4) usize {
1058	for (0..weight_sorted_prefixed_symbols.len) \|i\| {
1059	weight_sorted_prefixed_symbols[i] = .{
1060	.symbol = @as(u8, @intCast(i)),
1061	.weight = undefined,
1062	.prefix = undefined,
1063	};
1064	}
1065
1066	std.mem.sort(
1067	PrefixedSymbol,
1068	weight_sorted_prefixed_symbols,
1069	weights,
1070	lessThanByWeight,
1071	);
1072
1073	var prefix: u16 = 0;
1074	var prefixed_symbol_count: usize = 0;
1075	var sorted_index: usize = 0;
1076	const symbol_count = weight_sorted_prefixed_symbols.len;
1077	while (sorted_index < symbol_count) {
1078	var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
1079	const weight = weights[symbol];
1080	if (weight == 0) {
1081	sorted_index += 1;
1082	continue;
1083	}
1084
1085	while (sorted_index < symbol_count) : ({
1086	sorted_index += 1;
1087	prefixed_symbol_count += 1;
1088	prefix += 1;
1089	}) {
1090	symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
1091	if (weights[symbol] != weight) {
1092	prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
1093	break;
1094	}
1095	weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
1096	weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
1097	weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
1098	}
1099	}
1100	return prefixed_symbol_count;
1101	}
1102
1103	fn build(weights: *[256]u4, symbol_count: usize) error{MalformedHuffmanTree}!HuffmanTree {
1104	var weight_power_sum_big: u32 = 0;
1105	for (weights[0 .. symbol_count - 1]) \|value\| {
1106	weight_power_sum_big += (@as(u16, 1) << value) >> 1;
1107	}
1108	if (weight_power_sum_big >= 1 << 11) return error.MalformedHuffmanTree;
1109	const weight_power_sum = @as(u16, @intCast(weight_power_sum_big));
1110
1111	// advance to next power of two (even if weight_power_sum is a power of 2)
1112	// TODO: is it valid to have weight_power_sum == 0?
1113	const max_number_of_bits = if (weight_power_sum == 0) 1 else std.math.log2_int(u16, weight_power_sum) + 1;
1114	const next_power_of_two = @as(u16, 1) << max_number_of_bits;
1115	weights[symbol_count - 1] = std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1;
1116
1117	var weight_sorted_prefixed_symbols: [256]PrefixedSymbol = undefined;
1118	const prefixed_symbol_count = assignSymbols(weight_sorted_prefixed_symbols[0..symbol_count], weights.*);
1119	const tree: HuffmanTree = .{
1120	.max_bit_count = max_number_of_bits,
1121	.symbol_count_minus_one = @as(u8, @intCast(prefixed_symbol_count - 1)),
1122	.nodes = weight_sorted_prefixed_symbols,
1123	};
1124	return tree;
1125	}
1126
1127	fn lessThanByWeight(
1128	weights: [256]u4,
1129	lhs: PrefixedSymbol,
1130	rhs: PrefixedSymbol,
1131	) bool {
1132	// NOTE: this function relies on the use of a stable sorting algorithm,
1133	// otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
1134	// should be added
1135	return weights[lhs.symbol] < weights[rhs.symbol];
1136	}
1137	};
1138
1139	pub const StreamCount = enum { one, four };
1140	pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
1141	return switch (block_type) {
1142	.raw, .rle => .one,
1143	.compressed, .treeless => if (size_format == 0) .one else .four,
1144	};
1145	}
1146
1147	pub const DecodeError = error{
1148	/// Invalid header.
1149	MalformedLiteralsHeader,
1150	/// Decoding errors.
1151	MalformedLiteralsSection,
1152	/// Compressed literals have invalid accuracy.
1153	MalformedAccuracyLog,
1154	/// Compressed literals have invalid FSE table.
1155	MalformedFseTable,
1156	/// Failed decoding a Huffamn tree.
1157	MalformedHuffmanTree,
1158	/// Not enough bytes to complete the section.
1159	EndOfStream,
1160	ReadFailed,
1161	MissingStartBit,
1162	};
1163
1164	pub fn decode(in: Reader, remaining: Limit, buffer: []u8) DecodeError!LiteralsSection {
1165	const header = try Header.decode(in, remaining);
1166	switch (header.block_type) {
1167	.raw => {
1168	if (buffer.len < header.regenerated_size) return error.MalformedLiteralsSection;
1169	remaining.* = remaining.subtract(header.regenerated_size) orelse return error.EndOfStream;
1170	try in.readSliceAll(buffer[0..header.regenerated_size]);
1171	return .{
1172	.header = header,
1173	.huffman_tree = null,
1174	.streams = .{ .one = buffer },
1175	};
1176	},
1177	.rle => {
1178	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
1179	buffer[0] = try in.takeByte();
1180	return .{
1181	.header = header,
1182	.huffman_tree = null,
1183	.streams = .{ .one = buffer[0..1] },
1184	};
1185	},
1186	.compressed, .treeless => {
1187	const before_remaining = remaining.*;
1188	const huffman_tree = if (header.block_type == .compressed)
1189	try HuffmanTree.decode(in, remaining)
1190	else
1191	null;
1192	const huffman_tree_size = @intFromEnum(before_remaining) - @intFromEnum(remaining.*);
1193	const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
1194	return error.MalformedLiteralsSection;
1195	if (total_streams_size > buffer.len) return error.MalformedLiteralsSection;
1196	remaining.* = remaining.subtract(total_streams_size) orelse return error.EndOfStream;
1197	try in.readSliceAll(buffer[0..total_streams_size]);
1198	const stream_data = buffer[0..total_streams_size];
1199	const streams = try Streams.decode(header.size_format, stream_data);
1200	return .{
1201	.header = header,
1202	.huffman_tree = huffman_tree,
1203	.streams = streams,
1204	};
1205	},
1206	}
1207	}
1208	};
1209
1210	pub const SequencesSection = struct {
1211	header: Header,
1212	literals_length_table: Table,
1213	offset_table: Table,
1214	match_length_table: Table,
1215
1216	pub const Header = struct {
1217	sequence_count: u24,
1218	match_lengths: Mode,
1219	offsets: Mode,
1220	literal_lengths: Mode,
1221
1222	pub const Mode = enum(u2) {
1223	predefined,
1224	rle,
1225	fse,
1226	repeat,
1227	};
1228
1229	pub const DecodeError = error{
1230	ReservedBitSet,
1231	EndOfStream,
1232	ReadFailed,
1233	};
1234
1235	pub fn decode(in: Reader, remaining: Limit) DecodeError!Header {
1236	var sequence_count: u24 = undefined;
1237
1238	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
1239	const byte0 = try in.takeByte();
1240	if (byte0 == 0) {
1241	return .{
1242	.sequence_count = 0,
1243	.offsets = undefined,
1244	.match_lengths = undefined,
1245	.literal_lengths = undefined,
1246	};
1247	} else if (byte0 < 128) {
1248	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
1249	sequence_count = byte0;
1250	} else if (byte0 < 255) {
1251	remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
1252	sequence_count = (@as(u24, (byte0 - 128)) << 8) + try in.takeByte();
1253	} else {
1254	remaining.* = remaining.subtract(3) orelse return error.EndOfStream;
1255	sequence_count = (try in.takeByte()) + (@as(u24, try in.takeByte()) << 8) + 0x7F00;
1256	}
1257
1258	const compression_modes = try in.takeByte();
1259
1260	const matches_mode: Header.Mode = @enumFromInt((compression_modes & 0b00001100) >> 2);
1261	const offsets_mode: Header.Mode = @enumFromInt((compression_modes & 0b00110000) >> 4);
1262	const literal_mode: Header.Mode = @enumFromInt((compression_modes & 0b11000000) >> 6);
1263	if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
1264
1265	return .{
1266	.sequence_count = sequence_count,
1267	.offsets = offsets_mode,
1268	.match_lengths = matches_mode,
1269	.literal_lengths = literal_mode,
1270	};
1271	}
1272	};
1273	};
1274
1275	pub const Table = union(enum) {
1276	fse: []const Fse,
1277	rle: u8,
1278
1279	pub const Fse = struct {
1280	symbol: u8,
1281	baseline: u16,
1282	bits: u8,
1283	};
1284
1285	pub fn decode(
1286	bit_reader: *BitReader,
1287	expected_symbol_count: usize,
1288	max_accuracy_log: u4,
1289	entries: []Table.Fse,
1290	) !usize {
1291	const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
1292	if (accuracy_log_biased > max_accuracy_log -\| 5) return error.MalformedAccuracyLog;
1293	const accuracy_log = accuracy_log_biased + 5;
1294
1295	var values: [256]u16 = undefined;
1296	var value_count: usize = 0;
1297
1298	const total_probability = @as(u16, 1) << accuracy_log;
1299	var accumulated_probability: u16 = 0;
1300
1301	while (accumulated_probability < total_probability) {
1302	// WARNING: The RFC is poorly worded, and would suggest std.math.log2_int_ceil is correct here,
1303	// but power of two (remaining probabilities + 1) need max bits set to 1 more.
1304	const max_bits = std.math.log2_int(u16, total_probability - accumulated_probability + 1) + 1;
1305	const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
1306
1307	const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
1308
1309	const value = if (small < cutoff)
1310	small
1311	else value: {
1312	const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
1313	break :value if (value_read < @as(u16, 1) << (max_bits - 1))
1314	value_read
1315	else
1316	value_read - cutoff;
1317	};
1318
1319	accumulated_probability += if (value != 0) value - 1 else 1;
1320
1321	values[value_count] = value;
1322	value_count += 1;
1323
1324	if (value == 1) {
1325	while (true) {
1326	const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
1327	if (repeat_flag + value_count > 256) return error.MalformedFseTable;
1328	for (0..repeat_flag) \|_\| {
1329	values[value_count] = 1;
1330	value_count += 1;
1331	}
1332	if (repeat_flag < 3) break;
1333	}
1334	}
1335	if (value_count == 256) break;
1336	}
1337	bit_reader.alignToByte();
1338
1339	if (value_count < 2) return error.MalformedFseTable;
1340	if (accumulated_probability != total_probability) return error.MalformedFseTable;
1341	if (value_count > expected_symbol_count) return error.MalformedFseTable;
1342
1343	const table_size = total_probability;
1344
1345	try build(values[0..value_count], entries[0..table_size]);
1346	return table_size;
1347	}
1348
1349	pub fn build(values: []const u16, entries: []Table.Fse) !void {
1350	const total_probability = @as(u16, @intCast(entries.len));
1351	const accuracy_log = std.math.log2_int(u16, total_probability);
1352	assert(total_probability <= 1 << 9);
1353
1354	var less_than_one_count: usize = 0;
1355	for (values, 0..) \|value, i\| {
1356	if (value == 0) {
1357	entries[entries.len - 1 - less_than_one_count] = Table.Fse{
1358	.symbol = @as(u8, @intCast(i)),
1359	.baseline = 0,
1360	.bits = accuracy_log,
1361	};
1362	less_than_one_count += 1;
1363	}
1364	}
1365
1366	var position: usize = 0;
1367	var temp_states: [1 << 9]u16 = undefined;
1368	for (values, 0..) \|value, symbol\| {
1369	if (value == 0 or value == 1) continue;
1370	const probability = value - 1;
1371
1372	const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
1373	return error.MalformedFseTable;
1374	const share_size = @divExact(total_probability, state_share_dividend);
1375	const double_state_count = state_share_dividend - probability;
1376	const single_state_count = probability - double_state_count;
1377	const share_size_log = std.math.log2_int(u16, share_size);
1378
1379	for (0..probability) \|i\| {
1380	temp_states[i] = @as(u16, @intCast(position));
1381	position += (entries.len >> 1) + (entries.len >> 3) + 3;
1382	position &= entries.len - 1;
1383	while (position >= entries.len - less_than_one_count) {
1384	position += (entries.len >> 1) + (entries.len >> 3) + 3;
1385	position &= entries.len - 1;
1386	}
1387	}
1388	std.mem.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
1389	for (0..probability) \|i\| {
1390	entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
1391	.symbol = @as(u8, @intCast(symbol)),
1392	.bits = share_size_log + 1,
1393	.baseline = single_state_count * share_size + @as(u16, @intCast(i)) * 2 * share_size,
1394	} else Table.Fse{
1395	.symbol = @as(u8, @intCast(symbol)),
1396	.bits = share_size_log,
1397	.baseline = (@as(u16, @intCast(i)) - double_state_count) * share_size,
1398	};
1399	}
1400	}
1401	}
1402
1403	test build {
1404	const literals_length_default_values = [36]u16{
1405	5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
1406	3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
1407	0, 0, 0, 0,
1408	};
1409
1410	const match_lengths_default_values = [53]u16{
1411	2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
1412	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1413	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
1414	0, 0, 0, 0, 0,
1415	};
1416
1417	const offset_codes_default_values = [29]u16{
1418	2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
1419	2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
1420	};
1421
1422	var entries: [64]Table.Fse = undefined;
1423	try build(&literals_length_default_values, &entries);
1424	try std.testing.expectEqualSlices(Table.Fse, Table.predefined_literal.fse, &entries);
1425
1426	try build(&match_lengths_default_values, &entries);
1427	try std.testing.expectEqualSlices(Table.Fse, Table.predefined_match.fse, &entries);
1428
1429	try build(&offset_codes_default_values, entries[0..32]);
1430	try std.testing.expectEqualSlices(Table.Fse, Table.predefined_offset.fse, entries[0..32]);
1431	}
1432
1433	pub const predefined_literal: Table = .{
1434	.fse = &[64]Table.Fse{
1435	.{ .symbol = 0, .bits = 4, .baseline = 0 },
1436	.{ .symbol = 0, .bits = 4, .baseline = 16 },
1437	.{ .symbol = 1, .bits = 5, .baseline = 32 },
1438	.{ .symbol = 3, .bits = 5, .baseline = 0 },
1439	.{ .symbol = 4, .bits = 5, .baseline = 0 },
1440	.{ .symbol = 6, .bits = 5, .baseline = 0 },
1441	.{ .symbol = 7, .bits = 5, .baseline = 0 },
1442	.{ .symbol = 9, .bits = 5, .baseline = 0 },
1443	.{ .symbol = 10, .bits = 5, .baseline = 0 },
1444	.{ .symbol = 12, .bits = 5, .baseline = 0 },
1445	.{ .symbol = 14, .bits = 6, .baseline = 0 },
1446	.{ .symbol = 16, .bits = 5, .baseline = 0 },
1447	.{ .symbol = 18, .bits = 5, .baseline = 0 },
1448	.{ .symbol = 19, .bits = 5, .baseline = 0 },
1449	.{ .symbol = 21, .bits = 5, .baseline = 0 },
1450	.{ .symbol = 22, .bits = 5, .baseline = 0 },
1451	.{ .symbol = 24, .bits = 5, .baseline = 0 },
1452	.{ .symbol = 25, .bits = 5, .baseline = 32 },
1453	.{ .symbol = 26, .bits = 5, .baseline = 0 },
1454	.{ .symbol = 27, .bits = 6, .baseline = 0 },
1455	.{ .symbol = 29, .bits = 6, .baseline = 0 },
1456	.{ .symbol = 31, .bits = 6, .baseline = 0 },
1457	.{ .symbol = 0, .bits = 4, .baseline = 32 },
1458	.{ .symbol = 1, .bits = 4, .baseline = 0 },
1459	.{ .symbol = 2, .bits = 5, .baseline = 0 },
1460	.{ .symbol = 4, .bits = 5, .baseline = 32 },
1461	.{ .symbol = 5, .bits = 5, .baseline = 0 },
1462	.{ .symbol = 7, .bits = 5, .baseline = 32 },
1463	.{ .symbol = 8, .bits = 5, .baseline = 0 },
1464	.{ .symbol = 10, .bits = 5, .baseline = 32 },
1465	.{ .symbol = 11, .bits = 5, .baseline = 0 },
1466	.{ .symbol = 13, .bits = 6, .baseline = 0 },
1467	.{ .symbol = 16, .bits = 5, .baseline = 32 },
1468	.{ .symbol = 17, .bits = 5, .baseline = 0 },
1469	.{ .symbol = 19, .bits = 5, .baseline = 32 },
1470	.{ .symbol = 20, .bits = 5, .baseline = 0 },
1471	.{ .symbol = 22, .bits = 5, .baseline = 32 },
1472	.{ .symbol = 23, .bits = 5, .baseline = 0 },
1473	.{ .symbol = 25, .bits = 4, .baseline = 0 },
1474	.{ .symbol = 25, .bits = 4, .baseline = 16 },
1475	.{ .symbol = 26, .bits = 5, .baseline = 32 },
1476	.{ .symbol = 28, .bits = 6, .baseline = 0 },
1477	.{ .symbol = 30, .bits = 6, .baseline = 0 },
1478	.{ .symbol = 0, .bits = 4, .baseline = 48 },
1479	.{ .symbol = 1, .bits = 4, .baseline = 16 },
1480	.{ .symbol = 2, .bits = 5, .baseline = 32 },
1481	.{ .symbol = 3, .bits = 5, .baseline = 32 },
1482	.{ .symbol = 5, .bits = 5, .baseline = 32 },
1483	.{ .symbol = 6, .bits = 5, .baseline = 32 },
1484	.{ .symbol = 8, .bits = 5, .baseline = 32 },
1485	.{ .symbol = 9, .bits = 5, .baseline = 32 },
1486	.{ .symbol = 11, .bits = 5, .baseline = 32 },
1487	.{ .symbol = 12, .bits = 5, .baseline = 32 },
1488	.{ .symbol = 15, .bits = 6, .baseline = 0 },
1489	.{ .symbol = 17, .bits = 5, .baseline = 32 },
1490	.{ .symbol = 18, .bits = 5, .baseline = 32 },
1491	.{ .symbol = 20, .bits = 5, .baseline = 32 },
1492	.{ .symbol = 21, .bits = 5, .baseline = 32 },
1493	.{ .symbol = 23, .bits = 5, .baseline = 32 },
1494	.{ .symbol = 24, .bits = 5, .baseline = 32 },
1495	.{ .symbol = 35, .bits = 6, .baseline = 0 },
1496	.{ .symbol = 34, .bits = 6, .baseline = 0 },
1497	.{ .symbol = 33, .bits = 6, .baseline = 0 },
1498	.{ .symbol = 32, .bits = 6, .baseline = 0 },
1499	},
1500	};
1501
1502	pub const predefined_match: Table = .{
1503	.fse = &[64]Table.Fse{
1504	.{ .symbol = 0, .bits = 6, .baseline = 0 },
1505	.{ .symbol = 1, .bits = 4, .baseline = 0 },
1506	.{ .symbol = 2, .bits = 5, .baseline = 32 },
1507	.{ .symbol = 3, .bits = 5, .baseline = 0 },
1508	.{ .symbol = 5, .bits = 5, .baseline = 0 },
1509	.{ .symbol = 6, .bits = 5, .baseline = 0 },
1510	.{ .symbol = 8, .bits = 5, .baseline = 0 },
1511	.{ .symbol = 10, .bits = 6, .baseline = 0 },
1512	.{ .symbol = 13, .bits = 6, .baseline = 0 },
1513	.{ .symbol = 16, .bits = 6, .baseline = 0 },
1514	.{ .symbol = 19, .bits = 6, .baseline = 0 },
1515	.{ .symbol = 22, .bits = 6, .baseline = 0 },
1516	.{ .symbol = 25, .bits = 6, .baseline = 0 },
1517	.{ .symbol = 28, .bits = 6, .baseline = 0 },
1518	.{ .symbol = 31, .bits = 6, .baseline = 0 },
1519	.{ .symbol = 33, .bits = 6, .baseline = 0 },
1520	.{ .symbol = 35, .bits = 6, .baseline = 0 },
1521	.{ .symbol = 37, .bits = 6, .baseline = 0 },
1522	.{ .symbol = 39, .bits = 6, .baseline = 0 },
1523	.{ .symbol = 41, .bits = 6, .baseline = 0 },
1524	.{ .symbol = 43, .bits = 6, .baseline = 0 },
1525	.{ .symbol = 45, .bits = 6, .baseline = 0 },
1526	.{ .symbol = 1, .bits = 4, .baseline = 16 },
1527	.{ .symbol = 2, .bits = 4, .baseline = 0 },
1528	.{ .symbol = 3, .bits = 5, .baseline = 32 },
1529	.{ .symbol = 4, .bits = 5, .baseline = 0 },
1530	.{ .symbol = 6, .bits = 5, .baseline = 32 },
1531	.{ .symbol = 7, .bits = 5, .baseline = 0 },
1532	.{ .symbol = 9, .bits = 6, .baseline = 0 },
1533	.{ .symbol = 12, .bits = 6, .baseline = 0 },
1534	.{ .symbol = 15, .bits = 6, .baseline = 0 },
1535	.{ .symbol = 18, .bits = 6, .baseline = 0 },
1536	.{ .symbol = 21, .bits = 6, .baseline = 0 },
1537	.{ .symbol = 24, .bits = 6, .baseline = 0 },
1538	.{ .symbol = 27, .bits = 6, .baseline = 0 },
1539	.{ .symbol = 30, .bits = 6, .baseline = 0 },
1540	.{ .symbol = 32, .bits = 6, .baseline = 0 },
1541	.{ .symbol = 34, .bits = 6, .baseline = 0 },
1542	.{ .symbol = 36, .bits = 6, .baseline = 0 },
1543	.{ .symbol = 38, .bits = 6, .baseline = 0 },
1544	.{ .symbol = 40, .bits = 6, .baseline = 0 },
1545	.{ .symbol = 42, .bits = 6, .baseline = 0 },
1546	.{ .symbol = 44, .bits = 6, .baseline = 0 },
1547	.{ .symbol = 1, .bits = 4, .baseline = 32 },
1548	.{ .symbol = 1, .bits = 4, .baseline = 48 },
1549	.{ .symbol = 2, .bits = 4, .baseline = 16 },
1550	.{ .symbol = 4, .bits = 5, .baseline = 32 },
1551	.{ .symbol = 5, .bits = 5, .baseline = 32 },
1552	.{ .symbol = 7, .bits = 5, .baseline = 32 },
1553	.{ .symbol = 8, .bits = 5, .baseline = 32 },
1554	.{ .symbol = 11, .bits = 6, .baseline = 0 },
1555	.{ .symbol = 14, .bits = 6, .baseline = 0 },
1556	.{ .symbol = 17, .bits = 6, .baseline = 0 },
1557	.{ .symbol = 20, .bits = 6, .baseline = 0 },
1558	.{ .symbol = 23, .bits = 6, .baseline = 0 },
1559	.{ .symbol = 26, .bits = 6, .baseline = 0 },
1560	.{ .symbol = 29, .bits = 6, .baseline = 0 },
1561	.{ .symbol = 52, .bits = 6, .baseline = 0 },
1562	.{ .symbol = 51, .bits = 6, .baseline = 0 },
1563	.{ .symbol = 50, .bits = 6, .baseline = 0 },
1564	.{ .symbol = 49, .bits = 6, .baseline = 0 },
1565	.{ .symbol = 48, .bits = 6, .baseline = 0 },
1566	.{ .symbol = 47, .bits = 6, .baseline = 0 },
1567	.{ .symbol = 46, .bits = 6, .baseline = 0 },
1568	},
1569	};
1570
1571	pub const predefined_offset: Table = .{
1572	.fse = &[32]Table.Fse{
1573	.{ .symbol = 0, .bits = 5, .baseline = 0 },
1574	.{ .symbol = 6, .bits = 4, .baseline = 0 },
1575	.{ .symbol = 9, .bits = 5, .baseline = 0 },
1576	.{ .symbol = 15, .bits = 5, .baseline = 0 },
1577	.{ .symbol = 21, .bits = 5, .baseline = 0 },
1578	.{ .symbol = 3, .bits = 5, .baseline = 0 },
1579	.{ .symbol = 7, .bits = 4, .baseline = 0 },
1580	.{ .symbol = 12, .bits = 5, .baseline = 0 },
1581	.{ .symbol = 18, .bits = 5, .baseline = 0 },
1582	.{ .symbol = 23, .bits = 5, .baseline = 0 },
1583	.{ .symbol = 5, .bits = 5, .baseline = 0 },
1584	.{ .symbol = 8, .bits = 4, .baseline = 0 },
1585	.{ .symbol = 14, .bits = 5, .baseline = 0 },
1586	.{ .symbol = 20, .bits = 5, .baseline = 0 },
1587	.{ .symbol = 2, .bits = 5, .baseline = 0 },
1588	.{ .symbol = 7, .bits = 4, .baseline = 16 },
1589	.{ .symbol = 11, .bits = 5, .baseline = 0 },
1590	.{ .symbol = 17, .bits = 5, .baseline = 0 },
1591	.{ .symbol = 22, .bits = 5, .baseline = 0 },
1592	.{ .symbol = 4, .bits = 5, .baseline = 0 },
1593	.{ .symbol = 8, .bits = 4, .baseline = 16 },
1594	.{ .symbol = 13, .bits = 5, .baseline = 0 },
1595	.{ .symbol = 19, .bits = 5, .baseline = 0 },
1596	.{ .symbol = 1, .bits = 5, .baseline = 0 },
1597	.{ .symbol = 6, .bits = 4, .baseline = 16 },
1598	.{ .symbol = 10, .bits = 5, .baseline = 0 },
1599	.{ .symbol = 16, .bits = 5, .baseline = 0 },
1600	.{ .symbol = 28, .bits = 5, .baseline = 0 },
1601	.{ .symbol = 27, .bits = 5, .baseline = 0 },
1602	.{ .symbol = 26, .bits = 5, .baseline = 0 },
1603	.{ .symbol = 25, .bits = 5, .baseline = 0 },
1604	.{ .symbol = 24, .bits = 5, .baseline = 0 },
1605	},
1606	};
1607	};
1608
1609	const low_bit_mask = [9]u8{
1610	0b00000000,
1611	0b00000001,
1612	0b00000011,
1613	0b00000111,
1614	0b00001111,
1615	0b00011111,
1616	0b00111111,
1617	0b01111111,
1618	0b11111111,
1619	};
1620
1621	fn Bits(comptime T: type) type {
1622	return struct { T, u16 };
1623	}
1624
1625	/// For reading the reversed bit streams used to encode FSE compressed data.
1626	const ReverseBitReader = struct {
1627	bytes: []const u8,
1628	remaining: usize,
1629	bits: u8,
1630	count: u4,
1631
1632	fn init(bytes: []const u8) error{MissingStartBit}!ReverseBitReader {
1633	var result: ReverseBitReader = .{
1634	.bytes = bytes,
1635	.remaining = bytes.len,
1636	.bits = 0,
1637	.count = 0,
1638	};
1639	if (bytes.len == 0) return result;
1640	for (0..8) \|_\| if (0 != (result.readBitsNoEof(u1, 1) catch unreachable)) return result;
1641	return error.MissingStartBit;
1642	}
1643
1644	fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) {
1645	const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1646	return .{
1647	@bitCast(@as(UT, @intCast(out))),
1648	num,
1649	};
1650	}
1651
1652	fn readBitsNoEof(self: *ReverseBitReader, comptime T: type, num: u16) error{EndOfStream}!T {
1653	const b, const c = try self.readBitsTuple(T, num);
1654	if (c < num) return error.EndOfStream;
1655	return b;
1656	}
1657
1658	fn readBits(self: ReverseBitReader, comptime T: type, num: u16, out_bits: u16) !T {
1659	const b, const c = try self.readBitsTuple(T, num);
1660	out_bits.* = c;
1661	return b;
1662	}
1663
1664	fn readBitsTuple(self: *ReverseBitReader, comptime T: type, num: u16) !Bits(T) {
1665	const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1666	const U = if (@bitSizeOf(T) < 8) u8 else UT;
1667
1668	if (num <= self.count) return initBits(T, self.removeBits(@intCast(num)), num);
1669
1670	var out_count: u16 = self.count;
1671	var out: U = self.removeBits(self.count);
1672
1673	const full_bytes_left = (num - out_count) / 8;
1674
1675	for (0..full_bytes_left) \|_\| {
1676	const byte = takeByte(self) catch \|err\| switch (err) {
1677	error.EndOfStream => return initBits(T, out, out_count),
1678	};
1679	if (U == u8) out = 0 else out <<= 8;
1680	out \|= byte;
1681	out_count += 8;
1682	}
1683
1684	const bits_left = num - out_count;
1685	const keep = 8 - bits_left;
1686
1687	if (bits_left == 0) return initBits(T, out, out_count);
1688
1689	const final_byte = takeByte(self) catch \|err\| switch (err) {
1690	error.EndOfStream => return initBits(T, out, out_count),
1691	};
1692
1693	out <<= @intCast(bits_left);
1694	out \|= final_byte >> @intCast(keep);
1695	self.bits = final_byte & low_bit_mask[keep];
1696
1697	self.count = @intCast(keep);
1698	return initBits(T, out, num);
1699	}
1700
1701	fn takeByte(rbr: *ReverseBitReader) error{EndOfStream}!u8 {
1702	if (rbr.remaining == 0) return error.EndOfStream;
1703	rbr.remaining -= 1;
1704	return rbr.bytes[rbr.remaining];
1705	}
1706
1707	fn isEmpty(self: *const ReverseBitReader) bool {
1708	return self.remaining == 0 and self.count == 0;
1709	}
1710
1711	fn removeBits(self: *ReverseBitReader, num: u4) u8 {
1712	if (num == 8) {
1713	self.count = 0;
1714	return self.bits;
1715	}
1716
1717	const keep = self.count - num;
1718	const bits = self.bits >> @intCast(keep);
1719	self.bits &= low_bit_mask[keep];
1720
1721	self.count = keep;
1722	return bits;
1723	}
1724	};
1725
1726	const BitReader = struct {
1727	bytes: []const u8,
1728	index: usize = 0,
1729	bits: u8 = 0,
1730	count: u4 = 0,
1731
1732	fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) {
1733	const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1734	return .{
1735	@bitCast(@as(UT, @intCast(out))),
1736	num,
1737	};
1738	}
1739
1740	fn readBitsNoEof(self: *@This(), comptime T: type, num: u16) !T {
1741	const b, const c = try self.readBitsTuple(T, num);
1742	if (c < num) return error.EndOfStream;
1743	return b;
1744	}
1745
1746	fn readBits(self: @This(), comptime T: type, num: u16, out_bits: u16) !T {
1747	const b, const c = try self.readBitsTuple(T, num);
1748	out_bits.* = c;
1749	return b;
1750	}
1751
1752	fn readBitsTuple(self: *@This(), comptime T: type, num: u16) !Bits(T) {
1753	const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1754	const U = if (@bitSizeOf(T) < 8) u8 else UT;
1755
1756	if (num <= self.count) return initBits(T, self.removeBits(@intCast(num)), num);
1757
1758	var out_count: u16 = self.count;
1759	var out: U = self.removeBits(self.count);
1760
1761	const full_bytes_left = (num - out_count) / 8;
1762
1763	for (0..full_bytes_left) \|_\| {
1764	const byte = takeByte(self) catch \|err\| switch (err) {
1765	error.EndOfStream => return initBits(T, out, out_count),
1766	};
1767
1768	const pos = @as(U, byte) << @intCast(out_count);
1769	out \|= pos;
1770	out_count += 8;
1771	}
1772
1773	const bits_left = num - out_count;
1774	const keep = 8 - bits_left;
1775
1776	if (bits_left == 0) return initBits(T, out, out_count);
1777
1778	const final_byte = takeByte(self) catch \|err\| switch (err) {
1779	error.EndOfStream => return initBits(T, out, out_count),
1780	};
1781
1782	const pos = @as(U, final_byte & low_bit_mask[bits_left]) << @intCast(out_count);
1783	out \|= pos;
1784	self.bits = final_byte >> @intCast(bits_left);
1785
1786	self.count = @intCast(keep);
1787	return initBits(T, out, num);
1788	}
1789
1790	fn takeByte(br: *BitReader) error{EndOfStream}!u8 {
1791	if (br.bytes.len - br.index == 0) return error.EndOfStream;
1792	const result = br.bytes[br.index];
1793	br.index += 1;
1794	return result;
1795	}
1796
1797	fn removeBits(self: *@This(), num: u4) u8 {
1798	if (num == 8) {
1799	self.count = 0;
1800	return self.bits;
1801	}
1802
1803	const keep = self.count - num;
1804	const bits = self.bits & low_bit_mask[num];
1805	self.bits >>= @intCast(num);
1806	self.count = keep;
1807	return bits;
1808	}
1809
1810	fn alignToByte(self: *@This()) void {
1811	self.bits = 0;
1812	self.count = 0;
1813	}
1814	};
1815
1816	test {
1817	_ = Table;
1818	}

lib/std/fs/File.zig+13-1

...	@@ -2026,7 +2026,19 @@ pub const Writer = struct {	...	@@ -2026,7 +2026,19 @@ pub const Writer = struct {
2026	/// along with other write failures.	2026	/// along with other write failures.
2027	pub fn end(w: *Writer) EndError!void {	2027	pub fn end(w: *Writer) EndError!void {
2028	try w.interface.flush();	2028	try w.interface.flush();
2029	return w.file.setEndPos(w.pos);	2029	switch (w.mode) {
		2030	.positional,
		2031	.positional_reading,
		2032	=> w.file.setEndPos(w.pos) catch \|err\| switch (err) {
		2033	error.NonResizable => return,
		2034	else => \|e\| return e,
		2035	},
		2036
		2037	.streaming,
		2038	.streaming_reading,
		2039	.failure,
		2040	=> {},
		2041	}
2030	}	2042	}
2031	};	2043	};
2032		2044

lib/std/http/Server.zig+1-2

 @@ -129,11 +129,10 @@ pub const Request = struct {
     pub const Compression = union(enum) {
         pub const DeflateDecompressor = std.compress.zlib.Decompressor(std.io.AnyReader);
         pub const GzipDecompressor = std.compress.gzip.Decompressor(std.io.AnyReader);
-        pub const ZstdDecompressor = std.compress.zstd.Decompressor(std.io.AnyReader);
         deflate: DeflateDecompressor,
         gzip: GzipDecompressor,
-        zstd: ZstdDecompressor,
+        zstd: std.compress.zstd.Decompress,
         none: void,
     };

lib/std/posix.zig+3-2

src/Package/Fetch.zig+8-9

 @@ -1198,7 +1198,8 @@ fn unpackResource(
     switch (file_type) {
         .tar => {
-            var adapter = resource.reader().adaptToNewApi();
+            var adapter_buffer: [1024]u8 = undefined;
+            var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
             return unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
         },
         .@"tar.gz" => {
 @@ -1225,16 +1226,14 @@ fn unpackResource(
             return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
         },
         .@"tar.zst" => {
-            const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;
+            const window_size = std.compress.zstd.default_window_len;
             const window_buffer = try f.arena.allocator().create([window_size]u8);
-            const reader = resource.reader();
+            var adapter_buffer: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined;
-            var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
+            var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
-            var dcp = std.compress.zstd.decompressor(br.reader(), .{
+            var decompress: std.compress.zstd.Decompress = .init(&adapter.new_interface, window_buffer, .{
-                .window_buffer = window_buffer,
+                .verify_checksum = false,
             });
-            var adapter_buffer: [1024]u8 = undefined;
+            return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
-            var adapter = dcp.reader().adaptToNewApi(&adapter_buffer);
-            return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
         },
         .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) {
             error.FetchFailed => return error.FetchFailed,

src/Package/Fetch/git.zig+60-5

...	@@ -1281,7 +1281,7 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in	...	@@ -1281,7 +1281,7 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
1281	}	1281	}
1282	@memset(fan_out_table[fan_out_index..], count);	1282	@memset(fan_out_table[fan_out_index..], count);
1283		1283
1284	var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format));	1284	var index_hashed_writer = hashedWriter(index_writer, Oid.Hasher.init(format));
1285	const writer = index_hashed_writer.writer();	1285	const writer = index_hashed_writer.writer();
1286	try writer.writeAll(IndexHeader.signature);	1286	try writer.writeAll(IndexHeader.signature);
1287	try writer.writeInt(u32, IndexHeader.supported_version, .big);	1287	try writer.writeInt(u32, IndexHeader.supported_version, .big);
...	@@ -1331,7 +1331,7 @@ fn indexPackFirstPass(	...	@@ -1331,7 +1331,7 @@ fn indexPackFirstPass(
1331	) !Oid {	1331	) !Oid {
1332	var pack_buffered_reader = std.io.bufferedReader(pack.deprecatedReader());	1332	var pack_buffered_reader = std.io.bufferedReader(pack.deprecatedReader());
1333	var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());	1333	var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());
1334	var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));	1334	var pack_hashed_reader = hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));
1335	const pack_reader = pack_hashed_reader.reader();	1335	const pack_reader = pack_hashed_reader.reader();
1336		1336
1337	const pack_header = try PackHeader.read(pack_reader);	1337	const pack_header = try PackHeader.read(pack_reader);
...	@@ -1339,13 +1339,13 @@ fn indexPackFirstPass(	...	@@ -1339,13 +1339,13 @@ fn indexPackFirstPass(
1339	var current_entry: u32 = 0;	1339	var current_entry: u32 = 0;
1340	while (current_entry < pack_header.total_objects) : (current_entry += 1) {	1340	while (current_entry < pack_header.total_objects) : (current_entry += 1) {
1341	const entry_offset = pack_counting_reader.bytes_read;	1341	const entry_offset = pack_counting_reader.bytes_read;
1342	var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init());	1342	var entry_crc32_reader = hashedReader(pack_reader, std.hash.Crc32.init());
1343	const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader());	1343	const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader());
1344	switch (entry_header) {	1344	switch (entry_header) {
1345	.commit, .tree, .blob, .tag => \|object\| {	1345	.commit, .tree, .blob, .tag => \|object\| {
1346	var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());	1346	var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
1347	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());	1347	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1348	var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format));	1348	var entry_hashed_writer = hashedWriter(std.io.null_writer, Oid.Hasher.init(format));
1349	const entry_writer = entry_hashed_writer.writer();	1349	const entry_writer = entry_hashed_writer.writer();
1350	// The object header is not included in the pack data but is	1350	// The object header is not included in the pack data but is
1351	// part of the object's ID	1351	// part of the object's ID
...	@@ -1432,7 +1432,7 @@ fn indexPackHashDelta(	...	@@ -1432,7 +1432,7 @@ fn indexPackHashDelta(
1432	const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);	1432	const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);
1433		1433
1434	var entry_hasher: Oid.Hasher = .init(format);	1434	var entry_hasher: Oid.Hasher = .init(format);
1435	var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher);	1435	var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher);
1436	try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });	1436	try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });
1437	entry_hasher.update(base_data);	1437	entry_hasher.update(base_data);
1438	return entry_hasher.finalResult();	1438	return entry_hasher.finalResult();
...	@@ -1703,3 +1703,58 @@ pub fn main() !void {	...	@@ -1703,3 +1703,58 @@ pub fn main() !void {
1703	std.debug.print("Diagnostic: {}\n", .{err});	1703	std.debug.print("Diagnostic: {}\n", .{err});
1704	}	1704	}
1705	}	1705	}
		1706
		1707	/// Deprecated
		1708	fn hashedReader(reader: anytype, hasher: anytype) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
		1709	return .{ .child_reader = reader, .hasher = hasher };
		1710	}
		1711
		1712	/// Deprecated
		1713	fn HashedReader(ReaderType: type, HasherType: type) type {
		1714	return struct {
		1715	child_reader: ReaderType,
		1716	hasher: HasherType,
		1717
		1718	pub const Error = ReaderType.Error;
		1719	pub const Reader = std.io.GenericReader(*@This(), Error, read);
		1720
		1721	pub fn read(self: *@This(), buf: []u8) Error!usize {
		1722	const amt = try self.child_reader.read(buf);
		1723	self.hasher.update(buf[0..amt]);
		1724	return amt;
		1725	}
		1726
		1727	pub fn reader(self: *@This()) Reader {
		1728	return .{ .context = self };
		1729	}
		1730	};
		1731	}
		1732
		1733	/// Deprecated
		1734	pub fn HashedWriter(WriterType: type, HasherType: type) type {
		1735	return struct {
		1736	child_writer: WriterType,
		1737	hasher: HasherType,
		1738
		1739	pub const Error = WriterType.Error;
		1740	pub const Writer = std.io.GenericWriter(*@This(), Error, write);
		1741
		1742	pub fn write(self: *@This(), buf: []const u8) Error!usize {
		1743	const amt = try self.child_writer.write(buf);
		1744	self.hasher.update(buf[0..amt]);
		1745	return amt;
		1746	}
		1747
		1748	pub fn writer(self: *@This()) Writer {
		1749	return .{ .context = self };
		1750	}
		1751	};
		1752	}
		1753
		1754	/// Deprecated
		1755	pub fn hashedWriter(
		1756	writer: anytype,
		1757	hasher: anytype,
		1758	) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
		1759	return .{ .child_writer = writer, .hasher = hasher };
		1760	}

...	@@ -1052,6 +1052,7 @@ pub const TruncateError = error{	...	@@ -1052,6 +1052,7 @@ pub const TruncateError = error{
1052	FileBusy,	1052	FileBusy,
1053	AccessDenied,	1053	AccessDenied,
1054	PermissionDenied,	1054	PermissionDenied,
		1055	NonResizable,
1055	} \|\| UnexpectedError;	1056	} \|\| UnexpectedError;
1056		1057
1057	/// Length must be positive when treated as an i64.	1058	/// Length must be positive when treated as an i64.
...	@@ -1091,7 +1092,7 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void {	...	@@ -1091,7 +1092,7 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void {
1091	.PERM => return error.PermissionDenied,	1092	.PERM => return error.PermissionDenied,
1092	.TXTBSY => return error.FileBusy,	1093	.TXTBSY => return error.FileBusy,
1093	.BADF => unreachable, // Handle not open for writing	1094	.BADF => unreachable, // Handle not open for writing
1094	.INVAL => unreachable, // Handle not open for writing, negative length, or non-resizable handle	1095	.INVAL => return error.NonResizable,
1095	.NOTCAPABLE => return error.AccessDenied,	1096	.NOTCAPABLE => return error.AccessDenied,
1096	else => \|err\| return unexpectedErrno(err),	1097	else => \|err\| return unexpectedErrno(err),
1097	}	1098	}
...	@@ -1107,7 +1108,7 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void {	...	@@ -1107,7 +1108,7 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void {
1107	.PERM => return error.PermissionDenied,	1108	.PERM => return error.PermissionDenied,
1108	.TXTBSY => return error.FileBusy,	1109	.TXTBSY => return error.FileBusy,
1109	.BADF => unreachable, // Handle not open for writing	1110	.BADF => unreachable, // Handle not open for writing
1110	.INVAL => unreachable, // Handle not open for writing, negative length, or non-resizable handle	1111	.INVAL => return error.NonResizable, // This is returned for /dev/null for example.
1111	else => \|err\| return unexpectedErrno(err),	1112	else => \|err\| return unexpectedErrno(err),
1112	}	1113	}
1113	}	1114	}